tsan: better diagnostics if thread finishes with ignores enabled

print thread creation stack
and stacks where ignores were enabled.

llvm-svn: 195836

compiler-rt/lib/tsan/CMakeLists.txt

@@ -27,6 +27,7 @@ set(TSAN_SOURCES
   rtl/tsan_clock.cc
   rtl/tsan_flags.cc
   rtl/tsan_fd.cc
+  rtl/tsan_ignoreset.cc
   rtl/tsan_interceptors.cc
   rtl/tsan_interface_ann.cc
   rtl/tsan_interface_atomic.cc

compiler-rt/lib/tsan/lit_tests/thread_end_with_ignore.cc

@@ -1,4 +1,4 @@
-// RUN: %clangxx_tsan -O1 %s -o %t && %t 2>&1 | FileCheck %s
+// RUN: %clangxx_tsan -O1 %s -o %t && not %t 2>&1 | FileCheck %s
 #include <pthread.h>
 #include <stdio.h>
 
@@ -15,5 +15,10 @@ int main() {
   pthread_join(t, 0);
 }
 
-// CHECK: ThreadSanitizer: thread T1 finished with ignores enabled
+// CHECK: ThreadSanitizer: thread T1  finished with ignores enabled, created at:
+// CHECK:     #0 pthread_create
+// CHECK:     #1 main
+// CHECK:   Ignore was enabled at:
+// CHECK:     #0 AnnotateIgnoreReadsBegin
+// CHECK:     #1 Thread
 

compiler-rt/lib/tsan/lit_tests/thread_end_with_ignore2.cc

@@ -1,9 +1,12 @@
-// RUN: %clangxx_tsan -O1 %s -o %t && %t 2>&1 | FileCheck %s
+// RUN: %clangxx_tsan -O1 %s -o %t && not %t 2>&1 | FileCheck %s
 extern "C" void AnnotateIgnoreWritesBegin(const char *f, int l);
 
 int main() {
   AnnotateIgnoreWritesBegin("", 0);
 }
 
-// CHECK: ThreadSanitizer: thread T0 finished with ignores enabled
+// CHECK: ThreadSanitizer: main thread finished with ignores enabled
+// CHECK:   Ignore was enabled at:
+// CHECK:     #0 AnnotateIgnoreWritesBegin
+// CHECK:     #1 main
 

compiler-rt/lib/tsan/lit_tests/thread_end_with_ignore3.cc

@@ -0,0 +1,22 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && not %t 2>&1 | FileCheck %s
+extern "C" void AnnotateIgnoreReadsBegin(const char *f, int l);
+extern "C" void AnnotateIgnoreReadsEnd(const char *f, int l);
+
+int main() {
+  AnnotateIgnoreReadsBegin("", 0);
+  AnnotateIgnoreReadsBegin("", 0);
+  AnnotateIgnoreReadsEnd("", 0);
+  AnnotateIgnoreReadsEnd("", 0);
+  AnnotateIgnoreReadsBegin("", 0);
+  AnnotateIgnoreReadsBegin("", 0);
+  AnnotateIgnoreReadsEnd("", 0);
+}
+
+// CHECK: ThreadSanitizer: main thread finished with ignores enabled
+// CHECK:   Ignore was enabled at:
+// CHECK:     #0 AnnotateIgnoreReadsBegin
+// CHECK:     #1 main {{.*}}thread_end_with_ignore3.cc:10
+// CHECK:   Ignore was enabled at:
+// CHECK:     #0 AnnotateIgnoreReadsBegin
+// CHECK:     #1 main {{.*}}thread_end_with_ignore3.cc:11
+

compiler-rt/lib/tsan/rtl/tsan_defs.h

@@ -154,6 +154,7 @@ struct MD5Hash {
 MD5Hash md5_hash(const void *data, uptr size);
 
 struct ThreadState;
+class ThreadContext;
 struct Context;
 struct ReportStack;
 class ReportDesc;

compiler-rt/lib/tsan/rtl/tsan_ignoreset.cc

@@ -0,0 +1,47 @@
+//===-- tsan_ignoreset.cc -------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_ignoreset.h"
+
+namespace __tsan {
+
+const uptr IgnoreSet::kMaxSize;
+
+IgnoreSet::IgnoreSet()
+    : size_() {
+}
+
+void IgnoreSet::Add(u32 stack_id) {
+  if (size_ == kMaxSize)
+    return;
+  for (uptr i = 0; i < size_; i++) {
+    if (stacks_[i] == stack_id)
+      return;
+  }
+  stacks_[size_++] = stack_id;
+}
+
+void IgnoreSet::Reset() {
+  size_ = 0;
+}
+
+uptr IgnoreSet::Size() const {
+  return size_;
+}
+
+u32 IgnoreSet::At(uptr i) const {
+  CHECK_LT(i, size_);
+  CHECK_LE(size_, kMaxSize);
+  return stacks_[i];
+}
+
+}  // namespace __tsan

compiler-rt/lib/tsan/rtl/tsan_ignoreset.h

@@ -0,0 +1,38 @@
+//===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// IgnoreSet holds a set of stack traces where ignores were enabled.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_IGNORESET_H
+#define TSAN_IGNORESET_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class IgnoreSet {
+ public:
+  static const uptr kMaxSize = 16;
+
+  IgnoreSet();
+  void Add(u32 stack_id);
+  void Reset();
+  uptr Size() const;
+  u32 At(uptr i) const;
+
+ private:
+  uptr size_;
+  u32 stacks_[kMaxSize];
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_IGNORESET_H

compiler-rt/lib/tsan/rtl/tsan_interceptors.cc

@@ -160,6 +160,7 @@ class ScopedInterceptor {
   ~ScopedInterceptor();
  private:
   ThreadState *const thr_;
+  const uptr pc_;
   const int in_rtl_;
   bool in_ignored_lib_;
 };
@@ -167,6 +168,7 @@ class ScopedInterceptor {
 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
                                      uptr pc)
     : thr_(thr)
+    , pc_(pc)
     , in_rtl_(thr->in_rtl)
     , in_ignored_lib_(false) {
   if (thr_->in_rtl == 0) {
@@ -180,14 +182,14 @@ ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
   if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
     in_ignored_lib_ = true;
     thr_->in_ignored_lib = true;
-    ThreadIgnoreBegin(thr_);
+    ThreadIgnoreBegin(thr_, pc_);
   }
 }
 
 ScopedInterceptor::~ScopedInterceptor() {
   if (in_ignored_lib_) {
     thr_->in_ignored_lib = false;
-    ThreadIgnoreEnd(thr_);
+    ThreadIgnoreEnd(thr_, pc_);
   }
   thr_->in_rtl--;
   if (thr_->in_rtl == 0) {
@@ -360,9 +362,9 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
   if (dso) {
     // Memory allocation in __cxa_atexit will race with free during exit,
     // because we do not see synchronization around atexit callback list.
-    ThreadIgnoreBegin(thr);
+    ThreadIgnoreBegin(thr, pc);
     int res = REAL(__cxa_atexit)(f, arg, dso);
-    ThreadIgnoreEnd(thr);
+    ThreadIgnoreEnd(thr, pc);
     return res;
   }
   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg);
@@ -1768,13 +1770,13 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
   // We miss atomic synchronization in getaddrinfo,
   // and can report false race between malloc and free
   // inside of getaddrinfo. So ignore memory accesses.
-  ThreadIgnoreBegin(thr);
+  ThreadIgnoreBegin(thr, pc);
   // getaddrinfo calls fopen, which can be intercepted by user.
   thr->in_rtl--;
   CHECK_EQ(thr->in_rtl, 0);
   int res = REAL(getaddrinfo)(node, service, hints, rv);
   thr->in_rtl++;
-  ThreadIgnoreEnd(thr);
+  ThreadIgnoreEnd(thr, pc);
   return res;
 }
 

compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc

@@ -55,11 +55,11 @@ class ScopedAnnotation {
     if (!flags()->enable_annotations) \
       return; \
     ThreadState *thr = cur_thread(); \
-    const uptr pc = (uptr)__builtin_return_address(0); \
+    const uptr caller_pc = (uptr)__builtin_return_address(0); \
     StatInc(thr, StatAnnotation); \
     StatInc(thr, Stat##typ); \
-    ScopedAnnotation sa(thr, __FUNCTION__, f, l, \
-        (uptr)__builtin_return_address(0)); \
+    ScopedAnnotation sa(thr, __FUNCTION__, f, l, caller_pc); \
+    const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
     (void)pc; \
 /**/
 
@@ -383,32 +383,32 @@ void INTERFACE_ATTRIBUTE AnnotateBenignRace(
 
 void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreReadsBegin);
-  ThreadIgnoreBegin(thr);
+  ThreadIgnoreBegin(thr, pc);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd);
-  ThreadIgnoreEnd(thr);
+  ThreadIgnoreEnd(thr, pc);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreWritesBegin);
-  ThreadIgnoreBegin(thr);
+  ThreadIgnoreBegin(thr, pc);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd);
-  ThreadIgnoreEnd(thr);
+  ThreadIgnoreEnd(thr, pc);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreSyncBegin);
-  ThreadIgnoreSyncBegin(thr);
+  ThreadIgnoreSyncBegin(thr, pc);
 }
 
 void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd);
-  ThreadIgnoreSyncEnd(thr);
+  ThreadIgnoreSyncEnd(thr, pc);
 }
 
 void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange(

compiler-rt/lib/tsan/rtl/tsan_mutexset.h

@@ -62,4 +62,4 @@ MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
 
 }  // namespace __tsan
 
-#endif  // TSAN_REPORT_H
+#endif  // TSAN_MUTEXSET_H

compiler-rt/lib/tsan/rtl/tsan_rtl.cc

@@ -697,31 +697,45 @@ void FuncExit(ThreadState *thr) {
   thr->shadow_stack_pos--;
 }
 
-void ThreadIgnoreBegin(ThreadState *thr) {
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
   thr->ignore_reads_and_writes++;
   CHECK_GT(thr->ignore_reads_and_writes, 0);
   thr->fast_state.SetIgnoreBit();
+#ifndef TSAN_GO
+  thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
+#endif
 }
 
-void ThreadIgnoreEnd(ThreadState *thr) {
+void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
   thr->ignore_reads_and_writes--;
   CHECK_GE(thr->ignore_reads_and_writes, 0);
-  if (thr->ignore_reads_and_writes == 0)
+  if (thr->ignore_reads_and_writes == 0) {
     thr->fast_state.ClearIgnoreBit();
+#ifndef TSAN_GO
+    thr->mop_ignore_set.Reset();
+#endif
+  }
 }
 
-void ThreadIgnoreSyncBegin(ThreadState *thr) {
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
   thr->ignore_sync++;
   CHECK_GT(thr->ignore_sync, 0);
+#ifndef TSAN_GO
+  thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
+#endif
 }
 
-void ThreadIgnoreSyncEnd(ThreadState *thr) {
+void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
   thr->ignore_sync--;
   CHECK_GE(thr->ignore_sync, 0);
+#ifndef TSAN_GO
+  if (thr->ignore_sync == 0)
+    thr->mop_ignore_set.Reset();
+#endif
 }
 
 bool MD5Hash::operator==(const MD5Hash &other) const {

compiler-rt/lib/tsan/rtl/tsan_rtl.h

@@ -41,6 +41,7 @@
 #include "tsan_report.h"
 #include "tsan_platform.h"
 #include "tsan_mutexset.h"
+#include "tsan_ignoreset.h"
 
 #if SANITIZER_WORDSIZE != 64
 # error "ThreadSanitizer is supported only on 64-bit platforms"
@@ -413,6 +414,11 @@ struct ThreadState {
   // for better performance.
   int ignore_reads_and_writes;
   int ignore_sync;
+  // Go does not support ignores.
+#ifndef TSAN_GO
+  IgnoreSet mop_ignore_set;
+  IgnoreSet sync_ignore_set;
+#endif
   // C/C++ uses fixed size shadow stack embed into Trace.
   // Go uses malloc-allocated shadow stack with dynamic size.
   uptr *shadow_stack;
@@ -440,6 +446,7 @@ struct ThreadState {
   const uptr stk_size;
   const uptr tls_addr;
   const uptr tls_size;
+  ThreadContext *tctx;
 
   DeadlockDetector deadlock_detector;
 
@@ -627,6 +634,7 @@ ReportStack *SkipTsanInternalFrames(ReportStack *ent);
 #endif
 
 u32 CurrentStackId(ThreadState *thr, uptr pc);
+ReportStack *SymbolizeStackId(u32 stack_id);
 void PrintCurrentStack(ThreadState *thr, uptr pc);
 void PrintCurrentStackSlow();  // uses libunwind
 
@@ -678,10 +686,10 @@ void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
 
-void ThreadIgnoreBegin(ThreadState *thr);
-void ThreadIgnoreEnd(ThreadState *thr);
-void ThreadIgnoreSyncBegin(ThreadState *thr);
-void ThreadIgnoreSyncEnd(ThreadState *thr);
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
 
 void FuncEntry(ThreadState *thr, uptr pc);
 void FuncExit(ThreadState *thr);

compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc

@@ -101,6 +101,18 @@ static void StackStripMain(ReportStack *stack) {
 #endif
 }
 
+#ifndef TSAN_GO
+ReportStack *SymbolizeStackId(u32 stack_id) {
+  uptr ssz = 0;
+  const uptr *stack = StackDepotGet(stack_id, &ssz);
+  if (stack == 0)
+    return 0;
+  StackTrace trace;
+  trace.Init(stack, ssz);
+  return SymbolizeStack(trace);
+}
+#endif
+
 static ReportStack *SymbolizeStack(const StackTrace& trace) {
   if (trace.IsEmpty())
     return 0;
@@ -203,13 +215,7 @@ void ScopedReport::AddThread(const ThreadContext *tctx) {
 #ifdef TSAN_GO
   rt->stack = SymbolizeStack(tctx->creation_stack);
 #else
-  uptr ssz = 0;
-  const uptr *stack = StackDepotGet(tctx->creation_stack_id, &ssz);
-  if (stack) {
-    StackTrace trace;
-    trace.Init(stack, ssz);
-    rt->stack = SymbolizeStack(trace);
-  }
+  rt->stack = SymbolizeStackId(tctx->creation_stack_id);
 #endif
 }
 
@@ -272,13 +278,7 @@ void ScopedReport::AddMutex(const SyncVar *s) {
   rm->destroyed = false;
   rm->stack = 0;
 #ifndef TSAN_GO
-  uptr ssz = 0;
-  const uptr *stack = StackDepotGet(s->creation_stack_id, &ssz);
-  if (stack) {
-    StackTrace trace;
-    trace.Init(stack, ssz);
-    rm->stack = SymbolizeStack(trace);
-  }
+  rm->stack = SymbolizeStackId(s->creation_stack_id);
 #endif
 }
 
@@ -310,13 +310,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
     loc->type = ReportLocationFD;
     loc->fd = fd;
     loc->tid = creat_tid;
-    uptr ssz = 0;
-    const uptr *stack = StackDepotGet(creat_stack, &ssz);
-    if (stack) {
-      StackTrace trace;
-      trace.Init(stack, ssz);
-      loc->stack = SymbolizeStack(trace);
-    }
+    loc->stack = SymbolizeStackId(creat_stack);
     ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
     if (tctx)
       AddThread(tctx);
@@ -337,13 +331,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
     loc->file = 0;
     loc->line = 0;
     loc->stack = 0;
-    uptr ssz = 0;
-    const uptr *stack = StackDepotGet(b->StackId(), &ssz);
-    if (stack) {
-      StackTrace trace;
-      trace.Init(stack, ssz);
-      loc->stack = SymbolizeStack(trace);
-    }
+    loc->stack = SymbolizeStackId(b->StackId());
     if (tctx)
       AddThread(tctx);
     return;
@@ -367,13 +355,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
 
 #ifndef TSAN_GO
 void ScopedReport::AddSleep(u32 stack_id) {
-  uptr ssz = 0;
-  const uptr *stack = StackDepotGet(stack_id, &ssz);
-  if (stack) {
-    StackTrace trace;
-    trace.Init(stack, ssz);
-    rep_->sleep = SymbolizeStack(trace);
-  }
+  rep_->sleep = SymbolizeStackId(stack_id);
 }
 #endif
 

compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc

@@ -160,17 +160,32 @@ static void MaybeReportThreadLeak(ThreadContextBase *tctx_base, void *arg) {
 }
 #endif
 
-static void ThreadCheckIgnore(ThreadState *thr) {
-  if (thr->ignore_reads_and_writes) {
-    Printf("ThreadSanitizer: thread T%d finished with ignores enabled.\n",
-           thr->tid);
+#ifndef TSAN_GO
+static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
+  if (tctx->tid == 0) {
+    Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
+  } else {
+    Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled,"
+      " created at:\n", tctx->tid, tctx->name);
+    PrintStack(SymbolizeStackId(tctx->creation_stack_id));
   }
-  if (thr->ignore_sync) {
-    Printf("ThreadSanitizer: thread T%d finished with sync ignores enabled.\n",
-           thr->tid);
+  for (uptr i = 0; i < set->Size(); i++) {
+    Printf("  Ignore was enabled at:\n");
+    PrintStack(SymbolizeStackId(set->At(i)));
   }
+  Die();
 }
 
+static void ThreadCheckIgnore(ThreadState *thr) {
+  if (thr->ignore_reads_and_writes)
+    ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set);
+  if (thr->ignore_sync)
+    ReportIgnoresEnabled(thr->tctx, &thr->sync_ignore_set);
+}
+#else
+static void ThreadCheckIgnore(ThreadState *thr) {}
+#endif
+
 void ThreadFinalize(ThreadState *thr) {
   CHECK_GT(thr->in_rtl, 0);
   ThreadCheckIgnore(thr);
@@ -210,6 +225,7 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
 }
 
 void ThreadStart(ThreadState *thr, int tid, uptr os_id) {
+  Context *ctx = CTX();
   CHECK_GT(thr->in_rtl, 0);
   uptr stk_addr = 0;
   uptr stk_size = 0;
@@ -236,8 +252,13 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) {
     }
   }
 
+  ThreadRegistry *tr = ctx->thread_registry;
   OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
-  CTX()->thread_registry->StartThread(tid, os_id, &args);
+  tr->StartThread(tid, os_id, &args);
+
+  tr->Lock();
+  thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid);
+  tr->Unlock();
 }
 
 void ThreadFinish(ThreadState *thr) {