[libsanitizer] StopTheWorld in sanitizer_common

StopTheWorld puts the process in a suspended state before running the
user-supplied callback. To be used in TSan and in leak checking code.

Linux implementation provided.

Patch by Sergey Matveev (earthdok@google.com)

llvm-svn: 177156

compiler-rt/lib/sanitizer_common/CMakeLists.txt

@@ -13,6 +13,7 @@ set(SANITIZER_SOURCES
   sanitizer_printf.cc
   sanitizer_stackdepot.cc
   sanitizer_stacktrace.cc
+  sanitizer_stoptheworld_linux.cc
   sanitizer_symbolizer.cc
   sanitizer_symbolizer_itanium.cc
   sanitizer_symbolizer_linux.cc

compiler-rt/lib/sanitizer_common/sanitizer_common.cc

@@ -17,6 +17,7 @@
 namespace __sanitizer {
 
 const char *SanitizerToolName = "SanitizerTool";
+uptr SanitizerVerbosity = 0;
 
 uptr GetPageSizeCached() {
   static uptr PageSize;

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -33,6 +33,7 @@ const uptr kCacheLineSize = 64;
 #endif
 
 extern const char *SanitizerToolName;  // Can be changed by the tool.
+extern uptr SanitizerVerbosity;
 
 uptr GetPageSize();
 uptr GetPageSizeCached();

compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld.h

@@ -0,0 +1,67 @@
+//===-- sanitizer_stoptheworld.h --------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the StopTheWorld function which suspends the execution of the current
+// process and runs the user-supplied callback in the same address space.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_STOPTHEWORLD_H
+#define SANITIZER_STOPTHEWORLD_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+typedef int SuspendedThreadID;
+
+// Holds the list of suspended threads. Also provides register dumping
+// functionality (to be implemented).
+class SuspendedThreadsList {
+ public:
+  SuspendedThreadsList()
+    : thread_ids_(1024) {}
+  SuspendedThreadID GetThreadID(uptr index) {
+    CHECK_LT(index, thread_ids_.size());
+    return thread_ids_[index];
+  }
+  void DumpRegisters(uptr index) const {
+    UNIMPLEMENTED();
+  }
+  uptr thread_count() { return thread_ids_.size(); }
+  bool Contains(SuspendedThreadID thread_id) {
+    for (uptr i = 0; i < thread_ids_.size(); i++) {
+      if (thread_ids_[i] == thread_id)
+        return true;
+    }
+    return false;
+  }
+  void Append(SuspendedThreadID thread_id) {
+    thread_ids_.push_back(thread_id);
+  }
+
+ private:
+  InternalVector<SuspendedThreadID> thread_ids_;
+
+  // Prohibit copy and assign.
+  SuspendedThreadsList(const SuspendedThreadsList&);
+  void operator=(const SuspendedThreadsList&);
+};
+
+typedef void (*StopTheWorldCallback)(
+    const SuspendedThreadsList &suspended_threads_list,
+    void *argument);
+
+// Suspend all threads in the current process and run the callback on the list
+// of suspended threads. This function will resume the threads before returning.
+// The callback should not call any libc functions.
+void StopTheWorld(StopTheWorldCallback callback, void *argument);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_STOPTHEWORLD_H

compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux.cc

@@ -0,0 +1,326 @@
+//===-- sanitizer_stoptheworld_linux.cc -----------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __linux__
+
+#include "sanitizer_stoptheworld.h"
+
+#include <errno.h>
+#include <sched.h> // for clone
+#include <stddef.h>
+#include <sys/prctl.h> // for PR_* definitions
+#include <sys/ptrace.h> // for PTRACE_* definitions
+#include <sys/types.h> // for pid_t
+#include <sys/wait.h> // for signal-related stuff
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mutex.h"
+
+// This module works by spawning a Linux task which then attaches to every
+// thread in the caller process with ptrace. This suspends the threads, and
+// PTRACE_GETREGS can then be used to obtain their register state. The callback
+// supplied to StopTheWorld() is run in the tracer task while the threads are
+// suspended.
+// The tracer task must be placed in a different thread group for ptrace to
+// work, so it cannot be spawned as a pthread. Instead, we use the low-level
+// clone() interface (we want to share the address space with the caller
+// process, so we prefer clone() over fork()).
+//
+// We avoid the use of libc for two reasons:
+// 1. calling a library function while threads are suspended could cause a
+// deadlock, if one of the treads happens to be holding a libc lock;
+// 2. it's generally not safe to call libc functions from the tracer task,
+// because clone() does not set up a thread-local storage for it. Any
+// thread-local variables used by libc will be shared between the tracer task
+// and the thread which spawned it.
+//
+// We deal with this by replacing libc calls with calls to our own
+// implementations defined in sanitizer_libc.h and sanitizer_linux.h. However,
+// there are still some libc functions which are used here:
+//
+// * All of the system calls ultimately go through the libc syscall() function.
+// We're operating under the assumption that syscall()'s implementation does
+// not acquire any locks or use any thread-local data (except for the errno
+// variable, which we handle separately).
+//
+// * We lack custom implementations of sigfillset() and sigaction(), so we use
+// the libc versions instead. The same assumptions as above apply.
+//
+// * It is safe to call libc functions before the cloned thread is spawned or
+// after it has exited. The following functions are used in this manner:
+// sigdelset()
+// sigprocmask()
+// clone()
+
+COMPILER_CHECK(sizeof(SuspendedThreadID) == sizeof(pid_t));
+
+namespace __sanitizer {
+// This class handles thread suspending/unsuspending in the tracer thread.
+class ThreadSuspender {
+ public:
+  explicit ThreadSuspender(pid_t pid)
+    : pid_(pid) {
+      CHECK_GE(pid, 0);
+    }
+  bool SuspendAllThreads();
+  void ResumeAllThreads();
+  void KillAllThreads();
+  SuspendedThreadsList &suspended_threads_list() {
+    return suspended_threads_list_;
+  }
+ private:
+  SuspendedThreadsList suspended_threads_list_;
+  pid_t pid_;
+  bool SuspendThread(SuspendedThreadID thread_id);
+};
+
+bool ThreadSuspender::SuspendThread(SuspendedThreadID thread_id) {
+  // Are we already attached to this thread?
+  // Currently this check takes linear time, however the number of threads is
+  // usually small.
+  if (suspended_threads_list_.Contains(thread_id))
+    return false;
+  if (internal_ptrace(PTRACE_ATTACH, thread_id, NULL, NULL) != 0) {
+    // Either the thread is dead, or something prevented us from attaching.
+    // Log this event and move on.
+    Report("Could not attach to thread %d (errno %d).\n", thread_id, errno);
+    return false;
+  } else {
+    if (SanitizerVerbosity > 0)
+      Report("Attached to thread %d.\n", thread_id);
+    // The thread is not guaranteed to stop before ptrace returns, so we must
+    // wait on it.
+    int waitpid_status;
+    HANDLE_EINTR(waitpid_status, internal_waitpid(thread_id, NULL, __WALL));
+    if (waitpid_status < 0) {
+      // Got a ECHILD error. I don't think this situation is possible, but it
+      // doesn't hurt to report it.
+      Report("Waiting on thread %d failed, detaching (errno %d).\n", thread_id,
+             errno);
+      internal_ptrace(PTRACE_DETACH, thread_id, NULL, NULL);
+      return false;
+    }
+    suspended_threads_list_.Append(thread_id);
+    return true;
+  }
+}
+
+void ThreadSuspender::ResumeAllThreads() {
+  for (uptr i = 0; i < suspended_threads_list_.thread_count(); i++) {
+    pid_t tid = suspended_threads_list_.GetThreadID(i);
+    if (internal_ptrace(PTRACE_DETACH, tid, NULL, NULL) == 0) {
+      if (SanitizerVerbosity > 0)
+        Report("Detached from thread %d.\n", tid);
+    } else {
+      // Either the thread is dead, or we are already detached.
+      // The latter case is possible, for instance, if this function was called
+      // from a signal handler.
+      Report("Could not detach from thread %d (errno %d).\n", tid, errno);
+    }
+  }
+}
+
+void ThreadSuspender::KillAllThreads() {
+  for (uptr i = 0; i < suspended_threads_list_.thread_count(); i++)
+    internal_ptrace(PTRACE_KILL, suspended_threads_list_.GetThreadID(i),
+                    NULL, NULL);
+}
+
+bool ThreadSuspender::SuspendAllThreads() {
+  ThreadLister thread_lister(pid_);
+  bool added_threads;
+  do {
+    // Run through the directory entries once.
+    added_threads = false;
+    pid_t tid = thread_lister.GetNextTID();
+    while (tid >= 0) {
+      if (SuspendThread(tid))
+        added_threads = true;
+      tid = thread_lister.GetNextTID();
+    }
+    if (thread_lister.error()) {
+      // Detach threads and fail.
+      ResumeAllThreads();
+      return false;
+    }
+    thread_lister.Reset();
+  } while (added_threads);
+  return true;
+}
+
+// Pointer to the ThreadSuspender instance for use in signal handler.
+static ThreadSuspender *thread_suspender_instance = NULL;
+
+// Signals that should not be blocked (this is used in the parent thread as well
+// as the tracer thread).
+static const int kUnblockedSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV,
+                                         SIGBUS, SIGXCPU, SIGXFSZ };
+
+// Structure for passing arguments into the tracer thread.
+struct TracerThreadArgument {
+  StopTheWorldCallback callback;
+  void *callback_argument;
+};
+
+// Signal handler to wake up suspended threads when the tracer thread dies.
+void TracerThreadSignalHandler(int signum, siginfo_t *siginfo, void *) {
+  if (thread_suspender_instance != NULL) {
+    if (signum == SIGABRT)
+      thread_suspender_instance->KillAllThreads();
+    else
+      thread_suspender_instance->ResumeAllThreads();
+  }
+  internal__exit((signum == SIGABRT) ? 1 : 2);
+}
+
+// The tracer thread waits on this mutex while the parent finished its
+// preparations.
+static BlockingMutex tracer_init_mutex(LINKER_INITIALIZED);
+
+// Size of alternative stack for signal handlers in the tracer thread.
+static const int kHandlerStackSize = 4096;
+
+// This function will be run as a cloned task.
+int TracerThread(void* argument) {
+  TracerThreadArgument *tracer_thread_argument =
+      (TracerThreadArgument *)argument;
+
+  // Wait for the parent thread to finish preparations.
+  tracer_init_mutex.Lock();
+  tracer_init_mutex.Unlock();
+
+  ThreadSuspender thread_suspender(internal_getppid());
+  // Global pointer for the signal handler.
+  thread_suspender_instance = &thread_suspender;
+
+  // Alternate stack for signal handling.
+  InternalScopedBuffer<char> handler_stack_memory(kHandlerStackSize);
+  struct sigaltstack handler_stack;
+  internal_memset(&handler_stack, 0, sizeof(handler_stack));
+  handler_stack.ss_sp = handler_stack_memory.data();
+  handler_stack.ss_size = kHandlerStackSize;
+  internal_sigaltstack(&handler_stack, NULL);
+
+  // Install our handler for fatal signals. Other signals should be blocked by
+  // the mask we inherited from the caller thread.
+  for (uptr signal_index = 0; signal_index < ARRAY_SIZE(kUnblockedSignals);
+       signal_index++) {
+    struct sigaction new_sigaction;
+    internal_memset(&new_sigaction, 0, sizeof(new_sigaction));
+    new_sigaction.sa_sigaction = TracerThreadSignalHandler;
+    new_sigaction.sa_flags = SA_ONSTACK | SA_SIGINFO;
+    sigfillset(&new_sigaction.sa_mask);
+    sigaction(kUnblockedSignals[signal_index], &new_sigaction, NULL);
+  }
+
+  int exit_code = 0;
+  if (!thread_suspender.SuspendAllThreads()) {
+    Report("Failed suspending threads.\n");
+    exit_code = 3;
+  } else {
+    tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
+                                     tracer_thread_argument->callback_argument);
+    thread_suspender.ResumeAllThreads();
+    exit_code = 0;
+  }
+  thread_suspender_instance = NULL;
+  handler_stack.ss_flags = SS_DISABLE;
+  internal_sigaltstack(&handler_stack, NULL);
+  return exit_code;
+}
+
+static BlockingMutex stoptheworld_mutex(LINKER_INITIALIZED);
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  BlockingMutexLock lock(&stoptheworld_mutex);
+  // Block all signals that can be blocked safely, and install default handlers
+  // for the remaining signals.
+  // We cannot allow user-defined handlers to run while the ThreadSuspender
+  // thread is active, because they could conceivably call some libc functions
+  // which modify errno (which is shared between the two threads).
+  sigset_t blocked_sigset;
+  sigfillset(&blocked_sigset);
+  struct sigaction old_sigactions[ARRAY_SIZE(kUnblockedSignals)];
+  for (uptr signal_index = 0; signal_index < ARRAY_SIZE(kUnblockedSignals);
+       signal_index++) {
+    // Remove the signal from the set of blocked signals.
+    sigdelset(&blocked_sigset, kUnblockedSignals[signal_index]);
+    // Install the default handler.
+    struct sigaction new_sigaction;
+    internal_memset(&new_sigaction, 0, sizeof(new_sigaction));
+    new_sigaction.sa_handler = SIG_DFL;
+    sigfillset(&new_sigaction.sa_mask);
+    sigaction(kUnblockedSignals[signal_index], &new_sigaction,
+                    &old_sigactions[signal_index]);
+  }
+  sigset_t old_sigset;
+  int sigprocmask_status = sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
+  CHECK_EQ(sigprocmask_status, 0); // sigprocmask should never fail
+  // Make this process dumpable. Processes that are not dumpable cannot be
+  // attached to.
+  int process_was_dumpable = internal_prctl(PR_GET_DUMPABLE, 0, 0, 0, 0);
+  if (!process_was_dumpable)
+    internal_prctl(PR_SET_DUMPABLE, 1, 0, 0, 0);
+  // Block the execution of TracerThread until after we have set ptrace
+  // permissions.
+  tracer_init_mutex.Lock();
+  // Prepare the arguments for TracerThread.
+  struct TracerThreadArgument tracer_thread_argument;
+  tracer_thread_argument.callback = callback;
+  tracer_thread_argument.callback_argument = argument;
+  // The tracer thread will run on the same stack, so we must reserve some
+  // stack space for the caller thread to run in as it waits on the tracer.
+  const uptr kReservedStackSize = 4096;
+  // Get a 16-byte aligned pointer for stack.
+  int a_local_variable __attribute__((__aligned__(16)));
+  pid_t tracer_pid = clone(TracerThread,
+                          (char *)&a_local_variable - kReservedStackSize,
+                          CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED,
+                          &tracer_thread_argument, 0, 0, 0);
+  if (tracer_pid < 0) {
+    Report("Failed spawning a tracer thread (errno %d).\n", errno);
+    tracer_init_mutex.Unlock();
+  } else {
+    // On some systems we have to explicitly declare that we want to be traced
+    // by the tracer thread.
+#ifdef PR_SET_PTRACER
+    internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0);
+#endif
+    // Allow the tracer thread to start.
+    tracer_init_mutex.Unlock();
+    // Since errno is shared between this thread and the tracer thread, we
+    // must avoid using errno while the tracer thread is running.
+    // At this point, any signal will either be blocked or kill us, so waitpid
+    // should never return (and set errno) while the tracer thread is alive.
+    int waitpid_status = internal_waitpid(tracer_pid, NULL, __WALL);
+    if (waitpid_status < 0)
+      Report("Waiting on the tracer thread failed (errno %d).\n", errno);
+  }
+  // Restore the dumpable flag.
+  if (!process_was_dumpable)
+    internal_prctl(PR_SET_DUMPABLE, 0, 0, 0, 0);
+  // Restore the signal handlers.
+  for (uptr signal_index = 0; signal_index < ARRAY_SIZE(kUnblockedSignals);
+       signal_index++) {
+    sigaction(kUnblockedSignals[signal_index],
+              &old_sigactions[signal_index], NULL);
+  }
+  sigprocmask(SIG_SETMASK, &old_sigset, &old_sigset);
+}
+
+}  // namespace __sanitizer
+
+#endif  // __linux__

compiler-rt/lib/sanitizer_common/tests/CMakeLists.txt

@@ -12,6 +12,7 @@ set(SANITIZER_UNITTESTS
   sanitizer_scanf_interceptor_test.cc
   sanitizer_stackdepot_test.cc
   sanitizer_stacktrace_test.cc
+  sanitizer_stoptheworld_test.cc
   sanitizer_test_main.cc
   sanitizer_thread_registry_test.cc
   )

compiler-rt/lib/sanitizer_common/tests/sanitizer_stoptheworld_test.cc

@@ -0,0 +1,193 @@
+//===-- sanitizer_stoptheworld_test.cc ------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Tests for sanitizer_stoptheworld.h
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __linux__
+
+#include "sanitizer_common/sanitizer_stoptheworld.h"
+#include "gtest/gtest.h"
+
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+#include <pthread.h>
+#include <sched.h>
+
+namespace __sanitizer {
+
+static pthread_mutex_t incrementer_thread_exit_mutex;
+
+struct CallbackArgument {
+  volatile int counter;
+  volatile bool threads_stopped;
+  volatile bool callback_executed;
+  CallbackArgument()
+    : counter(0),
+      threads_stopped(false),
+      callback_executed(false) {}
+};
+
+void *IncrementerThread(void *argument) {
+  CallbackArgument *callback_argument = (CallbackArgument *)argument;
+  while (true) {
+    __sync_fetch_and_add(&callback_argument->counter, 1);
+    if (pthread_mutex_trylock(&incrementer_thread_exit_mutex) == 0) {
+      pthread_mutex_unlock(&incrementer_thread_exit_mutex);
+      return NULL;
+    } else {
+      sched_yield();
+    }
+  }
+}
+
+// This callback checks that IncrementerThread is suspended at the time of its
+// execution.
+void Callback(const SuspendedThreadsList &suspended_threads_list,
+              void *argument) {
+  CallbackArgument *callback_argument = (CallbackArgument *)argument;
+  callback_argument->callback_executed = true;
+  int counter_at_init = __sync_fetch_and_add(&callback_argument->counter, 0);
+  for (uptr i = 0; i < 1000; i++) {
+    sched_yield();
+    if (__sync_fetch_and_add(&callback_argument->counter, 0) !=
+          counter_at_init) {
+      callback_argument->threads_stopped = false;
+      return;
+    }
+  }
+  callback_argument->threads_stopped = true;
+}
+
+TEST(StopTheWorld, SuspendThreadsSimple) {
+  pthread_mutex_init(&incrementer_thread_exit_mutex, NULL);
+  CallbackArgument argument;
+  pthread_t thread_id;
+  int pthread_create_result;
+  pthread_mutex_lock(&incrementer_thread_exit_mutex);
+  pthread_create_result = pthread_create(&thread_id, NULL, IncrementerThread,
+                                         &argument);
+  ASSERT_EQ(0, pthread_create_result);
+  StopTheWorld(&Callback, &argument);
+  pthread_mutex_unlock(&incrementer_thread_exit_mutex);
+  EXPECT_TRUE(argument.callback_executed);
+  EXPECT_TRUE(argument.threads_stopped);
+  // argument is on stack, so we have to wait for the incrementer thread to
+  // terminate before we can return from this function.
+  ASSERT_EQ(0, pthread_join(thread_id, NULL));
+  pthread_mutex_destroy(&incrementer_thread_exit_mutex);
+}
+
+// A more comprehensive test where we spawn a bunch of threads while executing
+// StopTheWorld in parallel.
+static const uptr kThreadCount = 50;
+static const uptr kStopWorldAfter = 10; // let this many threads spawn first
+
+static pthread_mutex_t advanced_incrementer_thread_exit_mutex;
+
+struct AdvancedCallbackArgument {
+  volatile uptr thread_index;
+  volatile int counters[kThreadCount];
+  pthread_t thread_ids[kThreadCount];
+  volatile bool threads_stopped;
+  volatile bool callback_executed;
+  volatile bool fatal_error;
+  AdvancedCallbackArgument()
+    : thread_index(0),
+      threads_stopped(false),
+      callback_executed(false),
+      fatal_error(false) {}
+};
+
+void *AdvancedIncrementerThread(void *argument) {
+  AdvancedCallbackArgument *callback_argument =
+      (AdvancedCallbackArgument *)argument;
+  uptr this_thread_index = __sync_fetch_and_add(&callback_argument->thread_index,
+                                               1);
+  // Spawn the next thread.
+  int pthread_create_result;
+  if (this_thread_index + 1 < kThreadCount) {
+    pthread_create_result =
+        pthread_create(&callback_argument->thread_ids[this_thread_index + 1],
+                       NULL, AdvancedIncrementerThread, argument);
+    // Cannot use ASSERT_EQ in non-void-returning functions. If there's a
+    // problem, defer failing to the main thread.
+    if (pthread_create_result != 0) {
+      callback_argument->fatal_error = true;
+      __sync_fetch_and_add(&callback_argument->thread_index,
+                           kThreadCount - callback_argument->thread_index);
+    }
+  }
+  // Do the actual work.
+  while (true) {
+    __sync_fetch_and_add(&callback_argument->counters[this_thread_index], 1);
+    if (pthread_mutex_trylock(&advanced_incrementer_thread_exit_mutex) == 0) {
+      pthread_mutex_unlock(&advanced_incrementer_thread_exit_mutex);
+      return NULL;
+    } else {
+      sched_yield();
+    }
+  }
+}
+
+void AdvancedCallback(const SuspendedThreadsList &suspended_threads_list,
+                             void *argument) {
+  AdvancedCallbackArgument *callback_argument =
+      (AdvancedCallbackArgument *)argument;
+  callback_argument->callback_executed = true;
+
+  int counters_at_init[kThreadCount];
+  for (uptr j = 0; j < kThreadCount; j++)
+    counters_at_init[j] = __sync_fetch_and_add(&callback_argument->counters[j],
+                                               0);
+  for (uptr i = 0; i < 10; i++) {
+    sched_yield();
+    for (uptr j = 0; j < kThreadCount; j++)
+      if (__sync_fetch_and_add(&callback_argument->counters[j], 0) !=
+            counters_at_init[j]) {
+        callback_argument->threads_stopped = false;
+        return;
+      }
+  }
+  callback_argument->threads_stopped = true;
+}
+
+TEST(StopTheWorld, SuspendThreadsAdvanced) {
+  pthread_mutex_init(&advanced_incrementer_thread_exit_mutex, NULL);
+  AdvancedCallbackArgument argument;
+
+  pthread_mutex_lock(&advanced_incrementer_thread_exit_mutex);
+  int pthread_create_result;
+  pthread_create_result = pthread_create(&argument.thread_ids[0], NULL,
+                                         AdvancedIncrementerThread,
+                                         &argument);
+  ASSERT_EQ(0, pthread_create_result);
+  // Wait for several threads to spawn before proceeding.
+  while (__sync_fetch_and_add(&argument.thread_index, 0) < kStopWorldAfter)
+    sched_yield();
+  StopTheWorld(&AdvancedCallback, &argument);
+  EXPECT_TRUE(argument.callback_executed);
+  EXPECT_TRUE(argument.threads_stopped);
+
+  // Wait for all threads to spawn before we start terminating them.
+  while (__sync_fetch_and_add(&argument.thread_index, 0) < kThreadCount)
+    sched_yield();
+  ASSERT_FALSE(argument.fatal_error); // a pthread_create has failed 
+  // Signal the threads to terminate.
+  pthread_mutex_unlock(&advanced_incrementer_thread_exit_mutex);
+  for (uptr i = 0; i < kThreadCount; i++)
+    ASSERT_EQ(0, pthread_join(argument.thread_ids[i], NULL));
+  pthread_mutex_destroy(&advanced_incrementer_thread_exit_mutex);
+}
+
+}  // namespace __sanitizer
+
+#endif  // __linux__

compiler-rt/lib/sanitizer_common/tests/sanitizer_stoptheworld_testlib.cc

@@ -0,0 +1,52 @@
+//===-- sanitizer_stoptheworld_testlib.cc ---------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Dynamic library to test StopTheWorld functionality.
+// When loaded with LD_PRELOAD, it will periodically suspend all threads.
+//===----------------------------------------------------------------------===//
+/* Usage:
+clang++ -fno-exceptions -g -fPIC -I. \
+ sanitizer_common/tests/sanitizer_stoptheworld_testlib.cc \
+ sanitizer_common/sanitizer_*.cc -shared -lpthread -o teststoptheworld.so
+LD_PRELOAD=`pwd`/teststoptheworld.so /your/app
+*/
+
+#ifdef __linux__
+
+#include <dlfcn.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <pthread.h>
+#include <unistd.h>
+
+#include "sanitizer_common/sanitizer_stoptheworld.h"
+
+namespace {
+const uptr kSuspendDuration = 3;
+const uptr kRunDuration = 3;
+
+void Callback(const SuspendedThreadsList &suspended_threads_list,
+              void *argument) {
+  sleep(kSuspendDuration);
+}
+
+void *SuspenderThread(void *argument) {
+  while (true) {
+    sleep(kRunDuration);
+    StopTheWorld(Callback, NULL);
+  }
+  return NULL;
+}
+
+__attribute__((constructor)) void StopTheWorldTestLibConstructor(void) {
+  pthread_t thread_id;
+  pthread_create(&thread_id, NULL, SuspenderThread, NULL);
+}
+}  // namespace
+
+#endif  // __linux__