[sanitizer] Make DTLS_on_tls_get_addr signal safer

Avoid relocating DTV table and use linked list of mmap-ed pages.

Reviewed By: eugenis

Differential Revision: https://reviews.llvm.org/D92428

compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp

@@ -12,6 +12,7 @@
 
 #include "sanitizer_tls_get_addr.h"
 
+#include "sanitizer_atomic.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_platform_interceptors.h"
 
@@ -42,39 +43,54 @@ static atomic_uintptr_t number_of_live_dtls;
 
 static const uptr kDestroyedThread = -1;
 
-static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
-  if (!size) return;
-  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
-  UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
+static void DTLS_Deallocate(DTLS::DTVBlock *block) {
+  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
+  UnmapOrDie(block, sizeof(DTLS::DTVBlock));
   atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
 }
 
-static inline void DTLS_Resize(uptr new_size) {
-  if (dtls.dtv_size >= new_size) return;
-  new_size = RoundUpToPowerOfTwo(new_size);
-  new_size = Max(new_size, 4096UL / sizeof(DTLS::DTV));
-  DTLS::DTV *new_dtv =
-      (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
+static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
+  uptr v = atomic_load(cur, memory_order_acquire);
+  if (v == kDestroyedThread)
+    return nullptr;
+  DTLS::DTVBlock *next = (DTLS::DTVBlock *)v;
+  if (next)
+    return next;
+  DTLS::DTVBlock *new_dtv =
+      (DTLS::DTVBlock *)MmapOrDie(sizeof(DTLS::DTVBlock), "DTLS_NextBlock");
+  uptr prev = 0;
+  if (!atomic_compare_exchange_strong(cur, &prev, (uptr)new_dtv,
+                                      memory_order_seq_cst)) {
+    UnmapOrDie(new_dtv, sizeof(DTLS::DTVBlock));
+    return (DTLS::DTVBlock *)prev;
+  }
   uptr num_live_dtls =
       atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
-  VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
-  CHECK_LT(num_live_dtls, 1 << 20);
-  uptr old_dtv_size = dtls.dtv_size;
-  DTLS::DTV *old_dtv = dtls.dtv;
-  if (old_dtv_size)
-    internal_memcpy(new_dtv, dtls.dtv, dtls.dtv_size * sizeof(DTLS::DTV));
-  dtls.dtv = new_dtv;
-  dtls.dtv_size = new_size;
-  if (old_dtv_size)
-    DTLS_Deallocate(old_dtv, old_dtv_size);
+  VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+  return new_dtv;
+}
+
+static DTLS::DTV *DTLS_Find(uptr id) {
+  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+  static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
+  DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
+  if (!cur)
+    return nullptr;
+  for (; id >= kPerBlock; id -= kPerBlock) cur = DTLS_NextBlock(&cur->next);
+  return cur->dtvs + id;
 }
 
 void DTLS_Destroy() {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
-  uptr s = dtls.dtv_size;
-  dtls.dtv_size = kDestroyedThread;  // Do this before unmap for AS-safety.
-  DTLS_Deallocate(dtls.dtv, s);
+  VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+  DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
+      &dtls.dtv_block, kDestroyedThread, memory_order_release);
+  while (block) {
+    DTLS::DTVBlock *next =
+        (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+    DTLS_Deallocate(block);
+    block = next;
+  }
 }
 
 #if defined(__powerpc64__) || defined(__mips__)
@@ -96,9 +112,9 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   if (!common_flags()->intercept_tls_get_addr) return 0;
   TlsGetAddrParam *arg = reinterpret_cast<TlsGetAddrParam *>(arg_void);
   uptr dso_id = arg->dso_id;
-  if (dtls.dtv_size == kDestroyedThread) return 0;
-  DTLS_Resize(dso_id + 1);
-  if (dtls.dtv[dso_id].beg) return 0;
+  DTLS::DTV *dtv = DTLS_Find(dso_id);
+  if (!dtv || dtv->beg)
+    return 0;
   uptr tls_size = 0;
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
   VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
@@ -126,9 +142,9 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
     // This may happen inside the DTOR of main thread, so just ignore it.
     tls_size = 0;
   }
-  dtls.dtv[dso_id].beg = tls_beg;
-  dtls.dtv[dso_id].size = tls_size;
-  return dtls.dtv + dso_id;
+  dtv->beg = tls_beg;
+  dtv->size = tls_size;
+  return dtv;
 }
 
 void DTLS_on_libc_memalign(void *ptr, uptr size) {
@@ -141,7 +157,8 @@ void DTLS_on_libc_memalign(void *ptr, uptr size) {
 DTLS *DTLS_Get() { return &dtls; }
 
 bool DTLSInDestruction(DTLS *dtls) {
-  return dtls->dtv_size == kDestroyedThread;
+  return atomic_load(&dtls->dtv_block, memory_order_relaxed) ==
+         kDestroyedThread;
 }
 
 #else

compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h

@@ -28,6 +28,7 @@
 #ifndef SANITIZER_TLS_GET_ADDR_H
 #define SANITIZER_TLS_GET_ADDR_H
 
+#include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
 
 namespace __sanitizer {
@@ -38,9 +39,14 @@ struct DTLS {
   struct DTV {
     uptr beg, size;
   };
+  struct DTVBlock {
+    atomic_uintptr_t next;
+    DTV dtvs[(4096UL - sizeof(next)) / sizeof(DTLS::DTV)];
+  };
 
-  uptr dtv_size;
-  DTV *dtv;  // dtv_size elements, allocated by MmapOrDie.
+  static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size");
+
+  atomic_uintptr_t dtv_block;
 
   // Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
   uptr last_memalign_size;
@@ -49,7 +55,13 @@ struct DTLS {
 
 template <typename Fn>
 void ForEachDVT(DTLS *dtls, const Fn &fn) {
-  for (uptr j = 0; j < dtls->dtv_size; ++j) fn(dtls->dtv[j], j);
+  DTLS::DTVBlock *block =
+      (DTLS::DTVBlock *)atomic_load(&dtls->dtv_block, memory_order_acquire);
+  while (block) {
+    int id = 0;
+    for (auto &d : block->dtvs) fn(d, id++);
+    block = (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+  }
 }
 
 // Returns pointer and size of a linker-allocated TLS block.

compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp

@@ -0,0 +1,55 @@
+// RUN: %clangxx %s -DBUILD_DSO -fPIC -shared -o %t.so
+// RUN: %clangxx --std=c++11 %s -o %t
+// RUN: %env_tool_opts=verbosity=2 %run %t 2>&1 | FileCheck %s
+
+// Does not call __tls_get_addr
+// UNSUPPORTED: i386-linux
+
+// Do not intercept __tls_get_addr
+// UNSUPPORTED: lsan, ubsan, android
+
+#include <string.h>
+
+#ifndef BUILD_DSO
+
+#include <assert.h>
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+char buff[10000];
+
+int main(int argc, char *argv[]) {
+  sprintf(buff, "rm -f %s.so.*", argv[0]);
+  system(buff);
+
+  void *prev_handle = 0;
+  for (int i = 0; i < 300; ++i) {
+    sprintf(buff, "cp %s.so %s.so.%d", argv[0], argv[0], i);
+    system(buff);
+
+    sprintf(buff, "%s.so.%d", argv[0], i);
+    void *handle = dlopen(buff, RTLD_LAZY);
+    assert(handle != 0);
+    assert(handle != prev_handle);
+    prev_handle = handle;
+
+    typedef void (*FnType)(char c);
+    ((FnType)dlsym(handle, "StoreToTLS"))(i);
+  }
+  return 0;
+}
+
+#else  // BUILD_DSO
+__thread char huge_thread_local_array[1 << 12];
+
+extern "C" void StoreToTLS(char c) {
+  memset(huge_thread_local_array, c, sizeof(huge_thread_local_array));
+}
+#endif // BUILD_DSO
+
+// CHECK:      DTLS_Find [[DTLS:0x[a-f0-9]+]] {{[0-9]+}}
+// CHECK-NEXT: DTLS_NextBlock [[DTLS]] 0
+// CHECK:      DTLS_Find [[DTLS:0x[a-f0-9]+]] 255
+// CHECK-NEXT: DTLS_NextBlock [[DTLS]] 1
+// CHECK-NOT:  DTLS_NextBlock