Fix PR41395 - __cxa_vec_new may overflow in allocation size calculation.

llvm-svn: 357814

libcxxabi/src/cxa_vector.cpp

@@ -13,6 +13,9 @@
 #include "cxxabi.h"
 
 #include <exception>        // for std::terminate
+#include <new>              // for std::bad_alloc
+
+#include "abort_message.h"
 
 namespace __cxxabiv1 {
 
@@ -107,6 +110,32 @@ namespace {
 #pragma mark --Externally visible routines--
 #endif
 
+namespace {
+_LIBCXXABI_NORETURN
+void throw_bad_array_new_length() {
+#ifndef _LIBCXXABI_NO_EXCEPTIONS
+  throw std::bad_array_new_length();
+#else
+  abort_message("__cxa_vec_new failed to allocate memory");
+#endif
+}
+
+size_t calculate_allocation_size_or_throw(size_t element_count,
+                                          size_t element_size,
+                                          size_t padding_size) {
+  const size_t element_heap_size = element_count * element_size;
+  if (element_heap_size / element_count != element_size)
+    throw_bad_array_new_length();
+
+  const size_t allocation_size = element_heap_size + padding_size;
+  if (allocation_size < element_heap_size)
+    throw_bad_array_new_length();
+
+  return allocation_size;
+}
+
+} // namespace
+
 extern "C" {
 
 // Equivalent to
@@ -121,7 +150,6 @@ __cxa_vec_new(size_t element_count, size_t element_size, size_t padding_size,
 }
 
 
-
 // Given the number and size of elements for an array and the non-negative
 // size of prefix padding for a cookie, allocate space (using alloc) for
 // the array preceded by the specified padding, initialize the cookie if
@@ -142,12 +170,13 @@ _LIBCXXABI_FUNC_VIS void *
 __cxa_vec_new2(size_t element_count, size_t element_size, size_t padding_size,
                void (*constructor)(void *), void (*destructor)(void *),
                void *(*alloc)(size_t), void (*dealloc)(void *)) {
-    const size_t heap_size = element_count * element_size + padding_size;
-    char * const heap_block = static_cast<char *> ( alloc ( heap_size ));
-    char *vec_base = heap_block;
-    
-    if ( NULL != vec_base ) {
-        st_heap_block2 heap ( dealloc, heap_block );
+  const size_t heap_size = calculate_allocation_size_or_throw(
+      element_count, element_size, padding_size);
+  char* const heap_block = static_cast<char*>(alloc(heap_size));
+  char* vec_base = heap_block;
+
+  if (NULL != vec_base) {
+    st_heap_block2 heap(dealloc, heap_block);
 
     //  put the padding before the array elements
         if ( 0 != padding_size ) {
@@ -170,12 +199,13 @@ _LIBCXXABI_FUNC_VIS void *
 __cxa_vec_new3(size_t element_count, size_t element_size, size_t padding_size,
                void (*constructor)(void *), void (*destructor)(void *),
                void *(*alloc)(size_t), void (*dealloc)(void *, size_t)) {
-    const size_t heap_size = element_count * element_size + padding_size;
-    char * const heap_block = static_cast<char *> ( alloc ( heap_size ));
-    char *vec_base = heap_block;
-    
-    if ( NULL != vec_base ) {
-        st_heap_block3 heap ( dealloc, heap_block, heap_size );
+  const size_t heap_size = calculate_allocation_size_or_throw(
+      element_count, element_size, padding_size);
+  char* const heap_block = static_cast<char*>(alloc(heap_size));
+  char* vec_base = heap_block;
+
+  if (NULL != vec_base) {
+    st_heap_block3 heap(dealloc, heap_block, heap_size);
 
     //  put the padding before the array elements
         if ( 0 != padding_size ) {

libcxxabi/test/cxa_vec_new_overflow_PR41395.pass.cpp

@@ -0,0 +1,122 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: libcxxabi-no-exceptions
+
+#include "cxxabi.h"
+#include <new>
+#include <cassert>
+
+void dummy_ctor(void*) { assert(false && "should not be called"); }
+void dummy_dtor(void*) { assert(false && "should not be called"); }
+
+void *dummy_alloc(size_t) { assert(false && "should not be called"); }
+void dummy_dealloc(void*) { assert(false && "should not be called"); }
+void dummy_dealloc_sized(void*, size_t) { assert(false && "should not be called"); }
+
+
+bool check_mul_overflows(size_t x, size_t y) {
+  size_t tmp = x * y;
+  if (tmp / x != y)
+    return true;
+  return false;
+}
+
+bool check_add_overflows(size_t x, size_t y) {
+  size_t tmp = x + y;
+  if (tmp < x)
+    return true;
+
+  return false;
+}
+
+void test_overflow_in_multiplication() {
+  const size_t elem_count = std::size_t(1) << (sizeof(std::size_t) * 8 - 2);
+  const size_t elem_size = 8;
+  const size_t padding = 0;
+  assert(check_mul_overflows(elem_count, elem_size));
+
+  try {
+    __cxxabiv1::__cxa_vec_new(elem_count, elem_size, padding, dummy_ctor,
+                              dummy_dtor);
+    assert(false && "allocation should fail");
+  } catch (std::bad_array_new_length const&) {
+    // OK
+  } catch (...) {
+    assert(false && "unexpected exception");
+  }
+
+  try {
+    __cxxabiv1::__cxa_vec_new2(elem_count, elem_size, padding, dummy_ctor,
+                              dummy_dtor, &dummy_alloc, &dummy_dealloc);
+    assert(false && "allocation should fail");
+  } catch (std::bad_array_new_length const&) {
+    // OK
+  } catch (...) {
+    assert(false && "unexpected exception");
+  }
+
+  try {
+    __cxxabiv1::__cxa_vec_new3(elem_count, elem_size, padding, dummy_ctor,
+                               dummy_dtor, &dummy_alloc, &dummy_dealloc_sized);
+    assert(false && "allocation should fail");
+  } catch (std::bad_array_new_length const&) {
+    // OK
+  } catch (...) {
+    assert(false && "unexpected exception");
+  }
+}
+
+void test_overflow_in_addition() {
+  const size_t elem_size = 4;
+  const size_t elem_count = static_cast<size_t>(-1) / 4u;
+#if defined(_LIBCXXABI_ARM_EHABI)
+  const size_t padding = 8;
+#else
+  const size_t padding = sizeof(std::size_t);
+#endif
+  assert(!check_mul_overflows(elem_count, elem_size));
+  assert(check_add_overflows(elem_count * elem_size, padding));
+  try {
+    __cxxabiv1::__cxa_vec_new(elem_count, elem_size, padding, dummy_ctor,
+                              dummy_dtor);
+    assert(false && "allocation should fail");
+  } catch (std::bad_array_new_length const&) {
+    // OK
+  } catch (...) {
+    assert(false && "unexpected exception");
+  }
+
+
+  try {
+    __cxxabiv1::__cxa_vec_new2(elem_count, elem_size, padding, dummy_ctor,
+                               dummy_dtor, &dummy_alloc, &dummy_dealloc);
+    assert(false && "allocation should fail");
+  } catch (std::bad_array_new_length const&) {
+    // OK
+  } catch (...) {
+    assert(false && "unexpected exception");
+  }
+
+  try {
+    __cxxabiv1::__cxa_vec_new3(elem_count, elem_size, padding, dummy_ctor,
+                               dummy_dtor, &dummy_alloc, &dummy_dealloc_sized);
+    assert(false && "allocation should fail");
+  } catch (std::bad_array_new_length const&) {
+    // OK
+  } catch (...) {
+    assert(false && "unexpected exception");
+  }
+}
+
+int main(int, char**) {
+  test_overflow_in_multiplication();
+  test_overflow_in_addition();
+
+  return 0;
+}