maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Implement sized deallocation for std::allocator and friends. 

Summary:
C++14 sized deallocation is disabled by default due to ABI concerns. However, when a user manually enables it then libc++ should take advantage of it since sized deallocation can provide a significant performance win depending on the underlying malloc implementation. (Note that libc++'s definitions of sized delete don't do anything special yet, but users are free to provide their own).

This patch updates __libcpp_deallocate to selectively call sized operator delete when it's available. `__libcpp_deallocate_unsized` should be used when the size of the allocation is unknown.

On Apple this patch makes no attempt to determine if the sized operator delete is unavailable, only that the language feature is enabled. This could cause a compile error when using `std::allocator`, but the same compile error would occur whenever the user calls `new`, so I don't think it's a problem.

Reviewers: ldionne, mclow.lists

Reviewed By: ldionne

Subscribers: rsmith, ckennelly, libcxx-commits, christof

Differential Revision: https://reviews.llvm.org/D53120

llvm-svn: 345214
This commit is contained in:
Eric Fiselier 2018-10-24 22:44:01 +00:00
parent 7faf7ae0ad
commit 7e6448bb02
6 changed files with 378 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libcxx/include/__sso_allocator
View File

@ -57,12 +57,12 @@ public:
}
return static_cast<pointer>(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp)));
}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type)
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type __n)
{
if (__p == (pointer)&buf_)
__allocated_ = false;
else
_VSTD::__libcpp_deallocate(__p, __alignof(_Tp));
_VSTD::__libcpp_deallocate(__p, __n * sizeof(_Tp), __alignof(_Tp));
}
_LIBCPP_INLINE_VISIBILITY size_type max_size() const throw() {return size_type(~0) / sizeof(_Tp);}

10
libcxx/include/memory
View File

@ -1799,8 +1799,8 @@ public:
" 'n' exceeds maximum supported size");
return static_cast<pointer>(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp)));
}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) _NOEXCEPT
{_VSTD::__libcpp_deallocate((void*)__p, __alignof(_Tp));}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type __n) _NOEXCEPT
{_VSTD::__libcpp_deallocate((void*)__p, __n * sizeof(_Tp), __alignof(_Tp));}
_LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT
{return size_type(~0) / sizeof(_Tp);}
#if !defined(_LIBCPP_HAS_NO_RVALUE_REFERENCES) && !defined(_LIBCPP_HAS_NO_VARIADICS)
@ -1900,8 +1900,8 @@ public:
" 'n' exceeds maximum supported size");
return static_cast<pointer>(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp)));
}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) _NOEXCEPT
{_VSTD::__libcpp_deallocate((void*) const_cast<_Tp *>(__p), __alignof(_Tp));}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type __n) _NOEXCEPT
{_VSTD::__libcpp_deallocate((void*) const_cast<_Tp *>(__p), __n * sizeof(_Tp), __alignof(_Tp));}
_LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT
{return size_type(~0) / sizeof(_Tp);}
#if !defined(_LIBCPP_HAS_NO_RVALUE_REFERENCES) && !defined(_LIBCPP_HAS_NO_VARIADICS)
@ -2052,7 +2052,7 @@ template <class _Tp>
inline _LIBCPP_INLINE_VISIBILITY
void return_temporary_buffer(_Tp* __p) _NOEXCEPT
{
_VSTD::__libcpp_deallocate((void*)__p, __alignof(_Tp));
_VSTD::__libcpp_deallocate_unsized((void*)__p, __alignof(_Tp));
}
#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)

103
libcxx/include/new
View File

@ -104,12 +104,17 @@ void operator delete[](void* ptr, void*) noexcept;
#pragma GCC system_header
#endif
#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14
#if !defined(__cpp_sized_deallocation) || __cpp_sized_deallocation < 201309L
#define _LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION
#endif
#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14 && \
defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
# define _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
#endif
#if defined(_LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION) || \
!defined(__cpp_sized_deallocation) || __cpp_sized_deallocation < 201309
defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
# define _LIBCPP_HAS_NO_SIZED_DEALLOCATION
#endif
@ -255,24 +260,94 @@ inline _LIBCPP_INLINE_VISIBILITY void *__libcpp_allocate(size_t __size, size_t _
#endif
}
inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __align) {
#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
if (__is_overaligned_for_new(__align)) {
const align_val_t __align_val = static_cast<align_val_t>(__align);
# ifdef _LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE
return ::operator delete(__ptr, __align_val);
# else
return __builtin_operator_delete(__ptr, __align_val);
# endif
struct _DeallocateCaller {
static inline _LIBCPP_INLINE_VISIBILITY
void __do_deallocate_handle_size_align(void *__ptr, size_t __size, size_t __align) {
#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
((void)__align);
return __do_deallocate_handle_size(__ptr, __size);
#else
if (__is_overaligned_for_new(__align)) {
const align_val_t __align_val = static_cast<align_val_t>(__align);
return __do_deallocate_handle_size(__ptr, __size, __align_val);
} else {
return __do_deallocate_handle_size(__ptr, __size);
}
#endif
}
static inline _LIBCPP_INLINE_VISIBILITY
void __do_deallocate_handle_align(void *__ptr, size_t __align) {
#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
((void)__align);
return __do_call(__ptr);
#else
((void)__align);
if (__is_overaligned_for_new(__align)) {
const align_val_t __align_val = static_cast<align_val_t>(__align);
return __do_call(__ptr, __align_val);
} else {
return __do_call(__ptr);
}
#endif
}
private:
static inline void __do_deallocate_handle_size(void *__ptr, size_t __size) {
#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION
((void)__size);
return __do_call(__ptr);
#else
return __do_call(__ptr, __size);
#endif
}
#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
static inline void __do_deallocate_handle_size(void *__ptr, size_t __size, align_val_t __align) {
#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION
((void)__size);
return __do_call(__ptr, __align);
#else
return __do_call(__ptr, __size, __align);
#endif
}
#endif
private:
template <class _A1, class _A2>
static inline void __do_call(void *__ptr, _A1 __a1, _A2 __a2) {
#if defined(_LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE) || \
defined(_LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE)
return ::operator delete(__ptr, __a1, __a2);
#else
return __builtin_operator_delete(__ptr, __a1, __a2);
#endif
}
template <class _A1>
static inline void __do_call(void *__ptr, _A1 __a1) {
#if defined(_LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE) || \
defined(_LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE)
return ::operator delete(__ptr, __a1);
#else
return __builtin_operator_delete(__ptr, __a1);
#endif
}
static inline void __do_call(void *__ptr) {
#ifdef _LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE
return ::operator delete(__ptr);
return ::operator delete(__ptr);
#else
return __builtin_operator_delete(__ptr);
return __builtin_operator_delete(__ptr);
#endif
}
};
inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) {
_DeallocateCaller::__do_deallocate_handle_size_align(__ptr, __size, __align);
}
inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate_unsized(void* __ptr, size_t __align) {
_DeallocateCaller::__do_deallocate_handle_align(__ptr, __align);
}
#ifdef _LIBCPP_BAD_ARRAY_LENGTH_DEFINED

68
libcxx/include/valarray
View File

@ -1054,7 +1054,7 @@ private:
const _Up*
end(const valarray<_Up>& __v);
void __clear();
void __clear(size_t __capacity);
valarray& __assign_range(const value_type* __f, const value_type* __l);
};
@ -2762,13 +2762,13 @@ valarray<_Tp>::valarray(size_t __n)
try
{
#endif // _LIBCPP_NO_EXCEPTIONS
for (; __n; --__n, ++__end_)
for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
::new (__end_) value_type();
#ifndef _LIBCPP_NO_EXCEPTIONS
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2797,13 +2797,13 @@ valarray<_Tp>::valarray(const value_type* __p, size_t __n)
try
{
#endif // _LIBCPP_NO_EXCEPTIONS
for (; __n; ++__end_, ++__p, --__n)
for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left)
::new (__end_) value_type(*__p);
#ifndef _LIBCPP_NO_EXCEPTIONS
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2829,7 +2829,7 @@ valarray<_Tp>::valarray(const valarray& __v)
}
catch (...)
{
__clear();
__clear(__v.size());
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2852,7 +2852,7 @@ valarray<_Tp>::valarray(initializer_list<value_type> __il)
: __begin_(0),
__end_(0)
{
size_t __n = __il.size();
const size_t __n = __il.size();
if (__n)
{
__begin_ = __end_ = static_cast<value_type*>(
@ -2861,13 +2861,14 @@ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
try
{
#endif // _LIBCPP_NO_EXCEPTIONS
for (const value_type* __p = __il.begin(); __n; ++__end_, ++__p, --__n)
size_t __n_left = __n;
for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left)
::new (__end_) value_type(*__p);
#ifndef _LIBCPP_NO_EXCEPTIONS
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2881,7 +2882,7 @@ valarray<_Tp>::valarray(const slice_array<value_type>& __sa)
: __begin_(0),
__end_(0)
{
size_t __n = __sa.__size_;
const size_t __n = __sa.__size_;
if (__n)
{
__begin_ = __end_ = static_cast<value_type*>(
@ -2890,13 +2891,14 @@ valarray<_Tp>::valarray(const slice_array<value_type>& __sa)
try
{
#endif // _LIBCPP_NO_EXCEPTIONS
for (const value_type* __p = __sa.__vp_; __n; ++__end_, __p += __sa.__stride_, --__n)
size_t __n_left = __n;
for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left)
::new (__end_) value_type(*__p);
#ifndef _LIBCPP_NO_EXCEPTIONS
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2908,7 +2910,7 @@ valarray<_Tp>::valarray(const gslice_array<value_type>& __ga)
: __begin_(0),
__end_(0)
{
size_t __n = __ga.__1d_.size();
const size_t __n = __ga.__1d_.size();
if (__n)
{
__begin_ = __end_ = static_cast<value_type*>(
@ -2926,7 +2928,7 @@ valarray<_Tp>::valarray(const gslice_array<value_type>& __ga)
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2938,7 +2940,7 @@ valarray<_Tp>::valarray(const mask_array<value_type>& __ma)
: __begin_(0),
__end_(0)
{
size_t __n = __ma.__1d_.size();
const size_t __n = __ma.__1d_.size();
if (__n)
{
__begin_ = __end_ = static_cast<value_type*>(
@ -2956,7 +2958,7 @@ valarray<_Tp>::valarray(const mask_array<value_type>& __ma)
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2968,7 +2970,7 @@ valarray<_Tp>::valarray(const indirect_array<value_type>& __ia)
: __begin_(0),
__end_(0)
{
size_t __n = __ia.__1d_.size();
const size_t __n = __ia.__1d_.size();
if (__n)
{
__begin_ = __end_ = static_cast<value_type*>(
@ -2986,7 +2988,7 @@ valarray<_Tp>::valarray(const indirect_array<value_type>& __ia)
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS
@ -2997,7 +2999,7 @@ template <class _Tp>
inline
valarray<_Tp>::~valarray()
{
__clear();
__clear(size());
}
template <class _Tp>
@ -3007,7 +3009,7 @@ valarray<_Tp>::__assign_range(const value_type* __f, const value_type* __l)
size_t __n = __l - __f;
if (size() != __n)
{
__clear();
__clear(size());
__begin_ = static_cast<value_type*>(
_VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
__end_ = __begin_ + __n;
@ -3034,7 +3036,7 @@ inline
valarray<_Tp>&
valarray<_Tp>::operator=(valarray&& __v) _NOEXCEPT
{
__clear();
__clear(size());
__begin_ = __v.__begin_;
__end_ = __v.__end_;
__v.__begin_ = nullptr;
@ -3726,23 +3728,23 @@ valarray<_Tp>::apply(value_type __f(const value_type&)) const
}
template <class _Tp>
void
valarray<_Tp>::__clear()
inline _LIBCPP_INLINE_VISIBILITY
void valarray<_Tp>::__clear(size_t __capacity)
{
if (__begin_ != nullptr)
{
while (__end_ != __begin_)
(--__end_)->~value_type();
_VSTD::__libcpp_deallocate(__begin_, __alignof(value_type));
__begin_ = __end_ = nullptr;
}
if (__begin_ != nullptr)
{
while (__end_ != __begin_)
(--__end_)->~value_type();
_VSTD::__libcpp_deallocate(__begin_, __capacity * sizeof(value_type), __alignof(value_type));
__begin_ = __end_ = nullptr;
}
}
template <class _Tp>
void
valarray<_Tp>::resize(size_t __n, value_type __x)
{
__clear();
__clear(size());
if (__n)
{
__begin_ = __end_ = static_cast<value_type*>(
@ -3751,13 +3753,13 @@ valarray<_Tp>::resize(size_t __n, value_type __x)
try
{
#endif // _LIBCPP_NO_EXCEPTIONS
for (; __n; --__n, ++__end_)
for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
::new (__end_) value_type(__x);
#ifndef _LIBCPP_NO_EXCEPTIONS
}
catch (...)
{
__clear();
__clear(__n);
throw;
}
#endif // _LIBCPP_NO_EXCEPTIONS

4
libcxx/src/experimental/memory_resource.cpp
View File

@ -33,8 +33,8 @@ protected:
virtual void* do_allocate(size_t __size, size_t __align)
{ return _VSTD::__libcpp_allocate(__size, __align); /* FIXME */}
virtual void do_deallocate(void * __p, size_t, size_t __align)
{ _VSTD::__libcpp_deallocate(__p, __align); /* FIXME */ }
virtual void do_deallocate(void * __p, size_t __n, size_t __align)
{ _VSTD::__libcpp_deallocate(__p, __n, __align); /* FIXME */ }
virtual bool do_is_equal(memory_resource const & __other) const _NOEXCEPT
{ return &__other == this; }

245
libcxx/test/libcxx/language.support/support.dynamic/libcpp_deallocate.sh.cpp Normal file
View File

@ -0,0 +1,245 @@
//===----------------------------------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
// test libc++'s implementation of align_val_t, and the relevent new/delete
// overloads in all dialects when -faligned-allocation is present.
// Libc++ defers to the underlying MSVC library to provide the new/delete
// definitions, which does not yet provide aligned allocation
// XFAIL: LIBCXX-WINDOWS-FIXME
// XFAIL: with_system_cxx_lib=macosx10.12
// XFAIL: with_system_cxx_lib=macosx10.11
// XFAIL: with_system_cxx_lib=macosx10.10
// XFAIL: with_system_cxx_lib=macosx10.9
// XFAIL: with_system_cxx_lib=macosx10.7
// XFAIL: with_system_cxx_lib=macosx10.8
// RUN: %build -faligned-allocation -fsized-deallocation
// RUN: %run
// RUN: %build -faligned-allocation -fno-sized-deallocation -DNO_SIZE
// RUN: %run
// RUN: %build -fno-aligned-allocation -fsized-deallocation -DNO_ALIGN
// RUN: %run
// RUN: %build -fno-aligned-allocation -fno-sized-deallocation -DNO_ALIGN -DNO_SIZE
// RUN: %run
#include <new>
#include <typeinfo>
#include <string>
#include <cassert>
#include "test_macros.h"
struct alloc_stats {
alloc_stats() { reset(); }
int aligned_sized_called;
int aligned_called;
int sized_called;
int plain_called;
int last_size;
int last_align;
void reset() {
aligned_sized_called = aligned_called = sized_called = plain_called = 0;
last_align = last_size = -1;
}
bool expect_plain() const {
assert(aligned_sized_called == 0);
assert(aligned_called == 0);
assert(sized_called == 0);
assert(last_size == -1);
assert(last_align == -1);
return plain_called == 1;
}
bool expect_size(int n) const {
assert(plain_called == 0);
assert(aligned_sized_called == 0);
assert(aligned_called == 0);
assert(last_size == n);
assert(last_align == -1);
return sized_called == 1;
}
bool expect_align(int a) const {
assert(plain_called == 0);
assert(aligned_sized_called == 0);
assert(sized_called == 0);
assert(last_size == -1);
assert(last_align == a);
return aligned_called == 1;
}
bool expect_size_align(int n, int a) const {
assert(plain_called == 0);
assert(sized_called == 0);
assert(aligned_called == 0);
assert(last_size == n);
assert(last_align == a);
return aligned_sized_called == 1;
}
};
alloc_stats stats;
void operator delete(void *p) TEST_NOEXCEPT {
::free(p);
stats.plain_called++;
stats.last_size = stats.last_align = -1;
}
#ifndef NO_SIZE
void operator delete(void *p, size_t n) TEST_NOEXCEPT {
::free(p);
stats.sized_called++;
stats.last_size = n;
stats.last_align = -1;
}
#endif
#ifndef NO_ALIGN
void operator delete(void *p, std::align_val_t a) TEST_NOEXCEPT {
::free(p);
stats.aligned_called++;
stats.last_align = static_cast<int>(a);
stats.last_size = -1;
}
void operator delete(void * p, size_t n, std::align_val_t a) TEST_NOEXCEPT {
::free(p);
stats.aligned_sized_called++;
stats.last_align = static_cast<int>(a);
stats.last_size = n;
}
#endif
void test_libcpp_dealloc() {
void *p = nullptr;
size_t over_align_val = TEST_ALIGNOF(std::max_align_t) * 2;
size_t under_align_val = TEST_ALIGNOF(int);
size_t with_size_val = 2;
{
std::__libcpp_deallocate_unsized(p, under_align_val);
assert(stats.expect_plain());
}
stats.reset();
#if defined(NO_SIZE) && defined(NO_ALIGN)
{
std::__libcpp_deallocate(p, with_size_val, over_align_val);
assert(stats.expect_plain());
}
stats.reset();
#elif defined(NO_SIZE)
{
std::__libcpp_deallocate(p, with_size_val, over_align_val);
assert(stats.expect_align(over_align_val));
}
stats.reset();
#elif defined(NO_ALIGN)
{
std::__libcpp_deallocate(p, with_size_val, over_align_val);
assert(stats.expect_size(with_size_val));
}
stats.reset();
#else
{
std::__libcpp_deallocate(p, with_size_val, over_align_val);
assert(stats.expect_size_align(with_size_val, over_align_val));
}
stats.reset();
{
std::__libcpp_deallocate_unsized(p, over_align_val);
assert(stats.expect_align(over_align_val));
}
stats.reset();
{
std::__libcpp_deallocate(p, with_size_val, under_align_val);
assert(stats.expect_size(with_size_val));
}
stats.reset();
#endif
}
struct TEST_ALIGNAS(128) AlignedType {
AlignedType() : elem(0) {}
TEST_ALIGNAS(128) char elem;
};
void test_allocator_and_new_match() {
stats.reset();
#if defined(NO_SIZE) && defined(NO_ALIGN)
{
int *x = new int(42);
delete x;
assert(stats.expect_plain());
}
stats.reset();
{
AlignedType *a = new AlignedType();
delete a;
assert(stats.expect_plain());
}
stats.reset();
#elif defined(NO_SIZE)
stats.reset();
#if TEST_STD_VER >= 11
{
int *x = new int(42);
delete x;
assert(stats.expect_plain());
}
#endif
stats.reset();
{
AlignedType *a = new AlignedType();
delete a;
assert(stats.expect_align(TEST_ALIGNOF(AlignedType)));
}
stats.reset();
#elif defined(NO_ALIGN)
stats.reset();
{
int *x = new int(42);
delete x;
assert(stats.expect_size(sizeof(int)));
}
stats.reset();
{
AlignedType *a = new AlignedType();
delete a;
assert(stats.expect_size(sizeof(AlignedType)));
}
stats.reset();
#else
stats.reset();
{
int *x = new int(42);
delete x;
assert(stats.expect_size(sizeof(int)));
}
stats.reset();
{
AlignedType *a = new AlignedType();
delete a;
assert(stats.expect_size_align(sizeof(AlignedType), TEST_ALIGNOF(AlignedType)));
}
stats.reset();
#endif
}
int main() {
test_libcpp_dealloc();
test_allocator_and_new_match();
}