diff --git a/cmake/Jemalloc.cmake b/cmake/Jemalloc.cmake index c59a290d77..b2ce6ae1b9 100644 --- a/cmake/Jemalloc.cmake +++ b/cmake/Jemalloc.cmake @@ -11,43 +11,25 @@ if(NOT USE_JEMALLOC) return() endif() -add_definitions(-DUSE_JEMALLOC) -find_path(JEMALLOC_INCLUDE_DIR - NAMES - jemalloc/jemalloc.h - PATH_SUFFIXES - include - ) -find_library(JEMALLOC NAMES libjemalloc.a) -find_library(JEMALLOC_PIC NAMES libjemalloc_pic.a) add_library(im_jemalloc_pic STATIC IMPORTED) add_library(im_jemalloc STATIC IMPORTED) -if(JEMALLOC_INCLUDE_DIR AND JEMALLOC AND JEMALLOC_PIC) - set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_PIC}") - set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC}") - target_include_directories(jemalloc INTERFACE "${JEMALLOC_INCLUDE_DIR}") - # the ordering here is important: for dynamic libraries we have to use all - # symbols that are in the library which was compiled with PIC (for executables - # we could omit the pic-library) - target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc) -else() - include(ExternalProject) - set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc") - ExternalProject_add(Jemalloc_project - URL "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" - URL_HASH SHA256=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 - BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h" - "${JEMALLOC_DIR}/lib/libjemalloc.a" - "${JEMALLOC_DIR}/lib/libjemalloc_pic.a" - CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof - BUILD_IN_SOURCE ON - BUILD_COMMAND make - INSTALL_DIR "${JEMALLOC_DIR}" - INSTALL_COMMAND make install) - add_dependencies(im_jemalloc Jemalloc_project) - add_dependencies(im_jemalloc_pic Jemalloc_project) - set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a") - set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a") - target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include") - target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc) -endif() +include(ExternalProject) +set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc") +ExternalProject_add(Jemalloc_project + URL "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" + URL_HASH SHA256=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 + BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h" + "${JEMALLOC_DIR}/lib/libjemalloc.a" + "${JEMALLOC_DIR}/lib/libjemalloc_pic.a" + PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/cmake/jemalloc.patch + CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof + BUILD_IN_SOURCE ON + BUILD_COMMAND make + INSTALL_DIR "${JEMALLOC_DIR}" + INSTALL_COMMAND make install) +add_dependencies(im_jemalloc Jemalloc_project) +add_dependencies(im_jemalloc_pic Jemalloc_project) +set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a") +set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a") +target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include") +target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc) \ No newline at end of file diff --git a/cmake/jemalloc.patch b/cmake/jemalloc.patch new file mode 100644 index 0000000000..660849041c --- /dev/null +++ b/cmake/jemalloc.patch @@ -0,0 +1,38 @@ +diff --git a/include/jemalloc/internal/sz.h b/include/jemalloc/internal/sz.h +index 68e558ab..87bb2280 100644 +--- a/include/jemalloc/internal/sz.h ++++ b/include/jemalloc/internal/sz.h +@@ -266,7 +266,7 @@ sz_sa2u(size_t size, size_t alignment) { + assert(alignment != 0 && ((alignment - 1) & alignment) == 0); + + /* Try for a small size class. */ +- if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) { ++ if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) { + /* + * Round size up to the nearest multiple of alignment. + * +diff --git a/src/arena.c b/src/arena.c +index ba50e410..dc7646e6 100644 +--- a/src/arena.c ++++ b/src/arena.c +@@ -1533,10 +1533,17 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, + bool zero, tcache_t *tcache) { + void *ret; + +- if (usize <= SC_SMALL_MAXCLASS +- && (alignment < PAGE +- || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { ++ if (usize <= SC_SMALL_MAXCLASS) { + /* Small; alignment doesn't require special slab placement. */ ++ ++ /* usize should be a result of sz_sa2u() */ ++ assert((usize & (alignment - 1)) == 0); ++ ++ /* ++ * Small usize can't come from an alignment larger than a page. ++ */ ++ assert(alignment <= PAGE); ++ + ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), + zero, tcache, true); + } else { diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp index 3fe8d6af31..d344c4cd21 100644 --- a/flow/FastAlloc.cpp +++ b/flow/FastAlloc.cpp @@ -24,6 +24,7 @@ #include "flow/Trace.h" #include "flow/Error.h" #include "flow/Knobs.h" +#include "flow/UnitTest.h" #include "flow/crc32c.h" #include "flow/flow.h" @@ -588,3 +589,20 @@ template class FastAllocator<2048>; template class FastAllocator<4096>; template class FastAllocator<8192>; template class FastAllocator<16384>; + +#ifdef USE_JEMALLOC +#include +TEST_CASE("/jemalloc/4k_aligned_usable_size") { + for (int i = 1; i < 4; ++i) { + auto* ptr = aligned_alloc(4096, i * 4096); + try { + ASSERT_EQ(malloc_usable_size(ptr), i * 4096); + } catch (...) { + aligned_free(ptr); + throw; + } + aligned_free(ptr); + } + return Void(); +} +#endif \ No newline at end of file diff --git a/flow/config.h.cmake b/flow/config.h.cmake index 35a3c147fa..864d7c68fb 100644 --- a/flow/config.h.cmake +++ b/flow/config.h.cmake @@ -28,4 +28,5 @@ # endif # cmakedefine DTRACE_PROBES # cmakedefine HAS_ALIGNED_ALLOC +# cmakedefine USE_JEMALLOC #endif // WIN32