Apply jemalloc fix, and always build jemalloc (#6610)

* Apply jemalloc fix, and always build jemalloc

See https://github.com/jemalloc/jemalloc/pull/1924

* Move USE_JEMALLOC to flow/config.h

* Add unit test to validate fix

cmake/Jemalloc.cmake

@@ -11,43 +11,25 @@ if(NOT USE_JEMALLOC)
   return()
 endif()
 
-add_definitions(-DUSE_JEMALLOC)
-find_path(JEMALLOC_INCLUDE_DIR
-  NAMES
-  jemalloc/jemalloc.h
-  PATH_SUFFIXES
-  include
-  )
-find_library(JEMALLOC NAMES libjemalloc.a)
-find_library(JEMALLOC_PIC NAMES libjemalloc_pic.a)
 add_library(im_jemalloc_pic STATIC IMPORTED)
 add_library(im_jemalloc STATIC IMPORTED)
-if(JEMALLOC_INCLUDE_DIR AND JEMALLOC AND JEMALLOC_PIC)
-  set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_PIC}")
-  set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC}")
-  target_include_directories(jemalloc INTERFACE "${JEMALLOC_INCLUDE_DIR}")
-  # the ordering here is important: for dynamic libraries we have to use all
-  # symbols that are in the library which was compiled with PIC (for executables
-  # we could omit the pic-library)
-  target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)
-else()
-  include(ExternalProject)
-  set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc")
-  ExternalProject_add(Jemalloc_project
-    URL "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
-    URL_HASH SHA256=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
-    BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
-    "${JEMALLOC_DIR}/lib/libjemalloc.a"
-    "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
-    CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
-    BUILD_IN_SOURCE ON
-    BUILD_COMMAND make
-    INSTALL_DIR "${JEMALLOC_DIR}"
-    INSTALL_COMMAND make install)
-  add_dependencies(im_jemalloc Jemalloc_project)
-  add_dependencies(im_jemalloc_pic Jemalloc_project)
-  set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a")
-  set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a")
-  target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include")
-  target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)
-endif()
+include(ExternalProject)
+set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc")
+ExternalProject_add(Jemalloc_project
+  URL "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
+  URL_HASH SHA256=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6
+  BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
+  "${JEMALLOC_DIR}/lib/libjemalloc.a"
+  "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
+  PATCH_COMMAND patch -p1 < ${CMAKE_SOURCE_DIR}/cmake/jemalloc.patch
+  CONFIGURE_COMMAND ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
+  BUILD_IN_SOURCE ON
+  BUILD_COMMAND make
+  INSTALL_DIR "${JEMALLOC_DIR}"
+  INSTALL_COMMAND make install)
+add_dependencies(im_jemalloc Jemalloc_project)
+add_dependencies(im_jemalloc_pic Jemalloc_project)
+set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a")
+set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a")
+target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include")
+target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)

cmake/jemalloc.patch

@@ -0,0 +1,38 @@
+diff --git a/include/jemalloc/internal/sz.h b/include/jemalloc/internal/sz.h
+index 68e558ab..87bb2280 100644
+--- a/include/jemalloc/internal/sz.h
++++ b/include/jemalloc/internal/sz.h
+@@ -266,7 +266,7 @@ sz_sa2u(size_t size, size_t alignment) {
+ 	assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
+ 
+ 	/* Try for a small size class. */
+-	if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) {
++	if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) {
+ 		/*
+ 		 * Round size up to the nearest multiple of alignment.
+ 		 *
+diff --git a/src/arena.c b/src/arena.c
+index ba50e410..dc7646e6 100644
+--- a/src/arena.c
++++ b/src/arena.c
+@@ -1533,10 +1533,17 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
+     bool zero, tcache_t *tcache) {
+ 	void *ret;
+ 
+-	if (usize <= SC_SMALL_MAXCLASS
+-	    && (alignment < PAGE
+-	    || (alignment == PAGE && (usize & PAGE_MASK) == 0))) {
++	if (usize <= SC_SMALL_MAXCLASS) {
+ 		/* Small; alignment doesn't require special slab placement. */
++
++		/* usize should be a result of sz_sa2u() */
++		assert((usize & (alignment - 1)) == 0);
++
++		/*
++		 * Small usize can't come from an alignment larger than a page.
++		 */
++		assert(alignment <= PAGE);
++
+ 		ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize),
+ 		    zero, tcache, true);
+ 	} else {

flow/FastAlloc.cpp

@@ -24,6 +24,7 @@
 #include "flow/Trace.h"
 #include "flow/Error.h"
 #include "flow/Knobs.h"
+#include "flow/UnitTest.h"
 #include "flow/crc32c.h"
 #include "flow/flow.h"
 
@@ -588,3 +589,20 @@ template class FastAllocator<2048>;
 template class FastAllocator<4096>;
 template class FastAllocator<8192>;
 template class FastAllocator<16384>;
+
+#ifdef USE_JEMALLOC
+#include <jemalloc/jemalloc.h>
+TEST_CASE("/jemalloc/4k_aligned_usable_size") {
+	for (int i = 1; i < 4; ++i) {
+		auto* ptr = aligned_alloc(4096, i * 4096);
+		try {
+			ASSERT_EQ(malloc_usable_size(ptr), i * 4096);
+		} catch (...) {
+			aligned_free(ptr);
+			throw;
+		}
+		aligned_free(ptr);
+	}
+	return Void();
+}
+#endif

flow/config.h.cmake

@@ -28,4 +28,5 @@
 # endif
 # cmakedefine DTRACE_PROBES
 # cmakedefine HAS_ALIGNED_ALLOC
+# cmakedefine USE_JEMALLOC
 #endif // WIN32