Use of aligned_alloc() for 4k pages causes too much wasted virtual memory. Added new 4k-aligned fast allocator, and changed Arena::allocatedAlignedBuffer() to be 4k-specific, now called Arena::allocate4kAlignedBuffer().

fdbrpc/dsltest.actor.cpp

@@ -632,8 +632,8 @@ void showArena(ArenaBlock* a, ArenaBlock* parent) {
 			ArenaBlockRef* r = (ArenaBlockRef*)((char*)a->getData() + o);
 
 			// If alignedBuffer is valid then print its pointer and size, else recurse
-			if (r->alignedBufferSize != 0) {
-				printf("AlignedBuffer %p (<-%p) %u bytes\n", r->alignedBuffer, a, r->alignedBufferSize);
+			if (r->aligned4kBufferSize != 0) {
+				printf("AlignedBuffer %p (<-%p) %u bytes\n", r->aligned4kBuffer, a, r->aligned4kBufferSize);
 			} else {
 				showArena(r->next, a);
 			}

fdbserver/IPager.h

@@ -43,7 +43,7 @@ public:
 	// The page's logical size includes an opaque checksum, use size() to get usable size
 	ArenaPage(int logicalSize, int bufferSize) : logicalSize(logicalSize), bufferSize(bufferSize), userData(nullptr) {
 		if (bufferSize > 0) {
-			buffer = (uint8_t*)arena.allocateAlignedBuffer(4096, bufferSize);
+			buffer = (uint8_t*)arena.allocate4kAlignedBuffer(bufferSize);
 
 			// Mark any unused page portion defined
 			VALGRIND_MAKE_MEM_DEFINED(buffer + logicalSize, bufferSize - logicalSize);
@@ -56,6 +56,9 @@ public:
 		if (userData != nullptr && userDataDestructor != nullptr) {
 			userDataDestructor(userData);
 		}
+		if(buffer != 0) {
+			VALGRIND_MAKE_MEM_UNDEFINED(buffer, bufferSize);
+		}
 	}
 
 	uint8_t const* begin() const { return (uint8_t*)buffer; }

flow/Arena.cpp

@@ -102,8 +102,8 @@ void Arena::dependsOn(const Arena& p) {
 	}
 }
 
-void* Arena::allocateAlignedBuffer(size_t alignment, size_t size) {
-	return ArenaBlock::dependOnAlignedBuffer(impl, alignment, size);
+void* Arena::allocate4kAlignedBuffer(size_t size) {
+	return ArenaBlock::dependOn4kAlignedBuffer(impl, size);
 }
 
 size_t Arena::getSize() const {
@@ -177,8 +177,8 @@ size_t ArenaBlock::totalSize() {
 	while (o) {
 		ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + o);
 		makeDefined(r, sizeof(ArenaBlockRef));
-		if (r->alignedBufferSize != 0) {
-			s += r->alignedBufferSize;
+		if (r->aligned4kBufferSize != 0) {
+			s += r->aligned4kBufferSize;
 		} else {
 			allowAccess(r->next);
 			s += r->next->totalSize();
@@ -201,7 +201,7 @@ void ArenaBlock::getUniqueBlocks(std::set<ArenaBlock*>& a) {
 		makeDefined(r, sizeof(ArenaBlockRef));
 
 		// If next is valid recursively count its blocks
-		if (r->alignedBufferSize == 0) {
+		if (r->aligned4kBufferSize == 0) {
 			r->next->getUniqueBlocks(a);
 		}
 
@@ -226,7 +226,7 @@ int ArenaBlock::addUsed(int bytes) {
 void ArenaBlock::makeReference(ArenaBlock* next) {
 	ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + bigUsed);
 	makeDefined(r, sizeof(ArenaBlockRef));
-	r->alignedBufferSize = 0;
+	r->aligned4kBufferSize = 0;
 	r->next = next;
 	r->nextBlockOffset = nextBlockOffset;
 	makeNoAccess(r, sizeof(ArenaBlockRef));
@@ -234,17 +234,17 @@ void ArenaBlock::makeReference(ArenaBlock* next) {
 	bigUsed += sizeof(ArenaBlockRef);
 }
 
-void* ArenaBlock::makeAlignedBuffer(size_t alignment, size_t size) {
+void* ArenaBlock::make4kAlignedBuffer(size_t size) {
 	ArenaBlockRef* r = (ArenaBlockRef*)((char*)getData() + bigUsed);
 	makeDefined(r, sizeof(ArenaBlockRef));
-	r->alignedBufferSize = size;
-	r->alignedBuffer = aligned_alloc(alignment, size);
-	// printf("Arena::alignedBuffer alloc %p\n", r->alignedBuffer);
+	r->aligned4kBufferSize = size;
+	r->aligned4kBuffer = allocateFast4kAligned(size);
+	//printf("Arena::aligned4kBuffer alloc size=%u ptr=%p\n", size, r->aligned4kBuffer);
 	r->nextBlockOffset = nextBlockOffset;
 	makeNoAccess(r, sizeof(ArenaBlockRef));
 	nextBlockOffset = bigUsed;
 	bigUsed += sizeof(ArenaBlockRef);
-	return r->alignedBuffer;
+	return r->aligned4kBuffer;
 }
 
 void ArenaBlock::dependOn(Reference<ArenaBlock>& self, ArenaBlock* other) {
@@ -255,11 +255,11 @@ void ArenaBlock::dependOn(Reference<ArenaBlock>& self, ArenaBlock* other) {
 		self->makeReference(other);
 }
 
-void* ArenaBlock::dependOnAlignedBuffer(Reference<ArenaBlock>& self, size_t alignment, size_t size) {
+void* ArenaBlock::dependOn4kAlignedBuffer(Reference<ArenaBlock>& self, size_t size) {
 	if (!self || self->isTiny() || self->unused() < sizeof(ArenaBlockRef)) {
-		return create(SMALL, self)->makeAlignedBuffer(alignment, size);
+		return create(SMALL, self)->make4kAlignedBuffer(size);
 	} else {
-		return self->makeAlignedBuffer(alignment, size);
+		return self->make4kAlignedBuffer(size);
 	}
 }
 
@@ -396,10 +396,10 @@ void ArenaBlock::destroy() {
 				ArenaBlockRef* br = (ArenaBlockRef*)((char*)b->getData() + o);
 				makeDefined(br, sizeof(ArenaBlockRef));
 
-				// If alignedBuffer is valid, free it
-				if (br->alignedBufferSize != 0) {
-					// printf("Arena::alignedBuffer free %p\n", br->alignedBuffer);
-					aligned_free(br->alignedBuffer);
+				// If aligned4kBuffer is valid, free it
+				if (br->aligned4kBufferSize != 0) {
+					//printf("Arena::aligned4kBuffer free %p\n", br->aligned4kBuffer);
+					freeFast4kAligned(br->aligned4kBufferSize, br->aligned4kBuffer);
 				} else {
 					allowAccess(br->next);
 					if (br->next->delref_no_destroy())

flow/Arena.h

@@ -102,7 +102,7 @@ public:
 	Arena& operator=(Arena&&) noexcept;
 
 	void dependsOn(const Arena& p);
-	void* allocateAlignedBuffer(size_t alignment, size_t size);
+	void* allocate4kAlignedBuffer(size_t size);
 	size_t getSize() const;
 
 	bool hasFree(size_t size, const void* address);
@@ -130,12 +130,12 @@ struct scalar_traits<Arena> : std::true_type {
 };
 
 struct ArenaBlockRef {
-	// If alignedBufferSize is not 0, alignedBuffer is valid and must be freed with aligned_free()
-	// Otherwise, next is valid
-	size_t alignedBufferSize;
+	// Only one of (next, aligned4kBuffer) are valid at any one time, as they occupy the same space.
+	// If aligned4kBufferSize is not 0, aligned4kBuffer is valid, otherwise next is valid.
+	size_t aligned4kBufferSize;
 	union {
 		ArenaBlock* next;
-		void* alignedBuffer;
+		void* aligned4kBuffer;
 	};
 	uint32_t nextBlockOffset;
 };
@@ -167,9 +167,9 @@ struct ArenaBlock : NonCopyable, ThreadSafeReferenceCounted<ArenaBlock> {
 	void getUniqueBlocks(std::set<ArenaBlock*>& a);
 	int addUsed(int bytes);
 	void makeReference(ArenaBlock* next);
-	void* makeAlignedBuffer(size_t alignment, size_t size);
+	void* make4kAlignedBuffer(size_t size);
 	static void dependOn(Reference<ArenaBlock>& self, ArenaBlock* other);
-	static void* dependOnAlignedBuffer(Reference<ArenaBlock>& self, size_t alignment, size_t size);
+	static void* dependOn4kAlignedBuffer(Reference<ArenaBlock>& self, size_t size);
 	static void* allocate(Reference<ArenaBlock>& self, int bytes);
 	// Return an appropriately-sized ArenaBlock to store the given data
 	static ArenaBlock* create(int dataSize, Reference<ArenaBlock>& next);

flow/FastAlloc.h

@@ -266,4 +266,24 @@ inline void freeFast(int size, void* ptr) {
 	delete[](uint8_t*) ptr;
 }
 
+[[nodiscard]] inline void* allocateFast4kAligned(int size) {
+	if (size <= 4096)
+		return FastAllocator<4096>::allocate();
+	if (size <= 8192)
+		return FastAllocator<8192>::allocate();
+	if (size <= 16384)
+		return FastAllocator<16384>::allocate();
+	return aligned_alloc(4096, size);
+}
+
+inline void freeFast4kAligned(int size, void* ptr) {
+	if (size <= 4096)
+		return FastAllocator<4096>::release(ptr);
+	if (size <= 8192)
+		return FastAllocator<8192>::release(ptr);
+	if (size <= 16384)
+		return FastAllocator<16384>::release(ptr);
+	aligned_free(ptr);
+}
+
 #endif