Merge pull request #1336 from tclinken/fast-allocate-ptree-nodes

Create 96-byte fast allocator for storage queue PTree nodes

fdbclient/VersionedMap.h

@@ -483,7 +483,8 @@ public:
 		return r->second;
 	}
 
-	static const int overheadPerItem = 128*4;
+	// For each item in the versioned map, 4 PTree nodes are potentially allocated:
+	static const int overheadPerItem = NextFastAllocatedSize<sizeof(PTreeT)>::Result*4;
 	struct iterator;
 
 	VersionedMap() : oldestVersion(0), latestVersion(0) {

fdbserver/storageserver.actor.cpp

@@ -3695,7 +3695,7 @@ void versionedMapTest() {
 	printf("SS Ptree node is %zu bytes\n", sizeof( StorageServer::VersionedData::PTreeT ) );
 
 	const int NSIZE = sizeof(VersionedMap<int,int>::PTreeT);
-	const int ASIZE = NSIZE<=64 ? 64 : NextPowerOfTwo<NSIZE>::Result;
+	const int ASIZE = NSIZE<=64 ? 64 : NextFastAllocatedSize<NSIZE>::Result;
 
 	auto before = FastAllocator< ASIZE >::getTotalMemory();
 

flow/FastAlloc.cpp

@@ -239,14 +239,15 @@ static int64_t getSizeCode(int i) {
 		case 16: return 1;
 		case 32: return 2;
 		case 64: return 3;
-		case 128: return 4;
-		case 256: return 5;
-		case 512: return 6;
-		case 1024: return 7;
-		case 2048: return 8;
-		case 4096: return 9;
-		case 8192: return 10;
-		default: return 11;
+		case 96: return 4;
+		case 128: return 5;
+		case 256: return 6;
+		case 512: return 7;
+		case 1024: return 8;
+		case 2048: return 9;
+		case 4096: return 10;
+		case 8192: return 11;
+		default: return 12;
 	}
 }
 #endif
@@ -496,6 +497,7 @@ void releaseAllThreadMagazines() {
 	FastAllocator<16>::releaseThreadMagazines();
 	FastAllocator<32>::releaseThreadMagazines();
 	FastAllocator<64>::releaseThreadMagazines();
+	FastAllocator<96>::releaseThreadMagazines();
 	FastAllocator<128>::releaseThreadMagazines();
 	FastAllocator<256>::releaseThreadMagazines();
 	FastAllocator<512>::releaseThreadMagazines();
@@ -511,6 +513,7 @@ int64_t getTotalUnusedAllocatedMemory() {
 	unusedMemory += FastAllocator<16>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<32>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<64>::getApproximateMemoryUnused();
+	unusedMemory += FastAllocator<96>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<128>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<256>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<512>::getApproximateMemoryUnused();
@@ -525,6 +528,7 @@ int64_t getTotalUnusedAllocatedMemory() {
 template class FastAllocator<16>;
 template class FastAllocator<32>;
 template class FastAllocator<64>;
+template class FastAllocator<96>;
 template class FastAllocator<128>;
 template class FastAllocator<256>;
 template class FastAllocator<512>;

flow/FastAlloc.h

@@ -158,7 +158,7 @@ int64_t getTotalUnusedAllocatedMemory();
 void setFastAllocatorThreadInitFunction( void (*)() );  // The given function will be called at least once in each thread that allocates from a FastAllocator.  Currently just one such function is tracked.
 
 template<int X>
-class NextPowerOfTwo {
+class NextFastAllocatedSize {
 	static const int A = X-1;
 	static const int B = A | (A>>1);
 	static const int C = B | (B>>2);
@@ -166,7 +166,7 @@ class NextPowerOfTwo {
 	static const int E = D | (D>>8);
 	static const int F = E | (E>>16);
 public:
-	static const int Result = F+1;
+	static const int Result = (X > 64 && X <= 96) ? 96 : F+1;
 };
 
 template <class Object>
@@ -175,13 +175,13 @@ public:
 	static void* operator new(size_t s) {
 		if (s != sizeof(Object)) abort();
 		INSTRUMENT_ALLOCATE(typeid(Object).name());
-		void* p = FastAllocator<sizeof(Object)<=64 ? 64 : NextPowerOfTwo<sizeof(Object)>::Result>::allocate();		
+		void* p = FastAllocator<sizeof(Object)<=64 ? 64 : NextFastAllocatedSize<sizeof(Object)>::Result>::allocate();
 		return p;
 	}
 
 	static void operator delete(void* s) {
 		INSTRUMENT_RELEASE(typeid(Object).name());
-		FastAllocator<sizeof(Object)<=64 ? 64 : NextPowerOfTwo<sizeof(Object)>::Result>::release(s);
+		FastAllocator<sizeof(Object)<=64 ? 64 : NextFastAllocatedSize<sizeof(Object)>::Result>::release(s);
 	}
 	// Redefine placement new so you can still use it
 	static void* operator new( size_t, void* p ) { return p; }
@@ -192,6 +192,7 @@ static void* allocateFast(int size) {
 	if (size <= 16) return FastAllocator<16>::allocate();
 	if (size <= 32) return FastAllocator<32>::allocate();
 	if (size <= 64) return FastAllocator<64>::allocate();
+	if (size <= 96) return FastAllocator<96>::allocate();
 	if (size <= 128) return FastAllocator<128>::allocate();
 	if (size <= 256) return FastAllocator<256>::allocate();
 	if (size <= 512) return FastAllocator<512>::allocate();
@@ -202,6 +203,7 @@ static void freeFast(int size, void* ptr) {
 	if (size <= 16) return FastAllocator<16>::release(ptr);
 	if (size <= 32) return FastAllocator<32>::release(ptr);
 	if (size <= 64) return FastAllocator<64>::release(ptr);
+	if (size <= 96) return FastAllocator<96>::release(ptr);
 	if (size <= 128) return FastAllocator<128>::release(ptr);
 	if (size <= 256) return FastAllocator<256>::release(ptr);
 	if (size <= 512) return FastAllocator<512>::release(ptr);

flow/Platform.cpp

@@ -2477,6 +2477,7 @@ void outOfMemory() {
 	TRACEALLOCATOR(16);
 	TRACEALLOCATOR(32);
 	TRACEALLOCATOR(64);
+	TRACEALLOCATOR(96);
 	TRACEALLOCATOR(128);
 	TRACEALLOCATOR(256);
 	TRACEALLOCATOR(512);

flow/SystemMonitor.cpp

@@ -108,6 +108,7 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta
 				.DETAILALLOCATORMEMUSAGE(16)
 				.DETAILALLOCATORMEMUSAGE(32)
 				.DETAILALLOCATORMEMUSAGE(64)
+				.DETAILALLOCATORMEMUSAGE(96)
 				.DETAILALLOCATORMEMUSAGE(128)
 				.DETAILALLOCATORMEMUSAGE(256)
 				.DETAILALLOCATORMEMUSAGE(512)
@@ -259,6 +260,7 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta
 			TRACEALLOCATOR(16);
 			TRACEALLOCATOR(32);
 			TRACEALLOCATOR(64);
+			TRACEALLOCATOR(96);
 			TRACEALLOCATOR(128);
 			TRACEALLOCATOR(256);
 			TRACEALLOCATOR(512);