Granule file performance benchmark and improvements (#7742)

* added cpu microbenchmark for blob granule files

* Added edge case read benchmarks, and sorting memory deltas

* Sorted merge for granule files

* key block comparison optimization in granule files

* More performance improvements to granule file read

* fixing zlib not supported build

* fixing formatting

* Added debug macro for new debugging prints

* review comments

* more strict compression size validation assert

fdbclient/ServerKnobs.cpp

@@ -914,7 +914,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES,               64*1024 ); if ( randomize && BUGGIFY ) BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES = BG_SNAPSHOT_FILE_TARGET_BYTES / (1 << deterministicRandom()->randomInt(0, 8));
 	init( BG_DELTA_BYTES_BEFORE_COMPACT, BG_SNAPSHOT_FILE_TARGET_BYTES/2 );
 	init( BG_DELTA_FILE_TARGET_BYTES,   BG_DELTA_BYTES_BEFORE_COMPACT/10 );
-	init( BG_DELTA_FILE_TARGET_CHUNK_BYTES,                  64*1024 ); if ( randomize && BUGGIFY ) BG_DELTA_FILE_TARGET_CHUNK_BYTES = BG_DELTA_FILE_TARGET_BYTES / (1 << deterministicRandom()->randomInt(0, 7));
+	init( BG_DELTA_FILE_TARGET_CHUNK_BYTES,                  32*1024 ); if ( randomize && BUGGIFY ) BG_DELTA_FILE_TARGET_CHUNK_BYTES = BG_DELTA_FILE_TARGET_BYTES / (1 << deterministicRandom()->randomInt(0, 7));
 	init( BG_MAX_SPLIT_FANOUT,                                    10 ); if( randomize && BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom()->randomInt(5, 15);
 	init( BG_MAX_MERGE_FANIN,                                     10 ); if( randomize && BUGGIFY ) BG_MAX_MERGE_FANIN = deterministicRandom()->randomInt(2, 15);
 	init( BG_HOT_SNAPSHOT_VERSIONS,                          5000000 );

fdbclient/include/fdbclient/BlobGranuleFiles.h

@@ -27,13 +27,13 @@
 #include "flow/CompressionUtils.h"
 
 Value serializeChunkedSnapshot(const Standalone<StringRef>& fileNameRef,
-                               Standalone<GranuleSnapshot> snapshot,
+                               const Standalone<GranuleSnapshot>& snapshot,
                                int chunkSize,
                                Optional<CompressionFilter> compressFilter,
                                Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx = {});
 
 Value serializeChunkedDeltaFile(const Standalone<StringRef>& fileNameRef,
-                                Standalone<GranuleDeltas> deltas,
+                                const Standalone<GranuleDeltas>& deltas,
                                 const KeyRangeRef& fileRange,
                                 int chunkSize,
                                 Optional<CompressionFilter> compressFilter,

fdbserver/include/fdbserver/DeltaTree.h

@@ -34,40 +34,6 @@
 #define deltatree_printf(...)
 #endif
 
-typedef uint64_t Word;
-// Get the number of prefix bytes that are the same between a and b, up to their common length of cl
-static inline int commonPrefixLength(uint8_t const* ap, uint8_t const* bp, int cl) {
-	int i = 0;
-	const int wordEnd = cl - sizeof(Word) + 1;
-
-	for (; i < wordEnd; i += sizeof(Word)) {
-		Word a = *(Word*)ap;
-		Word b = *(Word*)bp;
-		if (a != b) {
-			return i + ctzll(a ^ b) / 8;
-		}
-		ap += sizeof(Word);
-		bp += sizeof(Word);
-	}
-
-	for (; i < cl; i++) {
-		if (*ap != *bp) {
-			return i;
-		}
-		++ap;
-		++bp;
-	}
-	return cl;
-}
-
-static inline int commonPrefixLength(const StringRef& a, const StringRef& b) {
-	return commonPrefixLength(a.begin(), b.begin(), std::min(a.size(), b.size()));
-}
-
-static inline int commonPrefixLength(const StringRef& a, const StringRef& b, int skipLen) {
-	return commonPrefixLength(a.begin() + skipLen, b.begin() + skipLen, std::min(a.size(), b.size()) - skipLen);
-}
-
 // This appears to be the fastest version
 static int lessOrEqualPowerOfTwo(int n) {
 	int p;

flow/CompressionUtils.cpp

@@ -128,4 +128,25 @@ TEST_CASE("/CompressionUtils/gzipCompression") {
 
 	return Void();
 }
+
+TEST_CASE("/CompressionUtils/gzipCompression2") {
+	Arena arena;
+	const int size = deterministicRandom()->randomInt(512, 1024);
+	std::string s(size, 'x');
+	Standalone<StringRef> uncompressed = Standalone<StringRef>(StringRef(s));
+	printf("Size before: %d\n", (int)uncompressed.size());
+
+	Standalone<StringRef> compressed = CompressionUtils::compress(CompressionFilter::GZIP, uncompressed, arena);
+	ASSERT_NE(compressed.compare(uncompressed), 0);
+	printf("Size after: %d\n", (int)compressed.size());
+	// Assert compressed size is less than half.
+	ASSERT(compressed.size() * 2 < uncompressed.size());
+
+	StringRef verify = CompressionUtils::decompress(CompressionFilter::GZIP, compressed, arena);
+	ASSERT_EQ(verify.compare(uncompressed), 0);
+
+	TraceEvent("GzipCompression_Done").log();
+
+	return Void();
+}
 #endif

flow/include/flow/Arena.h

@@ -853,6 +853,40 @@ inline bool operator>=(const StringRef& lhs, const StringRef& rhs) {
 	return !(lhs < rhs);
 }
 
+typedef uint64_t Word;
+// Get the number of prefix bytes that are the same between a and b, up to their common length of cl
+static inline int commonPrefixLength(uint8_t const* ap, uint8_t const* bp, int cl) {
+	int i = 0;
+	const int wordEnd = cl - sizeof(Word) + 1;
+
+	for (; i < wordEnd; i += sizeof(Word)) {
+		Word a = *(Word*)ap;
+		Word b = *(Word*)bp;
+		if (a != b) {
+			return i + ctzll(a ^ b) / 8;
+		}
+		ap += sizeof(Word);
+		bp += sizeof(Word);
+	}
+
+	for (; i < cl; i++) {
+		if (*ap != *bp) {
+			return i;
+		}
+		++ap;
+		++bp;
+	}
+	return cl;
+}
+
+static inline int commonPrefixLength(const StringRef& a, const StringRef& b) {
+	return commonPrefixLength(a.begin(), b.begin(), std::min(a.size(), b.size()));
+}
+
+static inline int commonPrefixLength(const StringRef& a, const StringRef& b, int skipLen) {
+	return commonPrefixLength(a.begin() + skipLen, b.begin() + skipLen, std::min(a.size(), b.size()) - skipLen);
+}
+
 // This trait is used by VectorRef to determine if deep copy constructor should recursively
 // call deep copies of each element.
 //