From 007abbc45b4dbf2208ea7f47b99b0c567c94b2ce Mon Sep 17 00:00:00 2001
From: Trevor Clinkenbeard <trevorclinkenbeard@gmail.com>
Date: Wed, 20 Mar 2019 10:11:12 -0700
Subject: [PATCH 001/118] Added 96-byte FastAllocator

Since storage queue nodes account for a large portion of memory usage,
we can save space by only allocating 96 bytes instead of 128 bytes for
each node.
---
 fdbclient/VersionedMap.h          |  2 +-
 fdbserver/storageserver.actor.cpp |  2 +-
 flow/Arena.h                      |  6 ++++--
 flow/FastAlloc.cpp                | 20 ++++++++++++--------
 flow/FastAlloc.h                  | 10 ++++++----
 5 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index f0dc5e2b46..fae6d85b13 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -483,7 +483,7 @@ public:
 		return r->second;
 	}
 
-	static const int overheadPerItem = 128*4;
+	static const int overheadPerItem = 96*4;
 	struct iterator;
 
 	VersionedMap() : oldestVersion(0), latestVersion(0) {
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index 9d0f07f025..7c09e8f6c1 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -3693,7 +3693,7 @@ void versionedMapTest() {
 	printf("SS Ptree node is %zu bytes\n", sizeof( StorageServer::VersionedData::PTreeT ) );
 
 	const int NSIZE = sizeof(VersionedMap<int,int>::PTreeT);
-	const int ASIZE = NSIZE<=64 ? 64 : NextPowerOfTwo<NSIZE>::Result;
+	const int ASIZE = NSIZE<=64 ? 64 : NextFastAllocatedSize<NSIZE>::Result;
 
 	auto before = FastAllocator< ASIZE >::getTotalMemory();
 
diff --git a/flow/Arena.h b/flow/Arena.h
index 258be4d3cb..263074d2b8 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -222,7 +222,8 @@ struct ArenaBlock : NonCopyable, ThreadSafeReferenceCounted<ArenaBlock>
 			}
 
 			if (reqSize < LARGE) {
-				if (reqSize <= 128) { b = (ArenaBlock*)FastAllocator<128>::allocate(); b->bigSize = 128; INSTRUMENT_ALLOCATE("Arena128"); }
+				if (reqSize <= 96) { b = (ArenaBlock*)FastAllocator<96>::allocate(); b->bigSize = 96; INSTRUMENT_ALLOCATE("Arena96"); }
+				else if (reqSize <= 128) { b = (ArenaBlock*)FastAllocator<128>::allocate(); b->bigSize = 128; INSTRUMENT_ALLOCATE("Arena128"); }
 				else if (reqSize <= 256) { b = (ArenaBlock*)FastAllocator<256>::allocate(); b->bigSize = 256; INSTRUMENT_ALLOCATE("Arena256"); }
 				else if (reqSize <= 512) { b = (ArenaBlock*)FastAllocator<512>::allocate(); b->bigSize = 512; INSTRUMENT_ALLOCATE("Arena512"); }
 				else if (reqSize <= 1024) { b = (ArenaBlock*)FastAllocator<1024>::allocate(); b->bigSize = 1024; INSTRUMENT_ALLOCATE("Arena1024"); }
@@ -264,7 +265,8 @@ struct ArenaBlock : NonCopyable, ThreadSafeReferenceCounted<ArenaBlock>
 			else if (tinySize <= 32) { FastAllocator<32>::release(this); INSTRUMENT_RELEASE("Arena32"); }
 			else { FastAllocator<64>::release(this); INSTRUMENT_RELEASE("Arena64"); }
 		} else {
-			if (bigSize <= 128) { FastAllocator<128>::release(this); INSTRUMENT_RELEASE("Arena128"); }
+			if (bigSize <= 96) { FastAllocator<96>::release(this); INSTRUMENT_RELEASE("Arena96"); }
+			else if (bigSize <= 128) { FastAllocator<128>::release(this); INSTRUMENT_RELEASE("Arena128"); }
 			else if (bigSize <= 256) { FastAllocator<256>::release(this); INSTRUMENT_RELEASE("Arena256"); }
 			else if (bigSize <= 512) { FastAllocator<512>::release(this); INSTRUMENT_RELEASE("Arena512"); }
 			else if (bigSize <= 1024) { FastAllocator<1024>::release(this); INSTRUMENT_RELEASE("Arena1024"); }
diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp
index 83766000ac..d2230cf395 100644
--- a/flow/FastAlloc.cpp
+++ b/flow/FastAlloc.cpp
@@ -219,14 +219,15 @@ static int64_t getSizeCode(int i) {
 		case 16: return 1;
 		case 32: return 2;
 		case 64: return 3;
-		case 128: return 4;
-		case 256: return 5;
-		case 512: return 6;
-		case 1024: return 7;
-		case 2048: return 8;
-		case 4096: return 9;
-		case 8192: return 10;
-		default: return 11;
+		case 96: return 4;
+		case 128: return 5;
+		case 256: return 6;
+		case 512: return 7;
+		case 1024: return 8;
+		case 2048: return 9;
+		case 4096: return 10;
+		case 8192: return 11;
+		default: return 12;
 	}
 }
 
@@ -475,6 +476,7 @@ void releaseAllThreadMagazines() {
 	FastAllocator<16>::releaseThreadMagazines();
 	FastAllocator<32>::releaseThreadMagazines();
 	FastAllocator<64>::releaseThreadMagazines();
+	FastAllocator<96>::releaseThreadMagazines();
 	FastAllocator<128>::releaseThreadMagazines();
 	FastAllocator<256>::releaseThreadMagazines();
 	FastAllocator<512>::releaseThreadMagazines();
@@ -490,6 +492,7 @@ int64_t getTotalUnusedAllocatedMemory() {
 	unusedMemory += FastAllocator<16>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<32>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<64>::getApproximateMemoryUnused();
+	unusedMemory += FastAllocator<96>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<128>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<256>::getApproximateMemoryUnused();
 	unusedMemory += FastAllocator<512>::getApproximateMemoryUnused();
@@ -504,6 +507,7 @@ int64_t getTotalUnusedAllocatedMemory() {
 template class FastAllocator<16>;
 template class FastAllocator<32>;
 template class FastAllocator<64>;
+template class FastAllocator<96>;
 template class FastAllocator<128>;
 template class FastAllocator<256>;
 template class FastAllocator<512>;
diff --git a/flow/FastAlloc.h b/flow/FastAlloc.h
index 06f35711c3..b522f5bf0b 100644
--- a/flow/FastAlloc.h
+++ b/flow/FastAlloc.h
@@ -156,7 +156,7 @@ int64_t getTotalUnusedAllocatedMemory();
 void setFastAllocatorThreadInitFunction( void (*)() );  // The given function will be called at least once in each thread that allocates from a FastAllocator.  Currently just one such function is tracked.
 
 template<int X>
-class NextPowerOfTwo {
+class NextFastAllocatedSize {
 	static const int A = X-1;
 	static const int B = A | (A>>1);
 	static const int C = B | (B>>2);
@@ -164,7 +164,7 @@ class NextPowerOfTwo {
 	static const int E = D | (D>>8);
 	static const int F = E | (E>>16);
 public:
-	static const int Result = F+1;
+	static const int Result = (X > 64 && X <= 96) ? 96 : F+1;
 };
 
 template <class Object>
@@ -173,13 +173,13 @@ public:
 	static void* operator new(size_t s) {
 		if (s != sizeof(Object)) abort();
 		INSTRUMENT_ALLOCATE(typeid(Object).name());
-		void* p = FastAllocator<sizeof(Object)<=64 ? 64 : NextPowerOfTwo<sizeof(Object)>::Result>::allocate();		
+		void* p = FastAllocator<sizeof(Object)<=64 ? 64 : NextFastAllocatedSize<sizeof(Object)>::Result>::allocate();
 		return p;
 	}
 
 	static void operator delete(void* s) {
 		INSTRUMENT_RELEASE(typeid(Object).name());
-		FastAllocator<sizeof(Object)<=64 ? 64 : NextPowerOfTwo<sizeof(Object)>::Result>::release(s);
+		FastAllocator<sizeof(Object)<=64 ? 64 : NextFastAllocatedSize<sizeof(Object)>::Result>::release(s);
 	}
 	// Redefine placement new so you can still use it
 	static void* operator new( size_t, void* p ) { return p; }
@@ -190,6 +190,7 @@ static void* allocateFast(int size) {
 	if (size <= 16) return FastAllocator<16>::allocate();
 	if (size <= 32) return FastAllocator<32>::allocate();
 	if (size <= 64) return FastAllocator<64>::allocate();
+	if (size <= 96) return FastAllocator<96>::allocate();
 	if (size <= 128) return FastAllocator<128>::allocate();
 	if (size <= 256) return FastAllocator<256>::allocate();
 	if (size <= 512) return FastAllocator<512>::allocate();
@@ -200,6 +201,7 @@ static void freeFast(int size, void* ptr) {
 	if (size <= 16) return FastAllocator<16>::release(ptr);
 	if (size <= 32) return FastAllocator<32>::release(ptr);
 	if (size <= 64) return FastAllocator<64>::release(ptr);
+	if (size <= 96) return FastAllocator<96>::release(ptr);
 	if (size <= 128) return FastAllocator<128>::release(ptr);
 	if (size <= 256) return FastAllocator<256>::release(ptr);
 	if (size <= 512) return FastAllocator<512>::release(ptr);

From 4a3467296f3c92ea67f290d96ee72091f55ab183 Mon Sep 17 00:00:00 2001
From: Trevor Clinkenbeard <trevorclinkenbeard@gmail.com>
Date: Wed, 20 Mar 2019 17:11:53 -0700
Subject: [PATCH 002/118] Removed Arena96

---
 flow/Arena.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/flow/Arena.h b/flow/Arena.h
index 263074d2b8..258be4d3cb 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -222,8 +222,7 @@ struct ArenaBlock : NonCopyable, ThreadSafeReferenceCounted<ArenaBlock>
 			}
 
 			if (reqSize < LARGE) {
-				if (reqSize <= 96) { b = (ArenaBlock*)FastAllocator<96>::allocate(); b->bigSize = 96; INSTRUMENT_ALLOCATE("Arena96"); }
-				else if (reqSize <= 128) { b = (ArenaBlock*)FastAllocator<128>::allocate(); b->bigSize = 128; INSTRUMENT_ALLOCATE("Arena128"); }
+				if (reqSize <= 128) { b = (ArenaBlock*)FastAllocator<128>::allocate(); b->bigSize = 128; INSTRUMENT_ALLOCATE("Arena128"); }
 				else if (reqSize <= 256) { b = (ArenaBlock*)FastAllocator<256>::allocate(); b->bigSize = 256; INSTRUMENT_ALLOCATE("Arena256"); }
 				else if (reqSize <= 512) { b = (ArenaBlock*)FastAllocator<512>::allocate(); b->bigSize = 512; INSTRUMENT_ALLOCATE("Arena512"); }
 				else if (reqSize <= 1024) { b = (ArenaBlock*)FastAllocator<1024>::allocate(); b->bigSize = 1024; INSTRUMENT_ALLOCATE("Arena1024"); }
@@ -265,8 +264,7 @@ struct ArenaBlock : NonCopyable, ThreadSafeReferenceCounted<ArenaBlock>
 			else if (tinySize <= 32) { FastAllocator<32>::release(this); INSTRUMENT_RELEASE("Arena32"); }
 			else { FastAllocator<64>::release(this); INSTRUMENT_RELEASE("Arena64"); }
 		} else {
-			if (bigSize <= 96) { FastAllocator<96>::release(this); INSTRUMENT_RELEASE("Arena96"); }
-			else if (bigSize <= 128) { FastAllocator<128>::release(this); INSTRUMENT_RELEASE("Arena128"); }
+			if (bigSize <= 128) { FastAllocator<128>::release(this); INSTRUMENT_RELEASE("Arena128"); }
 			else if (bigSize <= 256) { FastAllocator<256>::release(this); INSTRUMENT_RELEASE("Arena256"); }
 			else if (bigSize <= 512) { FastAllocator<512>::release(this); INSTRUMENT_RELEASE("Arena512"); }
 			else if (bigSize <= 1024) { FastAllocator<1024>::release(this); INSTRUMENT_RELEASE("Arena1024"); }

From dbcf1d717c28837f02427e684e6f70c3f1a3fcd8 Mon Sep 17 00:00:00 2001
From: Trevor Clinkenbeard <trevorclinkenbeard@gmail.com>
Date: Thu, 21 Mar 2019 13:18:14 -0700
Subject: [PATCH 003/118] Added calls to TRACEALLOCATOR(96) and
 DETAILALLOCATORMEMUSAGE(96)

---
 flow/Platform.cpp      | 1 +
 flow/SystemMonitor.cpp | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/flow/Platform.cpp b/flow/Platform.cpp
index e08527eaa5..c9815300cd 100644
--- a/flow/Platform.cpp
+++ b/flow/Platform.cpp
@@ -2370,6 +2370,7 @@ void outOfMemory() {
 	TRACEALLOCATOR(16);
 	TRACEALLOCATOR(32);
 	TRACEALLOCATOR(64);
+	TRACEALLOCATOR(96);
 	TRACEALLOCATOR(128);
 	TRACEALLOCATOR(256);
 	TRACEALLOCATOR(512);
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 207e5c30d7..18a6c1b6d2 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -105,6 +105,7 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta
 				.DETAILALLOCATORMEMUSAGE(16)
 				.DETAILALLOCATORMEMUSAGE(32)
 				.DETAILALLOCATORMEMUSAGE(64)
+				.DETAILALLOCATORMEMUSAGE(96)
 				.DETAILALLOCATORMEMUSAGE(128)
 				.DETAILALLOCATORMEMUSAGE(256)
 				.DETAILALLOCATORMEMUSAGE(512)
@@ -254,6 +255,7 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta
 			TRACEALLOCATOR(16);
 			TRACEALLOCATOR(32);
 			TRACEALLOCATOR(64);
+			TRACEALLOCATOR(96);
 			TRACEALLOCATOR(128);
 			TRACEALLOCATOR(256);
 			TRACEALLOCATOR(512);

From 4293a3e230f20fe90346b54c391d86a1bec56127 Mon Sep 17 00:00:00 2001
From: Trevor Clinkenbeard <trevorclinkenbeard@gmail.com>
Date: Thu, 21 Mar 2019 13:59:20 -0700
Subject: [PATCH 004/118] Generalized VersionedMap<K, T>::overheadPerItem
 calculation

---
 fdbclient/VersionedMap.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbclient/VersionedMap.h b/fdbclient/VersionedMap.h
index fae6d85b13..a9b1eb00ab 100644
--- a/fdbclient/VersionedMap.h
+++ b/fdbclient/VersionedMap.h
@@ -483,7 +483,8 @@ public:
 		return r->second;
 	}
 
-	static const int overheadPerItem = 96*4;
+	// For each item in the versioned map, 4 PTree nodes are potentially allocated:
+	static const int overheadPerItem = NextFastAllocatedSize<sizeof(PTreeT)>::Result*4;
 	struct iterator;
 
 	VersionedMap() : oldestVersion(0), latestVersion(0) {

From d6a4a0a0bb0d79f71ab896dada40a33ef0c679cb Mon Sep 17 00:00:00 2001
From: Amir Abu Shareb <yields@icloud.com>
Date: Wed, 3 Apr 2019 18:47:18 +0300
Subject: [PATCH 005/118] bindings/go: keep futures alive on use

---
 bindings/go/src/fdb/futures.go | 36 ++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go
index 747a1135e7..fe7b946c75 100644
--- a/bindings/go/src/fdb/futures.go
+++ b/bindings/go/src/fdb/futures.go
@@ -100,15 +100,18 @@ func fdb_future_block_until_ready(f *C.FDBFuture) {
 	m.Lock()
 }
 
-func (f future) BlockUntilReady() {
+func (f *future) BlockUntilReady() {
+	defer runtime.KeepAlive(f)
 	fdb_future_block_until_ready(f.ptr)
 }
 
-func (f future) IsReady() bool {
+func (f *future) IsReady() bool {
+	defer runtime.KeepAlive(f)
 	return C.fdb_future_is_ready(f.ptr) != 0
 }
 
-func (f future) Cancel() {
+func (f *future) Cancel() {
+	defer runtime.KeepAlive(f)
 	C.fdb_future_cancel(f.ptr)
 }
 
@@ -140,6 +143,8 @@ type futureByteSlice struct {
 
 func (f *futureByteSlice) Get() ([]byte, error) {
 	f.o.Do(func() {
+		defer runtime.KeepAlive(f.future)
+
 		var present C.fdb_bool_t
 		var value *C.uint8_t
 		var length C.int
@@ -195,6 +200,8 @@ type futureKey struct {
 
 func (f *futureKey) Get() (Key, error) {
 	f.o.Do(func() {
+		defer runtime.KeepAlive(f.future)
+
 		var value *C.uint8_t
 		var length C.int
 
@@ -241,7 +248,9 @@ type futureNil struct {
 	*future
 }
 
-func (f futureNil) Get() error {
+func (f *futureNil) Get() error {
+	defer runtime.KeepAlive(f.future)
+
 	f.BlockUntilReady()
 	if err := C.fdb_future_get_error(f.ptr); err != 0 {
 		return Error{int(err)}
@@ -250,7 +259,7 @@ func (f futureNil) Get() error {
 	return nil
 }
 
-func (f futureNil) MustGet() {
+func (f *futureNil) MustGet() {
 	if err := f.Get(); err != nil {
 		panic(err)
 	}
@@ -272,7 +281,9 @@ func stringRefToSlice(ptr unsafe.Pointer) []byte {
 	return C.GoBytes(src, size)
 }
 
-func (f futureKeyValueArray) Get() ([]KeyValue, bool, error) {
+func (f *futureKeyValueArray) Get() ([]KeyValue, bool, error) {
+	defer runtime.KeepAlive(f.future)
+
 	f.BlockUntilReady()
 
 	var kvs *C.FDBKeyValue
@@ -316,17 +327,20 @@ type futureInt64 struct {
 	*future
 }
 
-func (f futureInt64) Get() (int64, error) {
+func (f *futureInt64) Get() (int64, error) {
+	defer runtime.KeepAlive(f.future)
+
 	f.BlockUntilReady()
 
 	var ver C.int64_t
 	if err := C.fdb_future_get_version(f.ptr, &ver); err != 0 {
 		return 0, Error{int(err)}
 	}
+
 	return int64(ver), nil
 }
 
-func (f futureInt64) MustGet() int64 {
+func (f *futureInt64) MustGet() int64 {
 	val, err := f.Get()
 	if err != nil {
 		panic(err)
@@ -356,7 +370,9 @@ type futureStringSlice struct {
 	*future
 }
 
-func (f futureStringSlice) Get() ([]string, error) {
+func (f *futureStringSlice) Get() ([]string, error) {
+	defer runtime.KeepAlive(f.future)
+
 	f.BlockUntilReady()
 
 	var strings **C.char
@@ -375,7 +391,7 @@ func (f futureStringSlice) Get() ([]string, error) {
 	return ret, nil
 }
 
-func (f futureStringSlice) MustGet() []string {
+func (f *futureStringSlice) MustGet() []string {
 	val, err := f.Get()
 	if err != nil {
 		panic(err)

From 2462d292357d320013a0b8c0a3828fd707f59329 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Tue, 30 Apr 2019 11:33:12 +0200
Subject: [PATCH 006/118] AsyncFileCached: switch from a random to an LRU cache
 eviction policy

---
 fdbrpc/AsyncFileCached.actor.cpp | 11 ++++++-----
 fdbrpc/AsyncFileCached.actor.h   | 31 ++++++++++++++++++++++---------
 2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.actor.cpp
index e8fb62e2e3..3242215652 100644
--- a/fdbrpc/AsyncFileCached.actor.cpp
+++ b/fdbrpc/AsyncFileCached.actor.cpp
@@ -33,11 +33,8 @@ EvictablePage::~EvictablePage() {
 		else
 			aligned_free(data);
 	}
-	if (index > -1) {
-		pageCache->pages[index] = pageCache->pages.back();
-		pageCache->pages[index]->index = index;
-		pageCache->pages.pop_back();
-	}
+	// remove it from the LRU
+	pageCache->lruPages.erase(EvictablePageCache::List::s_iterator_to(*this));
 }
 
 std::map< std::string, OpenFileInfo > AsyncFileCached::openFiles;
@@ -97,6 +94,8 @@ Future<Void> AsyncFileCached::read_write_impl( AsyncFileCached* self, void* data
 		if ( p == self->pages.end() ) {
 			AFCPage* page = new AFCPage( self, pageOffset );
 			p = self->pages.insert( std::make_pair(pageOffset, page) ).first;
+		} else {
+			self->pageCache->updateHit(p->second);
 		}
 
 		int bytesInPage = std::min(self->pageCache->pageSize - offsetInPage, remaining);
@@ -133,6 +132,8 @@ Future<Void> AsyncFileCached::readZeroCopy( void** data, int* length, int64_t of
 	if ( p == pages.end() ) {
 		AFCPage* page = new AFCPage( this, offset );
 		p = pages.insert( std::make_pair(offset, page) ).first;
+	} else {
+		p->second->pageCache->updateHit(p->second);
 	}
 
 	*data = p->second->data;
diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h
index 51c04abc2e..5f2ca3a2a7 100644
--- a/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/AsyncFileCached.actor.h
@@ -27,6 +27,8 @@
 #elif !defined(FLOW_ASYNCFILECACHED_ACTOR_H)
 	#define FLOW_ASYNCFILECACHED_ACTOR_H
 
+#include <boost/intrusive/list.hpp>
+
 #include "flow/flow.h"
 #include "fdbrpc/IAsyncFile.h"
 #include "flow/Knobs.h"
@@ -34,18 +36,21 @@
 #include "flow/network.h"
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
+namespace bi = boost::intrusive;
 struct EvictablePage {
 	void* data;
-	int index;
 	class Reference<struct EvictablePageCache> pageCache;
+	bi::list_member_hook<> member_hook;
 
 	virtual bool evict() = 0; // true if page was evicted, false if it isn't immediately evictable (but will be evicted regardless if possible)
 
-	EvictablePage(Reference<EvictablePageCache> pageCache) : data(0), index(-1), pageCache(pageCache) {}
+	EvictablePage(Reference<EvictablePageCache> pageCache) : data(0), pageCache(pageCache) {}
 	virtual ~EvictablePage();
 };
 
 struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
+	using List = bi::list< EvictablePage, bi::member_hook< EvictablePage, bi::list_member_hook<>, &EvictablePage::member_hook>>;
+
 	EvictablePageCache() : pageSize(0), maxPages(0) {}
 	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize) {}
 
@@ -53,21 +58,29 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 		try_evict();
 		try_evict();
 		page->data = pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096,pageSize);
-		page->index = pages.size();
-		pages.push_back(page);
+		lruPages.push_back(*page); // new page is considered the most recently used (placed at LRU tail)
+	}
+
+	void updateHit(EvictablePage* page) {
+		// on a hit, update page's location in the LRU so that it's most recent (tail)
+		lruPages.erase(List::s_iterator_to(*page));
+		lruPages.push_back(*page);
 	}
 
 	void try_evict() {
-		if (pages.size() >= (uint64_t)maxPages && !pages.empty()) {
-			for (int i = 0; i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS; i++) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
-				int toEvict = g_random->randomInt(0, pages.size());
-				if (pages[toEvict]->evict())
+		if (lruPages.size() >= (uint64_t)maxPages) {
+			int i = 0;
+			// try the least recently used pages first (starting at head of the LRU list)
+			for (List::iterator it = lruPages.begin();
+			     it != lruPages.end() && i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS;
+			     ++it, ++i) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
+				if (it->evict())
 					break;
 			}
 		}
 	}
 
-	std::vector<EvictablePage*> pages;
+	List lruPages;
 	int pageSize;
 	int64_t maxPages;
 };

From b2280e15bfa46d1ccfb6077eaf9baffd90cc1e64 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Thu, 2 May 2019 17:35:30 +0200
Subject: [PATCH 007/118] AsyncFileCached: support for lru cache eviction
 policy

- Added a knob to control page cache eviction policy (0=random default, 1=lru)
- Added page cache hit, miss, and eviction stats
---
 fdbrpc/AsyncFileCached.actor.cpp | 12 +++++-
 fdbrpc/AsyncFileCached.actor.h   | 63 +++++++++++++++++++++++++-------
 flow/Knobs.cpp                   |  1 +
 flow/Knobs.h                     |  1 +
 4 files changed, 61 insertions(+), 16 deletions(-)

diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.actor.cpp
index 3242215652..7daecd99e2 100644
--- a/fdbrpc/AsyncFileCached.actor.cpp
+++ b/fdbrpc/AsyncFileCached.actor.cpp
@@ -33,8 +33,16 @@ EvictablePage::~EvictablePage() {
 		else
 			aligned_free(data);
 	}
-	// remove it from the LRU
-	pageCache->lruPages.erase(EvictablePageCache::List::s_iterator_to(*this));
+	if (EvictablePageCache::RANDOM == FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+		if (index > -1) {
+			pageCache->pages[index] = pageCache->pages.back();
+			pageCache->pages[index]->index = index;
+			pageCache->pages.pop_back();
+		}
+	} else {
+		// remove it from the LRU
+		pageCache->lruPages.erase(EvictablePageCache::List::s_iterator_to(*this));
+	}
 }
 
 std::map< std::string, OpenFileInfo > AsyncFileCached::openFiles;
diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h
index 5f2ca3a2a7..55e6892775 100644
--- a/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/AsyncFileCached.actor.h
@@ -39,50 +39,85 @@
 namespace bi = boost::intrusive;
 struct EvictablePage {
 	void* data;
+	int index;
 	class Reference<struct EvictablePageCache> pageCache;
 	bi::list_member_hook<> member_hook;
 
 	virtual bool evict() = 0; // true if page was evicted, false if it isn't immediately evictable (but will be evicted regardless if possible)
 
-	EvictablePage(Reference<EvictablePageCache> pageCache) : data(0), pageCache(pageCache) {}
+	EvictablePage(Reference<EvictablePageCache> pageCache) : data(0), index(-1), pageCache(pageCache) {}
 	virtual ~EvictablePage();
 };
 
 struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	using List = bi::list< EvictablePage, bi::member_hook< EvictablePage, bi::list_member_hook<>, &EvictablePage::member_hook>>;
 
-	EvictablePageCache() : pageSize(0), maxPages(0) {}
+	EvictablePageCache() : pageSize(0), maxPages(0)	{
+		cacheHits.init(LiteralStringRef("EvictablePageCache.CacheHits"));
+		cacheMisses.init(LiteralStringRef("EvictablePageCache.CacheMisses"));
+		cacheEvictions.init(LiteralStringRef("EvictablePageCache.CacheEviction"));
+	}
+
 	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize) {}
 
 	void allocate(EvictablePage* page) {
 		try_evict();
 		try_evict();
 		page->data = pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096,pageSize);
-		lruPages.push_back(*page); // new page is considered the most recently used (placed at LRU tail)
+		if (RANDOM == FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+			page->index = pages.size();
+			pages.push_back(page);
+		} else {
+			lruPages.push_back(*page); // new page is considered the most recently used (placed at LRU tail)
+		}
+		cacheMisses++;
 	}
 
 	void updateHit(EvictablePage* page) {
-		// on a hit, update page's location in the LRU so that it's most recent (tail)
-		lruPages.erase(List::s_iterator_to(*page));
-		lruPages.push_back(*page);
+		if (RANDOM != FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+			// on a hit, update page's location in the LRU so that it's most recent (tail)
+			lruPages.erase(List::s_iterator_to(*page));
+			lruPages.push_back(*page);
+		}
+		cacheHits++;
 	}
 
 	void try_evict() {
-		if (lruPages.size() >= (uint64_t)maxPages) {
-			int i = 0;
-			// try the least recently used pages first (starting at head of the LRU list)
-			for (List::iterator it = lruPages.begin();
-			     it != lruPages.end() && i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS;
-			     ++it, ++i) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
-				if (it->evict())
-					break;
+		if (RANDOM == FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+			if (pages.size() >= (uint64_t)maxPages && !pages.empty()) {
+				for (int i = 0; i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS; i++) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
+					int toEvict = g_random->randomInt(0, pages.size());
+					if (pages[toEvict]->evict()) {
+						cacheEvictions++;
+						break;
+					}
+				}
+			}
+		} else {
+			// For now, LRU is the only other CACHE_EVICTION option
+			if (lruPages.size() >= (uint64_t)maxPages) {
+				int i = 0;
+				// try the least recently used pages first (starting at head of the LRU list)
+				for (List::iterator it = lruPages.begin();
+				     it != lruPages.end() && i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS;
+				     ++it, ++i) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
+					if (it->evict()) {
+						cacheEvictions++;
+						break;
+					}
+				}
 			}
 		}
 	}
 
+	std::vector<EvictablePage*> pages;
 	List lruPages;
 	int pageSize;
 	int64_t maxPages;
+	Int64MetricHandle cacheHits;
+	Int64MetricHandle cacheMisses;
+	Int64MetricHandle cacheEvictions;
+	enum CacheEvictionType { RANDOM = 0, LRU = 1 };
 };
 
 struct OpenFileInfo : NonCopyable {
diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 75e7001bc8..64049c72aa 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -77,6 +77,7 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
 	init( BUGGIFY_SIM_PAGE_CACHE_4K,                           1e6 );
 	init( BUGGIFY_SIM_PAGE_CACHE_64K,                          1e6 );
 	init( MAX_EVICT_ATTEMPTS,                                  100 ); if( randomize && BUGGIFY ) MAX_EVICT_ATTEMPTS = 2;
+	init( CACHE_EVICTION_POLICY,                                 0 );
 	init( PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION,                 0.1 ); if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 0.0; else if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 1.0;
 
 	//AsyncFileKAIO
diff --git a/flow/Knobs.h b/flow/Knobs.h
index 9becb6f052..5490752ad7 100644
--- a/flow/Knobs.h
+++ b/flow/Knobs.h
@@ -93,6 +93,7 @@ public:
 	int64_t SIM_PAGE_CACHE_64K;
 	int64_t BUGGIFY_SIM_PAGE_CACHE_4K;
 	int64_t BUGGIFY_SIM_PAGE_CACHE_64K;
+	int CACHE_EVICTION_POLICY; // 0 RANDOM (default), 1 LRU
 	int MAX_EVICT_ATTEMPTS;
 	double PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION;
 	double TOO_MANY_CONNECTIONS_CLOSED_RESET_DELAY;

From fdb39929902dc818021374a32f89bd0d5af63d39 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Fri, 3 May 2019 14:04:43 +0200
Subject: [PATCH 008/118] AsyncFileCached: switch to string for eviction policy
 knob

---
 fdbrpc/AsyncFileCached.actor.cpp |  2 +-
 fdbrpc/AsyncFileCached.actor.h   | 22 +++++++++++++++-------
 flow/Knobs.cpp                   |  2 +-
 flow/Knobs.h                     |  2 +-
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/fdbrpc/AsyncFileCached.actor.cpp b/fdbrpc/AsyncFileCached.actor.cpp
index 7daecd99e2..ec5b6f6e73 100644
--- a/fdbrpc/AsyncFileCached.actor.cpp
+++ b/fdbrpc/AsyncFileCached.actor.cpp
@@ -33,7 +33,7 @@ EvictablePage::~EvictablePage() {
 		else
 			aligned_free(data);
 	}
-	if (EvictablePageCache::RANDOM == FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+	if (EvictablePageCache::RANDOM == pageCache->cacheEvictionType) {
 		if (index > -1) {
 			pageCache->pages[index] = pageCache->pages.back();
 			pageCache->pages[index]->index = index;
diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h
index 55e6892775..84a1d29849 100644
--- a/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/AsyncFileCached.actor.h
@@ -52,19 +52,26 @@ struct EvictablePage {
 struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	using List = bi::list< EvictablePage, bi::member_hook< EvictablePage, bi::list_member_hook<>, &EvictablePage::member_hook>>;
 
-	EvictablePageCache() : pageSize(0), maxPages(0)	{
+	EvictablePageCache() : pageSize(0), maxPages(0) {}
+
+	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize) {
+		std::string cep = FLOW_KNOBS->CACHE_EVICTION_POLICY;
+		std::transform(cep.begin(), cep.end(), cep.begin(), ::tolower);
+		if (0 == cep.compare("random"))
+			cacheEvictionType = RANDOM;
+		else
+			cacheEvictionType = LRU;
+
 		cacheHits.init(LiteralStringRef("EvictablePageCache.CacheHits"));
 		cacheMisses.init(LiteralStringRef("EvictablePageCache.CacheMisses"));
 		cacheEvictions.init(LiteralStringRef("EvictablePageCache.CacheEviction"));
 	}
 
-	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize) {}
-
 	void allocate(EvictablePage* page) {
 		try_evict();
 		try_evict();
 		page->data = pageSize == 4096 ? FastAllocator<4096>::allocate() : aligned_alloc(4096,pageSize);
-		if (RANDOM == FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+		if (RANDOM == cacheEvictionType) {
 			page->index = pages.size();
 			pages.push_back(page);
 		} else {
@@ -74,7 +81,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	}
 
 	void updateHit(EvictablePage* page) {
-		if (RANDOM != FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+		if (RANDOM != cacheEvictionType) {
 			// on a hit, update page's location in the LRU so that it's most recent (tail)
 			lruPages.erase(List::s_iterator_to(*page));
 			lruPages.push_back(*page);
@@ -83,7 +90,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	}
 
 	void try_evict() {
-		if (RANDOM == FLOW_KNOBS->CACHE_EVICTION_POLICY) {
+		if (RANDOM == cacheEvictionType) {
 			if (pages.size() >= (uint64_t)maxPages && !pages.empty()) {
 				for (int i = 0; i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS; i++) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
 					int toEvict = g_random->randomInt(0, pages.size());
@@ -110,6 +117,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 		}
 	}
 
+	enum CacheEvictionType { RANDOM = 0, LRU = 1 };
 	std::vector<EvictablePage*> pages;
 	List lruPages;
 	int pageSize;
@@ -117,7 +125,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	Int64MetricHandle cacheHits;
 	Int64MetricHandle cacheMisses;
 	Int64MetricHandle cacheEvictions;
-	enum CacheEvictionType { RANDOM = 0, LRU = 1 };
+	CacheEvictionType cacheEvictionType;
 };
 
 struct OpenFileInfo : NonCopyable {
diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index 64049c72aa..0b7df45839 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -77,7 +77,7 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
 	init( BUGGIFY_SIM_PAGE_CACHE_4K,                           1e6 );
 	init( BUGGIFY_SIM_PAGE_CACHE_64K,                          1e6 );
 	init( MAX_EVICT_ATTEMPTS,                                  100 ); if( randomize && BUGGIFY ) MAX_EVICT_ATTEMPTS = 2;
-	init( CACHE_EVICTION_POLICY,                                 0 );
+	init( CACHE_EVICTION_POLICY,                          "random" );
 	init( PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION,                 0.1 ); if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 0.0; else if( randomize && BUGGIFY ) PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION = 1.0;
 
 	//AsyncFileKAIO
diff --git a/flow/Knobs.h b/flow/Knobs.h
index 5490752ad7..fd3c61368d 100644
--- a/flow/Knobs.h
+++ b/flow/Knobs.h
@@ -93,7 +93,7 @@ public:
 	int64_t SIM_PAGE_CACHE_64K;
 	int64_t BUGGIFY_SIM_PAGE_CACHE_4K;
 	int64_t BUGGIFY_SIM_PAGE_CACHE_64K;
-	int CACHE_EVICTION_POLICY; // 0 RANDOM (default), 1 LRU
+	std::string CACHE_EVICTION_POLICY; // for now, "random", "lru", are supported
 	int MAX_EVICT_ATTEMPTS;
 	double PAGE_CACHE_TRUNCATE_LOOKUP_FRACTION;
 	double TOO_MANY_CONNECTIONS_CLOSED_RESET_DELAY;

From d6170210e71d8458781b26e64bf42e42338796f7 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Mon, 6 May 2019 10:11:46 +0200
Subject: [PATCH 009/118] AsyncFileCached: throw (new) exception, through
 helper static fn, if cache eviction polity not recognized.

---
 fdbrpc/AsyncFileCached.actor.h | 24 ++++++++++++++----------
 flow/error_definitions.h       |  1 +
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h
index 84a1d29849..3efa52d9fa 100644
--- a/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/AsyncFileCached.actor.h
@@ -51,17 +51,22 @@ struct EvictablePage {
 
 struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	using List = bi::list< EvictablePage, bi::member_hook< EvictablePage, bi::list_member_hook<>, &EvictablePage::member_hook>>;
+	enum CacheEvictionType { RANDOM = 0, LRU = 1 };
 
-	EvictablePageCache() : pageSize(0), maxPages(0) {}
-
-	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize) {
-		std::string cep = FLOW_KNOBS->CACHE_EVICTION_POLICY;
+	static CacheEvictionType evictionPolicyStringToEnum(const std::string &policy) {
+		std::string cep = policy;
 		std::transform(cep.begin(), cep.end(), cep.begin(), ::tolower);
-		if (0 == cep.compare("random"))
-			cacheEvictionType = RANDOM;
-		else
-			cacheEvictionType = LRU;
+		if (cep != "random" && cep != "lru")
+			throw invalid_cache_eviction_policy();
 
+		if (cep == "random")
+			return RANDOM;
+		return LRU;
+	}
+
+	EvictablePageCache() : pageSize(0), maxPages(0), cacheEvictionType(RANDOM) {}
+
+	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize), cacheEvictionType(evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY)) {
 		cacheHits.init(LiteralStringRef("EvictablePageCache.CacheHits"));
 		cacheMisses.init(LiteralStringRef("EvictablePageCache.CacheMisses"));
 		cacheEvictions.init(LiteralStringRef("EvictablePageCache.CacheEviction"));
@@ -117,7 +122,6 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 		}
 	}
 
-	enum CacheEvictionType { RANDOM = 0, LRU = 1 };
 	std::vector<EvictablePage*> pages;
 	List lruPages;
 	int pageSize;
@@ -125,7 +129,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	Int64MetricHandle cacheHits;
 	Int64MetricHandle cacheMisses;
 	Int64MetricHandle cacheEvictions;
-	CacheEvictionType cacheEvictionType;
+	const CacheEvictionType cacheEvictionType;
 };
 
 struct OpenFileInfo : NonCopyable {
diff --git a/flow/error_definitions.h b/flow/error_definitions.h
index d57e75c032..82752ca938 100755
--- a/flow/error_definitions.h
+++ b/flow/error_definitions.h
@@ -131,6 +131,7 @@ ERROR( transaction_invalid_version, 2020, "Transaction does not have a valid com
 ERROR( no_commit_version, 2021, "Transaction is read-only and therefore does not have a commit version" )
 ERROR( environment_variable_network_option_failed, 2022, "Environment variable network option could not be set" )
 ERROR( transaction_read_only, 2023, "Attempted to commit a transaction specified as read-only" )
+ERROR( invalid_cache_eviction_policy, 2024, "Invalid cache eviction policy, only random and lru are supported" )
 
 ERROR( incompatible_protocol_version, 2100, "Incompatible protocol version" )
 ERROR( transaction_too_large, 2101, "Transaction exceeds byte limit" )

From 8033a0e7173e7406aba4726260ec7d58ad1f57d6 Mon Sep 17 00:00:00 2001
From: Colin Adler <cadler@coder.com>
Date: Thu, 14 Feb 2019 17:47:30 -0600
Subject: [PATCH 010/118] bindings/go: format lines over 120 characters long

---
 bindings/go/src/fdb/database.go    |  5 +-
 bindings/go/src/fdb/fdb.go         |  5 +-
 bindings/go/src/fdb/transaction.go | 77 ++++++++++++++++++++++++++----
 3 files changed, 77 insertions(+), 10 deletions(-)

diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go
index db888307b4..34b90ee926 100644
--- a/bindings/go/src/fdb/database.go
+++ b/bindings/go/src/fdb/database.go
@@ -216,7 +216,10 @@ func (d Database) LocalityGetBoundaryKeys(er ExactRange, limit int, readVersion
 	tr.Options().SetLockAware()
 
 	bk, ek := er.FDBRangeKeys()
-	ffer := KeyRange{append(Key("\xFF/keyServers/"), bk.FDBKey()...), append(Key("\xFF/keyServers/"), ek.FDBKey()...)}
+	ffer := KeyRange{
+		append(Key("\xFF/keyServers/"), bk.FDBKey()...),
+		append(Key("\xFF/keyServers/"), ek.FDBKey()...),
+	}
 
 	kvs, e := tr.Snapshot().GetRange(ffer, RangeOptions{Limit: limit}).GetSliceWithError()
 	if e != nil {
diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go
index a65a9552f0..3566c7f2ce 100644
--- a/bindings/go/src/fdb/fdb.go
+++ b/bindings/go/src/fdb/fdb.go
@@ -139,7 +139,10 @@ func APIVersion(version int) error {
 			if e == 2203 {
 				maxSupportedVersion := C.fdb_get_max_api_version()
 				if headerVersion > int(maxSupportedVersion) {
-					return fmt.Errorf("This version of the FoundationDB Go binding is not supported by the installed FoundationDB C library. The binding requires a library that supports API version %d, but the installed library supports a maximum version of %d.", headerVersion, maxSupportedVersion)
+					return fmt.Errorf("This version of the FoundationDB Go binding is "+
+						"not supported by the installed FoundationDB C library. "+
+						"The binding requires a library that supports API version %d, "+
+						"but the installed library supports a maximum version of %d.", headerVersion, maxSupportedVersion)
 				}
 				return fmt.Errorf("API version %d is not supported by the installed FoundationDB C library.", version)
 			}
diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go
index 79a8011fa5..2f9c270044 100644
--- a/bindings/go/src/fdb/transaction.go
+++ b/bindings/go/src/fdb/transaction.go
@@ -236,7 +236,14 @@ func (t Transaction) Watch(key KeyConvertible) FutureNil {
 }
 
 func (t *transaction) get(key []byte, snapshot int) FutureByteSlice {
-	return &futureByteSlice{future: newFuture(C.fdb_transaction_get(t.ptr, byteSliceToPtr(key), C.int(len(key)), C.fdb_bool_t(snapshot)))}
+	return &futureByteSlice{
+		newFuture(C.fdb_transaction_get(
+			t.ptr,
+			byteSliceToPtr(key),
+			C.int(len(key)),
+			C.fdb_bool_t(snapshot),
+		)),
+	}
 }
 
 // Get returns the (future) value associated with the specified key. The read is
@@ -253,7 +260,24 @@ func (t *transaction) doGetRange(r Range, options RangeOptions, snapshot bool, i
 	bkey := bsel.Key.FDBKey()
 	ekey := esel.Key.FDBKey()
 
-	return futureKeyValueArray{newFuture(C.fdb_transaction_get_range(t.ptr, byteSliceToPtr(bkey), C.int(len(bkey)), C.fdb_bool_t(boolToInt(bsel.OrEqual)), C.int(bsel.Offset), byteSliceToPtr(ekey), C.int(len(ekey)), C.fdb_bool_t(boolToInt(esel.OrEqual)), C.int(esel.Offset), C.int(options.Limit), C.int(0), C.FDBStreamingMode(options.Mode-1), C.int(iteration), C.fdb_bool_t(boolToInt(snapshot)), C.fdb_bool_t(boolToInt(options.Reverse))))}
+	return futureKeyValueArray{
+		newFuture(C.fdb_transaction_get_range(
+			t.ptr,
+			byteSliceToPtr(bkey),
+			C.int(len(bkey)),
+			C.fdb_bool_t(boolToInt(bsel.OrEqual)),
+			C.int(bsel.Offset),
+			byteSliceToPtr(ekey),
+			C.int(len(ekey)),
+			C.fdb_bool_t(boolToInt(esel.OrEqual)),
+			C.int(esel.Offset),
+			C.int(options.Limit),
+			C.int(0),
+			C.FDBStreamingMode(options.Mode-1),
+			C.int(iteration),
+			C.fdb_bool_t(boolToInt(snapshot)),
+			C.fdb_bool_t(boolToInt(options.Reverse)),
+		))}
 }
 
 func (t *transaction) getRange(r Range, options RangeOptions, snapshot bool) RangeResult {
@@ -358,7 +382,16 @@ func boolToInt(b bool) int {
 
 func (t *transaction) getKey(sel KeySelector, snapshot int) FutureKey {
 	key := sel.Key.FDBKey()
-	return &futureKey{future: newFuture(C.fdb_transaction_get_key(t.ptr, byteSliceToPtr(key), C.int(len(key)), C.fdb_bool_t(boolToInt(sel.OrEqual)), C.int(sel.Offset), C.fdb_bool_t(snapshot)))}
+	return &futureKey{
+		newFuture(C.fdb_transaction_get_key(
+			t.ptr,
+			byteSliceToPtr(key),
+			C.int(len(key)),
+			C.fdb_bool_t(boolToInt(sel.OrEqual)),
+			C.int(sel.Offset),
+			C.fdb_bool_t(snapshot),
+		)),
+	}
 }
 
 // GetKey returns the future key referenced by the provided key selector. The
@@ -375,14 +408,28 @@ func (t Transaction) GetKey(sel Selectable) FutureKey {
 }
 
 func (t Transaction) atomicOp(key []byte, param []byte, code int) {
-	C.fdb_transaction_atomic_op(t.ptr, byteSliceToPtr(key), C.int(len(key)), byteSliceToPtr(param), C.int(len(param)), C.FDBMutationType(code))
+	C.fdb_transaction_atomic_op(
+		t.ptr,
+		byteSliceToPtr(key),
+		C.int(len(key)),
+		byteSliceToPtr(param),
+		C.int(len(param)),
+		C.FDBMutationType(code),
+	)
 }
 
 func addConflictRange(t *transaction, er ExactRange, crtype conflictRangeType) error {
 	begin, end := er.FDBRangeKeys()
 	bkb := begin.FDBKey()
 	ekb := end.FDBKey()
-	if err := C.fdb_transaction_add_conflict_range(t.ptr, byteSliceToPtr(bkb), C.int(len(bkb)), byteSliceToPtr(ekb), C.int(len(ekb)), C.FDBConflictRangeType(crtype)); err != 0 {
+	if err := C.fdb_transaction_add_conflict_range(
+		t.ptr,
+		byteSliceToPtr(bkb),
+		C.int(len(bkb)),
+		byteSliceToPtr(ekb),
+		C.int(len(ekb)),
+		C.FDBConflictRangeType(crtype),
+	); err != 0 {
 		return Error{int(err)}
 	}
 
@@ -414,7 +461,11 @@ func copyAndAppend(orig []byte, b byte) []byte {
 // For more information on conflict ranges, see
 // https://apple.github.io/foundationdb/developer-guide.html#conflict-ranges.
 func (t Transaction) AddReadConflictKey(key KeyConvertible) error {
-	return addConflictRange(t.transaction, KeyRange{key, Key(copyAndAppend(key.FDBKey(), 0x00))}, conflictRangeTypeRead)
+	return addConflictRange(
+		t.transaction,
+		KeyRange{key, Key(copyAndAppend(key.FDBKey(), 0x00))},
+		conflictRangeTypeRead,
+	)
 }
 
 // AddWriteConflictRange adds a range of keys to the transactions write
@@ -435,7 +486,11 @@ func (t Transaction) AddWriteConflictRange(er ExactRange) error {
 // For more information on conflict ranges, see
 // https://apple.github.io/foundationdb/developer-guide.html#conflict-ranges.
 func (t Transaction) AddWriteConflictKey(key KeyConvertible) error {
-	return addConflictRange(t.transaction, KeyRange{key, Key(copyAndAppend(key.FDBKey(), 0x00))}, conflictRangeTypeWrite)
+	return addConflictRange(
+		t.transaction,
+		KeyRange{key, Key(copyAndAppend(key.FDBKey(), 0x00))},
+		conflictRangeTypeWrite,
+	)
 }
 
 // Options returns a TransactionOptions instance suitable for setting options
@@ -446,7 +501,13 @@ func (t Transaction) Options() TransactionOptions {
 
 func localityGetAddressesForKey(t *transaction, key KeyConvertible) FutureStringSlice {
 	kb := key.FDBKey()
-	return &futureStringSlice{newFuture(C.fdb_transaction_get_addresses_for_key(t.ptr, byteSliceToPtr(kb), C.int(len(kb))))}
+	return &futureStringSlice{
+		newFuture(C.fdb_transaction_get_addresses_for_key(
+			t.ptr,
+			byteSliceToPtr(kb),
+			C.int(len(kb)),
+		)),
+	}
 }
 
 // LocalityGetAddressesForKey returns the (future) public network addresses of

From 0d7748d59fbef8930cd157451bda5363e030b89e Mon Sep 17 00:00:00 2001
From: Colin Adler <cadler@coder.com>
Date: Thu, 14 Feb 2019 19:03:03 -0600
Subject: [PATCH 011/118] bindings/go: use binary.LittleEndian.PutUint64
 instead of binary.Write

Benchmarks can be found in bindings/go/bench
---
 bindings/go/bench/int64ToBytes_bench_test.go  | 41 +++++++++++++++++++
 .../go/src/_util/translate_fdb_options.go     | 10 ++---
 bindings/go/src/fdb/generated.go              | 24 +++++------
 3 files changed, 56 insertions(+), 19 deletions(-)
 create mode 100644 bindings/go/bench/int64ToBytes_bench_test.go

diff --git a/bindings/go/bench/int64ToBytes_bench_test.go b/bindings/go/bench/int64ToBytes_bench_test.go
new file mode 100644
index 0000000000..95c2e1f420
--- /dev/null
+++ b/bindings/go/bench/int64ToBytes_bench_test.go
@@ -0,0 +1,41 @@
+package bench
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+)
+
+var result []byte
+
+func Benchmark_Int64ToBytesBuffer(b *testing.B) {
+	b.ReportAllocs()
+
+	var r []byte
+	for n := 0; n < b.N; n++ {
+		buf := new(bytes.Buffer)
+		if err := binary.Write(buf, binary.LittleEndian, int64(n)); err != nil {
+			b.Error("failed to write int64:", err)
+		}
+
+		b.SetBytes(int64(buf.Len()))
+		r = buf.Bytes()
+	}
+
+	result = r
+}
+
+func Benchmark_Int64ToBytesPut(b *testing.B) {
+	b.ReportAllocs()
+
+	var r []byte
+	for n := 0; n < b.N; n++ {
+		buf := make([]byte, 8)
+		binary.LittleEndian.PutUint64(buf, uint64(n))
+
+		b.SetBytes(int64(len(buf)))
+		r = buf
+	}
+
+	result = r
+}
diff --git a/bindings/go/src/_util/translate_fdb_options.go b/bindings/go/src/_util/translate_fdb_options.go
index 418a6b2e1c..327e716648 100644
--- a/bindings/go/src/_util/translate_fdb_options.go
+++ b/bindings/go/src/_util/translate_fdb_options.go
@@ -209,12 +209,10 @@ import (
 	"encoding/binary"
 )
 
-func int64ToBytes(i int64) ([]byte, error) {
-	buf := new(bytes.Buffer)
-	if e := binary.Write(buf, binary.LittleEndian, i); e != nil {
-		return nil, e
-	}
-	return buf.Bytes(), nil
+func int64ToBytes(i int64) []byte {
+	buf := make([]byte, 8)
+	binary.LittleEndian.PutUint64(buf, uint64(i))
+	return buf
 }
 `)
 
diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index f6b3ccc18b..9f5ea8dda9 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -34,12 +34,10 @@ import (
 	"encoding/binary"
 )
 
-func int64ToBytes(i int64) ([]byte, error) {
-	buf := new(bytes.Buffer)
-	if e := binary.Write(buf, binary.LittleEndian, i); e != nil {
-		return nil, e
-	}
-	return buf.Bytes(), nil
+func int64ToBytes(i int64) []byte {
+	buf := make([]byte, 8)
+	binary.LittleEndian.PutUint64(buf, uint64(i))
+	return buf
 }
 
 // Deprecated
@@ -495,7 +493,7 @@ const (
 	// Infrequently used. The client has passed a specific row limit and wants
 	// that many rows delivered in a single batch. Because of iterator operation
 	// in client drivers make request batches transparent to the user, consider
-	// ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this
+	// “WANT_ALL“ StreamingMode instead. A row limit must be specified if this
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
@@ -612,15 +610,15 @@ type ErrorPredicate int
 
 const (
 
-	// Returns ``true`` if the error indicates the operations in the
-	// transactions should be retried because of transient error.
+	// Returns “true“ if the error indicates the operations in the transactions
+	// should be retried because of transient error.
 	ErrorPredicateRetryable ErrorPredicate = 50000
 
-	// Returns ``true`` if the error indicates the transaction may have
-	// succeeded, though not in a way the system can verify.
+	// Returns “true“ if the error indicates the transaction may have succeeded,
+	// though not in a way the system can verify.
 	ErrorPredicateMaybeCommitted ErrorPredicate = 50001
 
-	// Returns ``true`` if the error indicates the transaction has not
-	// committed, though in a way that can be retried.
+	// Returns “true“ if the error indicates the transaction has not committed,
+	// though in a way that can be retried.
 	ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002
 )

From 5793b1a55e7d93eea9ce8de36ddf3d44843e4449 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Tue, 7 May 2019 08:32:57 +0200
Subject: [PATCH 012/118] Validate cache eviction policy value after knob args
 have been set.

---
 fdbserver/fdbserver.actor.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index b6281d2a5b..e8148e5927 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -54,6 +54,7 @@
 #include "fdbrpc/TLSConnection.h"
 #include "fdbrpc/Net2FileSystem.h"
 #include "fdbrpc/Platform.h"
+#include "fdbrpc/AsyncFileCached.actor.h"
 #include "fdbserver/CoroFlow.h"
 #include "flow/SignalSafeUnwind.h"
 #if defined(CMAKE_BUILD) || !defined(WIN32)
@@ -1426,6 +1427,11 @@ int main(int argc, char* argv[]) {
 		}
 		if (!serverKnobs->setKnob("server_mem_limit", std::to_string(memLimit))) ASSERT(false);
 
+		if (EvictablePageCache::RANDOM != EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY) &&
+			EvictablePageCache::LRU != EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY)) {
+			ASSERT(false);
+		}
+
 		if (role == SkipListTest) {
 			skipListTest();
 			flushAndExit(FDB_EXIT_SUCCESS);

From c2827f4fa3e980a99ddf41b689b970ff704a83d6 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Wed, 8 May 2019 15:41:17 +0200
Subject: [PATCH 013/118] Add page cache hit, miss, and eviction stats to
 SystemMonitor

---
 fdbrpc/AsyncFileCached.actor.h | 10 +++++-----
 flow/SystemMonitor.cpp         |  3 +++
 flow/SystemMonitor.h           |  6 ++++++
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fdbrpc/AsyncFileCached.actor.h b/fdbrpc/AsyncFileCached.actor.h
index 3efa52d9fa..9c874d30c2 100644
--- a/fdbrpc/AsyncFileCached.actor.h
+++ b/fdbrpc/AsyncFileCached.actor.h
@@ -69,7 +69,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 	explicit EvictablePageCache(int pageSize, int64_t maxSize) : pageSize(pageSize), maxPages(maxSize / pageSize), cacheEvictionType(evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY)) {
 		cacheHits.init(LiteralStringRef("EvictablePageCache.CacheHits"));
 		cacheMisses.init(LiteralStringRef("EvictablePageCache.CacheMisses"));
-		cacheEvictions.init(LiteralStringRef("EvictablePageCache.CacheEviction"));
+		cacheEvictions.init(LiteralStringRef("EvictablePageCache.CacheEvictions"));
 	}
 
 	void allocate(EvictablePage* page) {
@@ -82,7 +82,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 		} else {
 			lruPages.push_back(*page); // new page is considered the most recently used (placed at LRU tail)
 		}
-		cacheMisses++;
+		++cacheMisses;
 	}
 
 	void updateHit(EvictablePage* page) {
@@ -91,7 +91,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 			lruPages.erase(List::s_iterator_to(*page));
 			lruPages.push_back(*page);
 		}
-		cacheHits++;
+		++cacheHits;
 	}
 
 	void try_evict() {
@@ -100,7 +100,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 				for (int i = 0; i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS; i++) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
 					int toEvict = g_random->randomInt(0, pages.size());
 					if (pages[toEvict]->evict()) {
-						cacheEvictions++;
+						++cacheEvictions;
 						break;
 					}
 				}
@@ -114,7 +114,7 @@ struct EvictablePageCache : ReferenceCounted<EvictablePageCache> {
 				     it != lruPages.end() && i < FLOW_KNOBS->MAX_EVICT_ATTEMPTS;
 				     ++it, ++i) { // If we don't manage to evict anything, just go ahead and exceed the cache limit
 					if (it->evict()) {
-						cacheEvictions++;
+						++cacheEvictions;
 						break;
 					}
 				}
diff --git a/flow/SystemMonitor.cpp b/flow/SystemMonitor.cpp
index 3e9b010a9e..08639c357a 100644
--- a/flow/SystemMonitor.cpp
+++ b/flow/SystemMonitor.cpp
@@ -89,6 +89,9 @@ SystemStatistics customSystemMonitor(std::string eventName, StatisticsState *sta
 				.detail("CachePageReadsMerged", netData.countFileCachePageReadsMerged - statState->networkState.countFileCachePageReadsMerged)
 				.detail("CacheWrites", netData.countFileCacheWrites - statState->networkState.countFileCacheWrites)
 				.detail("CacheReads", netData.countFileCacheReads - statState->networkState.countFileCacheReads)
+				.detail("CacheHits", netData.countFilePageCacheHits - statState->networkState.countFilePageCacheHits)
+				.detail("CacheMisses", netData.countFilePageCacheMisses - statState->networkState.countFilePageCacheMisses)
+				.detail("CacheEvictions", netData.countFilePageCacheEvictions - statState->networkState.countFilePageCacheEvictions)
 				.detail("ZoneID", machineState.zoneId)
 				.detail("MachineID", machineState.machineId)
 				.detail("AIOSubmitCount", netData.countAIOSubmit - statState->networkState.countAIOSubmit)
diff --git a/flow/SystemMonitor.h b/flow/SystemMonitor.h
index 8c7d41c5ba..4c0585cd69 100644
--- a/flow/SystemMonitor.h
+++ b/flow/SystemMonitor.h
@@ -74,6 +74,9 @@ struct NetworkData {
 	int64_t countFileCachePageReadsMerged;
 	int64_t countFileCacheFinds;
 	int64_t countFileCacheReadBytes;
+	int64_t countFilePageCacheHits;
+	int64_t countFilePageCacheMisses;
+	int64_t countFilePageCacheEvictions;
 	int64_t countConnEstablished;
 	int64_t countConnClosedWithError;
 	int64_t countConnClosedWithoutError;
@@ -121,6 +124,9 @@ struct NetworkData {
 		countFileCachePageReadsMerged = getValue(LiteralStringRef("AsyncFile.CountCachePageReadsMerged"));
 		countFileCacheFinds = getValue(LiteralStringRef("AsyncFile.CountCacheFinds"));
 		countFileCacheReadBytes = getValue(LiteralStringRef("AsyncFile.CountCacheReadBytes"));
+		countFilePageCacheHits = getValue(LiteralStringRef("EvictablePageCache.CacheHits"));
+		countFilePageCacheMisses = getValue(LiteralStringRef("EvictablePageCache.CacheMisses"));
+		countFilePageCacheEvictions = getValue(LiteralStringRef("EvictablePageCache.CacheEvictions"));
 	}
 };
 

From b50926c79262455c4c98daaedf551cb853372ffc Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Wed, 8 May 2019 21:22:14 -1000
Subject: [PATCH 014/118] replaceFile is truncate(0) on windows

---
 fdbserver/DiskQueue.actor.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index 7097698710..9e5e9a4021 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -283,6 +283,13 @@ public:
 		TraceEvent("DiskQueueReplaceTruncateEnded").detail("Filename", file->getFilename());
 	}
 
+#if defined(_WIN32)
+	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
+		// Windows doesn't support a rename over an open file.
+		wait( toReplace->truncate(0) );
+		return toReplace;
+	}
+#else
 	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
 		incrementalTruncate( toReplace );
 
@@ -292,6 +299,7 @@ public:
 
 		return replacement;
 	}
+#endif
 
 	Future<Void> push(StringRef pageData, vector<Reference<SyncQueue>>* toSync) {
 		return push( this, pageData, toSync );

From 0d0f54d1e6f2b68efe2c68a9f89c6cfd5c7d846b Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Wed, 8 May 2019 21:22:40 -1000
Subject: [PATCH 015/118] Fix IAsyncFileSystem::open() flags to stop a crash.

OPEN_ATOMIC_WRITE_AND_CREATE was missing a required OPEN_CREATE.

I'm honestly baffled how this was missed in testing.
---
 fdbserver/DiskQueue.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index 9e5e9a4021..d01f304102 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -293,7 +293,7 @@ public:
 	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
 		incrementalTruncate( toReplace );
 
-		Reference<IAsyncFile> _replacement = wait( IAsyncFileSystem::filesystem()->open( toReplace->getFilename(), IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK, 0 ) );
+		Reference<IAsyncFile> _replacement = wait( IAsyncFileSystem::filesystem()->open( toReplace->getFilename(), IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, 0 ) );
 		state Reference<IAsyncFile> replacement = _replacement;
 		wait( replacement->sync() );
 

From c6c33a4daa83776c1508eaedc9b0c32a9964741e Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Wed, 8 May 2019 21:23:42 -1000
Subject: [PATCH 016/118] Make replaceFile more likely to be tested.

---
 fdbserver/DiskQueue.actor.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index d01f304102..eb5b11fe6d 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -339,8 +339,9 @@ public:
 				if (self->files[1].size > desiredMaxFileSize) {
 					// Either shrink self->files[1] to the size of self->files[0], or chop off fileShrinkBytes
 					int64_t maxShrink = std::max( pageFloor(self->files[1].size - desiredMaxFileSize), self->fileShrinkBytes );
-					if (maxShrink / SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES >
-							SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) {
+					if ((maxShrink / SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES >
+							 SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) ||
+					    BUGGIFY_WITH_PROB(0.1)) {
 						TEST(true);  // Replacing DiskQueue file
 						TraceEvent("DiskQueueReplaceFile", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", self->files[1].size).detail("ElidedTruncateSize", maxShrink);
 						Reference<IAsyncFile> newFile = wait( replaceFile(self->files[1].f) );

From 510b0b2fcd64197dfcda9295833acd419dc6569b Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Wed, 8 May 2019 23:08:25 -1000
Subject: [PATCH 017/118] Fix DiskQueue not replaceFile'ing frequently enough
 for the final time.

---
 fdbrpc/sim2.actor.cpp         | 2 +-
 fdbserver/DiskQueue.actor.cpp | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 7aa8166b27..4368904a98 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -608,7 +608,7 @@ private:
 			wait( waitUntilDiskReady( self->diskParameters, 0 ) );
 
 		if( _chsize( self->h, (long) size ) == -1 ) {
-			TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 6);
+			TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 6).detail("Filename", self->filename).detail("Error", strerror(errno)).detail("Size", size).detail("Fd", self->h);
 			throw io_error();
 		}
 
diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index eb5b11fe6d..d48e323a09 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -336,19 +336,20 @@ public:
 
 				const int64_t activeDataVolume = pageCeiling(self->files[0].size - self->files[0].popped + self->fileExtensionBytes + self->fileShrinkBytes);
 				const int64_t desiredMaxFileSize = std::max( activeDataVolume, SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES * 2 );
-				if (self->files[1].size > desiredMaxFileSize) {
+				const bool frivolouslyTruncate = BUGGIFY_WITH_PROB(0.001);
+				if (self->files[1].size > desiredMaxFileSize || frivolouslyTruncate) {
 					// Either shrink self->files[1] to the size of self->files[0], or chop off fileShrinkBytes
 					int64_t maxShrink = std::max( pageFloor(self->files[1].size - desiredMaxFileSize), self->fileShrinkBytes );
 					if ((maxShrink / SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES >
 							 SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) ||
-					    BUGGIFY_WITH_PROB(0.1)) {
+					    (frivolouslyTruncate && g_random->random01() < 0.3)) {
 						TEST(true);  // Replacing DiskQueue file
 						TraceEvent("DiskQueueReplaceFile", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", self->files[1].size).detail("ElidedTruncateSize", maxShrink);
 						Reference<IAsyncFile> newFile = wait( replaceFile(self->files[1].f) );
 						self->files[1].setFile(newFile);
 						self->files[1].size = 0;
 					} else {
-						self->files[1].size -= maxShrink;
+						self->files[1].size -= std::min(maxShrink, self->files[1].size);
 						waitfor.push_back( self->files[1].f->truncate( self->files[1].size ) );
 					}
 				}

From fbc4e7b3510ae8225a3f65077a613b98012eaf4d Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Wed, 8 May 2019 23:10:18 -1000
Subject: [PATCH 018/118] Add a release note

---
 documentation/sphinx/source/release-notes.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index d04214a2b7..31613f92fd 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -126,6 +126,7 @@ Fixes only impacting 6.1.0+
 * The transaction log spill-by-reference policy could read too much data from disk. [6.1.5] `(PR #1527) <https://github.com/apple/foundationdb/pull/1527>`_
 * Memory tracking trace events could cause the program to crash when called from inside a trace event. [6.1.5] `(PR #1541) <https://github.com/apple/foundationdb/pull/1541>`_
 * TLogs will replace a large file with an empty file rather than doing a large truncate operation. [6.1.5] `(PR #1545) <https://github.com/apple/foundationdb/pull/1545>`_
+* Fix PR #1545 to work on Windows and Linux. [6.1.6] `(PR #1556) <https://github.com/apple/foundationdb/pull/1556>`_
 
 Earlier release notes
 ---------------------

From 4a794923e648513d3e4eaea24a3f51f59c9c10c2 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Fri, 10 May 2019 10:49:32 -0700
Subject: [PATCH 019/118] Added environmental variable representing the docker
 version Added Java Home environmental variable Removed unnecessary packages

---
 build/Dockerfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index 895858f7a0..b710225fb5 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -1,5 +1,6 @@
 FROM centos:6
-LABEL version=0.1.2
+LABEL version=0.1.3
+ENV DOCKER_IMAGEVER=0.1.3
 
 # Install dependencies for developer tools, bindings,\
 # documentation, actorcompiler, and packaging tools\
@@ -8,7 +9,7 @@ RUN yum install -y yum-utils &&\
 	yum -y install centos-release-scl epel-release &&\
 	yum -y install devtoolset-7 mono-core java-1.8.0-openjdk-devel \
 		rh-python36-python-devel rh-ruby24 golang python27 \
-		rpm-build debbuild python-pip npm ccache distcc &&\
+		rpm-build debbuild python-pip npm &&\
 	pip install boto3==1.1.1
 
 USER root
@@ -39,4 +40,5 @@ RUN curl -L https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz >
     cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- make -j`nproc` install &&\
     rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz
 
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
 CMD scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash

From c3dcbefeb21788a4918118dcfe653c5b6f137f0b Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Fri, 10 May 2019 10:56:31 -0700
Subject: [PATCH 020/118] Updated docker compose to use latest build image

---
 build/docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml
index a3a281ee78..b7fecb150b 100644
--- a/build/docker-compose.yaml
+++ b/build/docker-compose.yaml
@@ -2,7 +2,7 @@ version: "3"
 
 services:
   common: &common
-    image: foundationdb/foundationdb-build:0.1.2
+    image: foundationdb/foundationdb-build:0.1.3
 
   build-setup: &build-setup
     <<: *common

From 1cfd94f30ba43f809ecbd29c1f5c4afa5ca8e5bb Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Fri, 10 May 2019 11:01:21 -0700
Subject: [PATCH 021/118] Modified docker compose to use make rather than cmake
 for bindings, packages, and docs

---
 build/docker-compose.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml
index b7fecb150b..c19fa81337 100644
--- a/build/docker-compose.yaml
+++ b/build/docker-compose.yaml
@@ -26,16 +26,16 @@ services:
 
   build-docs:
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" package_html'
+    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage'
 
 
   release-packages: &release-packages
     <<: *release-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=1 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" packages preinstall && cpack'
+    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
 
   snapshot-packages: &snapshot-packages
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" packages preinstall && cpack'
+    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
 
   prb-packages:
     <<: *snapshot-packages
@@ -43,11 +43,11 @@ services:
 
   release-bindings: &release-bindings
     <<: *release-setup
-    command: bash -c 'make -j "$${MAKEJOBS}" bindings'
+    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
 
   snapshot-bindings: &snapshot-bindings
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" python_binding'
+    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
 
   prb-bindings:
     <<: *snapshot-bindings

From 1d041222ec38cc0a17d8c04b0df15ba8e356488f Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Fri, 10 May 2019 12:58:10 -0700
Subject: [PATCH 022/118] Suppress error message for missing tool binaries

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 177922a31a..78b83afe80 100644
--- a/Makefile
+++ b/Makefile
@@ -15,14 +15,14 @@ else
   $(error Not prepared to compile on $(ARCH))
 endif
 
-MONO := $(shell which mono)
+MONO := $(shell which mono 2>/dev/null)
 ifeq ($(MONO),)
   MONO := /usr/bin/mono
 endif
 
-MCS := $(shell which mcs)
+MCS := $(shell which mcs 2>/dev/null)
 ifeq ($(MCS),)
-  MCS := $(shell which dmcs) 
+  MCS := $(shell which dmcs 2>/dev/null)
 endif
 ifeq ($(MCS),)
   MCS := /usr/bin/mcs
@@ -70,7 +70,7 @@ else
 endif
 BOOSTDIR ?= ${BOOST_BASEDIR}/${BOOST_BASENAME}
 
-CCACHE := $(shell which ccache)
+CCACHE := $(shell which ccache 2>/dev/null)
 ifneq ($(CCACHE),)
   CCACHE_CC := $(CCACHE) $(CC)
   CCACHE_CXX := $(CCACHE) $(CXX)

From 93c1dced1695d795bcdfc7ab10d02e7f885b52ef Mon Sep 17 00:00:00 2001
From: Colin Adler <cadler@coder.com>
Date: Fri, 10 May 2019 15:18:44 -0500
Subject: [PATCH 023/118] bindings/go: add labels for futures

---
 bindings/go/src/fdb/transaction.go | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go
index 2f9c270044..4cd0186404 100644
--- a/bindings/go/src/fdb/transaction.go
+++ b/bindings/go/src/fdb/transaction.go
@@ -184,7 +184,9 @@ func (t Transaction) Snapshot() Snapshot {
 // Typical code will not use OnError directly. (Database).Transact uses
 // OnError internally to implement a correct retry loop.
 func (t Transaction) OnError(e Error) FutureNil {
-	return &futureNil{newFuture(C.fdb_transaction_on_error(t.ptr, C.fdb_error_t(e.Code)))}
+	return &futureNil{
+		future: newFuture(C.fdb_transaction_on_error(t.ptr, C.fdb_error_t(e.Code))),
+	}
 }
 
 // Commit attempts to commit the modifications made in the transaction to the
@@ -198,7 +200,9 @@ func (t Transaction) OnError(e Error) FutureNil {
 // see
 // https://apple.github.io/foundationdb/developer-guide.html#transactions-with-unknown-results.
 func (t Transaction) Commit() FutureNil {
-	return &futureNil{newFuture(C.fdb_transaction_commit(t.ptr))}
+	return &futureNil{
+		future: newFuture(C.fdb_transaction_commit(t.ptr)),
+	}
 }
 
 // Watch creates a watch and returns a FutureNil that will become ready when the
@@ -232,12 +236,14 @@ func (t Transaction) Commit() FutureNil {
 // cancelled by calling (FutureNil).Cancel on its returned future.
 func (t Transaction) Watch(key KeyConvertible) FutureNil {
 	kb := key.FDBKey()
-	return &futureNil{newFuture(C.fdb_transaction_watch(t.ptr, byteSliceToPtr(kb), C.int(len(kb))))}
+	return &futureNil{
+		future: newFuture(C.fdb_transaction_watch(t.ptr, byteSliceToPtr(kb), C.int(len(kb)))),
+	}
 }
 
 func (t *transaction) get(key []byte, snapshot int) FutureByteSlice {
 	return &futureByteSlice{
-		newFuture(C.fdb_transaction_get(
+		future: newFuture(C.fdb_transaction_get(
 			t.ptr,
 			byteSliceToPtr(key),
 			C.int(len(key)),
@@ -261,7 +267,7 @@ func (t *transaction) doGetRange(r Range, options RangeOptions, snapshot bool, i
 	ekey := esel.Key.FDBKey()
 
 	return futureKeyValueArray{
-		newFuture(C.fdb_transaction_get_range(
+		future: newFuture(C.fdb_transaction_get_range(
 			t.ptr,
 			byteSliceToPtr(bkey),
 			C.int(len(bkey)),
@@ -302,7 +308,9 @@ func (t Transaction) GetRange(r Range, options RangeOptions) RangeResult {
 }
 
 func (t *transaction) getReadVersion() FutureInt64 {
-	return &futureInt64{newFuture(C.fdb_transaction_get_read_version(t.ptr))}
+	return &futureInt64{
+		future: newFuture(C.fdb_transaction_get_read_version(t.ptr)),
+	}
 }
 
 // (Infrequently used) GetReadVersion returns the (future) transaction read version. The read is
@@ -383,7 +391,7 @@ func boolToInt(b bool) int {
 func (t *transaction) getKey(sel KeySelector, snapshot int) FutureKey {
 	key := sel.Key.FDBKey()
 	return &futureKey{
-		newFuture(C.fdb_transaction_get_key(
+		future: newFuture(C.fdb_transaction_get_key(
 			t.ptr,
 			byteSliceToPtr(key),
 			C.int(len(key)),
@@ -502,7 +510,7 @@ func (t Transaction) Options() TransactionOptions {
 func localityGetAddressesForKey(t *transaction, key KeyConvertible) FutureStringSlice {
 	kb := key.FDBKey()
 	return &futureStringSlice{
-		newFuture(C.fdb_transaction_get_addresses_for_key(
+		future: newFuture(C.fdb_transaction_get_addresses_for_key(
 			t.ptr,
 			byteSliceToPtr(kb),
 			C.int(len(kb)),

From 9a85c19063640b8f47ce486dd525991b5bc0390d Mon Sep 17 00:00:00 2001
From: Colin <colin.adler@yahoo.com>
Date: Thu, 9 May 2019 01:34:07 -0500
Subject: [PATCH 024/118] bindings/go: fix generated int64ToBytes calls

---
 .../go/src/_util/translate_fdb_options.go     |  7 +-
 bindings/go/src/fdb/generated.go              | 87 ++++---------------
 2 files changed, 20 insertions(+), 74 deletions(-)

diff --git a/bindings/go/src/_util/translate_fdb_options.go b/bindings/go/src/_util/translate_fdb_options.go
index 327e716648..033d0e770e 100644
--- a/bindings/go/src/_util/translate_fdb_options.go
+++ b/bindings/go/src/_util/translate_fdb_options.go
@@ -66,11 +66,7 @@ func writeOptBytes(w io.Writer, receiver string, function string, opt Option) {
 
 func writeOptInt(w io.Writer, receiver string, function string, opt Option) {
 	fmt.Fprintf(w, `func (o %s) %s(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(%d, b)
+	return o.setOpt(%d, int64ToBytes(param))
 }
 `, receiver, function, opt.Code)
 }
@@ -205,7 +201,6 @@ func main() {
 package fdb
 
 import (
-	"bytes"
 	"encoding/binary"
 )
 
diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index 9f5ea8dda9..6842072df8 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -30,7 +30,6 @@
 package fdb
 
 import (
-	"bytes"
 	"encoding/binary"
 )
 
@@ -65,22 +64,14 @@ func (o NetworkOptions) SetTraceEnable(param string) error {
 //
 // Parameter: max size of a single trace output file
 func (o NetworkOptions) SetTraceRollSize(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(31, b)
+	return o.setOpt(31, int64ToBytes(param))
 }
 
 // Sets the maximum size of all the trace output files put together. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time.
 //
 // Parameter: max total size of trace files
 func (o NetworkOptions) SetTraceMaxLogsSize(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(32, b)
+	return o.setOpt(32, int64ToBytes(param))
 }
 
 // Sets the 'LogGroup' attribute with the specified value for all events in the trace output files. The default log group is 'default'.
@@ -160,22 +151,14 @@ func (o NetworkOptions) SetBuggifyDisable() error {
 //
 // Parameter: probability expressed as a percentage between 0 and 100
 func (o NetworkOptions) SetBuggifySectionActivatedProbability(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(50, b)
+	return o.setOpt(50, int64ToBytes(param))
 }
 
 // Set the probability of an active BUGGIFY section being fired
 //
 // Parameter: probability expressed as a percentage between 0 and 100
 func (o NetworkOptions) SetBuggifySectionFiredProbability(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(51, b)
+	return o.setOpt(51, int64ToBytes(param))
 }
 
 // Set the ca bundle
@@ -242,22 +225,14 @@ func (o NetworkOptions) SetEnableSlowTaskProfiling() error {
 //
 // Parameter: Max location cache entries
 func (o DatabaseOptions) SetLocationCacheSize(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(10, b)
+	return o.setOpt(10, int64ToBytes(param))
 }
 
 // Set the maximum number of watches allowed to be outstanding on a database connection. Increasing this number could result in increased resource usage. Reducing this number will not cancel any outstanding watches. Defaults to 10000 and cannot be larger than 1000000.
 //
 // Parameter: Max outstanding watches
 func (o DatabaseOptions) SetMaxWatches(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(20, b)
+	return o.setOpt(20, int64ToBytes(param))
 }
 
 // Specify the machine ID that was passed to fdbserver processes running on the same machine as this client, for better location-aware load balancing.
@@ -278,33 +253,21 @@ func (o DatabaseOptions) SetDatacenterId(param string) error {
 //
 // Parameter: value in milliseconds of timeout
 func (o DatabaseOptions) SetTransactionTimeout(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(500, b)
+	return o.setOpt(500, int64ToBytes(param))
 }
 
 // Set a timeout in milliseconds which, when elapsed, will cause a transaction automatically to be cancelled. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information.
 //
 // Parameter: number of times to retry
 func (o DatabaseOptions) SetTransactionRetryLimit(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(501, b)
+	return o.setOpt(501, int64ToBytes(param))
 }
 
 // Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. This sets the ``max_retry_delay`` option of each transaction created by this database. See the transaction option description for more information.
 //
 // Parameter: value in milliseconds of maximum delay
 func (o DatabaseOptions) SetTransactionMaxRetryDelay(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(502, b)
+	return o.setOpt(502, int64ToBytes(param))
 }
 
 // Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior.
@@ -415,33 +378,21 @@ func (o TransactionOptions) SetLogTransaction() error {
 //
 // Parameter: value in milliseconds of timeout
 func (o TransactionOptions) SetTimeout(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(500, b)
+	return o.setOpt(500, int64ToBytes(param))
 }
 
 // Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. Valid parameter values are ``[-1, INT_MAX]``. If set to -1, will disable the retry limit. Prior to API version 610, like all other transaction options, the retry limit must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the retry limit each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
 //
 // Parameter: number of times to retry
 func (o TransactionOptions) SetRetryLimit(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(501, b)
+	return o.setOpt(501, int64ToBytes(param))
 }
 
 // Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. Defaults to 1000 ms. Valid parameter values are ``[0, INT_MAX]``. If the maximum retry delay is less than the current retry delay of the transaction, then the current retry delay will be clamped to the maximum retry delay. Prior to API version 610, like all other transaction options, the maximum retry delay must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the maximum retry delay each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option.
 //
 // Parameter: value in milliseconds of maximum delay
 func (o TransactionOptions) SetMaxRetryDelay(param int64) error {
-	b, e := int64ToBytes(param)
-	if e != nil {
-		return e
-	}
-	return o.setOpt(502, b)
+	return o.setOpt(502, int64ToBytes(param))
 }
 
 // Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior.
@@ -493,7 +444,7 @@ const (
 	// Infrequently used. The client has passed a specific row limit and wants
 	// that many rows delivered in a single batch. Because of iterator operation
 	// in client drivers make request batches transparent to the user, consider
-	// “WANT_ALL“ StreamingMode instead. A row limit must be specified if this
+	// ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
@@ -610,15 +561,15 @@ type ErrorPredicate int
 
 const (
 
-	// Returns “true“ if the error indicates the operations in the transactions
-	// should be retried because of transient error.
+	// Returns ``true`` if the error indicates the operations in the
+	// transactions should be retried because of transient error.
 	ErrorPredicateRetryable ErrorPredicate = 50000
 
-	// Returns “true“ if the error indicates the transaction may have succeeded,
-	// though not in a way the system can verify.
+	// Returns ``true`` if the error indicates the transaction may have
+	// succeeded, though not in a way the system can verify.
 	ErrorPredicateMaybeCommitted ErrorPredicate = 50001
 
-	// Returns “true“ if the error indicates the transaction has not committed,
-	// though in a way that can be retried.
+	// Returns ``true`` if the error indicates the transaction has not
+	// committed, though in a way that can be retried.
 	ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002
 )

From c328b15d366a2f867f31646b608f7c8fbeba5723 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 10 May 2019 14:51:20 -0700
Subject: [PATCH 025/118] TLS was creating trace events with invalid types
 (containing spaces).

---
 FDBLibTLS/FDBLibTLSSession.cpp | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp
index 680e2aae6c..352e4ae00b 100644
--- a/FDBLibTLS/FDBLibTLSSession.cpp
+++ b/FDBLibTLS/FDBLibTLSSession.cpp
@@ -245,11 +245,11 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 	// Verify the certificate.
 	if ((store_ctx = X509_STORE_CTX_new()) == NULL) {
 		TraceEvent(SevError, "FDBLibTLSOutOfMemory", uid);
-		reason = "FDBLibTLSOutOfMemory";
+		reason = "Out of memory";
 		goto err;
 	}
 	if (!X509_STORE_CTX_init(store_ctx, NULL, sk_X509_value(certs, 0), certs)) {
-		reason = "FDBLibTLSStoreCtxInit";
+		reason = "Store ctx init";
 		goto err;
 	}
 	X509_STORE_CTX_trusted_stack(store_ctx, policy->roots);
@@ -258,31 +258,31 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 		X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME);
 	if (X509_verify_cert(store_ctx) <= 0) {
 		const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
-		reason = "FDBLibTLSVerifyCert VerifyError " + std::string(errstr);
+		reason = "Verify cert error: " + std::string(errstr);
 		goto err;
 	}
 
 	// Check subject criteria.
 	cert = sk_X509_value(store_ctx->chain, 0);
 	if ((subject = X509_get_subject_name(cert)) == NULL) {
-		reason = "FDBLibTLSCertSubjectError";
+		reason = "Cert subject error";
 		goto err;
 	}
 	for (auto &pair: verify->subject_criteria) {
 		if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
-			reason = "FDBLibTLSCertSubjectMatchFailure";
+			reason = "Cert subject match failure";
 			goto err;
 		}
 	}
 
 	// Check issuer criteria.
 	if ((issuer = X509_get_issuer_name(cert)) == NULL) {
-		reason = "FDBLibTLSCertIssuerError";
+		reason = "Cert issuer error";
 		goto err;
 	}
 	for (auto &pair: verify->issuer_criteria) {
 		if (!match_criteria(cert, issuer, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
-			reason = "FDBLibTLSCertIssuerMatchFailure";
+			reason = "Cert issuer match failure";
 			goto err;
 		}
 	}
@@ -290,12 +290,12 @@ std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSV
 	// Check root criteria - this is the subject of the final certificate in the stack.
 	cert = sk_X509_value(store_ctx->chain, sk_X509_num(store_ctx->chain) - 1);
 	if ((subject = X509_get_subject_name(cert)) == NULL) {
-		reason = "FDBLibTLSRootSubjectError";
+		reason = "Root subject error";
 		goto err;
 	}
 	for (auto &pair: verify->root_criteria) {
 		if (!match_criteria(cert, subject, pair.first, pair.second.criteria, pair.second.match_type, pair.second.location)) {
-			reason = "FDBLibTLSRootSubjectMatchFailure";
+			reason = "Root subject match failure";
 			goto err;
 		}
 	}
@@ -345,7 +345,7 @@ bool FDBLibTLSSession::verify_peer() {
 	if (!rc) {
 		// log the various failure reasons
 		for (std::string reason : verify_failure_reasons) {
-			TraceEvent(reason.c_str(), uid).suppressFor(1.0);
+			TraceEvent("FDBLibTLSVerifyFailure", uid).detail("Reason", reason).suppressFor(1.0);
 		}
 	}
 

From 4d32909837fc155a91c7bb183f11cbbc8c821510 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Fri, 10 May 2019 15:22:41 -0700
Subject: [PATCH 026/118] Moved standard libraries to LDFLAGS rather than LIBS

---
 bindings/flow/tester/local.mk | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bindings/flow/tester/local.mk b/bindings/flow/tester/local.mk
index 2ef4fcb753..6e59625c5f 100644
--- a/bindings/flow/tester/local.mk
+++ b/bindings/flow/tester/local.mk
@@ -35,8 +35,7 @@ _fdb_flow_tester_clean:
 	@rm -rf bindings/flow/bin
 
 ifeq ($(PLATFORM),linux)
-  fdb_flow_tester_LIBS += -ldl -lpthread -lrt
-  fdb_flow_tester_LDFLAGS += -static-libstdc++ -static-libgcc
+  fdb_flow_tester_LDFLAGS += -static-libstdc++ -static-libgcc -ldl -lpthread -lrt
 else ifeq ($(PLATFORM),osx)
   fdb_flow_tester_LDFLAGS += -lc++
 endif

From ab4618c3f9c7f77bbf644f46716e11cd55f84a65 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 10 May 2019 13:15:06 -0700
Subject: [PATCH 027/118] Don't access TraceEvent::eventCounts outside the
 network thread. Make net2liveness an atomic counter.

---
 flow/Net2.actor.cpp | 4 ++--
 flow/Platform.cpp   | 9 +++++----
 flow/Trace.cpp      | 5 ++++-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index 6b5bd1ac42..fe081e9131 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -65,7 +65,7 @@ static_assert(currentProtocolVersion < 0x0FDB00B100000000LL, "Unexpected protoco
 #if defined(__linux__)
 #include <execinfo.h>
 
-volatile double net2liveness = 0;
+std::atomic<int64_t> net2liveness(0);
 
 volatile size_t net2backtraces_max = 10000;
 volatile void** volatile net2backtraces = NULL;
@@ -689,7 +689,7 @@ void Net2::run() {
 			}
 
 			// to keep the thread liveness check happy
-			net2liveness = g_nondeterministic_random->random01();
+			net2liveness.fetch_add(1);
 		}
 #endif
 
diff --git a/flow/Platform.cpp b/flow/Platform.cpp
index a854c30d44..d303e48d80 100644
--- a/flow/Platform.cpp
+++ b/flow/Platform.cpp
@@ -2742,7 +2742,7 @@ extern volatile size_t net2backtraces_offset;
 extern volatile size_t net2backtraces_max;
 extern volatile bool net2backtraces_overflow;
 extern volatile int net2backtraces_count;
-extern volatile double net2liveness;
+extern std::atomic<int64_t> net2liveness;
 extern volatile thread_local int profilingEnabled;
 extern void initProfiling();
 
@@ -2789,12 +2789,13 @@ void* checkThread(void *arg) {
 	pthread_t mainThread = *(pthread_t*)arg;
 	free(arg);
 
-	double lastValue = net2liveness;
+	int64_t lastValue = net2liveness.load();
 	double lastSignal = 0;
 	double logInterval = FLOW_KNOBS->SLOWTASK_PROFILING_INTERVAL;
 	while(true) {
 		threadSleep(FLOW_KNOBS->SLOWTASK_PROFILING_INTERVAL);
-		if(lastValue == net2liveness) {
+		int64_t currentLiveness = net2liveness.load();
+		if(lastValue == currentLiveness) {
 			double t = timer();
 			if(lastSignal == 0 || t - lastSignal >= logInterval) {
 				if(lastSignal > 0) {
@@ -2806,10 +2807,10 @@ void* checkThread(void *arg) {
 			}
 		}
 		else {
+			lastValue = currentLiveness;
 			lastSignal = 0;
 			logInterval = FLOW_KNOBS->SLOWTASK_PROFILING_INTERVAL;
 		}
-		lastValue = net2liveness;
 	}
 	return NULL;
 #else
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 60eb571c86..306885fa36 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -921,7 +921,10 @@ TraceEvent::~TraceEvent() {
 				severity = SevError;
 			}
 
-			TraceEvent::eventCounts[severity/10]++;
+			if(isNetworkThread()) {
+				TraceEvent::eventCounts[severity/10]++;
+			}
+
 			g_traceLog.writeEvent( fields, trackingKey, severity > SevWarnAlways );
 
 			if (g_traceLog.isOpen()) {

From c502ed3d152e966affcbf0487d824a9fa46472ba Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Fri, 10 May 2019 14:46:26 -1000
Subject: [PATCH 028/118] Fix a variety of problems stemming from a wait()
 being added to push().

And that this code was previously insufficiently tested.
---
 fdbrpc/AsyncFileKAIO.actor.h  |  2 +-
 fdbrpc/sim2.actor.cpp         |  7 ++++++-
 fdbserver/DiskQueue.actor.cpp | 32 +++++++++++++++++++++-----------
 3 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.actor.h
index d9fb6de2f8..882e112ed7 100644
--- a/fdbrpc/AsyncFileKAIO.actor.h
+++ b/fdbrpc/AsyncFileKAIO.actor.h
@@ -265,7 +265,7 @@ public:
 			result = fallocate( fd, 0, 0, size);
 			if (result != 0) {
 				int fallocateErrCode = errno;
-				TraceEvent("AsyncFileKAIOAllocateError").detail("Fd",fd).detail("Filename", filename).GetLastError();
+				TraceEvent("AsyncFileKAIOAllocateError").detail("Fd",fd).detail("Filename", filename).detail("Size", size).GetLastError();
 				if ( fallocateErrCode == EOPNOTSUPP ) {
 					// Mark fallocate as unsupported. Try again with truncate.
 					ctx.fallocateSupported = false;
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 4368904a98..5bc89266f9 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -604,11 +604,16 @@ private:
 		if (randLog)
 			fprintf( randLog, "SFT1 %s %s %s %lld\n", self->dbgId.shortString().c_str(), self->filename.c_str(), opId.shortString().c_str(), size );
 
+		if (size == 0) {
+			// KAIO will return EINVAL, as len==0 is an error.
+			throw io_error();
+		}
+
 		if(self->delayOnWrite)
 			wait( waitUntilDiskReady( self->diskParameters, 0 ) );
 
 		if( _chsize( self->h, (long) size ) == -1 ) {
-			TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 6).detail("Filename", self->filename).detail("Error", strerror(errno)).detail("Size", size).detail("Fd", self->h);
+			TraceEvent(SevWarn, "SimpleFileIOError").detail("Location", 6).detail("Filename", self->filename).detail("Size", size).detail("Fd", self->h).GetLastError();
 			throw io_error();
 		}
 
diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index d48e323a09..49e46bbf4c 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -158,13 +158,13 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 public:
 	RawDiskQueue_TwoFiles( std::string basename, std::string fileExtension, UID dbgid, int64_t fileSizeWarningLimit )
 		: basename(basename), fileExtension(fileExtension), onError(delayed(error.getFuture())), onStopped(stopped.getFuture()),
-		readingFile(-1), readingPage(-1), writingPos(-1), dbgid(dbgid),
+		readingFile(-1), readingPage(-1), pushlock(1), writingPos(-1), dbgid(dbgid),
 		dbg_file0BeginSeq(0), fileExtensionBytes(SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES),
 		fileShrinkBytes(SERVER_KNOBS->DISK_QUEUE_FILE_SHRINK_BYTES), readingBuffer( dbgid ),
 		readyToPush(Void()), fileSizeWarningLimit(fileSizeWarningLimit), lastCommit(Void()), isFirstCommit(true)
 	{
 		if (BUGGIFY)
-			fileExtensionBytes = 1<<10 * g_random->randomSkewedUInt32( 1, 40<<10 );
+			fileExtensionBytes = _PAGE_SIZE * g_random->randomSkewedUInt32( 1, 10<<10 );
 		if (BUGGIFY)
 			fileShrinkBytes = _PAGE_SIZE * g_random->randomSkewedUInt32( 1, 10<<10 );
 		files[0].dbgFilename = filename(0);
@@ -261,6 +261,7 @@ public:
 	int readingFile;  // i if the next page after readingBuffer should be read from files[i], 2 if recovery is complete
 	int64_t readingPage;  // Page within readingFile that is the next page after readingBuffer
 
+	FlowLock pushlock;
 	int64_t writingPos;  // Position within files[1] that will be next written
 
 	int64_t fileExtensionBytes;
@@ -293,7 +294,7 @@ public:
 	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
 		incrementalTruncate( toReplace );
 
-		Reference<IAsyncFile> _replacement = wait( IAsyncFileSystem::filesystem()->open( toReplace->getFilename(), IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, 0 ) );
+		Reference<IAsyncFile> _replacement = wait( IAsyncFileSystem::filesystem()->open( toReplace->getFilename(), IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_LOCK, 0600 ) );
 		state Reference<IAsyncFile> replacement = _replacement;
 		wait( replacement->sync() );
 
@@ -306,6 +307,11 @@ public:
 	}
 
 	ACTOR static Future<Void> push(RawDiskQueue_TwoFiles* self, StringRef pageData, vector<Reference<SyncQueue>>* toSync) {
+		state TrackMe trackMe(self);
+
+		wait( self->pushlock.take(g_network->getCurrentTask(), 1) );
+		state FlowLock::Releaser releaser(self->pushlock, 1);
+
 		// Write the given data to the queue files, swapping or extending them if necessary.
 		// Don't do any syncs, but push the modified file(s) onto toSync.
 		ASSERT( self->readingFile == 2 );
@@ -333,23 +339,28 @@ public:
 				std::swap(self->firstPages[0], self->firstPages[1]);
 				self->files[1].popped = 0;
 				self->writingPos = 0;
+				*self->firstPages[1] = *(const Page*)pageData.begin();
 
 				const int64_t activeDataVolume = pageCeiling(self->files[0].size - self->files[0].popped + self->fileExtensionBytes + self->fileShrinkBytes);
-				const int64_t desiredMaxFileSize = std::max( activeDataVolume, SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES * 2 );
-				const bool frivolouslyTruncate = BUGGIFY_WITH_PROB(0.001);
+				const int64_t desiredMaxFileSize = pageCeiling( std::max( activeDataVolume, SERVER_KNOBS->TLOG_HARD_LIMIT_BYTES * 2 ) );
+				const bool frivolouslyTruncate = BUGGIFY_WITH_PROB(0.1);
 				if (self->files[1].size > desiredMaxFileSize || frivolouslyTruncate) {
 					// Either shrink self->files[1] to the size of self->files[0], or chop off fileShrinkBytes
-					int64_t maxShrink = std::max( pageFloor(self->files[1].size - desiredMaxFileSize), self->fileShrinkBytes );
+					int64_t maxShrink = pageFloor( std::max( self->files[1].size - desiredMaxFileSize, self->fileShrinkBytes ) );
 					if ((maxShrink / SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES >
-							 SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) ||
+					     SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) ||
 					    (frivolouslyTruncate && g_random->random01() < 0.3)) {
 						TEST(true);  // Replacing DiskQueue file
 						TraceEvent("DiskQueueReplaceFile", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", self->files[1].size).detail("ElidedTruncateSize", maxShrink);
 						Reference<IAsyncFile> newFile = wait( replaceFile(self->files[1].f) );
 						self->files[1].setFile(newFile);
-						self->files[1].size = 0;
+						waitfor.push_back( self->files[1].f->truncate( self->fileExtensionBytes ) );
+						self->files[1].size = self->fileExtensionBytes;
 					} else {
+						const int64_t startingSize = self->files[1].size;
 						self->files[1].size -= std::min(maxShrink, self->files[1].size);
+						self->files[1].size = std::max(self->files[1].size, self->fileExtensionBytes);
+						TraceEvent("DiskQueueTruncate", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", startingSize).detail("NewFileSize", self->files[1].size);
 						waitfor.push_back( self->files[1].f->truncate( self->files[1].size ) );
 					}
 				}
@@ -365,9 +376,8 @@ public:
 					TraceEvent(SevWarnAlways, "DiskQueueFileTooLarge", self->dbgid).suppressFor(1.0).detail("Filename", self->filename(1)).detail("Size", self->files[1].size);
 				}
 			}
-		}
-
-		if (self->writingPos == 0) {
+		} else if (self->writingPos == 0) {
+			// If this is the first write to a brand new disk queue file.
 			*self->firstPages[1] = *(const Page*)pageData.begin();
 		}
 

From c95d09f9fd5afb8daf0dcadd68cf0f39004cc90c Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Fri, 10 May 2019 14:53:46 -1000
Subject: [PATCH 029/118] Convert truncate(0) to truncate(4KB) on Windows.

Blindly, in case Windows doesn't like 0 length truncates too.
---
 fdbserver/DiskQueue.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index 49e46bbf4c..b1a12c21dd 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -287,7 +287,7 @@ public:
 #if defined(_WIN32)
 	ACTOR static Future<Reference<IAsyncFile>> replaceFile(Reference<IAsyncFile> toReplace) {
 		// Windows doesn't support a rename over an open file.
-		wait( toReplace->truncate(0) );
+		wait( toReplace->truncate(4<<10) );
 		return toReplace;
 	}
 #else

From ea12a54946930a0fb5e7284574ce3a30d04517e4 Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Fri, 10 May 2019 18:26:22 -1000
Subject: [PATCH 030/118] Rename DISK_QUEUE_MAX_TRUNCATE_EXTENTS -> ..._BYTES

So as to not make filesystem assumptions.  This knob did technically
appear in (only the) 6.1.5 release, but this feature was broken 6.1.5,
so thus impossible to use anyway.
---
 fdbserver/DiskQueue.actor.cpp | 3 +--
 fdbserver/Knobs.cpp           | 2 +-
 fdbserver/Knobs.h             | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index b1a12c21dd..5e5dffb443 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -347,8 +347,7 @@ public:
 				if (self->files[1].size > desiredMaxFileSize || frivolouslyTruncate) {
 					// Either shrink self->files[1] to the size of self->files[0], or chop off fileShrinkBytes
 					int64_t maxShrink = pageFloor( std::max( self->files[1].size - desiredMaxFileSize, self->fileShrinkBytes ) );
-					if ((maxShrink / SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES >
-					     SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_EXTENTS) ||
+					if ((maxShrink > SERVER_KNOBS->DISK_QUEUE_MAX_TRUNCATE_BYTES) ||
 					    (frivolouslyTruncate && g_random->random01() < 0.3)) {
 						TEST(true);  // Replacing DiskQueue file
 						TraceEvent("DiskQueueReplaceFile", self->dbgid).detail("Filename", self->files[1].f->getFilename()).detail("OldFileSize", self->files[1].size).detail("ElidedTruncateSize", maxShrink);
diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index 4c551d4791..e79839af5f 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -75,7 +75,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH,           16<<10 ); if ( randomize && BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH = 500;
 	init( DISK_QUEUE_FILE_EXTENSION_BYTES,                    10<<20 ); // BUGGIFYd per file within the DiskQueue
 	init( DISK_QUEUE_FILE_SHRINK_BYTES,                      100<<20 ); // BUGGIFYd per file within the DiskQueue
-	init( DISK_QUEUE_MAX_TRUNCATE_EXTENTS,                     1<<10 ); if ( randomize && BUGGIFY ) DISK_QUEUE_MAX_TRUNCATE_EXTENTS = 0;
+	init( DISK_QUEUE_MAX_TRUNCATE_BYTES,                       2<<30 ); if ( randomize && BUGGIFY ) DISK_QUEUE_MAX_TRUNCATE_BYTES = 0;
 	init( TLOG_DEGRADED_DELAY_COUNT,                               5 );
 	init( TLOG_DEGRADED_DURATION,                                5.0 );
 
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index ee91df4ad8..119656e3eb 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -79,7 +79,7 @@ public:
 	int64_t TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH;
 	int64_t DISK_QUEUE_FILE_EXTENSION_BYTES; // When we grow the disk queue, by how many bytes should it grow?
 	int64_t DISK_QUEUE_FILE_SHRINK_BYTES; // When we shrink the disk queue, by how many bytes should it shrink?
-	int DISK_QUEUE_MAX_TRUNCATE_EXTENTS;
+	int DISK_QUEUE_MAX_TRUNCATE_BYTES;  // A truncate larger than this will cause the file to be replaced instead.
 	int TLOG_DEGRADED_DELAY_COUNT;
 	double TLOG_DEGRADED_DURATION;
 

From 4a7e0319c7bc102ddc292bf63cb4861f07890ba9 Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Fri, 10 May 2019 20:21:37 -1000
Subject: [PATCH 031/118] Refactor away pushlock.

Pushing was already a serialized, sequential operation.

Instead make it explicit that there are two waits as part of a push:
1. The setup work to reserve a spot on in the file
2. The work of writing and sync'ing the data

And we return a Future<Future<Void>> to force these to be done sequentially.
---
 fdbserver/DiskQueue.actor.cpp | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index 5e5dffb443..3f5e2779cf 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -158,7 +158,7 @@ class RawDiskQueue_TwoFiles : public Tracked<RawDiskQueue_TwoFiles> {
 public:
 	RawDiskQueue_TwoFiles( std::string basename, std::string fileExtension, UID dbgid, int64_t fileSizeWarningLimit )
 		: basename(basename), fileExtension(fileExtension), onError(delayed(error.getFuture())), onStopped(stopped.getFuture()),
-		readingFile(-1), readingPage(-1), pushlock(1), writingPos(-1), dbgid(dbgid),
+		readingFile(-1), readingPage(-1), writingPos(-1), dbgid(dbgid),
 		dbg_file0BeginSeq(0), fileExtensionBytes(SERVER_KNOBS->DISK_QUEUE_FILE_EXTENSION_BYTES),
 		fileShrinkBytes(SERVER_KNOBS->DISK_QUEUE_FILE_SHRINK_BYTES), readingBuffer( dbgid ),
 		readyToPush(Void()), fileSizeWarningLimit(fileSizeWarningLimit), lastCommit(Void()), isFirstCommit(true)
@@ -261,7 +261,6 @@ public:
 	int readingFile;  // i if the next page after readingBuffer should be read from files[i], 2 if recovery is complete
 	int64_t readingPage;  // Page within readingFile that is the next page after readingBuffer
 
-	FlowLock pushlock;
 	int64_t writingPos;  // Position within files[1] that will be next written
 
 	int64_t fileExtensionBytes;
@@ -302,16 +301,11 @@ public:
 	}
 #endif
 
-	Future<Void> push(StringRef pageData, vector<Reference<SyncQueue>>* toSync) {
+	Future<Future<Void>> push(StringRef pageData, vector<Reference<SyncQueue>>* toSync) {
 		return push( this, pageData, toSync );
 	}
 
-	ACTOR static Future<Void> push(RawDiskQueue_TwoFiles* self, StringRef pageData, vector<Reference<SyncQueue>>* toSync) {
-		state TrackMe trackMe(self);
-
-		wait( self->pushlock.take(g_network->getCurrentTask(), 1) );
-		state FlowLock::Releaser releaser(self->pushlock, 1);
-
+	ACTOR static Future<Future<Void>> push(RawDiskQueue_TwoFiles* self, StringRef pageData, vector<Reference<SyncQueue>>* toSync) {
 		// Write the given data to the queue files, swapping or extending them if necessary.
 		// Don't do any syncs, but push the modified file(s) onto toSync.
 		ASSERT( self->readingFile == 2 );
@@ -387,8 +381,7 @@ public:
 		waitfor.push_back( self->files[1].f->write( pageData.begin(), pageData.size(), self->writingPos ) );
 		self->writingPos += pageData.size();
 
-		wait( waitForAll(waitfor) );
-		return Void();
+		return waitForAll(waitfor);
 	}
 
 	ACTOR static UNCANCELLABLE Future<Void> pushAndCommit(RawDiskQueue_TwoFiles* self, StringRef pageData, StringBuffer* pageMem, uint64_t poppedPages) {
@@ -415,11 +408,11 @@ public:
 
 			TEST( pageData.size() > sizeof(Page) ); // push more than one page of data
 
-			Future<Void> pushed = self->push( pageData, &syncFiles );
+			Future<Void> pushed = wait( self->push( pageData, &syncFiles ) );
 			pushing.send(Void());
-			wait( pushed );
 			ASSERT( syncFiles.size() >= 1 && syncFiles.size() <= 2 );
 			TEST(2==syncFiles.size());  // push spans both files
+			wait( pushed );
 
 			delete pageMem;
 			pageMem = 0;

From 0840271ddca5f61f104a8b945e7d34dcbe941dd5 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Sat, 11 May 2019 07:52:51 -0700
Subject: [PATCH 032/118] Added dos2unix package to build rpms

---
 build/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index b710225fb5..9efca4b0f0 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -9,7 +9,7 @@ RUN yum install -y yum-utils &&\
 	yum -y install centos-release-scl epel-release &&\
 	yum -y install devtoolset-7 mono-core java-1.8.0-openjdk-devel \
 		rh-python36-python-devel rh-ruby24 golang python27 \
-		rpm-build debbuild python-pip npm &&\
+		rpm-build debbuild python-pip npm dos2unix &&\
 	pip install boto3==1.1.1
 
 USER root

From cacd66fdde00c7497c4415ce3dbce36541c97a09 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Sat, 11 May 2019 07:53:22 -0700
Subject: [PATCH 033/118] Ensure that virtualenv is starting via python2

---
 documentation/sphinx/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/Makefile b/documentation/sphinx/Makefile
index d23b136f1f..3aab160be2 100644
--- a/documentation/sphinx/Makefile
+++ b/documentation/sphinx/Makefile
@@ -70,7 +70,7 @@ buildsphinx:
 		cd $(BUILDDIR); \
 		curl -OL $(VENV_URL); \
 		tar zxvf $(VENV_VERSION).tar.gz; \
-		./$(VENV_VERSION)/virtualenv.py venv; \
+		python2 ./$(VENV_VERSION)/virtualenv.py venv; \
 	fi
 	. $(VENVDIR)/bin/activate && \
 	cp .pip.conf $(VENVDIR)/pip.conf && \

From be571d3fd456ce6c2528a339c66f67a511b43136 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Sat, 11 May 2019 08:01:32 -0700
Subject: [PATCH 034/118] Changed volume mount for building docs in order to
 shorten directory length

---
 build/docker-compose.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml
index c19fa81337..28dea93408 100644
--- a/build/docker-compose.yaml
+++ b/build/docker-compose.yaml
@@ -26,6 +26,9 @@ services:
 
   build-docs:
     <<: *build-setup
+    volumes:
+      - ..:/foundationdb
+    working_dir: /foundationdb
     command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage'
 
 

From 59a9030ed64a261f61a904654017445b3ed9aa80 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 12 May 2019 19:02:09 -0700
Subject: [PATCH 035/118] updated documentation for 6.1.6

---
 documentation/sphinx/source/downloads.rst | 24 +++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst
index d4bb3bb93b..63a25896f6 100644
--- a/documentation/sphinx/source/downloads.rst
+++ b/documentation/sphinx/source/downloads.rst
@@ -10,38 +10,38 @@ macOS
 
 The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
 
-* `FoundationDB-6.1.5.pkg <https://www.foundationdb.org/downloads/6.1.5/macOS/installers/FoundationDB-6.1.5.pkg>`_
+* `FoundationDB-6.1.6.pkg <https://www.foundationdb.org/downloads/6.1.6/macOS/installers/FoundationDB-6.1.6.pkg>`_
 
 Ubuntu
 ------
 
 The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
 
-* `foundationdb-clients-6.1.5-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.5/ubuntu/installers/foundationdb-clients_6.1.5-1_amd64.deb>`_
-* `foundationdb-server-6.1.5-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.5/ubuntu/installers/foundationdb-server_6.1.5-1_amd64.deb>`_ (depends on the clients package)
+* `foundationdb-clients-6.1.6-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.6/ubuntu/installers/foundationdb-clients_6.1.6-1_amd64.deb>`_
+* `foundationdb-server-6.1.6-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.6/ubuntu/installers/foundationdb-server_6.1.6-1_amd64.deb>`_ (depends on the clients package)
 
 RHEL/CentOS EL6
 ---------------
 
 The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
 
-* `foundationdb-clients-6.1.5-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.5/rhel6/installers/foundationdb-clients-6.1.5-1.el6.x86_64.rpm>`_
-* `foundationdb-server-6.1.5-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.5/rhel6/installers/foundationdb-server-6.1.5-1.el6.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.1.6-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel6/installers/foundationdb-clients-6.1.6-1.el6.x86_64.rpm>`_
+* `foundationdb-server-6.1.6-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel6/installers/foundationdb-server-6.1.6-1.el6.x86_64.rpm>`_ (depends on the clients package)
 
 RHEL/CentOS EL7
 ---------------
 
 The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
 
-* `foundationdb-clients-6.1.5-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.5/rhel7/installers/foundationdb-clients-6.1.5-1.el7.x86_64.rpm>`_
-* `foundationdb-server-6.1.5-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.5/rhel7/installers/foundationdb-server-6.1.5-1.el7.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.1.6-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel7/installers/foundationdb-clients-6.1.6-1.el7.x86_64.rpm>`_
+* `foundationdb-server-6.1.6-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel7/installers/foundationdb-server-6.1.6-1.el7.x86_64.rpm>`_ (depends on the clients package)
 
 Windows
 -------
 
 The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
 
-* `foundationdb-6.1.5-x64.msi <https://www.foundationdb.org/downloads/6.1.5/windows/installers/foundationdb-6.1.5-x64.msi>`_
+* `foundationdb-6.1.6-x64.msi <https://www.foundationdb.org/downloads/6.1.6/windows/installers/foundationdb-6.1.6-x64.msi>`_
 
 API Language Bindings
 =====================
@@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
 
 If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package:
 
-* `foundationdb-6.1.5.tar.gz <https://www.foundationdb.org/downloads/6.1.5/bindings/python/foundationdb-6.1.5.tar.gz>`_
+* `foundationdb-6.1.6.tar.gz <https://www.foundationdb.org/downloads/6.1.6/bindings/python/foundationdb-6.1.6.tar.gz>`_
 
 Ruby 1.9.3/2.0.0+
 -----------------
 
-* `fdb-6.1.5.gem <https://www.foundationdb.org/downloads/6.1.5/bindings/ruby/fdb-6.1.5.gem>`_
+* `fdb-6.1.6.gem <https://www.foundationdb.org/downloads/6.1.6/bindings/ruby/fdb-6.1.6.gem>`_
 
 Java 8+
 -------
 
-* `fdb-java-6.1.5.jar <https://www.foundationdb.org/downloads/6.1.5/bindings/java/fdb-java-6.1.5.jar>`_
-* `fdb-java-6.1.5-javadoc.jar <https://www.foundationdb.org/downloads/6.1.5/bindings/java/fdb-java-6.1.5-javadoc.jar>`_
+* `fdb-java-6.1.6.jar <https://www.foundationdb.org/downloads/6.1.6/bindings/java/fdb-java-6.1.6.jar>`_
+* `fdb-java-6.1.6-javadoc.jar <https://www.foundationdb.org/downloads/6.1.6/bindings/java/fdb-java-6.1.6-javadoc.jar>`_
 
 Go 1.1+
 -------

From 84f8381905324d6082abe2fda7d3938117f72c04 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 12 May 2019 19:06:04 -0700
Subject: [PATCH 036/118] update installer WIX GUID following release

---
 packaging/msi/FDBInstaller.wxs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
index 9480cc2036..d29331e8c1 100644
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{32F74616-4B66-4A17-972F-765FF2C03728}'
+           Id='{D9581F18-4994-47E0-A16C-A2F97C4CC4D6}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'

From 23e1e355d54c2ce013ea2606e75a185487f962e7 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 12 May 2019 20:08:50 -0700
Subject: [PATCH 037/118] update versions target to 6.1.7

---
 versions.target | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.target b/versions.target
index 20408dae1b..872c948e7b 100644
--- a/versions.target
+++ b/versions.target
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
-    <Version>6.1.6</Version>
+    <Version>6.1.7</Version>
     <PackageName>6.1</PackageName>
   </PropertyGroup>
 </Project>

From 966668905e54d0cb0e69d62d7d11d95b882630df Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 12 May 2019 20:08:50 -0700
Subject: [PATCH 038/118] update installer WIX GUID following release

---
 packaging/msi/FDBInstaller.wxs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
index d29331e8c1..9947b6138a 100644
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{D9581F18-4994-47E0-A16C-A2F97C4CC4D6}'
+           Id='{92B71EC3-E4A1-4473-8D72-7A4D61435C8F}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'

From 62f5ebbd4bcca8ac5e6c44d57429ca2bc123b81f Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 12 May 2019 20:14:57 -0700
Subject: [PATCH 039/118] updated release notes header for 6.1.6

---
 documentation/sphinx/source/release-notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 31613f92fd..79720284ae 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -2,7 +2,7 @@
 Release Notes
 #############
 
-6.1.5
+6.1.6
 =====
 
 Features

From 067cdf9cdec57d803eece14e86ce8a235ae33585 Mon Sep 17 00:00:00 2001
From: Nikolas Ioannou <nio@zurich.ibm.com>
Date: Mon, 13 May 2019 08:50:04 +0200
Subject: [PATCH 040/118] Simplified cache eviction policy knob arg check.

---
 fdbserver/fdbserver.actor.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index e8148e5927..44c003d4df 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -1427,10 +1427,8 @@ int main(int argc, char* argv[]) {
 		}
 		if (!serverKnobs->setKnob("server_mem_limit", std::to_string(memLimit))) ASSERT(false);
 
-		if (EvictablePageCache::RANDOM != EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY) &&
-			EvictablePageCache::LRU != EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY)) {
-			ASSERT(false);
-		}
+		// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
+		EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY);
 
 		if (role == SkipListTest) {
 			skipListTest();

From 17a11b53f0a7ec0dea4603c668508653dc0a268a Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Mon, 13 May 2019 12:22:52 -0700
Subject: [PATCH 041/118] Upgraded to devtoolset-8 to support compilation of
 c++17

---
 build/Dockerfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index 9efca4b0f0..3a0f61908a 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -7,8 +7,8 @@ ENV DOCKER_IMAGEVER=0.1.3
 RUN yum install -y yum-utils &&\
 	yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
 	yum -y install centos-release-scl epel-release &&\
-	yum -y install devtoolset-7 mono-core java-1.8.0-openjdk-devel \
-		rh-python36-python-devel rh-ruby24 golang python27 \
+	yum -y install devtoolset-8 java-1.8.0-openjdk-devel \
+		rh-python36-python-devel mono-core rh-ruby24 golang python27 \
 		rpm-build debbuild python-pip npm dos2unix &&\
 	pip install boto3==1.1.1
 
@@ -36,9 +36,9 @@ RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.1
 RUN curl -L https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz > /tmp/libressl.tar.gz &&\
     cd /tmp && echo "b8cb31e59f1294557bfc80f2a662969bc064e83006ceef0574e2553a1c254fd5  libressl.tar.gz" > libressl-sha.txt &&\
     sha256sum -c libressl-sha.txt && tar xf libressl.tar.gz &&\
-    cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
-    cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- make -j`nproc` install &&\
+    cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-8 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\
+    cd /tmp/libressl-2.8.2 && scl enable devtoolset-8 -- make -j`nproc` install &&\
     rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz
 
 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
-CMD scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash
+CMD scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash

From 8e1927a5cba9f7c7317f89cba0960d9b2ff400d8 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Mon, 13 May 2019 13:25:01 -0700
Subject: [PATCH 042/118] Rename trace event field to avoid duplicating 'Time'
 field.

---
 flow/ActorCollection.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flow/ActorCollection.actor.cpp b/flow/ActorCollection.actor.cpp
index 23eae8a997..0a013f8e01 100644
--- a/flow/ActorCollection.actor.cpp
+++ b/flow/ActorCollection.actor.cpp
@@ -126,7 +126,7 @@ TEST_CASE("/flow/TraceEvent") {
 		wait(delay(0));
 	}
 	TraceEvent("TraceDuration")
-		.detail("Time", g_network->now() - startTime);
+		.detail("Duration", g_network->now() - startTime);
 	startTime = g_network->now();
 	for (i = 0; i < 1000000; ++i) {
 		for (unsigned j = 0; j < 100; ++j) {
@@ -142,7 +142,7 @@ TEST_CASE("/flow/TraceEvent") {
 		wait(delay(0));
 	}
 	TraceEvent("TraceDuration")
-		.detail("Time", g_network->now() - startTime);
+		.detail("Duration", g_network->now() - startTime);
 	printf("benchmark done\n");
 	wait(delay(10));
 	return Void();

From 8bbd28315e9dc66fa14c284ac66c7bcfe2243ba8 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 24 Jan 2019 13:54:28 -0800
Subject: [PATCH 043/118] Added Object serializer (does not yet compile)

---
 cmake/ConfigureCompiler.cmake |    2 +-
 flow/CMakeLists.txt           |    2 +
 flow/ObjectSerializer.h       |  128 ++++
 flow/flat_buffers.cpp         |  513 ++++++++++++++
 flow/flat_buffers.h           | 1241 +++++++++++++++++++++++++++++++++
 5 files changed, 1885 insertions(+), 1 deletion(-)
 create mode 100644 flow/ObjectSerializer.h
 create mode 100644 flow/flat_buffers.cpp
 create mode 100644 flow/flat_buffers.h

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 03af9c102e..e8f155a019 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -95,7 +95,7 @@ else()
     -mmmx
     -mavx
     -msse4.2)
-  add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-std=c++11>)
+  add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-std=c++17>)
   if (USE_VALGRIND)
     add_compile_options(-DVALGRIND -DUSE_VALGRIND)
   endif()
diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt
index 4af896d817..4d24b168c5 100644
--- a/flow/CMakeLists.txt
+++ b/flow/CMakeLists.txt
@@ -63,6 +63,8 @@ set(FLOW_SRCS
   XmlTraceLogFormatter.cpp
   actorcompiler.h
   error_definitions.h
+  flat_buffers.h
+  flat_buffers.cpp
   flow.cpp
   flow.h
   genericactors.actor.cpp
diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h
new file mode 100644
index 0000000000..03e68905a2
--- /dev/null
+++ b/flow/ObjectSerializer.h
@@ -0,0 +1,128 @@
+/*
+ * serialize.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/Error.h"
+#include "flow/Arena.h"
+#include "flow/flat_buffers.h"
+
+template <class Ar>
+struct LoadContext {
+	Ar& ar;
+	std::vector<std::function<void()>> doAfter;
+	LoadContext(Ar& ar) : ar(ar) {}
+	Arena& arena() { return ar.arena(); }
+
+	const uint8_t* tryReadZeroCopy(const uint8_t* ptr, unsigned len) {
+		if constexpr (Ar::ownsUnderlyingMemory) {
+			return ptr;
+		} else {
+			if (len == 0) return nullptr;
+			uint8_t* dat = new (arena()) uint8_t[len];
+			std::copy(ptr, ptr + len, dat);
+			return dat;
+		}
+	}
+
+	void done() const {
+		for (auto& f : doAfter) {
+			f();
+		}
+	}
+	void addArena(Arena& arena) { arena = ar.arena(); }
+};
+
+template <class ReaderImpl>
+class _ObjectReader {
+public:
+	template <class... Items>
+	void deserialize(flat_buffers::FileIdentifier file_identifier, Items&... items) {
+		const uint8_t* data = static_cast<ReaderImpl*>(this)->data();
+		LoadContext<ReaderImpl> context(*static_cast<ReaderImpl*>(this));
+		ASSERT(flat_buffers::read_file_identifier(data) == file_identifier);
+		flat_buffers::load_members(data, context, items...);
+		context.done();
+	}
+
+	template <class Item>
+	void deserialize(Item& item) {
+		deserialize(flat_buffers::FileIdentifierFor<Item>::value, item);
+	}
+};
+
+class ObjectReader : _ObjectReader<ObjectReader> {
+public:
+	static constexpr bool ownsUnderlyingMemory = false;
+
+	ObjectReader(const uint8_t* data) : _data(data) {}
+
+	const uint8_t* data() { return _data; }
+
+	Arena& arena() { return _arena; }
+
+private:
+	const uint8_t* _data;
+	Arena _arena;
+};
+
+class ArenaObjectReader : _ObjectReader<ArenaObjectReader> {
+public:
+	static constexpr bool ownsUnderlyingMemory = true;
+
+	ArenaObjectReader(Arena const& arena, const StringRef& input) : _data(input.begin()), _arena(arena) {}
+
+	const uint8_t* data() { return _data; }
+
+	Arena& arena() { return _arena; }
+
+private:
+	const uint8_t* _data;
+	Arena _arena;
+};
+
+class ObjectWriter {
+public:
+	template <class... Items>
+	void serialize(flat_buffers::FileIdentifier file_identifier, Items const&... items) {
+		ASSERT(data = nullptr); // object serializer can only serialize one object
+		int allocations = 0;
+		auto allocator = [this, &allocations](size_t size_) {
+			++allocations;
+			size = size_;
+			data = new uint8_t[size];
+			return data;
+		};
+		auto res = flat_buffers::save_members(allocator, file_identifier, items...);
+		ASSERT(allocations == 1);
+	}
+
+	template <class Item>
+	void serialize(Item const& item) {
+		serialize(flat_buffers::FileIdentifierFor<Item>::value, item);
+	}
+
+private:
+	uint8_t* data = nullptr;
+	int size = 0;
+};
+
+template <class Visitor, class... Items>
+std::enable_if<Visitor::is_object_serializer> serializer(Visitor& visitor, Items&... items) {
+	visitor(items...);
+}
diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
new file mode 100644
index 0000000000..29e2b6dd84
--- /dev/null
+++ b/flow/flat_buffers.cpp
@@ -0,0 +1,513 @@
+/*
+ * serialize.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flat_buffers.h"
+#include "UnitTest.h"
+#include "Arena.h"
+#include "serialize.h"
+
+#include <algorithm>
+#include <iomanip>
+#include <boost/variant.hpp>
+
+namespace flat_buffers {
+
+namespace detail {
+
+bool TraverseMessageTypes::vtableGeneratedBefore(const std::type_index& idx) {
+	return !f.known_types.insert(idx).second;
+}
+
+VTable generate_vtable(size_t numMembers, const std::vector<unsigned>& members,
+                       const std::vector<unsigned>& alignments) {
+	if (numMembers == 0) {
+		return VTable{ 4, 4 };
+	}
+	// first is index, second is size
+	std::vector<std::pair<unsigned, unsigned>> indexed;
+	indexed.reserve(members.size());
+	for (unsigned i = 0; i < members.size(); ++i) {
+		if (members[i] > 0) {
+			indexed.emplace_back(i, members[i]);
+		}
+	}
+	std::stable_sort(indexed.begin(), indexed.end(),
+	                 [](const std::pair<unsigned, unsigned>& lhs, const std::pair<unsigned, unsigned>& rhs) {
+		                 return lhs.second > rhs.second;
+	                 });
+	VTable result;
+	result.resize(members.size() + 2);
+	// size of the vtable is
+	// - 2 bytes per member +
+	// - 2 bytes for the size entry +
+	// - 2 bytes for the size of the object
+	result[0] = 2 * members.size() + 4;
+	int offset = 0;
+	for (auto p : indexed) {
+		auto align = alignments[p.first];
+		auto& res = result[p.first + 2];
+		res = offset % align == 0 ? offset : ((offset / align) + 1) * align;
+		offset = res + p.second;
+		res += 4;
+	}
+	result[1] = offset + 4;
+	return result;
+}
+
+} // namespace detail
+
+namespace unit_tests {
+
+TEST_CASE("flow/FlatBuffers/test") {
+	auto* vtable1 = detail::get_vtable<int>();
+	auto* vtable2 = detail::get_vtable<uint8_t, uint8_t, int, int64_t, int>();
+	auto* vtable3 = detail::get_vtable<uint8_t, uint8_t, int, int64_t, int>();
+	ASSERT(vtable1 != vtable2);
+	ASSERT(vtable2 == vtable3);
+	ASSERT(vtable1->size() == 3);
+	ASSERT(vtable2->size() == 7);
+	ASSERT((*vtable2)[0] == 14);
+	ASSERT((*vtable2)[1] == 22);
+	ASSERT(((*vtable2)[4] - 4) % 4 == 0);
+	ASSERT(((*vtable2)[5] - 4) % 8 == 0);
+	ASSERT(((*vtable2)[6] - 4) % 4 == 0);
+	return Void();
+}
+
+TEST_CASE("flow/FlatBuffers/emptyVtable") {
+	auto* vtable = detail::get_vtable<>();
+	ASSERT((*vtable)[0] == 4);
+	ASSERT((*vtable)[1] == 4);
+	return Void();
+}
+
+struct Table2 {
+	std::string m_p = {};
+	bool m_ujrnpumbfvc = {};
+	int64_t m_iwgxxt = {};
+	int64_t m_tjkuqo = {};
+	int16_t m_ed = {};
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, m_p, m_ujrnpumbfvc, m_iwgxxt, m_tjkuqo, m_ed);
+	}
+};
+
+struct Table3 {
+	uint16_t m_asbehdlquj = {};
+	uint16_t m_k = {};
+	uint16_t m_jib = {};
+	int64_t m_n = {};
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, m_asbehdlquj, m_k, m_jib, m_n);
+	}
+};
+
+TEST_CASE("flow/FlatBuffers/vtable2") {
+	const auto& vtable =
+	    *detail::get_vtable<uint64_t, bool, std::string, int64_t, std::vector<uint16_t>, Table2, Table3>();
+	ASSERT(!(vtable[2] <= vtable[4] && vtable[4] < vtable[2] + 8));
+	return Void();
+}
+
+struct Nested2 {
+	uint8_t a;
+	std::vector<std::string> b;
+	int c;
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, a, b, c);
+	}
+
+	friend bool operator==(const Nested2& lhs, const Nested2& rhs) {
+		return lhs.a == rhs.a && lhs.b == rhs.b && lhs.c == rhs.c;
+	}
+};
+
+struct Nested {
+	uint8_t a;
+	std::string b;
+	Nested2 nested;
+	std::vector<uint64_t> c;
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, a, b, nested, c);
+	}
+};
+
+struct Root {
+	uint8_t a;
+	std::vector<Nested2> b;
+	Nested c;
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, a, b, c);
+	}
+};
+
+TEST_CASE("flow/FlatBuffers/collectVTables") {
+	Root root;
+	const auto* vtables = detail::get_vtableset(root);
+	ASSERT(vtables == detail::get_vtableset(root));
+	ASSERT(vtables->offsets.size() == 3);
+	const auto& root_vtable = *detail::get_vtable<uint8_t, std::vector<Nested2>, Nested>();
+	const auto& nested_vtable = *detail::get_vtable<uint8_t, std::vector<std::string>, int>();
+	int root_offset = vtables->offsets.at(&root_vtable);
+	int nested_offset = vtables->offsets.at(&nested_vtable);
+	ASSERT(!memcmp((uint8_t*)&root_vtable[0], &vtables->packed_tables[root_offset], root_vtable.size()));
+	ASSERT(!memcmp((uint8_t*)&nested_vtable[0], &vtables->packed_tables[nested_offset], nested_vtable.size()));
+	return Void();
+}
+
+void print_buffer(const uint8_t* out, int len) {
+	std::cout << std::hex << std::setfill('0');
+	for (int i = 0; i < len; ++i) {
+		if (i % 8 == 0) {
+			std::cout << std::endl;
+			std::cout << std::setw(4) << i << ": ";
+		}
+		std::cout << std::setw(2) << (int)out[i] << " ";
+	}
+	std::cout << std::endl << std::dec;
+}
+
+struct Arena {
+	std::vector<std::pair<uint8_t*, size_t>> allocated;
+	~Arena() {
+		for (auto b : allocated) {
+			delete[] b.first;
+		}
+	}
+
+	uint8_t* operator()(size_t sz) {
+		auto res = new uint8_t[sz];
+		allocated.emplace_back(res, sz);
+		return res;
+	}
+
+	size_t get_size(const uint8_t* ptr) const {
+		for (auto& p : allocated) {
+			if (p.first == ptr) {
+				return p.second;
+			}
+		}
+		return -1;
+	}
+};
+
+struct DummyContext {
+	Arena a;
+	Arena& arena() { return a; }
+};
+
+TEST_CASE("flow/FlatBuffers/serializeDeserializeRoot") {
+	Root root{ 1,
+		       { { 13, { "ghi", "jkl" }, 15 }, { 16, { "mnop", "qrstuv" }, 18 } },
+		       { 3, "hello", { 6, { "abc", "def" }, 8 }, { 10, 11, 12 } } };
+	Root root2 = root;
+	Arena arena;
+	auto out = flat_buffers::detail::save(arena, root, flat_buffers::FileIdentifier{});
+
+	ASSERT(root.a == root2.a);
+	ASSERT(root.b == root2.b);
+	ASSERT(root.c.a == root2.c.a);
+	ASSERT(root.c.b == root2.c.b);
+	ASSERT(root.c.nested.a == root2.c.nested.a);
+	ASSERT(root.c.nested.b == root2.c.nested.b);
+	ASSERT(root.c.nested.c == root2.c.nested.c);
+	ASSERT(root.c.c == root2.c.c);
+
+	root2 = {};
+	DummyContext context;
+	flat_buffers::detail::load(root2, out, context);
+
+	ASSERT(root.a == root2.a);
+	ASSERT(root.b == root2.b);
+	ASSERT(root.c.a == root2.c.a);
+	ASSERT(root.c.b == root2.c.b);
+	ASSERT(root.c.nested.a == root2.c.nested.a);
+	ASSERT(root.c.nested.b == root2.c.nested.b);
+	ASSERT(root.c.nested.c == root2.c.nested.c);
+	ASSERT(root.c.c == root2.c.c);
+	return Void();
+}
+
+TEST_CASE("flow/FlatBuffers/serializeDeserializeMembers") {
+	Root root{ 1,
+		       { { 13, { "ghi", "jkl" }, 15 }, { 16, { "mnop", "qrstuv" }, 18 } },
+		       { 3, "hello", { 6, { "abc", "def" }, 8 }, { 10, 11, 12 } } };
+	Root root2 = root;
+	Arena arena;
+	const auto* out = save_members(arena, FileIdentifier{}, root.a, root.b, root.c);
+
+	ASSERT(root.a == root2.a);
+	ASSERT(root.b == root2.b);
+	ASSERT(root.c.a == root2.c.a);
+	ASSERT(root.c.b == root2.c.b);
+	ASSERT(root.c.nested.a == root2.c.nested.a);
+	ASSERT(root.c.nested.b == root2.c.nested.b);
+	ASSERT(root.c.nested.c == root2.c.nested.c);
+	ASSERT(root.c.c == root2.c.c);
+
+	root2 = {};
+	DummyContext context;
+	load_members(out, context, root2.a, root2.b, root2.c);
+
+	ASSERT(root.a == root2.a);
+	ASSERT(root.b == root2.b);
+	ASSERT(root.c.a == root2.c.a);
+	ASSERT(root.c.b == root2.c.b);
+	ASSERT(root.c.nested.a == root2.c.nested.a);
+	ASSERT(root.c.nested.b == root2.c.nested.b);
+	ASSERT(root.c.nested.c == root2.c.nested.c);
+	ASSERT(root.c.c == root2.c.c);
+	return Void();
+}
+
+} // namespace unit_tests
+
+template <class... Alternatives>
+struct union_like_traits<boost::variant<Alternatives...>> : std::true_type {
+	using Member = boost::variant<Alternatives...>;
+	using alternatives = pack<Alternatives...>;
+	static uint8_t index(const Member& variant) { return variant.which(); }
+	static bool empty(const Member& variant) { return false; }
+
+	template <int i>
+	static const index_t<i, alternatives>& get(const Member& variant) {
+		return boost::get<index_t<i, alternatives>>(variant);
+	}
+
+	template <size_t i, class Alternative>
+	static const void assign(Member& member, const Alternative& a) {
+		static_assert(std::is_same_v<index_t<i, alternatives>, Alternative>);
+		member = a;
+	}
+};
+
+namespace unit_tests {
+
+TEST_CASE("flow/FlatBuffers/variant") {
+	using V = boost::variant<int, double, Nested2>;
+	V v1;
+	V v2;
+	Arena arena;
+	DummyContext context;
+	const uint8_t* out;
+
+	v1 = 1;
+	out = save_members(arena, FileIdentifier{}, v1);
+	// print_buffer(out, arena.get_size(out));
+	load_members(out, context, v2);
+	ASSERT(boost::get<int>(v1) == boost::get<int>(v2));
+
+	v1 = 1.0;
+	out = save_members(arena, FileIdentifier{}, v1);
+	// print_buffer(out, arena.get_size(out));
+	load_members(out, context, v2);
+	ASSERT(boost::get<double>(v1) == boost::get<double>(v2));
+
+	v1 = Nested2{ 1, { "abc", "def" }, 2 };
+	out = save_members(arena, FileIdentifier{}, v1);
+	// print_buffer(out, arena.get_size(out));
+	load_members(out, context, v2);
+	ASSERT(boost::get<Nested2>(v1).a == boost::get<Nested2>(v2).a);
+	ASSERT(boost::get<Nested2>(v1).b == boost::get<Nested2>(v2).b);
+	ASSERT(boost::get<Nested2>(v1).c == boost::get<Nested2>(v2).c);
+	return Void();
+}
+
+TEST_CASE("flow/FlatBuffers/vectorBool") {
+	std::vector<bool> x1 = { true, false, true, false, true };
+	std::vector<bool> x2;
+	Arena arena;
+	DummyContext context;
+	const uint8_t* out;
+
+	out = save_members(arena, FileIdentifier{}, x1);
+	// print_buffer(out, arena.get_size(out));
+	load_members(out, context, x2);
+	ASSERT(x1 == x2);
+	return Void();
+}
+
+struct DynamicSizeThingy {
+	std::string x;
+	mutable int saves = 0;
+};
+
+} // namespace unit_tests
+
+template <>
+struct dynamic_size_traits<unit_tests::DynamicSizeThingy> : std::true_type {
+private:
+	using T = unit_tests::DynamicSizeThingy;
+
+public:
+	static WriteRawMemory save(const T& t) {
+		++t.saves;
+		T* t2 = new T(t);
+		return { { ownedPtr(reinterpret_cast<const uint8_t*>(t2->x.data()), [t2](auto*) { delete t2; }),
+			       t2->x.size() } };
+	}
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t* p, size_t n, T& t, Context&) {
+		t.x.assign(reinterpret_cast<const char*>(p), n);
+	}
+};
+
+namespace unit_tests {
+
+TEST_CASE("flow/FlatBuffers/dynamic_size_owned") {
+	DynamicSizeThingy x1 = { "abcdefg" };
+	DynamicSizeThingy x2;
+	Arena arena;
+	DummyContext context;
+	const uint8_t* out;
+
+	out = save_members(arena, FileIdentifier{}, x1);
+	ASSERT(x1.saves == 1);
+	// print_buffer(out, arena.get_size(out));
+	load_members(out, context, x2);
+	ASSERT(x1.x == x2.x);
+	return Void();
+}
+
+struct Y1 {
+	int a;
+
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, a);
+	}
+};
+
+struct Y2 {
+	int a;
+	boost::variant<int> b;
+
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, a, b);
+	}
+};
+
+template <class Y>
+struct X {
+	int a;
+	Y b;
+	int c;
+
+	template <class Archiver>
+	void serialize(Archiver& ar) {
+		return flat_buffers::serializer(ar, a, b, c);
+	}
+};
+
+TEST_CASE("flow/FlatBuffers/nestedCompat") {
+	X<Y1> x1 = { 1, { 2 }, 3 };
+	X<Y2> x2;
+	Arena arena;
+	DummyContext context;
+	const uint8_t* out;
+
+	out = save_members(arena, FileIdentifier{}, x1);
+	load_members(out, context, x2);
+	ASSERT(x1.a == x2.a);
+	ASSERT(x1.b.a == x2.b.a);
+	ASSERT(x1.c == x2.c);
+
+	x1 = {};
+	x2.b.b = 4;
+
+	out = save_members(arena, FileIdentifier{}, x2);
+	load_members(out, context, x1);
+	ASSERT(x1.a == x2.a);
+	ASSERT(x1.b.a == x2.b.a);
+	ASSERT(x1.c == x2.c);
+	return Void();
+}
+
+TEST_CASE("flow/FlatBuffers/struct") {
+	std::vector<std::tuple<int16_t, bool, int64_t>> x1 = { { 1, true, 2 }, { 3, false, 4 } };
+	decltype(x1) x2;
+	Arena arena;
+	DummyContext context;
+	const uint8_t* out;
+
+	out = save_members(arena, FileIdentifier{}, x1);
+	// print_buffer(out, arena.get_size(out));
+	load_members(out, context, x2);
+	ASSERT(x1 == x2);
+	return Void();
+}
+
+TEST_CASE("flow/FlatBuffers/file_identifier") {
+	Arena arena;
+	const uint8_t* out;
+	constexpr FileIdentifier file_identifier{ 1234 };
+	out = save_members(arena, file_identifier);
+	// print_buffer(out, arena.get_size(out));
+	ASSERT(read_file_identifier(out) == file_identifier);
+	return Void();
+}
+
+TEST_CASE("flow/FlatBuffers/VectorRef") {
+	// this test tests a few weird memory properties of
+	// serialized arenas. This is why it uses weird scoping
+
+	// first we construct the data to serialize/deserialize
+	// so we can compare it afterwards
+	std::vector<std::string> src;
+	src.push_back("Foo");
+	src.push_back("Bar");
+	::Arena vecArena;
+	VectorRef<StringRef> outVec;
+	{
+		::Arena readerArena;
+		StringRef serializedVector;
+		{
+			::Arena arena;
+			VectorRef<StringRef> vec;
+			for (const auto& str : src) {
+				vec.push_back(arena, str);
+			}
+			BinaryWriter writer(IncludeVersion());
+			::serialize_fake_root(writer, FileIdentifierFor<decltype(vec)>::value, arena, vec);
+			serializedVector = StringRef(readerArena, writer.toStringRef());
+		}
+		ArenaReader reader(readerArena, serializedVector, IncludeVersion());
+		::serialize_fake_root(reader, FileIdentifierFor<decltype(outVec)>::value, vecArena, outVec);
+	}
+	ASSERT(src.size() == outVec.size());
+	for (int i = 0; i < src.size(); ++i) {
+		auto str = outVec[i].toString();
+		ASSERT(str == src[i]);
+	}
+	return Void();
+}
+
+} // namespace unit_tests
+
+} // namespace flat_buffers
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
new file mode 100644
index 0000000000..adcbc6b8a9
--- /dev/null
+++ b/flow/flat_buffers.h
@@ -0,0 +1,1241 @@
+/*
+ * flat_buffers.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <cstring>
+#include <functional>
+#include <map>
+#include <memory>
+#include <set>
+#include <stdint.h>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <vector>
+#include <cstring>
+#include <array>
+#include <typeinfo>
+#include <typeindex>
+
+namespace flat_buffers {
+
+template <class... Ts>
+struct pack {};
+
+template <class T = pack<>, class...>
+struct concat {
+	using type = T;
+};
+template <class... T1, class... T2, class... Ts>
+struct concat<pack<T1...>, pack<T2...>, Ts...> : concat<pack<T1..., T2...>, Ts...> {};
+template <class... Ts>
+using concat_t = typename concat<Ts...>::type;
+
+template <class... Ts>
+constexpr auto pack_size(pack<Ts...>) {
+	return sizeof...(Ts);
+}
+
+template <int i, class... Ts>
+struct index;
+
+template <int i, class T, class... Ts>
+struct index<i, pack<T, Ts...>> {
+	using type = typename index<i - 1, pack<Ts...>>::type;
+};
+
+template <class T, class... Ts>
+struct index<0, pack<T, Ts...>> {
+	using type = T;
+};
+
+template <int i, class Pack>
+using index_t = typename index<i, Pack>::type;
+
+constexpr int RightAlign(int offset, int alignment) {
+	return offset % alignment == 0 ? offset : ((offset / alignment) + 1) * alignment;
+}
+
+using FileIdentifier = uint32_t;
+
+template <class T>
+struct FileIdentifierFor {
+	constexpr static FileIdentifier value = T::file_identifier;
+};
+
+template <>
+struct FileIdentifierFor<int> {
+	constexpr static FileIdentifier value = 1;
+};
+
+template <>
+struct FileIdentifierFor<unsigned> {
+	constexpr static FileIdentifier value = 2;
+};
+
+template <>
+struct FileIdentifierFor<long> {
+	constexpr static FileIdentifier value = 3;
+};
+
+template <>
+struct FileIdentifierFor<unsigned long> {
+	constexpr static FileIdentifier value = 4;
+};
+
+template <>
+struct FileIdentifierFor<long long> {
+	constexpr static FileIdentifier value = 5;
+};
+
+template <>
+struct FileIdentifierFor<unsigned long long> {
+	constexpr static FileIdentifier value = 6;
+};
+
+template <>
+struct FileIdentifierFor<short> {
+	constexpr static FileIdentifier value = 7;
+};
+
+template <>
+struct FileIdentifierFor<unsigned short> {
+	constexpr static FileIdentifier value = 8;
+};
+
+template <>
+struct FileIdentifierFor<signed char> {
+	constexpr static FileIdentifier value = 9;
+};
+
+template <>
+struct FileIdentifierFor<unsigned char> {
+	constexpr static FileIdentifier value = 10;
+};
+
+template <>
+struct FileIdentifierFor<bool> {
+	constexpr static FileIdentifier value = 11;
+};
+
+template <>
+struct FileIdentifierFor<float> {
+	constexpr static flat_buffers::FileIdentifier value = 7266212;
+};
+
+template <>
+struct FileIdentifierFor<double> {
+	constexpr static flat_buffers::FileIdentifier value = 9348150;
+};
+
+template <class K, class V, class Compare, class Allocator>
+struct FileIdentifierFor<std::map<K, V, Compare, Allocator>> {
+	constexpr static FileIdentifier value = FileIdentifierFor<std::pair<K, V>>::value;
+};
+
+template <class F, class S>
+struct FileIdentifierFor<std::pair<F, S>> {
+	constexpr static FileIdentifier value = FileIdentifierFor<F>::value ^ FileIdentifierFor<S>::value;
+};
+
+template <class T, class Allocator>
+struct FileIdentifierFor<std::vector<T, Allocator>> {
+	constexpr static FileIdentifier value = (0x10 << 24) | FileIdentifierFor<T>::value;
+};
+
+template <class CharT, class Traits, class Allocator>
+struct FileIdentifierFor<std::basic_string<CharT, Traits, Allocator>> {
+	constexpr static FileIdentifier value = 15694229;
+};
+
+template <class T>
+struct object_construction {
+	T obj;
+
+	object_construction() : obj() {}
+	object_construction(const T& o) : obj(o) {}
+	object_construction(T&& o) : obj(std::move(o)) {}
+
+	T& get() { return obj; }
+	const T& get() const { return obj; }
+	T move() { return std::move(obj); }
+};
+
+// A smart pointer that knows whether or not to delete itself.
+template <class T>
+using OwnershipErasedPtr = std::unique_ptr<T, std::function<void(T*)>>;
+
+// Creates an OwnershipErasedPtr<T> that will delete itself.
+template <class T, class Deleter = std::default_delete<T>>
+OwnershipErasedPtr<T> ownedPtr(T* t, Deleter&& d = Deleter{}) {
+	return OwnershipErasedPtr<T>{ t, std::forward<Deleter>(d) };
+}
+
+// Creates an OwnershipErasedPtr<T> that will not delete itself.
+template <class T>
+OwnershipErasedPtr<T> unownedPtr(T* t) {
+	return OwnershipErasedPtr<T>{ t, [](T*) {} };
+}
+
+template <class T, typename = void>
+struct scalar_traits : std::false_type {
+	constexpr static size_t size = 0;
+	static void save(uint8_t*, const T&);
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t*, T&, Context&);
+};
+
+struct WriteRawMemory {
+	using Block = std::pair<OwnershipErasedPtr<const uint8_t>, size_t>;
+	std::vector<Block> blocks;
+
+	WriteRawMemory() {}
+	WriteRawMemory(Block&& b) { blocks.emplace_back(std::move(b.first), b.second); }
+	WriteRawMemory(std::vector<Block>&& v) : blocks(std::move(v)) {}
+
+	WriteRawMemory(WriteRawMemory&&) = default;
+	WriteRawMemory& operator=(WriteRawMemory&&) = default;
+
+	size_t size() const {
+		size_t result = 0;
+		for (const auto& b : blocks) {
+			result += b.second;
+		}
+		return result;
+	}
+};
+
+template <class T>
+struct dynamic_size_traits : std::false_type {
+	static WriteRawMemory save(const T&);
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t*, size_t, T&, Context&);
+};
+
+template <class T>
+struct serializable_traits : std::false_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, T& v);
+};
+
+template <class VectorLike>
+struct vector_like_traits : std::false_type {
+	// Write this at the beginning of the buffer
+	using value_type = uint8_t;
+	using iterator = void;
+	using insert_iterator = void;
+
+	static size_t num_entries(VectorLike&);
+	template <class Context>
+	static void reserve(VectorLike&, size_t, Context&);
+
+	static insert_iterator insert(VectorLike&);
+	static iterator begin(const VectorLike&);
+	static void deserialization_done(VectorLike&); // Optional
+};
+
+template <class UnionLike>
+struct union_like_traits : std::false_type {
+	using Member = UnionLike;
+	using alternatives = pack<>;
+	static uint8_t index(const Member&);
+	static bool empty(const Member& variant);
+
+	template <int i>
+	static const index_t<i, alternatives>& get(const Member&);
+
+	template <int i, class Alternative>
+	static const void assign(Member&, const Alternative&);
+
+	template <class Context>
+	static void done(Member&, Context&);
+};
+
+// TODO(anoyes): Implement things that are currently using scalar traits with
+// struct-like traits.
+template <class StructLike>
+struct struct_like_traits : std::false_type {
+	using Member = StructLike;
+	using types = pack<>;
+
+	template <int i>
+	static const index_t<i, types>& get(const Member&);
+
+	template <int i>
+	static const void assign(Member&, const index_t<i, types>&);
+
+	template <class Context>
+	static void done(Member&, Context&);
+};
+
+template <class... Ts>
+struct struct_like_traits<std::tuple<Ts...>> : std::true_type {
+	using Member = std::tuple<Ts...>;
+	using types = pack<Ts...>;
+
+	template <int i>
+	static const index_t<i, types>& get(const Member& m) {
+		return std::get<i>(m);
+	}
+
+	template <int i, class Type>
+	static const void assign(Member& m, const Type& t) {
+		std::get<i>(m) = t;
+	}
+};
+
+template <class T>
+struct scalar_traits<T, std::enable_if_t<std::is_integral<T>::value || std::is_floating_point<T>::value>>
+  : std::true_type {
+	constexpr static size_t size = sizeof(T);
+	static void save(uint8_t* out, const T& t) { memcpy(out, &t, size); }
+	template <class Context>
+	static void load(const uint8_t* in, T& t, Context&) {
+		memcpy(&t, in, size);
+	}
+};
+
+template <class F, class... Items>
+void serializer(F& fun, Items&... items);
+
+template <class F, class S>
+struct serializable_traits<std::pair<F, S>> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, std::pair<F, S>& p) {
+		flat_buffers::serializer(ar, p.first, p.second);
+	}
+};
+
+template <class T, class Alloc>
+struct vector_like_traits<std::vector<T, Alloc>> : std::true_type {
+	using Vec = std::vector<T, Alloc>;
+	using value_type = typename Vec::value_type;
+	using iterator = typename Vec::const_iterator;
+	using insert_iterator = std::back_insert_iterator<Vec>;
+
+	static size_t num_entries(const Vec& v) { return v.size(); }
+	template <class Context>
+	static void reserve(Vec& v, size_t size, Context&) {
+		v.clear();
+		v.reserve(size);
+	}
+
+	static insert_iterator insert(Vec& v) { return std::back_inserter(v); }
+	static iterator begin(const Vec& v) { return v.begin(); }
+};
+
+template <class Key, class T, class Compare, class Allocator>
+struct vector_like_traits<std::map<Key, T, Compare, Allocator>> : std::true_type {
+	using Vec = std::map<Key, T, Compare, Allocator>;
+	using value_type = std::pair<Key, T>;
+	using iterator = typename Vec::const_iterator;
+	using insert_iterator = std::insert_iterator<Vec>;
+
+	static size_t num_entries(const Vec& v) { return v.size(); }
+	template <class Context>
+	static void reserve(Vec& v, size_t size, Context&) {}
+
+	static insert_iterator insert(Vec& v) { return std::inserter(v, v.end()); }
+	static iterator begin(const Vec& v) { return v.begin(); }
+};
+
+template <>
+struct dynamic_size_traits<std::string> : std::true_type {
+private:
+	using T = std::string;
+
+public:
+	static WriteRawMemory save(const T& t) {
+		return { { unownedPtr(reinterpret_cast<const uint8_t*>(t.data())), t.size() } };
+	};
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t* p, size_t n, T& t, Context&) {
+		t.assign(reinterpret_cast<const char*>(p), n);
+	}
+};
+
+namespace detail {
+
+template <class T>
+T interpret_as(const uint8_t* current) {
+	T t;
+	memcpy(&t, current, sizeof(t));
+	return t;
+}
+
+// Used to select an overload for |MessageWriter::write| that fixes relative
+// offsets.
+struct RelativeOffset {
+	int value;
+};
+static_assert(sizeof(RelativeOffset) == 4, "");
+
+template <class T>
+constexpr bool is_scalar = scalar_traits<T>::value;
+
+template <class T>
+constexpr bool is_dynamic_size = dynamic_size_traits<T>::value;
+
+template <class T>
+constexpr bool is_union_like = union_like_traits<T>::value;
+
+template <class T>
+constexpr bool is_vector_like = vector_like_traits<T>::value;
+
+template <class T>
+constexpr bool is_struct_like = struct_like_traits<T>::value;
+
+template <class T>
+constexpr bool expect_serialize_member =
+    !is_scalar<T> && !is_vector_like<T> && !is_union_like<T> && !is_dynamic_size<T> && !is_struct_like<T>;
+
+template <class T>
+constexpr bool use_indirection = !(is_scalar<T> || is_struct_like<T>);
+
+using VTable = std::vector<uint16_t>;
+
+template <class T>
+struct sfinae_true : std::true_type {};
+
+template <class T>
+auto test_deserialization_done(int) -> sfinae_true<decltype(T::deserialization_done)>;
+
+template <class T>
+auto test_deserialization_done(long) -> std::false_type;
+
+template <class T>
+struct has_deserialization_done : decltype(test_deserialization_done<T>(0)) {};
+
+template <class T>
+constexpr int fb_scalar_size = is_scalar<T> ? scalar_traits<T>::size : sizeof(RelativeOffset);
+
+template <size_t offset, size_t index, class... Ts>
+struct struct_offset_impl;
+
+template <size_t o, size_t index>
+struct struct_offset_impl<o, index> {
+	static_assert(index == 0);
+	static constexpr auto offset = o;
+};
+
+template <size_t o, size_t index, class T, class... Ts>
+struct struct_offset_impl<o, index, T, Ts...> {
+private:
+	static constexpr size_t offset_() {
+		if constexpr (index == 0) {
+			return RightAlign(o, fb_scalar_size<T>);
+		} else {
+			return struct_offset_impl<RightAlign(o, fb_scalar_size<T>) + fb_scalar_size<T>, index - 1, Ts...>::offset;
+		}
+	}
+
+public:
+	static_assert(!is_struct_like<T>, "Nested structs not supported yet");
+	static constexpr auto offset = offset_();
+};
+
+constexpr size_t AlignToPowerOfTwo(size_t s) {
+	if (s > 4) {
+		return 8;
+	} else if (s > 2) {
+		return 4;
+	} else if (s > 1) {
+		return 2;
+	} else {
+		return 1;
+	}
+}
+
+template <class... Ts>
+constexpr auto align_helper(pack<Ts...>) {
+	return std::max({ size_t{ 1 }, AlignToPowerOfTwo(fb_scalar_size<Ts>)... });
+}
+
+template <class... T>
+constexpr auto struct_size(pack<T...>) {
+	return std::max(1, RightAlign(struct_offset_impl<0, sizeof...(T), T...>::offset, align_helper(pack<T...>{})));
+}
+
+template <int i, class... T>
+constexpr auto struct_offset(pack<T...>) {
+	static_assert(i < sizeof...(T));
+	return struct_offset_impl<0, i, T...>::offset;
+}
+
+static_assert(struct_offset<0>(pack<int>{}) == 0);
+static_assert(struct_offset<1>(pack<int, bool>{}) == 4);
+static_assert(struct_offset<2>(pack<int, bool, double>{}) == 8);
+
+static_assert(struct_size(pack<>{}) == 1);
+static_assert(struct_size(pack<int>{}) == 4);
+static_assert(struct_size(pack<int, bool>{}) == 8);
+static_assert(struct_size(pack<int, bool, double>{}) == 16);
+
+template <class T>
+constexpr int fb_size = is_struct_like<T> ? struct_size(typename struct_like_traits<T>::types{}) : fb_scalar_size<T>;
+
+template <class T>
+constexpr int fb_align = is_struct_like<T> ? align_helper(typename struct_like_traits<T>::types{})
+                                           : AlignToPowerOfTwo(fb_scalar_size<T>);
+
+template <class T>
+struct _SizeOf {
+	static constexpr int size = fb_size<T>;
+	static constexpr int align = fb_align<T>;
+};
+
+struct PrecomputeSize {
+	// |offset| is measured from the end of the buffer. Precondition: len <=
+	// offset.
+	void write(const void*, int offset, int len) { current_buffer_size = std::max(current_buffer_size, offset); }
+
+	template <class ToRawMemory>
+	void writeRawMemory(ToRawMemory&& to_raw_memory) {
+		auto w = std::forward<ToRawMemory>(to_raw_memory)();
+		int start = RightAlign(current_buffer_size + w.size() + 4, 4);
+		write(nullptr, start, 4);
+		start -= 4;
+		for (auto& block : w.blocks) {
+			write(nullptr, start, block.second);
+			start -= block.second;
+		}
+		writeRawMemories.emplace_back(std::move(w));
+	}
+
+	struct Noop {
+		void write(const void* src, int offset, int len) {}
+		void writeTo(PrecomputeSize& writer, int offset) {
+
+			writer.write(nullptr, offset, size);
+			writer.writeToOffsets[writeToIndex] = offset;
+		}
+		void writeTo(PrecomputeSize& writer) { writeTo(writer, writer.current_buffer_size + size); }
+		int size;
+		int writeToIndex;
+	};
+
+	Noop getMessageWriter(int size) {
+		int writeToIndex = writeToOffsets.size();
+		writeToOffsets.push_back({});
+		return Noop{ size, writeToIndex };
+	}
+
+	int current_buffer_size = 0;
+
+	const int buffer_length = -1; // Dummy, the value of this should not affect anything.
+	const int vtable_start = -1; // Dummy, the value of this should not affect anything.
+	std::vector<int> writeToOffsets;
+	std::vector<WriteRawMemory> writeRawMemories;
+};
+
+template <class Member, class Context>
+void load_helper(Member&, const uint8_t*, Context&);
+
+class VTableSet;
+
+template <class T>
+struct is_array : std::false_type {};
+
+template <class T, size_t size>
+struct is_array<std::array<T, size>> : std::true_type {};
+
+struct WriteToBuffer {
+	// |offset| is measured from the end of the buffer. Precondition: len <=
+	// offset.
+	void write(const void* src, int offset, int len) {
+		copy_memory(src, offset, len);
+		current_buffer_size = std::max(current_buffer_size, offset);
+	}
+
+	template <class ToRawMemory>
+	void writeRawMemory(ToRawMemory&&) {
+		auto& w = *write_raw_memories_iter;
+		uint32_t size = w.size();
+		int start = RightAlign(current_buffer_size + size + 4, 4);
+		write(&size, start, 4);
+		start -= 4;
+		for (auto& p : w.blocks) {
+			if (p.second > 0) {
+				write(reinterpret_cast<const void*>(p.first.get()), start, p.second);
+			}
+			start -= p.second;
+		}
+		++write_raw_memories_iter;
+	}
+
+	WriteToBuffer(int buffer_length, int vtable_start, uint8_t* buffer, std::vector<int> writeToOffsets,
+	              std::vector<WriteRawMemory>::iterator write_raw_memories_iter)
+	  : buffer_length(buffer_length), vtable_start(vtable_start), buffer(buffer),
+	    writeToOffsets(std::move(writeToOffsets)), write_raw_memories_iter(write_raw_memories_iter) {}
+
+	struct MessageWriter {
+		template <class T>
+		void write(const T* src, int offset, size_t len) {
+			if constexpr (std::is_same_v<T, RelativeOffset>) {
+				uint32_t fixed_offset = finalLocation - offset - src->value;
+				writer.copy_memory(&fixed_offset, finalLocation - offset, len);
+			} else if constexpr (is_array<T>::value) {
+				writer.copy_memory(src, finalLocation - offset, std::min(src->size(), len));
+			} else {
+				writer.copy_memory(src, finalLocation - offset, len);
+			}
+		}
+		void writeTo(WriteToBuffer&) { writer.current_buffer_size += size; }
+		void writeTo(WriteToBuffer&, int offset) {
+			writer.current_buffer_size = std::max(writer.current_buffer_size, offset);
+		}
+		WriteToBuffer& writer;
+		int finalLocation;
+		int size;
+	};
+
+	MessageWriter getMessageWriter(int size) {
+		MessageWriter m{ *this, writeToOffsets[writeToIndex++], size };
+		return m;
+	}
+
+	const int buffer_length;
+	const int vtable_start;
+	int current_buffer_size = 0;
+
+private:
+	void copy_memory(const void* src, int offset, int len) {
+		memcpy(static_cast<void*>(&buffer[buffer_length - offset]), src, len);
+	}
+	std::vector<int> writeToOffsets;
+	std::vector<WriteRawMemory>::iterator write_raw_memories_iter;
+	int writeToIndex = 0;
+	uint8_t* buffer;
+};
+
+template <class Member>
+constexpr auto fields_helper() {
+	if constexpr (_SizeOf<Member>::size == 0) {
+		return pack<>{};
+	} else if constexpr (is_union_like<Member>) {
+		return pack<uint8_t, uint32_t>{};
+	} else {
+		return pack<Member>{};
+	}
+}
+
+template <class Member>
+using Fields = decltype(fields_helper<Member>());
+
+// TODO(anoyes): Make this `template <int... offsets>` so we can re-use
+// identical vtables even if they have different types.
+// Also, it's important that get_vtable always returns the same VTable pointer
+// so that we can decide equality by comparing the pointers.
+
+extern VTable generate_vtable(size_t numMembers, const std::vector<unsigned>& members,
+                              const std::vector<unsigned>& alignments);
+
+template <class... Members>
+VTable gen_vtable(pack<Members...> p) {
+	return generate_vtable(sizeof...(Members), std::vector<unsigned>{ { _SizeOf<Members>::size... } },
+	                       std::vector<unsigned>{ { _SizeOf<Members>::align... } });
+}
+
+template <class... Members>
+const VTable* get_vtable() {
+	static VTable table = gen_vtable(concat_t<Fields<Members>...>{});
+	return &table;
+}
+
+template <class F, class... Members>
+void for_each(F&& f, Members&&... members) {
+	(std::forward<F>(f)(std::forward<Members>(members)), ...);
+}
+
+struct VTableSet {
+	std::map<const VTable*, int> offsets;
+	std::vector<uint8_t> packed_tables;
+};
+
+struct InsertVTableLambda;
+
+struct TraverseMessageTypes {
+	InsertVTableLambda& f;
+
+	bool vtableGeneratedBefore(const std::type_index&);
+
+	template <class Member>
+	std::enable_if_t<expect_serialize_member<Member>> operator()(const Member& member) {
+		if (vtableGeneratedBefore(typeid(Member))) {
+			return;
+		}
+		if constexpr (serializable_traits<Member>::value) {
+			serializable_traits<Member>::serialize(f, const_cast<Member&>(member));
+		} else {
+			const_cast<Member&>(member).serialize(f);
+		}
+	};
+
+	template <class T>
+	std::enable_if_t<!expect_serialize_member<T> && !is_vector_like<T> && !is_union_like<T>> operator()(const T&) {}
+
+	template <class VectorLike>
+	std::enable_if_t<is_vector_like<VectorLike>> operator()(const VectorLike& members) {
+		using VectorTraits = vector_like_traits<VectorLike>;
+		using T = typename VectorTraits::value_type;
+		static_assert(!is_union_like<T>, "vector<union> not yet supported");
+		// we don't need to check for recursion here because the next call
+		// to operator() will do that and we don't generate a vtable for the
+		// vector-like type itself
+		object_construction<T> t;
+		(*this)(t.get());
+	}
+
+	template <class UnionLike>
+	std::enable_if_t<is_union_like<UnionLike>> operator()(const UnionLike& members) {
+		using UnionTraits = union_like_traits<UnionLike>;
+		static_assert(pack_size(typename UnionTraits::alternatives{}) <= 254,
+		              "Up to 254 alternatives are supported for unions");
+		union_helper(typename UnionTraits::alternatives{});
+	}
+
+private:
+	template <class T, class... Ts>
+	void union_helper(pack<T, Ts...>) {
+		object_construction<T> t;
+		(*this)(t.get());
+		union_helper(pack<Ts...>{});
+	}
+	void union_helper(pack<>) {}
+};
+
+struct InsertVTableLambda {
+	static constexpr bool isDeserializing = true;
+	std::set<const VTable*>& vtables;
+	std::set<std::type_index>& known_types;
+
+	template <class... Members>
+	void operator()(const Members&... members) {
+		vtables.insert(get_vtable<Members...>());
+		for_each(TraverseMessageTypes{ *this }, members...);
+	}
+};
+
+template <class T>
+int vec_bytes(const T& begin, const T& end) {
+	return sizeof(typename T::value_type) * (end - begin);
+}
+
+template <class Root>
+VTableSet get_vtableset_impl(const Root& root) {
+	std::set<const VTable*> vtables;
+	std::set<std::type_index> known_types;
+	InsertVTableLambda vlambda{ vtables, known_types };
+	if constexpr (serializable_traits<Root>::value) {
+		serializable_traits<Root>::serialize(vlambda, const_cast<Root&>(root));
+	} else {
+		const_cast<Root&>(root).serialize(vlambda);
+	}
+	size_t size = 0;
+	for (const auto* vtable : vtables) {
+		size += vec_bytes(vtable->begin(), vtable->end());
+	}
+	std::vector<uint8_t> packed_tables(size);
+	int i = 0;
+	std::map<const VTable*, int> offsets;
+	for (const auto* vtable : vtables) {
+		memcpy(&packed_tables[i], reinterpret_cast<const uint8_t*>(&(*vtable)[0]),
+		       vec_bytes(vtable->begin(), vtable->end()));
+		offsets[vtable] = i;
+		i += vec_bytes(vtable->begin(), vtable->end());
+	}
+	return VTableSet{ offsets, packed_tables };
+}
+
+template <class Root>
+const VTableSet* get_vtableset(const Root& root) {
+	static VTableSet result = get_vtableset_impl(root);
+	return &result;
+}
+
+template <class Root, class Writer>
+void save_with_vtables(const Root& root, const VTableSet* vtableset, Writer& writer, int* vtable_start,
+                       FileIdentifier file_identifier) {
+	auto vtable_writer = writer.getMessageWriter(vtableset->packed_tables.size());
+	vtable_writer.write(&vtableset->packed_tables[0], 0, vtableset->packed_tables.size());
+	RelativeOffset offset = save_helper(const_cast<Root&>(root), writer, vtableset);
+	vtable_writer.writeTo(writer);
+	*vtable_start = writer.current_buffer_size;
+	int root_writer_size = sizeof(uint32_t) + sizeof(file_identifier);
+	auto root_writer = writer.getMessageWriter(root_writer_size);
+	root_writer.write(&offset, 0, sizeof(offset));
+	root_writer.write(&file_identifier, sizeof(offset), sizeof(file_identifier));
+	root_writer.writeTo(writer, RightAlign(writer.current_buffer_size + root_writer_size, 8));
+}
+
+template <class Writer, class UnionTraits>
+struct SaveAlternative {
+	Writer& writer;
+	const VTableSet* vtables;
+
+	RelativeOffset save(uint8_t type_tag, const typename UnionTraits::Member& member) {
+		return save_<0>(type_tag, member);
+	}
+
+private:
+	template <uint8_t Alternative>
+	RelativeOffset save_(uint8_t type_tag, const typename UnionTraits::Member& member) {
+		if constexpr (Alternative < pack_size(typename UnionTraits::alternatives{})) {
+			if (type_tag == Alternative) {
+				auto result = save_helper(UnionTraits::template get<Alternative>(member), writer, vtables);
+				if constexpr (use_indirection<index_t<Alternative, typename UnionTraits::alternatives>>) {
+					return result;
+				}
+				writer.write(&result, writer.current_buffer_size + sizeof(result), sizeof(result));
+				return RelativeOffset{ writer.current_buffer_size };
+			} else {
+				return save_<Alternative + 1>(type_tag, member);
+			}
+		}
+		throw std::runtime_error("type_tag out of range. This should never happen.");
+	}
+};
+
+template <class Context, class UnionTraits>
+struct LoadAlternative {
+	Context& context;
+	const uint8_t* current;
+
+	void load(uint8_t type_tag, typename UnionTraits::Member& member) { return load_<0>(type_tag, member); }
+
+private:
+	template <uint8_t Alternative>
+	void load_(uint8_t type_tag, typename UnionTraits::Member& member) {
+		if constexpr (Alternative < pack_size(typename UnionTraits::alternatives{})) {
+			if (type_tag == Alternative) {
+				using AlternativeT = index_t<Alternative, typename UnionTraits::alternatives>;
+				object_construction<AlternativeT> alternative;
+				if constexpr (use_indirection<AlternativeT>) {
+					load_helper(alternative.get(), current, context);
+				} else {
+					uint32_t current_offset = interpret_as<uint32_t>(current);
+					current += current_offset;
+					load_helper(alternative.get(), current, context);
+				}
+				UnionTraits::template assign<Alternative>(member, std::move(alternative.move()));
+			} else {
+				load_<Alternative + 1>(type_tag, member);
+			}
+		}
+	}
+};
+
+template <class Writer>
+struct SaveVisitorLambda {
+	static constexpr bool isDeserializing = false;
+	const VTableSet* vtableset;
+	Writer& writer;
+
+	template <class... Members>
+	void operator()(const Members&... members) {
+		const auto& vtable = *get_vtable<Members...>();
+		auto self = writer.getMessageWriter(vtable[1] /* length */);
+		int i = 2;
+		for_each(
+		    [&](const auto& member) {
+			    using Member = std::decay_t<decltype(member)>;
+			    if constexpr (is_union_like<Member>) {
+				    using UnionTraits = union_like_traits<Member>;
+				    uint8_t type_tag = UnionTraits::index(member);
+				    uint8_t fb_type_tag = UnionTraits::empty(member) ? 0 : type_tag + 1; // Flatbuffers indexes from 1.
+				    self.write(&fb_type_tag, vtable[i++], sizeof(fb_type_tag));
+				    if (!UnionTraits::empty(member)) {
+					    RelativeOffset offset =
+					        (SaveAlternative<Writer, UnionTraits>{ writer, vtableset }).save(type_tag, member);
+					    self.write(&offset, vtable[i++], sizeof(offset));
+				    } else {
+					    ++i;
+				    }
+			    } else if constexpr (_SizeOf<Member>::size == 0) {
+				    save_helper(member, writer, vtableset);
+			    } else {
+				    auto result = save_helper(member, writer, vtableset);
+				    self.write(&result, vtable[i++], sizeof(result));
+			    }
+		    },
+		    members...);
+		int vtable_offset = writer.vtable_start - vtableset->offsets.at(&vtable);
+		int start = RightAlign(writer.current_buffer_size + vtable[1] - 4, std::max({ 1, fb_align<Members>... })) + 4;
+		int32_t relative = vtable_offset - start;
+		self.write(&relative, 0, sizeof(relative));
+		self.writeTo(writer, start);
+	}
+};
+
+template <class Context>
+struct LoadMember {
+	static constexpr bool isDeserializing = true;
+	const uint16_t* const vtable;
+	const uint8_t* const message;
+	const uint16_t vtable_length;
+	const uint16_t table_length;
+	int& i;
+	Context& context;
+	template <class Member>
+	void operator()(Member& member) {
+		if constexpr (is_union_like<Member>) {
+			if (!field_present()) {
+				i += 2;
+				return;
+			}
+			uint8_t fb_type_tag;
+			load_helper(fb_type_tag, &message[vtable[i]], context);
+			uint8_t type_tag = fb_type_tag - 1; // Flatbuffers indexes from 1.
+			++i;
+			if (field_present() && fb_type_tag > 0) {
+				(LoadAlternative<Context, union_like_traits<Member>>{ context, &message[vtable[i]] })
+				    .load(type_tag, member);
+			}
+			++i;
+		} else if constexpr (_SizeOf<Member>::size == 0) {
+			load_helper(member, nullptr, context);
+		} else {
+			if (field_present()) {
+				load_helper(member, &message[vtable[i]], context);
+			}
+			++i;
+		}
+	}
+
+private:
+	bool field_present() { return i < vtable_length && vtable[i] >= 4; }
+};
+
+template <size_t i>
+struct int_type {
+	static constexpr int value = i;
+};
+
+template <class F, size_t... I>
+void for_each_i_impl(F&& f, std::index_sequence<I...>) {
+	for_each(std::forward<F>(f), int_type<I>{}...);
+}
+
+template <size_t I, class F>
+void for_each_i(F&& f) {
+	for_each_i_impl(std::forward<F>(f), std::make_index_sequence<I>{});
+}
+
+template <class>
+struct LoadSaveHelper {
+	template <class U, class Context>
+	std::enable_if_t<is_scalar<U>> load(U& member, const uint8_t* current, Context& context) {
+		scalar_traits<U>::load(current, member, context);
+	}
+
+	template <class U, class Context>
+	std::enable_if_t<is_struct_like<U>> load(U& member, const uint8_t* current, Context& context) {
+		using StructTraits = struct_like_traits<U>;
+		using types = typename StructTraits::types;
+		constexpr auto size = struct_size(types{});
+		for_each_i<pack_size(types{})>([&](auto i_type) {
+			constexpr int i = decltype(i_type)::value;
+			using type = index_t<i, types>;
+			object_construction<type> t;
+			load_helper(t.get(), current + struct_offset<i>(types{}), context);
+			StructTraits::template assign<i>(member, t.move());
+		});
+	}
+
+	template <class U, class Context>
+	std::enable_if_t<is_dynamic_size<U>> load(U& member, const uint8_t* current, Context& context) {
+		uint32_t current_offset = interpret_as<uint32_t>(current);
+		current += current_offset;
+		uint32_t size = interpret_as<uint32_t>(current);
+		current += sizeof(size);
+		dynamic_size_traits<U>::load(current, size, member, context);
+	}
+
+	template <class Context>
+	struct SerializeFun {
+		static constexpr bool isDeserializing = true;
+
+		const uint16_t* vtable;
+		const uint8_t* current;
+		Context& context;
+
+		SerializeFun(const uint16_t* vtable, const uint8_t* current, Context& context)
+		  : vtable(vtable), current(current), context(context) {}
+
+		template <class... Args>
+		void operator()(Args&... members) {
+			int i = 0;
+			uint16_t vtable_length = vtable[i++] / sizeof(uint16_t);
+			uint16_t table_length = vtable[i++];
+			for_each(LoadMember<Context>{ vtable, current, vtable_length, table_length, i, context }, members...);
+		}
+	};
+
+	template <class Member, class Context>
+	std::enable_if_t<expect_serialize_member<Member>> load(Member& member, const uint8_t* current, Context& context) {
+		uint32_t current_offset = interpret_as<uint32_t>(current);
+		current += current_offset;
+		int32_t vtable_offset = interpret_as<int32_t>(current);
+		const uint16_t* vtable = reinterpret_cast<const uint16_t*>(current - vtable_offset);
+		SerializeFun<Context> fun(vtable, current, context);
+		if constexpr (serializable_traits<Member>::value) {
+			serializable_traits<Member>::serialize(fun, member);
+		} else {
+			member.serialize(fun);
+		}
+	}
+
+	template <class VectorLike, class Context>
+	std::enable_if_t<is_vector_like<VectorLike>> load(VectorLike& member, const uint8_t* current, Context& context) {
+		using VectorTraits = vector_like_traits<VectorLike>;
+		using T = typename VectorTraits::value_type;
+		uint32_t current_offset = interpret_as<uint32_t>(current);
+		current += current_offset;
+		uint32_t numEntries = interpret_as<uint32_t>(current);
+		current += sizeof(uint32_t);
+		VectorTraits::reserve(member, numEntries, context);
+		auto inserter = VectorTraits::insert(member);
+		for (int i = 0; i < numEntries; ++i) {
+			T value;
+			load_helper(value, current, context);
+			*inserter = std::move(value);
+			++inserter;
+			current += fb_size<T>;
+		}
+		if constexpr (has_deserialization_done<VectorTraits>::value) {
+			VectorTraits::deserialization_done(member);
+		}
+	}
+
+	template <class U, class Writer, typename = std::enable_if_t<is_scalar<U>>>
+	auto save(const U& message, Writer& writer, const VTableSet*) {
+		constexpr auto size = scalar_traits<U>::size;
+		std::array<uint8_t, size> result = {};
+		scalar_traits<U>::save(&result[0], message);
+		return result;
+	}
+
+	template <class U, class Writer>
+	auto save(const U& message, Writer& writer, const VTableSet* vtables,
+	          std::enable_if_t<is_struct_like<U>, int> _ = 0) {
+		using StructTraits = struct_like_traits<U>;
+		using types = typename StructTraits::types;
+		constexpr auto size = struct_size(types{});
+		std::array<uint8_t, size> struct_bytes = {};
+		for_each_i<pack_size(types{})>([&](auto i_type) {
+			constexpr int i = decltype(i_type)::value;
+			auto result = save_helper(StructTraits::template get<i>(message), writer, vtables);
+			memcpy(&struct_bytes[struct_offset<i>(types{})], &result, sizeof(result));
+		});
+		return struct_bytes;
+	}
+
+	template <class U, class Writer, typename = std::enable_if_t<is_dynamic_size<U>>>
+	RelativeOffset save(const U& message, Writer& writer, const VTableSet*,
+	                    std::enable_if_t<is_dynamic_size<U>, int> _ = 0) {
+		writer.writeRawMemory([&]() { return dynamic_size_traits<U>::save(message); });
+		return RelativeOffset{ writer.current_buffer_size };
+	}
+
+	template <class Member, class Writer>
+	RelativeOffset save(const Member& member, Writer& writer, const VTableSet* vtables,
+	                    std::enable_if_t<expect_serialize_member<Member>, int> _ = 0) {
+		SaveVisitorLambda<Writer> l{ vtables, writer };
+		if constexpr (serializable_traits<Member>::value) {
+			serializable_traits<Member>::serialize(l, const_cast<Member&>(member));
+		} else {
+			const_cast<Member&>(member).serialize(l);
+		}
+		return RelativeOffset{ writer.current_buffer_size };
+	}
+
+	template <class VectorLike, class Writer, typename = std::enable_if_t<is_vector_like<VectorLike>>>
+	RelativeOffset save(const VectorLike& members, Writer& writer, const VTableSet* vtables) {
+		using VectorTraits = vector_like_traits<VectorLike>;
+		using T = typename VectorTraits::value_type;
+		constexpr auto size = fb_size<T>;
+		uint32_t num_entries = VectorTraits::num_entries(members);
+		uint32_t len = num_entries * size;
+		auto self = writer.getMessageWriter(len);
+		auto iter = VectorTraits::begin(members);
+		for (int i = 0; i < num_entries; ++i) {
+			auto result = save_helper(*iter, writer, vtables);
+			self.write(&result, i * size, size);
+			++iter;
+		}
+		int start = RightAlign(writer.current_buffer_size + len, std::min(4, fb_align<T>)) + 4;
+		writer.write(&num_entries, start, sizeof(uint32_t));
+		self.writeTo(writer, start - sizeof(uint32_t));
+		return RelativeOffset{ writer.current_buffer_size };
+	}
+};
+
+template <class Alloc>
+struct LoadSaveHelper<std::vector<bool, Alloc>> {
+	template <class Context>
+	void load(std::vector<bool, Alloc>& member, const uint8_t* current, Context& context) {
+		uint32_t current_offset = interpret_as<uint32_t>(current);
+		current += current_offset;
+		uint32_t length = interpret_as<uint32_t>(current);
+		current += sizeof(uint32_t);
+		member.clear();
+		member.resize(length);
+		bool m;
+		for (int i = 0; i < length; ++i) {
+			load_helper(m, current, context);
+			member[i] = m;
+			current += fb_size<bool>;
+		}
+	}
+
+	template <class Writer>
+	RelativeOffset save(const std::vector<bool, Alloc>& members, Writer& writer, const VTableSet* vtables) {
+		uint32_t len = members.size();
+		int start = RightAlign(writer.current_buffer_size + sizeof(uint32_t) + len, sizeof(uint32_t));
+		writer.write(&len, start, sizeof(uint32_t));
+		int i = 0;
+		for (bool b : members) {
+			writer.write(&b, start - sizeof(uint32_t) - i++, 1);
+		}
+		return RelativeOffset{ writer.current_buffer_size };
+	}
+};
+
+template <class Member, class Context>
+void load_helper(Member& member, const uint8_t* current, Context& context) {
+	LoadSaveHelper<Member> helper;
+	helper.load(member, current, context);
+}
+
+template <class Member, class Writer>
+auto save_helper(const Member& member, Writer& writer, const VTableSet* vtables) {
+	LoadSaveHelper<Member> helper;
+	return helper.save(member, writer, vtables);
+}
+
+} // namespace detail
+
+template <class F, class... Items>
+void serializer(F& fun, Items&... items) {
+	fun(items...);
+}
+
+namespace detail {
+
+template <class... Members>
+struct FakeRoot {
+	std::tuple<Members&...> members;
+
+	FakeRoot(Members&... members) : members(members...) {}
+
+	template <class Archive>
+	void serialize(Archive& archive) {
+		serialize_impl(archive, std::index_sequence_for<Members...>{});
+	}
+
+private:
+	template <class Archive, size_t... is>
+	void serialize_impl(Archive& archive, std::index_sequence<is...>) {
+		flat_buffers::serializer(archive, std::get<is>(members)...);
+	}
+};
+
+template <class... Members>
+auto fake_root(Members&... members) {
+	return FakeRoot<Members...>(members...);
+}
+
+template <class Allocator, class Root>
+uint8_t* save(Allocator& allocator, const Root& root, FileIdentifier file_identifier) {
+	const auto* vtableset = get_vtableset(root);
+	PrecomputeSize precompute_size;
+	int vtable_start;
+	save_with_vtables(root, vtableset, precompute_size, &vtable_start, file_identifier);
+	uint8_t* out = allocator(precompute_size.current_buffer_size);
+	memset(out, 0, precompute_size.current_buffer_size);
+	WriteToBuffer writeToBuffer{ precompute_size.current_buffer_size, vtable_start, out,
+		                         std::move(precompute_size.writeToOffsets), precompute_size.writeRawMemories.begin() };
+	save_with_vtables(root, vtableset, writeToBuffer, &vtable_start, file_identifier);
+	return out;
+}
+
+template <class Root, class Context>
+void load(Root& root, const uint8_t* in, Context& context) {
+	flat_buffers::detail::load_helper(root, in, context);
+}
+
+} // namespace detail
+
+template <class Allocator, class... Members>
+uint8_t* save_members(Allocator& allocator, FileIdentifier file_identifier, Members&... members) {
+	const auto& root = flat_buffers::detail::fake_root(members...);
+	return detail::save(allocator, root, file_identifier);
+}
+
+template <class Context, class... Members>
+void load_members(const uint8_t* in, Context& context, Members&... members) {
+	auto root = flat_buffers::detail::fake_root(members...);
+	detail::load(root, in, context);
+}
+
+inline FileIdentifier read_file_identifier(const uint8_t* in) {
+	FileIdentifier result;
+	memcpy(&result, in + sizeof(result), sizeof(result));
+	return result;
+}
+
+// members of unions must be tables in flatbuffers, so you can use this to
+// introduce the indirection only when necessary.
+template <class T>
+struct EnsureTable {
+	constexpr static flat_buffers::FileIdentifier file_identifier = FileIdentifierFor<T>::value;
+	EnsureTable() = default;
+	EnsureTable(const object_construction<T>& t) : t(t) {}
+	EnsureTable(const T& t) : t(t) {}
+	template <class Archive>
+	void serialize(Archive& ar) {
+		if constexpr (detail::expect_serialize_member<T>) {
+			if constexpr (serializable_traits<T>::value) {
+				serializable_traits<T>::serialize(ar, t.get());
+			} else {
+				t.get().serialize(ar);
+			}
+		} else {
+			flat_buffers::serializer(ar, t.get());
+		}
+	}
+	T& asUnderlyingType() { return t.get(); }
+
+private:
+	object_construction<T> t;
+};
+
+} // namespace flat_buffers

From fe81454ec2d3f7dc724fce25a37c1ff78535251f Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 25 Jan 2019 15:10:20 -0800
Subject: [PATCH 044/118] basic functionality for object serializer

This commit includes:
- The flatbuffers implementation
- A draft on how it should be used for network messages
- A serializer that can be used independently

What is missing:
- All root objects will need a file identifier
- Many special classes can not be serialized yet as the
  corresponding traits are not yet implemented
- Object serialization can not yet be turned on (this will
  need a network option)
---
 fdbrpc/FlowTransport.actor.cpp |  36 +++--
 fdbrpc/FlowTransport.h         |   4 +
 fdbrpc/fdbrpc.h                |  11 ++
 fdbrpc/genericactors.actor.cpp |   5 -
 fdbrpc/networksender.actor.h   |  20 ++-
 flow/Arena.h                   |  43 ++++++
 flow/FileIdentifier.h          |  97 ++++++++++++
 flow/Net2.actor.cpp            |  12 ++
 flow/ObjectSerializer.h        |  28 ++--
 flow/ObjectSerializerTraits.h  | 164 +++++++++++++++++++++
 flow/flat_buffers.cpp          |  41 +++---
 flow/flat_buffers.h            | 260 +++------------------------------
 flow/network.h                 |   3 +
 flow/serialize.h               |  55 +++++--
 14 files changed, 473 insertions(+), 306 deletions(-)
 create mode 100644 flow/FileIdentifier.h
 create mode 100644 flow/ObjectSerializerTraits.h

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 70c0bfd37a..8f77027581 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -25,6 +25,7 @@
 #include "flow/Net2Packet.h"
 #include "flow/ActorCollection.h"
 #include "flow/TDMetric.actor.h"
+#include "flow/ObjectSerializer.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/crc32c.h"
 #include "fdbrpc/simulator.h"
@@ -371,7 +372,6 @@ struct Peer : NonCopyable {
 
 			// Send an (ignored) packet to make sure that, if our outgoing connection died before the peer made this connection attempt,
 			// we eventually find out that our connection is dead, close it, and then respond to the next connection reattempt from peer.
-			//sendPacket( self, SerializeSourceRaw(StringRef()), Endpoint(peer->address(), TOKEN_IGNORE_PACKET), false );
 		}
 	}
 
@@ -529,7 +529,8 @@ TransportData::~TransportData() {
 	}
 }
 
-ACTOR static void deliver( TransportData* self, Endpoint destination, ArenaReader reader, bool inReadSocket ) {
+ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader reader, bool inReadSocket,
+                          bool useFlatbuffers) {
 	int priority = self->endpoints.getPriority(destination.token);
 	if (priority < TaskReadSocket || !inReadSocket) {
 		wait( delay(0, priority) );
@@ -540,8 +541,13 @@ ACTOR static void deliver( TransportData* self, Endpoint destination, ArenaReade
 	auto receiver = self->endpoints.get(destination.token);
 	if (receiver) {
 		try {
-			g_currentDeliveryPeerAddress = destination.addresses;
-			receiver->receive( reader );
+			g_currentDeliveryPeerAddress = destination.address;
+			if (useFlatbuffers) {
+				ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll());
+				receiver->receive(objReader);
+			} else {
+                receiver->receive( reader );
+            }
 			g_currentDeliveryPeerAddress = {NetworkAddress()};
 		} catch (Error& e) {
 			g_currentDeliveryPeerAddress = {NetworkAddress()};
@@ -561,7 +567,8 @@ ACTOR static void deliver( TransportData* self, Endpoint destination, ArenaReade
 		g_network->setCurrentTask( TaskReadSocket );
 }
 
-static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin, uint8_t* e, Arena& arena, NetworkAddress const& peerAddress, uint64_t peerProtocolVersion ) {
+static void scanPackets(TransportData* transport, uint8_t*& unprocessed_begin, uint8_t* e, Arena& arena,
+                        NetworkAddress const& peerAddress, uint64_t peerProtocolVersion) {
 	// Find each complete packet in the given byte range and queue a ready task to deliver it.
 	// Remove the complete packets from the range by increasing unprocessed_begin.
 	// There won't be more than 64K of data plus one packet, so this shouldn't take a long time.
@@ -633,8 +640,9 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
 #if VALGRIND
 		VALGRIND_CHECK_MEM_IS_DEFINED(p, packetLen);
 #endif
-		ArenaReader reader( arena, StringRef(p, packetLen), AssumeVersion(peerProtocolVersion) );
-		UID token; reader >> token;
+		ArenaReader reader(arena, StringRef(p, packetLen), AssumeVersion(removeFlags(peerProtocolVersion)));
+		UID token;
+		reader >> token;
 
 		++transport->countPacketsReceived;
 
@@ -649,7 +657,7 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
 				transport->warnAlwaysForLargePacket = false;
 		}
 
-		deliver( transport, Endpoint( {peerAddress}, token ), std::move(reader), true );
+		deliver(transport, Endpoint({ peerAddress }, token), std::move(reader), true, hasObjectSerializerFlag(peerProtocolVersion));
 
 		unprocessed_begin = p = p + packetLen;
 	}
@@ -748,7 +756,8 @@ ACTOR static Future<Void> connectionReader(
 							TraceEvent("ConnectionEstablished", conn->getDebugID())
 								.suppressFor(1.0)
 								.detail("Peer", conn->getPeerAddress())
-								.detail("ConnectionId", connectionId);
+								.detail("ConnectionId", connectionId)
+								.detail("UseObjectSerializer", false);
 						}
 
 						if(connectionId > 1) {
@@ -994,13 +1003,18 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 		// SOMEDAY: Would it be better to avoid (de)serialization by doing this check in flow?
 
 		BinaryWriter wr( AssumeVersion(currentProtocolVersion) );
+		// we don't need to send using an object writer here. This is a loopback delivery
+		// and therefore it is guaranteed that both versions will have exactly the
+		// same structures - so the backwards compatability capabilities are never needed
+		// here.
 		what.serializeBinaryWriter(wr);
 		Standalone<StringRef> copy = wr.toValue();
 #if VALGRIND
 		VALGRIND_CHECK_MEM_IS_DEFINED(copy.begin(), copy.size());
 #endif
 
-		deliver( self, destination, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false );
+		deliver(self, destination, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false,
+		        false);
 
 		return (PacketID)NULL;
 	} else {
@@ -1039,7 +1053,7 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 
 		wr.writeAhead(packetInfoSize , &packetInfoBuffer);
 		wr << destination.token;
-		what.serializePacketWriter(wr);
+		what.serializePacketWriter(wr, g_network->useObjectSerializer());
 		pb = wr.finish();
 		len = wr.size() - packetInfoSize;
 
diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h
index 8be6fa6837..bcf9c334aa 100644
--- a/fdbrpc/FlowTransport.h
+++ b/fdbrpc/FlowTransport.h
@@ -85,9 +85,13 @@ public:
 };
 #pragma pack(pop)
 
+
+
+class ArenaObjectReader;
 class NetworkMessageReceiver {
 public:
 	virtual void receive( ArenaReader& ) = 0;
+	virtual void receive(ArenaObjectReader&) = 0;
 	virtual bool isStream() const { return false; }
 };
 
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index 986fe6525d..4a94af1543 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -92,6 +92,17 @@ struct NetSAV : SAV<T>, FlowReceiver, FastAllocated<NetSAV<T>> {
 			SAV<T>::sendErrorAndDelPromiseRef(error);
 		}
 	}
+	virtual void receive(ArenaObjectReader& reader) {
+		if (!SAV<T>::canBeSet()) return;
+		this->addPromiseRef();
+		ErrorOr<T> message;
+		reader.deserialize(message);
+		if (message.isError()) {
+			SAV<T>::sendErrorAndDelPromiseRef(message.getError());
+		} else {
+			SAV<T>::sendAndDelPromiseRef(message.get());
+		}
+	}
 };
 
 
diff --git a/fdbrpc/genericactors.actor.cpp b/fdbrpc/genericactors.actor.cpp
index 9b727021db..d0065668e4 100644
--- a/fdbrpc/genericactors.actor.cpp
+++ b/fdbrpc/genericactors.actor.cpp
@@ -24,11 +24,6 @@
 #include "fdbrpc/simulator.h"
 #include "flow/actorcompiler.h"
 
-ACTOR void simDeliverDuplicate( Standalone<StringRef> data, Endpoint destination ) {
-	wait( delay( g_random->random01() * FLOW_KNOBS->MAX_DELIVER_DUPLICATE_DELAY ) );
-	FlowTransport::transport().sendUnreliable( SerializeSourceRaw(data), destination );
-}
-
 ACTOR Future<Void> disableConnectionFailuresAfter( double time, std::string context ) {
 	wait( delay(time) );
 
diff --git a/fdbrpc/networksender.actor.h b/fdbrpc/networksender.actor.h
index bc108c01d6..629360d188 100644
--- a/fdbrpc/networksender.actor.h
+++ b/fdbrpc/networksender.actor.h
@@ -31,14 +31,22 @@
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
 ACTOR template <class T>
-void networkSender( Future<T> input, Endpoint endpoint ) {
+void networkSender(Future<T> input, Endpoint endpoint) {
 	try {
-		T value = wait( input );
-		FlowTransport::transport().sendUnreliable( SerializeBoolAnd<T>(true, value), endpoint, false );
+		T value = wait(input);
+		if (g_network->useObjectSerializer()) {
+			FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<T>>(ErrorOr<T>(value)), endpoint);
+		} else {
+			FlowTransport::transport().sendUnreliable(SerializeBoolAnd<T>(true, value), endpoint, false);
+		}
 	} catch (Error& err) {
-		//if (err.code() == error_code_broken_promise) return;
-		ASSERT( err.code() != error_code_actor_cancelled );
-		FlowTransport::transport().sendUnreliable( SerializeBoolAnd<Error>(false, err), endpoint, false );
+		// if (err.code() == error_code_broken_promise) return;
+		ASSERT(err.code() != error_code_actor_cancelled);
+		if (g_network->useObjectSerializer()) {
+			FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<T>>(ErrorOr<T>(err)), endpoint);
+		} else {
+			FlowTransport::transport().sendUnreliable(SerializeBoolAnd<Error>(false, err), endpoint, false);
+		}
 	}
 }
 #include "flow/unactorcompiler.h"
diff --git a/flow/Arena.h b/flow/Arena.h
index d93f09dafa..4c1e60c6a0 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -26,6 +26,7 @@
 #include "flow/FastRef.h"
 #include "flow/Error.h"
 #include "flow/Trace.h"
+#include "flow/ObjectSerializerTraits.h"
 #include <algorithm>
 #include <stdint.h>
 #include <string>
@@ -108,6 +109,18 @@ public:
 	Reference<struct ArenaBlock> impl;
 };
 
+template<>
+struct scalar_traits<Arena> : std::true_type {
+	constexpr static size_t size = 0;
+	static void save(uint8_t*, const Arena&) {}
+	// Context is an arbitrary type that is plumbed by reference throughout
+	// the load call tree.
+	template <class Context>
+	static void load(const uint8_t*, Arena& arena, Context& context) {
+		context.addArena(arena);
+	}
+};
+
 struct ArenaBlockRef {
 	ArenaBlock* next;
 	uint32_t nextBlockOffset;
@@ -723,6 +736,17 @@ inline void save( Archive& ar, const StringRef& value ) {
 	ar << (uint32_t)value.size();
 	ar.serializeBytes( value.begin(), value.size() );
 }
+
+template<>
+struct dynamic_size_traits<StringRef> : std::true_type {
+	static WriteRawMemory save(const StringRef& str) { return { { unownedPtr(str.begin()), str.size() } }; }
+
+	template <class Context>
+	static void load(const uint8_t* ptr, size_t sz, StringRef& str, Context& context) {
+		str = StringRef(context.tryReadZeroCopy(ptr, sz), sz);
+	}
+};
+
 inline bool operator == (const StringRef& lhs, const StringRef& rhs ) {
 	return lhs.size() == rhs.size() && !memcmp(lhs.begin(), rhs.begin(), lhs.size());
 }
@@ -753,6 +777,8 @@ struct memcpy_able<UID> : std::integral_constant<bool, true> {};
 template <class T>
 class VectorRef {
 public:
+	using value_type = T;
+
 	// T must be trivially destructible (and copyable)!
 	VectorRef() : data(0), m_size(0), m_capacity(0) {}
 
@@ -928,6 +954,23 @@ inline void save( Archive& ar, const VectorRef<T>& value ) {
 		ar << value[i];
 }
 
+template <class T>
+struct vector_like_traits<VectorRef<T>> : std::true_type {
+	using Vec = VectorRef<T>;
+	using value_type = typename Vec::value_type;
+	using iterator = const T*;
+	using insert_iterator = T*;
+
+	static size_t num_entries(const VectorRef<T>& v) { return v.size(); }
+	template <class Context>
+	static void reserve(VectorRef<T>& v, size_t s, Context& context) {
+		v.resize(context.arena(), s);
+	}
+
+	static insert_iterator insert(Vec& v) { return v.begin(); }
+	static iterator begin(const Vec& v) { return v.begin(); }
+};
+
  void ArenaBlock::destroy() {
 	// If the stack never contains more than one item, nothing will be allocated from stackArena.
 	// If stackArena is used, it will always be a linked list, so destroying *it* will not create another arena
diff --git a/flow/FileIdentifier.h b/flow/FileIdentifier.h
new file mode 100644
index 0000000000..4bbb1d9d14
--- /dev/null
+++ b/flow/FileIdentifier.h
@@ -0,0 +1,97 @@
+/*
+ * FileIdentifier.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include <cstdint>
+
+using FileIdentifier = uint32_t;
+
+template <class T>
+struct FileIdentifierFor {
+	//constexpr static FileIdentifier value = T::file_identifier;
+	// TODO: use file identifiers for different types
+	constexpr static FileIdentifier value = 0xffffff;
+};
+
+template <>
+struct FileIdentifierFor<int> {
+	constexpr static FileIdentifier value = 1;
+};
+
+template <>
+struct FileIdentifierFor<unsigned> {
+	constexpr static FileIdentifier value = 2;
+};
+
+template <>
+struct FileIdentifierFor<long> {
+	constexpr static FileIdentifier value = 3;
+};
+
+template <>
+struct FileIdentifierFor<unsigned long> {
+	constexpr static FileIdentifier value = 4;
+};
+
+template <>
+struct FileIdentifierFor<long long> {
+	constexpr static FileIdentifier value = 5;
+};
+
+template <>
+struct FileIdentifierFor<unsigned long long> {
+	constexpr static FileIdentifier value = 6;
+};
+
+template <>
+struct FileIdentifierFor<short> {
+	constexpr static FileIdentifier value = 7;
+};
+
+template <>
+struct FileIdentifierFor<unsigned short> {
+	constexpr static FileIdentifier value = 8;
+};
+
+template <>
+struct FileIdentifierFor<signed char> {
+	constexpr static FileIdentifier value = 9;
+};
+
+template <>
+struct FileIdentifierFor<unsigned char> {
+	constexpr static FileIdentifier value = 10;
+};
+
+template <>
+struct FileIdentifierFor<bool> {
+	constexpr static FileIdentifier value = 11;
+};
+
+template <>
+struct FileIdentifierFor<float> {
+	constexpr static FileIdentifier value = 7266212;
+};
+
+template <>
+struct FileIdentifierFor<double> {
+	constexpr static FileIdentifier value = 9348150;
+};
+
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index fe081e9131..5ee248a33e 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -58,6 +58,18 @@ using namespace boost::asio::ip;
 const uint64_t currentProtocolVersion        = 0x0FDB00B061070001LL;
 const uint64_t compatibleProtocolVersionMask = 0xffffffffffff0000LL;
 const uint64_t minValidProtocolVersion       = 0x0FDB00A200060001LL;
+const uint64_t objectSerializerFlag          = 0x1000000000000000LL;
+const uint64_t versionFlagMask               = 0x0FFFFFFFFFFFFFFFLL;
+
+uint64_t removeFlags(uint64_t version) {
+	return version & versionFlagMask;
+}
+uint64_t addObjectSerializerFlag(uint64_t version) {
+	return version | versionFlagMask;
+}
+bool hasObjectSerializerFlag(uint64_t version) {
+	return (version & objectSerializerFlag) > 0;
+}
 
 // This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to change when we reach version 10.
 static_assert(currentProtocolVersion < 0x0FDB00B100000000LL, "Unexpected protocol version");
diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h
index 03e68905a2..b10be8e835 100644
--- a/flow/ObjectSerializer.h
+++ b/flow/ObjectSerializer.h
@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#pragma once
 #include "flow/Error.h"
 #include "flow/Arena.h"
 #include "flow/flat_buffers.h"
@@ -52,21 +53,21 @@ template <class ReaderImpl>
 class _ObjectReader {
 public:
 	template <class... Items>
-	void deserialize(flat_buffers::FileIdentifier file_identifier, Items&... items) {
+	void deserialize(FileIdentifier file_identifier, Items&... items) {
 		const uint8_t* data = static_cast<ReaderImpl*>(this)->data();
 		LoadContext<ReaderImpl> context(*static_cast<ReaderImpl*>(this));
-		ASSERT(flat_buffers::read_file_identifier(data) == file_identifier);
-		flat_buffers::load_members(data, context, items...);
+		ASSERT(read_file_identifier(data) == file_identifier);
+		load_members(data, context, items...);
 		context.done();
 	}
 
 	template <class Item>
 	void deserialize(Item& item) {
-		deserialize(flat_buffers::FileIdentifierFor<Item>::value, item);
+		deserialize(FileIdentifierFor<Item>::value, item);
 	}
 };
 
-class ObjectReader : _ObjectReader<ObjectReader> {
+class ObjectReader : public _ObjectReader<ObjectReader> {
 public:
 	static constexpr bool ownsUnderlyingMemory = false;
 
@@ -81,7 +82,7 @@ private:
 	Arena _arena;
 };
 
-class ArenaObjectReader : _ObjectReader<ArenaObjectReader> {
+class ArenaObjectReader : public _ObjectReader<ArenaObjectReader> {
 public:
 	static constexpr bool ownsUnderlyingMemory = true;
 
@@ -99,7 +100,7 @@ private:
 class ObjectWriter {
 public:
 	template <class... Items>
-	void serialize(flat_buffers::FileIdentifier file_identifier, Items const&... items) {
+	void serialize(FileIdentifier file_identifier, Items const&... items) {
 		ASSERT(data = nullptr); // object serializer can only serialize one object
 		int allocations = 0;
 		auto allocator = [this, &allocations](size_t size_) {
@@ -108,21 +109,20 @@ public:
 			data = new uint8_t[size];
 			return data;
 		};
-		auto res = flat_buffers::save_members(allocator, file_identifier, items...);
+		auto res = save_members(allocator, file_identifier, items...);
 		ASSERT(allocations == 1);
 	}
 
 	template <class Item>
 	void serialize(Item const& item) {
-		serialize(flat_buffers::FileIdentifierFor<Item>::value, item);
+		serialize(FileIdentifierFor<Item>::value, item);
+	}
+
+	StringRef toStringRef() const {
+		return StringRef(data, size);
 	}
 
 private:
 	uint8_t* data = nullptr;
 	int size = 0;
 };
-
-template <class Visitor, class... Items>
-std::enable_if<Visitor::is_object_serializer> serializer(Visitor& visitor, Items&... items) {
-	visitor(items...);
-}
diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h
new file mode 100644
index 0000000000..9d1476e8e2
--- /dev/null
+++ b/flow/ObjectSerializerTraits.h
@@ -0,0 +1,164 @@
+/*
+ * ObjectSerializerTraits.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <type_traits>
+#include <cstdint>
+#include <cstddef>
+#include <memory>
+#include <functional>
+#include <vector>
+
+template <class... Ts>
+struct pack {};
+
+template <int i, class... Ts>
+struct index_impl;
+
+template <int i, class T, class... Ts>
+struct index_impl<i, pack<T, Ts...>> {
+	using type = typename index_impl<i - 1, pack<Ts...>>::type;
+};
+
+template <class T, class... Ts>
+struct index_impl<0, pack<T, Ts...>> {
+	using type = T;
+};
+
+template <int i, class Pack>
+using index_t = typename index_impl<i, Pack>::type;
+
+//  A smart pointer that knows whether or not to delete itself.
+template <class T>
+using OwnershipErasedPtr = std::unique_ptr<T, std::function<void(T*)>>;
+
+// Creates an OwnershipErasedPtr<T> that will delete itself.
+template <class T, class Deleter = std::default_delete<T>>
+OwnershipErasedPtr<T> ownedPtr(T* t, Deleter&& d = Deleter{}) {
+	return OwnershipErasedPtr<T>{ t, std::forward<Deleter>(d) };
+}
+
+// Creates an OwnershipErasedPtr<T> that will not delete itself.
+template <class T>
+OwnershipErasedPtr<T> unownedPtr(T* t) {
+	return OwnershipErasedPtr<T>{ t, [](T*) {} };
+}
+
+struct WriteRawMemory {
+	using Block = std::pair<OwnershipErasedPtr<const uint8_t>, size_t>;
+	std::vector<Block> blocks;
+
+	WriteRawMemory() {}
+	WriteRawMemory(Block&& b) { blocks.emplace_back(std::move(b.first), b.second); }
+	WriteRawMemory(std::vector<Block>&& v) : blocks(std::move(v)) {}
+
+	WriteRawMemory(WriteRawMemory&&) = default;
+	WriteRawMemory& operator=(WriteRawMemory&&) = default;
+
+	size_t size() const {
+		size_t result = 0;
+		for (const auto& b : blocks) {
+			result += b.second;
+		}
+		return result;
+	}
+};
+
+
+template <class T, typename = void>
+struct scalar_traits : std::false_type {
+	constexpr static size_t size = 0;
+	static void save(uint8_t*, const T&);
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t*, T&, Context&);
+};
+
+
+template <class T>
+struct dynamic_size_traits : std::false_type {
+	static WriteRawMemory save(const T&);
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t*, size_t, T&, Context&);
+};
+
+template <class T>
+struct serializable_traits : std::false_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, T& v);
+};
+
+template <class VectorLike>
+struct vector_like_traits : std::false_type {
+	// Write this at the beginning of the buffer
+	using value_type = uint8_t;
+	using iterator = void;
+	using insert_iterator = void;
+
+	static size_t num_entries(VectorLike&);
+	template <class Context>
+	static void reserve(VectorLike&, size_t, Context&);
+
+	static insert_iterator insert(VectorLike&);
+	static iterator begin(const VectorLike&);
+	static void deserialization_done(VectorLike&); // Optional
+};
+
+template <class UnionLike>
+struct union_like_traits : std::false_type {
+	using Member = UnionLike;
+	using alternatives = pack<>;
+	static uint8_t index(const Member&);
+	static bool empty(const Member& variant);
+
+	template <int i>
+	static const index_t<i, alternatives>& get(const Member&);
+
+	template <int i, class Alternative>
+	static const void assign(Member&, const Alternative&);
+
+	template <class Context>
+	static void done(Member&, Context&);
+};
+
+// TODO(anoyes): Implement things that are currently using scalar traits with
+// struct-like traits.
+template <class StructLike>
+struct struct_like_traits : std::false_type {
+	using Member = StructLike;
+	using types = pack<>;
+
+	template <int i>
+	static const index_t<i, types>& get(const Member&);
+
+	template <int i>
+	static const void assign(Member&, const index_t<i, types>&);
+
+	template <class Context>
+	static void done(Member&, Context&);
+};
+
+
diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
index 29e2b6dd84..53739e4b7b 100644
--- a/flow/flat_buffers.cpp
+++ b/flow/flat_buffers.cpp
@@ -18,17 +18,16 @@
  * limitations under the License.
  */
 
-#include "flat_buffers.h"
-#include "UnitTest.h"
-#include "Arena.h"
-#include "serialize.h"
+#include "flow/flat_buffers.h"
+#include "flow/UnitTest.h"
+#include "flow/Arena.h"
+#include "flow/serialize.h"
+#include "flow/ObjectSerializer.h"
 
 #include <algorithm>
 #include <iomanip>
 #include <boost/variant.hpp>
 
-namespace flat_buffers {
-
 namespace detail {
 
 bool TraverseMessageTypes::vtableGeneratedBefore(const std::type_index& idx) {
@@ -106,7 +105,7 @@ struct Table2 {
 	int16_t m_ed = {};
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, m_p, m_ujrnpumbfvc, m_iwgxxt, m_tjkuqo, m_ed);
+		serializer(ar, m_p, m_ujrnpumbfvc, m_iwgxxt, m_tjkuqo, m_ed);
 	}
 };
 
@@ -117,7 +116,7 @@ struct Table3 {
 	int64_t m_n = {};
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, m_asbehdlquj, m_k, m_jib, m_n);
+		serializer(ar, m_asbehdlquj, m_k, m_jib, m_n);
 	}
 };
 
@@ -134,7 +133,7 @@ struct Nested2 {
 	int c;
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, a, b, c);
+		serializer(ar, a, b, c);
 	}
 
 	friend bool operator==(const Nested2& lhs, const Nested2& rhs) {
@@ -149,7 +148,7 @@ struct Nested {
 	std::vector<uint64_t> c;
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, a, b, nested, c);
+		serializer(ar, a, b, nested, c);
 	}
 };
 
@@ -159,7 +158,7 @@ struct Root {
 	Nested c;
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, a, b, c);
+		serializer(ar, a, b, c);
 	}
 };
 
@@ -224,7 +223,7 @@ TEST_CASE("flow/FlatBuffers/serializeDeserializeRoot") {
 		       { 3, "hello", { 6, { "abc", "def" }, 8 }, { 10, 11, 12 } } };
 	Root root2 = root;
 	Arena arena;
-	auto out = flat_buffers::detail::save(arena, root, flat_buffers::FileIdentifier{});
+	auto out = detail::save(arena, root, FileIdentifier{});
 
 	ASSERT(root.a == root2.a);
 	ASSERT(root.b == root2.b);
@@ -237,7 +236,7 @@ TEST_CASE("flow/FlatBuffers/serializeDeserializeRoot") {
 
 	root2 = {};
 	DummyContext context;
-	flat_buffers::detail::load(root2, out, context);
+	detail::load(root2, out, context);
 
 	ASSERT(root.a == root2.a);
 	ASSERT(root.b == root2.b);
@@ -399,7 +398,7 @@ struct Y1 {
 
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, a);
+		serializer(ar, a);
 	}
 };
 
@@ -409,7 +408,7 @@ struct Y2 {
 
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, a, b);
+		serializer(ar, a, b);
 	}
 };
 
@@ -421,7 +420,7 @@ struct X {
 
 	template <class Archiver>
 	void serialize(Archiver& ar) {
-		return flat_buffers::serializer(ar, a, b, c);
+		serializer(ar, a, b, c);
 	}
 };
 
@@ -493,12 +492,12 @@ TEST_CASE("flow/FlatBuffers/VectorRef") {
 			for (const auto& str : src) {
 				vec.push_back(arena, str);
 			}
-			BinaryWriter writer(IncludeVersion());
-			::serialize_fake_root(writer, FileIdentifierFor<decltype(vec)>::value, arena, vec);
+			ObjectWriter writer;
+			writer.serialize(FileIdentifierFor<decltype(vec)>::value, arena, vec);
 			serializedVector = StringRef(readerArena, writer.toStringRef());
 		}
-		ArenaReader reader(readerArena, serializedVector, IncludeVersion());
-		::serialize_fake_root(reader, FileIdentifierFor<decltype(outVec)>::value, vecArena, outVec);
+		ArenaObjectReader reader(readerArena, serializedVector);
+		reader.deserialize(FileIdentifierFor<decltype(outVec)>::value, vecArena, outVec);
 	}
 	ASSERT(src.size() == outVec.size());
 	for (int i = 0; i < src.size(); ++i) {
@@ -509,5 +508,3 @@ TEST_CASE("flow/FlatBuffers/VectorRef") {
 }
 
 } // namespace unit_tests
-
-} // namespace flat_buffers
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index adcbc6b8a9..13637b704b 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -35,11 +35,8 @@
 #include <array>
 #include <typeinfo>
 #include <typeindex>
-
-namespace flat_buffers {
-
-template <class... Ts>
-struct pack {};
+#include "flow/FileIdentifier.h"
+#include "flow/ObjectSerializerTraits.h"
 
 template <class T = pack<>, class...>
 struct concat {
@@ -55,117 +52,23 @@ constexpr auto pack_size(pack<Ts...>) {
 	return sizeof...(Ts);
 }
 
-template <int i, class... Ts>
-struct index;
-
-template <int i, class T, class... Ts>
-struct index<i, pack<T, Ts...>> {
-	using type = typename index<i - 1, pack<Ts...>>::type;
-};
-
-template <class T, class... Ts>
-struct index<0, pack<T, Ts...>> {
-	using type = T;
-};
-
-template <int i, class Pack>
-using index_t = typename index<i, Pack>::type;
-
 constexpr int RightAlign(int offset, int alignment) {
 	return offset % alignment == 0 ? offset : ((offset / alignment) + 1) * alignment;
 }
 
-using FileIdentifier = uint32_t;
+template <class T, typename = void>
+struct is_fb_function_t : std::false_type {};
+
+template<class T>
+struct is_fb_function_t<T, typename std::enable_if<T::is_fb_visitor>::type> : std::true_type {};
 
 template <class T>
-struct FileIdentifierFor {
-	constexpr static FileIdentifier value = T::file_identifier;
-};
+constexpr bool is_fb_function = is_fb_function_t<T>::value;
 
-template <>
-struct FileIdentifierFor<int> {
-	constexpr static FileIdentifier value = 1;
-};
-
-template <>
-struct FileIdentifierFor<unsigned> {
-	constexpr static FileIdentifier value = 2;
-};
-
-template <>
-struct FileIdentifierFor<long> {
-	constexpr static FileIdentifier value = 3;
-};
-
-template <>
-struct FileIdentifierFor<unsigned long> {
-	constexpr static FileIdentifier value = 4;
-};
-
-template <>
-struct FileIdentifierFor<long long> {
-	constexpr static FileIdentifier value = 5;
-};
-
-template <>
-struct FileIdentifierFor<unsigned long long> {
-	constexpr static FileIdentifier value = 6;
-};
-
-template <>
-struct FileIdentifierFor<short> {
-	constexpr static FileIdentifier value = 7;
-};
-
-template <>
-struct FileIdentifierFor<unsigned short> {
-	constexpr static FileIdentifier value = 8;
-};
-
-template <>
-struct FileIdentifierFor<signed char> {
-	constexpr static FileIdentifier value = 9;
-};
-
-template <>
-struct FileIdentifierFor<unsigned char> {
-	constexpr static FileIdentifier value = 10;
-};
-
-template <>
-struct FileIdentifierFor<bool> {
-	constexpr static FileIdentifier value = 11;
-};
-
-template <>
-struct FileIdentifierFor<float> {
-	constexpr static flat_buffers::FileIdentifier value = 7266212;
-};
-
-template <>
-struct FileIdentifierFor<double> {
-	constexpr static flat_buffers::FileIdentifier value = 9348150;
-};
-
-template <class K, class V, class Compare, class Allocator>
-struct FileIdentifierFor<std::map<K, V, Compare, Allocator>> {
-	constexpr static FileIdentifier value = FileIdentifierFor<std::pair<K, V>>::value;
-};
-
-template <class F, class S>
-struct FileIdentifierFor<std::pair<F, S>> {
-	constexpr static FileIdentifier value = FileIdentifierFor<F>::value ^ FileIdentifierFor<S>::value;
-};
-
-template <class T, class Allocator>
-struct FileIdentifierFor<std::vector<T, Allocator>> {
-	constexpr static FileIdentifier value = (0x10 << 24) | FileIdentifierFor<T>::value;
-};
-
-template <class CharT, class Traits, class Allocator>
-struct FileIdentifierFor<std::basic_string<CharT, Traits, Allocator>> {
-	constexpr static FileIdentifier value = 15694229;
-};
+template <class Visitor, class... Items>
+typename std::enable_if<is_fb_function<Visitor>, void>::type serializer(Visitor& visitor, Items&... items) {
+	visitor(items...);
+}
 
 template <class T>
 struct object_construction {
@@ -180,119 +83,6 @@ struct object_construction {
 	T move() { return std::move(obj); }
 };
 
-// A smart pointer that knows whether or not to delete itself.
-template <class T>
-using OwnershipErasedPtr = std::unique_ptr<T, std::function<void(T*)>>;
-
-// Creates an OwnershipErasedPtr<T> that will delete itself.
-template <class T, class Deleter = std::default_delete<T>>
-OwnershipErasedPtr<T> ownedPtr(T* t, Deleter&& d = Deleter{}) {
-	return OwnershipErasedPtr<T>{ t, std::forward<Deleter>(d) };
-}
-
-// Creates an OwnershipErasedPtr<T> that will not delete itself.
-template <class T>
-OwnershipErasedPtr<T> unownedPtr(T* t) {
-	return OwnershipErasedPtr<T>{ t, [](T*) {} };
-}
-
-template <class T, typename = void>
-struct scalar_traits : std::false_type {
-	constexpr static size_t size = 0;
-	static void save(uint8_t*, const T&);
-
-	// Context is an arbitrary type that is plumbed by reference throughout the
-	// load call tree.
-	template <class Context>
-	static void load(const uint8_t*, T&, Context&);
-};
-
-struct WriteRawMemory {
-	using Block = std::pair<OwnershipErasedPtr<const uint8_t>, size_t>;
-	std::vector<Block> blocks;
-
-	WriteRawMemory() {}
-	WriteRawMemory(Block&& b) { blocks.emplace_back(std::move(b.first), b.second); }
-	WriteRawMemory(std::vector<Block>&& v) : blocks(std::move(v)) {}
-
-	WriteRawMemory(WriteRawMemory&&) = default;
-	WriteRawMemory& operator=(WriteRawMemory&&) = default;
-
-	size_t size() const {
-		size_t result = 0;
-		for (const auto& b : blocks) {
-			result += b.second;
-		}
-		return result;
-	}
-};
-
-template <class T>
-struct dynamic_size_traits : std::false_type {
-	static WriteRawMemory save(const T&);
-
-	// Context is an arbitrary type that is plumbed by reference throughout the
-	// load call tree.
-	template <class Context>
-	static void load(const uint8_t*, size_t, T&, Context&);
-};
-
-template <class T>
-struct serializable_traits : std::false_type {
-	template <class Archiver>
-	static void serialize(Archiver& ar, T& v);
-};
-
-template <class VectorLike>
-struct vector_like_traits : std::false_type {
-	// Write this at the beginning of the buffer
-	using value_type = uint8_t;
-	using iterator = void;
-	using insert_iterator = void;
-
-	static size_t num_entries(VectorLike&);
-	template <class Context>
-	static void reserve(VectorLike&, size_t, Context&);
-
-	static insert_iterator insert(VectorLike&);
-	static iterator begin(const VectorLike&);
-	static void deserialization_done(VectorLike&); // Optional
-};
-
-template <class UnionLike>
-struct union_like_traits : std::false_type {
-	using Member = UnionLike;
-	using alternatives = pack<>;
-	static uint8_t index(const Member&);
-	static bool empty(const Member& variant);
-
-	template <int i>
-	static const index_t<i, alternatives>& get(const Member&);
-
-	template <int i, class Alternative>
-	static const void assign(Member&, const Alternative&);
-
-	template <class Context>
-	static void done(Member&, Context&);
-};
-
-// TODO(anoyes): Implement things that are currently using scalar traits with
-// struct-like traits.
-template <class StructLike>
-struct struct_like_traits : std::false_type {
-	using Member = StructLike;
-	using types = pack<>;
-
-	template <int i>
-	static const index_t<i, types>& get(const Member&);
-
-	template <int i>
-	static const void assign(Member&, const index_t<i, types>&);
-
-	template <class Context>
-	static void done(Member&, Context&);
-};
-
 template <class... Ts>
 struct struct_like_traits<std::tuple<Ts...>> : std::true_type {
 	using Member = std::tuple<Ts...>;
@@ -320,14 +110,11 @@ struct scalar_traits<T, std::enable_if_t<std::is_integral<T>::value || std::is_f
 	}
 };
 
-template <class F, class... Items>
-void serializer(F& fun, Items&... items);
-
 template <class F, class S>
 struct serializable_traits<std::pair<F, S>> : std::true_type {
 	template <class Archiver>
 	static void serialize(Archiver& ar, std::pair<F, S>& p) {
-		flat_buffers::serializer(ar, p.first, p.second);
+		serializer(ar, p.first, p.second);
 	}
 };
 
@@ -734,6 +521,7 @@ private:
 
 struct InsertVTableLambda {
 	static constexpr bool isDeserializing = true;
+	static constexpr bool is_fb_visitor = true;
 	std::set<const VTable*>& vtables;
 	std::set<std::type_index>& known_types;
 
@@ -856,6 +644,7 @@ private:
 template <class Writer>
 struct SaveVisitorLambda {
 	static constexpr bool isDeserializing = false;
+	static constexpr bool is_fb_visitor = true;
 	const VTableSet* vtableset;
 	Writer& writer;
 
@@ -982,6 +771,7 @@ struct LoadSaveHelper {
 	template <class Context>
 	struct SerializeFun {
 		static constexpr bool isDeserializing = true;
+		static constexpr bool is_fb_visitor = true;
 
 		const uint16_t* vtable;
 		const uint8_t* current;
@@ -1143,11 +933,6 @@ auto save_helper(const Member& member, Writer& writer, const VTableSet* vtables)
 
 } // namespace detail
 
-template <class F, class... Items>
-void serializer(F& fun, Items&... items) {
-	fun(items...);
-}
-
 namespace detail {
 
 template <class... Members>
@@ -1164,7 +949,7 @@ struct FakeRoot {
 private:
 	template <class Archive, size_t... is>
 	void serialize_impl(Archive& archive, std::index_sequence<is...>) {
-		flat_buffers::serializer(archive, std::get<is>(members)...);
+		serializer(archive, std::get<is>(members)...);
 	}
 };
 
@@ -1189,20 +974,20 @@ uint8_t* save(Allocator& allocator, const Root& root, FileIdentifier file_identi
 
 template <class Root, class Context>
 void load(Root& root, const uint8_t* in, Context& context) {
-	flat_buffers::detail::load_helper(root, in, context);
+	detail::load_helper(root, in, context);
 }
 
 } // namespace detail
 
 template <class Allocator, class... Members>
 uint8_t* save_members(Allocator& allocator, FileIdentifier file_identifier, Members&... members) {
-	const auto& root = flat_buffers::detail::fake_root(members...);
+	const auto& root = detail::fake_root(members...);
 	return detail::save(allocator, root, file_identifier);
 }
 
 template <class Context, class... Members>
 void load_members(const uint8_t* in, Context& context, Members&... members) {
-	auto root = flat_buffers::detail::fake_root(members...);
+	auto root = detail::fake_root(members...);
 	detail::load(root, in, context);
 }
 
@@ -1216,7 +1001,7 @@ inline FileIdentifier read_file_identifier(const uint8_t* in) {
 // introduce the indirection only when necessary.
 template <class T>
 struct EnsureTable {
-	constexpr static flat_buffers::FileIdentifier file_identifier = FileIdentifierFor<T>::value;
+	constexpr static FileIdentifier file_identifier = FileIdentifierFor<T>::value;
 	EnsureTable() = default;
 	EnsureTable(const object_construction<T>& t) : t(t) {}
 	EnsureTable(const T& t) : t(t) {}
@@ -1229,7 +1014,7 @@ struct EnsureTable {
 				t.get().serialize(ar);
 			}
 		} else {
-			flat_buffers::serializer(ar, t.get());
+			serializer(ar, t.get());
 		}
 	}
 	T& asUnderlyingType() { return t.get(); }
@@ -1238,4 +1023,3 @@ private:
 	object_construction<T> t;
 };
 
-} // namespace flat_buffers
diff --git a/flow/network.h b/flow/network.h
index 2756ac963c..20124dcf66 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -399,6 +399,9 @@ public:
 	virtual bool isAddressOnThisHost( NetworkAddress const& addr ) = 0;
 	// Returns true if it is reasonably certain that a connection to the given address would be a fast loopback connection
 
+	virtual bool useObjectSerializer() { return false; }
+	// Whether or not the object serializer should be used when sending packets
+
 	// Shorthand for transport().getLocalAddress()
 	static NetworkAddress getLocalAddress()
 	{
diff --git a/flow/serialize.h b/flow/serialize.h
index 5e9c604e93..e9bf3ff84f 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -27,6 +27,8 @@
 #include <set>
 #include "flow/Error.h"
 #include "flow/Arena.h"
+#include "flow/FileIdentifier.h"
+#include "flow/ObjectSerializer.h"
 #include <algorithm>
 
 // Though similar, is_binary_serializable cannot be replaced by std::is_pod, as doing so would prefer
@@ -99,6 +101,11 @@ inline void load( Ar& ar, T& value ) {
 	Serializer<Ar,T>::serialize(ar, value);
 }
 
+template <class CharT, class Traits, class Allocator>
+struct FileIdentifierFor<std::basic_string<CharT, Traits, Allocator>> {
+	constexpr static FileIdentifier value = 15694229;
+};
+
 template <class Archive>
 inline void load( Archive& ar, std::string& value ) {
 	int32_t length;
@@ -123,6 +130,12 @@ public:
 	}
 };
 
+template <class F, class S>
+struct FileIdentifierFor<std::pair<F, S>> {
+	constexpr static FileIdentifier value = FileIdentifierFor<F>::value ^ FileIdentifierFor<S>::value;
+};
+
+
 template <class Archive, class T1, class T2>
 class Serializer< Archive, std::pair<T1,T2>, void > {
 public:
@@ -131,6 +144,11 @@ public:
 	}
 };
 
+template <class T, class Allocator>
+struct FileIdentifierFor<std::vector<T, Allocator>> {
+	constexpr static FileIdentifier value = (0x10 << 24) | FileIdentifierFor<T>::value;
+};
+
 template <class Archive, class T>
 inline void save( Archive& ar, const std::vector<T>& value ) {
 	ar << (int)value.size();
@@ -229,6 +247,11 @@ static inline bool valgrindCheck( const void* data, int bytes, const char* conte
 extern const uint64_t currentProtocolVersion;
 extern const uint64_t minValidProtocolVersion;
 extern const uint64_t compatibleProtocolVersionMask;
+extern const uint64_t objectSerializerFlag;
+
+extern uint64_t removeFlags(uint64_t version);
+extern uint64_t addObjectSerializerFlag(uint64_t version);
+extern bool hasObjectSerializerFlag(uint64_t version);
 
 struct _IncludeVersion {
 	uint64_t v;
@@ -504,6 +527,10 @@ public:
 		return (const uint8_t*)readBytes(bytes);
 	}
 
+	StringRef arenaReadAll() const {
+		return StringRef(reinterpret_cast<const uint8_t*>(begin), end - begin);
+	}
+
 	template <class T>
 	void serializeBinaryItem( T& t ) {
 		t = *(T*)readBytes(sizeof(T));
@@ -686,14 +713,22 @@ private:
 };
 
 struct ISerializeSource {
-	virtual void serializePacketWriter( PacketWriter& ) const = 0;
-	virtual void serializeBinaryWriter( BinaryWriter& ) const = 0;
+	virtual void serializePacketWriter(PacketWriter&, bool useObjectSerializer) const = 0;
+	virtual void serializeBinaryWriter(BinaryWriter&) const = 0;
 };
 
 template <class T>
 struct MakeSerializeSource : ISerializeSource {
-	virtual void serializePacketWriter( PacketWriter& w ) const { ((T const*)this)->serialize(w); }
-	virtual void serializeBinaryWriter( BinaryWriter& w ) const { ((T const*)this)->serialize(w); }
+	virtual void serializePacketWriter(PacketWriter& w, bool useObjectSerializer) const {
+		if (useObjectSerializer) {
+			ObjectWriter writer;
+			writer.serialize(get());
+		} else {
+			((T const*)this)->serialize(w);
+		}
+	}
+	virtual void serializeBinaryWriter(BinaryWriter& w) const { ((T const*)this)->serialize(w); }
+	virtual T const& get() const = 0;
 };
 
 template <class T>
@@ -701,6 +736,7 @@ struct SerializeSource : MakeSerializeSource<SerializeSource<T>> {
 	T const& value;
 	SerializeSource(T const& value) : value(value) {}
 	template <class Ar> void serialize(Ar& ar) const { ar << value; }
+	virtual T const& get() const { return value; }
 };
 
 template <class T>
@@ -709,12 +745,11 @@ struct SerializeBoolAnd : MakeSerializeSource<SerializeBoolAnd<T>> {
 	T const& value;
 	SerializeBoolAnd( bool b, T const& value ) : b(b), value(value) {}
 	template <class Ar> void serialize(Ar& ar) const { ar << b << value; }
-};
-
-struct SerializeSourceRaw : MakeSerializeSource<SerializeSourceRaw> {
-	StringRef data;
-	SerializeSourceRaw(StringRef data) : data(data) {}
-	template <class Ar> void serialize(Ar& ar) const { ar.serializeBytes(data); }
+	virtual T const& get() const {
+		// This is only used for the streaming serializer
+		ASSERT(false);
+		return value;
+	}
 };
 
 #endif

From ba83c458a68e9f4cabed045c8100c5ef21d1e137 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Mon, 28 Jan 2019 19:38:13 -0800
Subject: [PATCH 045/118] types implemented

---
 fdbclient/ClientDBInfo.h                |   4 +-
 fdbclient/ClusterInterface.h            |   6 +-
 fdbclient/FDBTypes.h                    |  36 +++++
 fdbclient/StorageServerInterface.h      |  13 +-
 fdbrpc/FlowTransport.actor.cpp          |  11 ++
 fdbrpc/FlowTransport.h                  |  22 +++
 fdbrpc/Locality.h                       |  63 ++++-----
 fdbrpc/ReplicationPolicy.cpp            |   2 +
 fdbrpc/ReplicationPolicy.h              | 178 +++++++++++++++++++++---
 fdbrpc/fdbrpc.h                         |  55 ++++++++
 fdbserver/ClusterRecruitmentInterface.h |  18 ++-
 fdbserver/LogSystemConfig.h             |  22 ++-
 fdbserver/MasterInterface.h             |   4 +-
 fdbserver/WorkerInterface.actor.h       |   4 +-
 flow/Error.h                            |   1 +
 flow/FastRef.h                          |   1 +
 flow/IRandom.h                          |  17 +++
 flow/ObjectSerializerTraits.h           |  15 ++
 flow/Trace.h                            |   5 +
 flow/flat_buffers.h                     |  17 +--
 flow/serialize.h                        |  15 +-
 21 files changed, 425 insertions(+), 84 deletions(-)

diff --git a/fdbclient/ClientDBInfo.h b/fdbclient/ClientDBInfo.h
index 7b6aa047f8..3468d0e2fa 100644
--- a/fdbclient/ClientDBInfo.h
+++ b/fdbclient/ClientDBInfo.h
@@ -38,7 +38,9 @@ struct ClientDBInfo {
 
 	template <class Archive>
 	void serialize(Archive& ar) {
-		ASSERT( ar.protocolVersion() >= 0x0FDB00A200040001LL );
+		if constexpr (!is_fb_function<Archive>) {
+			ASSERT(ar.protocolVersion() >= 0x0FDB00A200040001LL);
+		}
 		serializer(ar, proxies, id, clientTxnInfoSampleRate, clientTxnInfoSizeLimit);
 	}
 };
diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 61d6754b8f..28ab92be18 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -126,8 +126,10 @@ struct OpenDatabaseRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		ASSERT( ar.protocolVersion() >= 0x0FDB00A400040001LL );
-		serializer(ar, issues, supportedVersions, connectedCoordinatorsNum, traceLogGroup, knownClientInfoID, reply, arena);
+		if constexpr (!is_fb_function<Ar>) {
+                ASSERT( ar.protocolVersion() >= 0x0FDB00A400040001LL );
+		}
+        serializer(ar, issues, supportedVersions, connectedCoordinatorsNum, traceLogGroup, knownClientInfoID, reply, arena);
 	}
 };
 
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 790c70c6b6..eb24e84814 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -73,6 +73,23 @@ struct Tag {
 template <class Ar> void load( Ar& ar, Tag& tag ) { tag.serialize_unversioned(ar); }
 template <class Ar> void save( Ar& ar, Tag const& tag ) { const_cast<Tag&>(tag).serialize_unversioned(ar); }
 
+template<>
+struct scalar_traits<Tag> : std::true_type {
+	using locality_trait = scalar_traits<decltype(std::declval<Tag>().locality)>;
+	using id_trait = scalar_traits<decltype(std::declval<Tag>().id)>;
+	constexpr static size_t size = locality_trait::size + id_trait::size;
+	static void save(uint8_t* out, const Tag& tag) {
+		locality_trait::save(out, tag.locality);
+		id_trait::save(out + id_trait::size, tag.id);
+	}
+
+	template <class Context>
+	static void load(const uint8_t* in, Tag& tag, Context& context) {
+		locality_trait::load(in, tag.locality, context);
+		id_trait::load(in + locality_trait::size, tag.id, context);
+	}
+};
+
 static const Tag invalidTag {tagLocalitySpecial, 0};
 static const Tag txsTag {tagLocalitySpecial, 1};
 
@@ -727,6 +744,25 @@ struct AddressExclusion {
 	}
 };
 
+template <>
+struct scalar_traits<AddressExclusion> : std::true_type {
+	using ip_traits = scalar_traits<uint32_t>;
+	using port_traits = scalar_traits<int>;
+	constexpr static size_t size = ip_traits::size + port_traits::size;
+	static void save(uint8_t* buf, const AddressExclusion& value) {
+		ip_traits::save(buf, value.ip);
+		port_traits::save(buf + ip_traits::size, value.port);
+	}
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t* buf, AddressExclusion& value, Context& context) {
+		ip_traits::load<Context>(buf, value.ip, context);
+		port_traits::load<Context>(buf + ip_traits::size, value.port, context);
+	}
+};
+
 static bool addressExcluded( std::set<AddressExclusion> const& exclusions, NetworkAddress const& addr ) {
 	return exclusions.count( AddressExclusion(addr.ip, addr.port) ) || exclusions.count( AddressExclusion(addr.ip) );
 }
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index 9ea511e8dd..d535a81196 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -69,11 +69,16 @@ struct StorageServerInterface {
 	void serialize( Ar& ar ) {
 		// StorageServerInterface is persisted in the database and in the tLog's data structures, so changes here have to be
 		// versioned carefully!
-		serializer(ar, uniqueID, locality, getVersion, getValue, getKey, getKeyValues, getShardState, waitMetrics,
-			splitMetrics, getPhysicalMetrics, waitFailure, getQueuingMetrics, getKeyValueStoreType);
 
-		if( ar.protocolVersion() >= 0x0FDB00A200090001LL )
-			serializer(ar, watchValue);
+		if constexpr (!is_fb_function<Ar>) {
+			serializer(ar, uniqueID, locality, getVersion, getValue, getKey, getKeyValues, getShardState, waitMetrics,
+			           splitMetrics, getPhysicalMetrics, waitFailure, getQueuingMetrics, getKeyValueStoreType);
+			if (ar.protocolVersion() >= 0x0FDB00A200090001LL) serializer(ar, watchValue);
+		} else {
+			serializer(ar, uniqueID, locality, getVersion, getValue, getKey, getKeyValues, getShardState, waitMetrics,
+			           splitMetrics, getPhysicalMetrics, waitFailure, getQueuingMetrics, getKeyValueStoreType,
+			           watchValue);
+		}
 	}
 	bool operator == (StorageServerInterface const& s) const { return uniqueID == s.uniqueID; }
 	bool operator < (StorageServerInterface const& s) const { return uniqueID < s.uniqueID; }
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 8f77027581..60bff615dd 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -127,6 +127,12 @@ struct EndpointNotFoundReceiver : NetworkMessageReceiver {
 		Endpoint e; reader >> e;
 		IFailureMonitor::failureMonitor().endpointNotFound(e);
 	}
+
+	virtual void receive(ArenaObjectReader& reader) {
+		Endpoint e;
+		reader.deserialize(e);
+		IFailureMonitor::failureMonitor().endpointNotFound(e);
+	}
 };
 
 struct PingReceiver : NetworkMessageReceiver {
@@ -139,6 +145,11 @@ struct PingReceiver : NetworkMessageReceiver {
 		ReplyPromise<Void> reply; reader >> reply;
 		reply.send(Void());
 	}
+	virtual void receive(ArenaObjectReader& reader) {
+		ReplyPromise<Void> reply;
+		reader.deserialize(reply);
+		reply.send(Void());
+	}
 };
 
 class TransportData {
diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h
index bcf9c334aa..ef1f4a186c 100644
--- a/fdbrpc/FlowTransport.h
+++ b/fdbrpc/FlowTransport.h
@@ -86,6 +86,26 @@ public:
 #pragma pack(pop)
 
 
+template <>
+struct scalar_traits<Endpoint> : std::true_type {
+	using networkAddress = scalar_traits<NetworkAddress>;
+	using token = scalar_traits<UID>;
+
+	static constexpr size_t size = networkAddress::size + token::size;
+
+	static void save(uint8_t* out, const Endpoint& endpoint) {
+		networkAddress::save(out, endpoint.address);
+		out += networkAddress::size;
+		token::save(out, endpoint.token);
+	}
+
+	template <class C>
+	static void load(const uint8_t* in, Endpoint& endpoint, C& c) {
+		networkAddress::load(in, endpoint.address, c);
+		in += networkAddress::size;
+		token::load(in, endpoint.token, c);
+	}
+};
 
 class ArenaObjectReader;
 class NetworkMessageReceiver {
@@ -163,6 +183,8 @@ public:
 
 	Endpoint loadedEndpoint(const UID& token);
 
+	void loadedEndpoint(Endpoint&);
+
 private:
 	class TransportData* self;
 };
diff --git a/fdbrpc/Locality.h b/fdbrpc/Locality.h
index a05913460f..1b8b17ec2f 100644
--- a/fdbrpc/Locality.h
+++ b/fdbrpc/Locality.h
@@ -193,39 +193,40 @@ public:
 	void serialize(Ar& ar) {
 		// Locality is persisted in the database inside StorageServerInterface, so changes here have to be
 		// versioned carefully!
-		if (ar.protocolVersion() >= 0x0FDB00A446020001LL) {
-			Standalone<StringRef> key;
-			Optional<Standalone<StringRef>> value;
-			uint64_t mapSize = (uint64_t)_data.size();
-			serializer(ar, mapSize);
-			if (ar.isDeserializing) {
-				for (size_t i = 0; i < mapSize; i++) {
-					serializer(ar, key, value);
-					_data[key] = value;
+		if constexpr (is_fb_function<Ar>) {
+			serializer(ar, _data);
+		} else {
+			if (ar.protocolVersion() >= 0x0FDB00A446020001LL) {
+				Standalone<StringRef> key;
+				Optional<Standalone<StringRef>> value;
+				uint64_t mapSize = (uint64_t)_data.size();
+				serializer(ar, mapSize);
+				if (ar.isDeserializing) {
+					for (size_t i = 0; i < mapSize; i++) {
+						serializer(ar, key, value);
+						_data[key] = value;
+					}
+				} else {
+					for (auto it = _data.begin(); it != _data.end(); it++) {
+						key = it->first;
+						value = it->second;
+						serializer(ar, key, value);
+					}
 				}
-			}
-			else {
-				for (auto it = _data.begin(); it != _data.end(); it++) {
-					key = it->first;
-					value = it->second;
-					serializer(ar, key, value);
-				}
-			}
-		}
-		else {
-			ASSERT(ar.isDeserializing);
-			UID	zoneId, dcId, processId;
-			serializer(ar, zoneId, dcId);
-			set(keyZoneId, Standalone<StringRef>(zoneId.toString()));
-			set(keyDcId, Standalone<StringRef>(dcId.toString()));
+			} else {
+				ASSERT(ar.isDeserializing);
+				UID zoneId, dcId, processId;
+				serializer(ar, zoneId, dcId);
+				set(keyZoneId, Standalone<StringRef>(zoneId.toString()));
+				set(keyDcId, Standalone<StringRef>(dcId.toString()));
 
-			if (ar.protocolVersion() >= 0x0FDB00A340000001LL) {
-				serializer(ar, processId);
-				set(keyProcessId, Standalone<StringRef>(processId.toString()));
-			}
-			else {
-				int _machineClass = ProcessClass::UnsetClass;
-				serializer(ar, _machineClass);
+				if (ar.protocolVersion() >= 0x0FDB00A340000001LL) {
+					serializer(ar, processId);
+					set(keyProcessId, Standalone<StringRef>(processId.toString()));
+				} else {
+					int _machineClass = ProcessClass::UnsetClass;
+					serializer(ar, _machineClass);
+				}
 			}
 		}
 	}
diff --git a/fdbrpc/ReplicationPolicy.cpp b/fdbrpc/ReplicationPolicy.cpp
index 59b8f511d1..e31483cdd0 100644
--- a/fdbrpc/ReplicationPolicy.cpp
+++ b/fdbrpc/ReplicationPolicy.cpp
@@ -142,6 +142,8 @@ PolicyAcross::PolicyAcross(int count, std::string const& attribKey, Reference<IR
 	return;
 }
 
+PolicyAcross::PolicyAcross() : _policy(new PolicyOne()) {}
+
 PolicyAcross::~PolicyAcross()
 {
 	return;
diff --git a/fdbrpc/ReplicationPolicy.h b/fdbrpc/ReplicationPolicy.h
index 10b17d7041..3d6d969144 100644
--- a/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/ReplicationPolicy.h
@@ -58,6 +58,7 @@ struct IReplicationPolicy : public ReferenceCounted<IReplicationPolicy> {
 			serializeReplicationPolicy(ar, refThis);
 			refThis->delref_no_destroy();
 		}
+		virtual void deserializationDone() = 0;
 
 		// Utility functions
 		bool selectReplicas(
@@ -100,6 +101,7 @@ inline void save( Archive& ar, const Reference<IReplicationPolicy>& value ) {
 
 struct PolicyOne : IReplicationPolicy, public ReferenceCounted<PolicyOne> {
 	PolicyOne() {};
+	explicit PolicyOne(const PolicyOne& o) {}
 	virtual ~PolicyOne() {};
 	virtual std::string name() const { return "One"; }
 	virtual std::string info() const { return "1"; }
@@ -115,23 +117,22 @@ struct PolicyOne : IReplicationPolicy, public ReferenceCounted<PolicyOne> {
 	template <class Ar>
 	void serialize(Ar& ar) {
 	}
+	virtual void deserializationDone() {}
 	virtual void attributeKeys(std::set<std::string>* set) const override { return; }
 };
 
 struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross> {
-	PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy);
+	friend struct serializable_traits<PolicyAcross*>;
+	PolicyAcross(int count, std::string const& attribKey, IRepPolicyRef const policy);
+	explicit PolicyAcross();
+	explicit PolicyAcross(const PolicyAcross& other) : PolicyAcross(other._count, other._attribKey, other._policy) {}
 	virtual ~PolicyAcross();
 	virtual std::string name() const { return "Across"; }
-	virtual std::string info() const
-	{ return format("%s^%d x ", _attribKey.c_str(), _count) + _policy->info(); }
+	virtual std::string info() const { return format("%s^%d x ", _attribKey.c_str(), _count) + _policy->info(); }
 	virtual int maxResults() const { return _count * _policy->maxResults(); }
 	virtual int depth() const  { return 1 + _policy->depth(); }
-	virtual bool validate(
-		std::vector<LocalityEntry>	const&	solutionSet,
-		Reference<LocalitySet> const&				fromServers ) const;
-	virtual bool selectReplicas(
-		Reference<LocalitySet>	&						fromServers,
-		std::vector<LocalityEntry> const&		alsoServers,
+	virtual bool validate(std::vector<LocalityEntry> const& solutionSet, LocalitySetRef const& fromServers) const;
+	virtual bool selectReplicas(LocalitySetRef& fromServers, std::vector<LocalityEntry> const& alsoServers,
 		std::vector<LocalityEntry>	&				results );
 
 	template <class Ar>
@@ -140,31 +141,39 @@ struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross>
 		serializeReplicationPolicy(ar, _policy);
 	}
 
-	static bool compareAddedResults(const std::pair<int, int>& rhs, const std::pair<int, int>& lhs)
-	{ return (rhs.first < lhs.first) || (!(lhs.first < rhs.first) && (rhs.second < lhs.second)); }
+	virtual void deserializationDone() {}
 
-	virtual void attributeKeys(std::set<std::string> *set) const override
-	{ set->insert(_attribKey); _policy->attributeKeys(set); }
+	static bool compareAddedResults(const std::pair<int, int>& rhs, const std::pair<int, int>& lhs) {
+		return (rhs.first < lhs.first) || (!(lhs.first < rhs.first) && (rhs.second < lhs.second));
+	}
+
+	virtual void attributeKeys(std::set<std::string>* set) const override {
+		set->insert(_attribKey);
+		_policy->attributeKeys(set);
+	}
 
 protected:
 	int																_count;
 	std::string												_attribKey;
-	Reference<IReplicationPolicy>								_policy;
+	IRepPolicyRef _policy;
 
 	// Cache temporary members
 	std::vector<AttribValue>					_usedValues;
 	std::vector<LocalityEntry>				_newResults;
-	Reference<LocalitySet>										_selected;
+	LocalitySetRef _selected;
 	VectorRef<std::pair<int,int>>			_addedResults;
 	Arena															_arena;
 };
 
 struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
-	PolicyAnd(std::vector<Reference<IReplicationPolicy>> policies): _policies(policies), _sortedPolicies(policies)
+	friend struct serializable_traits<PolicyAnd*>;
+	PolicyAnd(std::vector<IRepPolicyRef> policies): _policies(policies), _sortedPolicies(policies)
 	{
 		// Sort the policy array
 		std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
 	}
+	explicit PolicyAnd(const PolicyAnd& other) : _policies(other._policies), _sortedPolicies(other._sortedPolicies) {}
+	explicit PolicyAnd() {}
 	virtual ~PolicyAnd() {}
 	virtual std::string name() const { return "And"; }
 	virtual std::string info() const {
@@ -218,6 +227,11 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
 		}
 	}
 
+	virtual void deserializationDone() {
+		_sortedPolicies = _policies;
+		std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
+	}
+
 	virtual void attributeKeys(std::set<std::string> *set) const override
 	{ for (const Reference<IReplicationPolicy>& r : _policies) { r->attributeKeys(set); } }
 
@@ -228,6 +242,138 @@ protected:
 
 extern int testReplication();
 
+#define POLICY_CONSTRUCTION_WRAPPER(t)                                                                                 \
+	template <>                                                                                                        \
+	struct object_construction<t*> {                                                                                   \
+		using type = t*;                                                                                               \
+		type obj;                                                                                                      \
+                                                                                                                       \
+		object_construction() : obj(new t()) {}                                                                        \
+		object_construction(object_construction&& other) : obj(other.obj) { other.obj = nullptr; }                     \
+		object_construction(const object_construction& other) : obj() {}                                               \
+                                                                                                                       \
+		object_construction& operator=(object_construction&& other) {                                                  \
+			if (obj != nullptr) {                                                                                      \
+				delete obj;                                                                                            \
+			}                                                                                                          \
+			obj = other.obj;                                                                                           \
+			other.obj = nullptr;                                                                                       \
+			return *this;                                                                                              \
+		}                                                                                                              \
+                                                                                                                       \
+		object_construction& operator=(const object_construction& other) {                                             \
+			if (obj != nullptr) {                                                                                      \
+				delete obj;                                                                                            \
+			}                                                                                                          \
+			obj = new t(*other.obj);                                                                                   \
+			return *this;                                                                                              \
+		}                                                                                                              \
+                                                                                                                       \
+		type& get() { return obj; }                                                                                    \
+		const type& get() const { return obj; }                                                                        \
+		type move() {                                                                                                  \
+			auto res = obj;                                                                                            \
+			obj = nullptr;                                                                                             \
+			return res;                                                                                                \
+		}                                                                                                              \
+	};
+
+template <>
+struct object_construction<IRepPolicyRef> {
+	using type = IRepPolicyRef;
+	type _impl;
+
+	object_construction() : _impl(new PolicyOne()){};
+
+	type& get() { return _impl; }
+	const type& get() const { return _impl; }
+
+	type move() { return std::move(_impl); }
+};
+
+POLICY_CONSTRUCTION_WRAPPER(PolicyOne);
+POLICY_CONSTRUCTION_WRAPPER(PolicyAcross);
+POLICY_CONSTRUCTION_WRAPPER(PolicyAnd);
+
+template <>
+struct FileIdentifierFor<IRepPolicyRef> {
+	static constexpr FileIdentifier value = 14695621;
+};
+
+template <>
+struct serializable_traits<PolicyOne*> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, PolicyOne*& p) {}
+};
+
+template <>
+struct serializable_traits<PolicyAcross*> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, PolicyAcross*& p) {
+		::serializer(ar, p->_count, p->_attribKey, p->_policy);
+	}
+};
+
+template <>
+struct serializable_traits<PolicyAnd*> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, PolicyAnd*& p) {
+		::serializer(ar, p->_policies);
+	}
+};
+
+template <>
+struct serializable_traits<IRepPolicyRef> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, IRepPolicyRef& policy) {
+		::serializer(ar, policy.changePtrUnsafe());
+	}
+};
+
+template <>
+struct union_like_traits<IReplicationPolicy*> : std::true_type {
+	using Member = IReplicationPolicy*;
+	using alternatives = pack<PolicyOne*, PolicyAcross*, PolicyAnd*>;
+
+	static uint8_t index(const Member& policy) {
+		if (policy->name() == "One") {
+			return 0;
+		} else if (policy->name() == "And") {
+			return 2;
+		} else {
+			return 1;
+		}
+	}
+
+	static bool empty(const Member& policy) { return policy == nullptr; }
+
+	template <int i>
+	static const index_t<i, alternatives>& get(IReplicationPolicy* const& member) {
+		if constexpr (i == 0) {
+			return reinterpret_cast<PolicyOne* const&>(member);
+		} else if constexpr (i == 1) {
+			return reinterpret_cast<PolicyAcross* const&>(member);
+		} else {
+			return reinterpret_cast<PolicyAnd* const&>(member);
+		}
+	}
+
+	template <int i, class Alternative>
+	static const void assign(Member& policy, const Alternative& impl) {
+		if (policy != nullptr) {
+			policy->delref();
+		}
+		policy = impl;
+	}
+
+	template <class Context>
+	static void done(Member& policy, Context&) {
+		if (policy != nullptr) {
+			policy->deserializationDone();
+		}
+	}
+};
+
 
 template <class Ar>
 void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index 4a94af1543..4d466d77ab 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -23,6 +23,7 @@
 #pragma once
 
 #include "flow/flow.h"
+#include "flow/serialize.h"
 #include "fdbrpc/FlowTransport.h" // NetworkMessageReceiver Endpoint
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbrpc/networksender.actor.h"
@@ -174,6 +175,30 @@ void load(Ar& ar, ReplyPromise<T>& value) {
 	networkSender(value.getFuture(), endpoint);
 }
 
+template <class T>
+struct scalar_traits<ReplyPromise<T>> : std::true_type {
+	using endpoint = scalar_traits<Endpoint>;
+
+	constexpr static size_t size = endpoint::size;
+
+	static void save(uint8_t* out, const ReplyPromise<T>& reply) {
+		auto const& ep = reply.getEndpoint();
+		endpoint::save(out, ep);
+		ASSERT(!ep.address.isValid() || ep.address.isPublic()); // No re-serializing non-public addresses
+		                                                        // (the reply connection won't be
+		                                                        // available to any other process)
+	}
+
+	template <class C>
+	static void load(const uint8_t* in, ReplyPromise<T>& reply, C& context) {
+		Endpoint ep;
+		endpoint::load(in, ep, context);
+		FlowTransport::transport().loadedEndpoint(ep);
+		reply = ReplyPromise<T>(ep);
+		networkSender(reply.getFuture(), ep);
+	}
+};
+
 
 template <class Reply>
 ReplyPromise<Reply> const& getReplyPromise(ReplyPromise<Reply> const& p) { return p; }
@@ -221,6 +246,13 @@ struct NetNotifiedQueue : NotifiedQueue<T>, FlowReceiver, FastAllocated<NetNotif
 		this->send(std::move(message));
 		this->delPromiseRef();
 	}
+	virtual void receive(ArenaObjectReader& reader) {
+		this->addPromiseRef();
+		T message;
+		reader.deserialize(message);
+		this->send(std::move(message));
+		this->delPromiseRef();
+	}
 	virtual bool isStream() const { return true; }
 };
 
@@ -390,5 +422,28 @@ void load(Ar& ar, RequestStream<T>& value) {
 	value = RequestStream<T>(endpoint);
 }
 
+template <class T>
+struct scalar_traits<RequestStream<T>> : std::true_type {
+	using endpointTraits = scalar_traits<Endpoint>;
+
+	static constexpr size_t size = endpointTraits::size;
+
+	static void save(uint8_t* out, const RequestStream<T>& stream) {
+		auto const& ep = stream.getEndpoint();
+		endpointTraits::save(out, ep);
+		UNSTOPPABLE_ASSERT(ep.address.isValid());
+	}
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t* in, RequestStream<T>& stream, Context& context) {
+		Endpoint endpoint;
+		endpointTraits::load(in, endpoint, context);
+		FlowTransport::transport().loadedEndpoint(endpoint);
+		stream = RequestStream<T>(endpoint);
+	}
+};
+
 #endif
 #include "fdbrpc/genericactors.actor.h"
diff --git a/fdbserver/ClusterRecruitmentInterface.h b/fdbserver/ClusterRecruitmentInterface.h
index f66b88032d..caf088c05f 100644
--- a/fdbserver/ClusterRecruitmentInterface.h
+++ b/fdbserver/ClusterRecruitmentInterface.h
@@ -60,9 +60,12 @@ struct ClusterControllerFullInterface {
 	}
 
 	template <class Ar>
-	void serialize( Ar& ar ) {
-		ASSERT( ar.protocolVersion() >= 0x0FDB00A200040001LL );
-		serializer(ar, clientInterface, recruitFromConfiguration, recruitRemoteFromConfiguration, recruitStorage, registerWorker, getWorkers, registerMaster, getServerDBInfo);
+	void serialize(Ar& ar) {
+		if constexpr (!is_fb_function<Ar>) {
+			ASSERT(ar.protocolVersion() >= 0x0FDB00A200040001LL);
+		}
+		serializer(ar, clientInterface, recruitFromConfiguration, recruitRemoteFromConfiguration, recruitStorage,
+		           registerWorker, getWorkers, registerMaster, getServerDBInfo);
 	}
 };
 
@@ -216,9 +219,12 @@ struct RegisterMasterRequest {
 	RegisterMasterRequest() {}
 
 	template <class Ar>
-	void serialize( Ar& ar ) {
-		ASSERT( ar.protocolVersion() >= 0x0FDB00A200040001LL );
-		serializer(ar, id, mi, logSystemConfig, proxies, resolvers, recoveryCount, registrationCount, configuration, priorCommittedLogServers, recoveryState, recoveryStalled, reply);
+	void serialize(Ar& ar) {
+		if constexpr (!is_fb_function<Ar>) {
+			ASSERT(ar.protocolVersion() >= 0x0FDB00A200040001LL);
+		}
+		serializer(ar, id, mi, logSystemConfig, proxies, resolvers, recoveryCount, registrationCount, configuration,
+		           priorCommittedLogServers, recoveryState, recoveryStalled, reply);
 	}
 };
 
diff --git a/fdbserver/LogSystemConfig.h b/fdbserver/LogSystemConfig.h
index bd06329384..2b5277e88a 100644
--- a/fdbserver/LogSystemConfig.h
+++ b/fdbserver/LogSystemConfig.h
@@ -28,6 +28,7 @@
 
 template <class Interface>
 struct OptionalInterface {
+	friend class serializable_traits<OptionalInterface<Interface>>;
 	// Represents an interface with a known id() and possibly known actual endpoints.
 	// For example, an OptionalInterface<TLogInterface> represents a particular tlog by id, which you might or might not presently know how to communicate with
 
@@ -58,6 +59,25 @@ protected:
 class LogSet;
 struct OldLogData;
 
+template <class Interface>
+struct serializable_traits<OptionalInterface<Interface>> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, OptionalInterface<Interface>& m) {
+		if constexpr (!Archiver::isDeserializing) {
+			if (m.iface.present()) {
+				m.ident = m.iface.get().id();
+			}
+		}
+		::serializer(ar, m.iface, m.ident);
+		if constexpr (Archiver::isDeserializing) {
+			if (m.iface.present()) {
+				m.ident = m.iface.get().id();
+			}
+		}
+	}
+};
+
+
 struct TLogSet {
 	std::vector<OptionalInterface<TLogInterface>> tLogs;
 	std::vector<OptionalInterface<TLogInterface>> logRouters;
@@ -136,7 +156,7 @@ struct OldTLogConf {
 	explicit OldTLogConf(const OldLogData&);
 
 	std::string toString() const {
-		return format("end: %d tags: %d %s", epochEnd, logRouterTags, describe(tLogs).c_str()); 
+		return format("end: %d tags: %d %s", epochEnd, logRouterTags, describe(tLogs).c_str());
 	}
 
 	bool operator == ( const OldTLogConf& rhs ) const {
diff --git a/fdbserver/MasterInterface.h b/fdbserver/MasterInterface.h
index 61f848b8c3..ac53d8ef2e 100644
--- a/fdbserver/MasterInterface.h
+++ b/fdbserver/MasterInterface.h
@@ -42,7 +42,9 @@ struct MasterInterface {
 	UID id() const { return changeCoordinators.getEndpoint().token; }
 	template <class Archive>
 	void serialize(Archive& ar) {
-		ASSERT( ar.protocolVersion() >= 0x0FDB00A200040001LL );
+		if constexpr (!is_fb_function<Archive>) {
+                ASSERT( ar.protocolVersion() >= 0x0FDB00A200040001LL );
+        }
 		serializer(ar, locality, waitFailure, tlogRejoin, changeCoordinators, getCommitVersion);
 	}
 
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 4c60fc27dd..d684109f0e 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -139,7 +139,9 @@ struct RecruitMasterRequest {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		ASSERT( ar.protocolVersion() >= 0x0FDB00A200040001LL );
+		if constexpr (!is_fb_function<Ar>) {
+			ASSERT(ar.protocolVersion() >= 0x0FDB00A200040001LL);
+		}
 		serializer(ar, lifetime, forceRecovery, reply, arena);
 	}
 };
diff --git a/flow/Error.h b/flow/Error.h
index ffaef85449..18a3a1c9e6 100644
--- a/flow/Error.h
+++ b/flow/Error.h
@@ -29,6 +29,7 @@
 #include <boost/preprocessor/control/if.hpp>
 #include "flow/Platform.h"
 #include "flow/Knobs.h"
+#include "flow/ObjectSerializerTraits.h"
 
 enum { invalid_error_code = 0xffff };
 
diff --git a/flow/FastRef.h b/flow/FastRef.h
index 3ddcdd55e9..380ad0ecda 100644
--- a/flow/FastRef.h
+++ b/flow/FastRef.h
@@ -156,6 +156,7 @@ public:
 
 	bool isValid() const { return ptr != NULL; }
 	explicit operator bool() const { return ptr != NULL; }
+	P*& changePtrUnsafe() { return ptr; }
 
 private:
 	P *ptr;
diff --git a/flow/IRandom.h b/flow/IRandom.h
index c5d60cb01d..dbf14ac619 100644
--- a/flow/IRandom.h
+++ b/flow/IRandom.h
@@ -23,6 +23,7 @@
 #pragma once
 
 #include "flow/Platform.h"
+#include "flow/ObjectSerializerTraits.h"
 #include <stdint.h>
 #if (defined(__APPLE__))
 #include <ext/hash_map>
@@ -59,6 +60,22 @@ public:
 template <class Ar> void load( Ar& ar, UID& uid ) { uid.serialize_unversioned(ar); }
 template <class Ar> void save( Ar& ar, UID const& uid ) { const_cast<UID&>(uid).serialize_unversioned(ar); }
 
+template <>
+struct scalar_traits<UID> : std::true_type {
+	constexpr static size_t size = sizeof(uint64_t[2]);
+	static void save(uint8_t* out, const UID& uid) {
+		uint64_t* outI = reinterpret_cast<uint64_t*>(out);
+		outI[0] = uid.first();
+		outI[1] = uid.second();
+	}
+
+	template <class Context>
+	static void load(const uint8_t* i, UID& out, Context& context) {
+		const uint64_t* in = reinterpret_cast<const uint64_t*>(i);
+		out = UID(in[0], in[1]);
+	}
+};
+
 namespace std {
 	template <>
 	class hash<UID> {
diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h
index 9d1476e8e2..069146ca9f 100644
--- a/flow/ObjectSerializerTraits.h
+++ b/flow/ObjectSerializerTraits.h
@@ -27,6 +27,21 @@
 #include <functional>
 #include <vector>
 
+template <class T, typename = void>
+struct is_fb_function_t : std::false_type {};
+
+template<class T>
+struct is_fb_function_t<T, typename std::enable_if<T::is_fb_visitor>::type> : std::true_type {};
+
+template <class T>
+constexpr bool is_fb_function = is_fb_function_t<T>::value;
+
+template <class Visitor, class... Items>
+typename std::enable_if<is_fb_function<Visitor>, void>::type serializer(Visitor& visitor, Items&... items) {
+	visitor(items...);
+}
+
+
 template <class... Ts>
 struct pack {};
 
diff --git a/flow/Trace.h b/flow/Trace.h
index 3ac5838852..c09e29374d 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -80,6 +80,11 @@ public:
 
 	std::string toString() const;
 	void validateFormat() const;
+	template<class Archiver>
+	void serialize(Archiver& ar) {
+		static_assert(is_fb_function<Archiver>, "Streaming serializer has to use load/save");
+		serializer(ar, fields);
+	}
 
 private:
 	FieldContainer fields;
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 13637b704b..502997e002 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -56,20 +56,6 @@ constexpr int RightAlign(int offset, int alignment) {
 	return offset % alignment == 0 ? offset : ((offset / alignment) + 1) * alignment;
 }
 
-template <class T, typename = void>
-struct is_fb_function_t : std::false_type {};
-
-template<class T>
-struct is_fb_function_t<T, typename std::enable_if<T::is_fb_visitor>::type> : std::true_type {};
-
-template <class T>
-constexpr bool is_fb_function = is_fb_function_t<T>::value;
-
-template <class Visitor, class... Items>
-typename std::enable_if<is_fb_function<Visitor>, void>::type serializer(Visitor& visitor, Items&... items) {
-	visitor(items...);
-}
-
 template <class T>
 struct object_construction {
 	T obj;
@@ -100,7 +86,8 @@ struct struct_like_traits<std::tuple<Ts...>> : std::true_type {
 };
 
 template <class T>
-struct scalar_traits<T, std::enable_if_t<std::is_integral<T>::value || std::is_floating_point<T>::value>>
+struct scalar_traits<
+    T, std::enable_if_t<std::is_integral<T>::value || std::is_floating_point<T>::value || std::is_enum<T>::value>>
   : std::true_type {
 	constexpr static size_t size = sizeof(T);
 	static void save(uint8_t* out, const T& t) { memcpy(out, &t, size); }
diff --git a/flow/serialize.h b/flow/serialize.h
index e9bf3ff84f..adaf56c560 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -717,22 +717,24 @@ struct ISerializeSource {
 	virtual void serializeBinaryWriter(BinaryWriter&) const = 0;
 };
 
-template <class T>
+template <class T, class V>
 struct MakeSerializeSource : ISerializeSource {
+	using value_type = V;
 	virtual void serializePacketWriter(PacketWriter& w, bool useObjectSerializer) const {
 		if (useObjectSerializer) {
 			ObjectWriter writer;
 			writer.serialize(get());
 		} else {
-			((T const*)this)->serialize(w);
+			static_cast<T const*>(this)->serialize(w);
 		}
 	}
-	virtual void serializeBinaryWriter(BinaryWriter& w) const { ((T const*)this)->serialize(w); }
-	virtual T const& get() const = 0;
+	virtual void serializeBinaryWriter(BinaryWriter& w) const { static_cast<T const*>(this)->serialize(w); }
+	virtual value_type const& get() const = 0;
 };
 
 template <class T>
-struct SerializeSource : MakeSerializeSource<SerializeSource<T>> {
+struct SerializeSource : MakeSerializeSource<SerializeSource<T>, T> {
+	using value_type = T;
 	T const& value;
 	SerializeSource(T const& value) : value(value) {}
 	template <class Ar> void serialize(Ar& ar) const { ar << value; }
@@ -740,7 +742,8 @@ struct SerializeSource : MakeSerializeSource<SerializeSource<T>> {
 };
 
 template <class T>
-struct SerializeBoolAnd : MakeSerializeSource<SerializeBoolAnd<T>> {
+struct SerializeBoolAnd : MakeSerializeSource<SerializeBoolAnd<T>, T> {
+	using value_type = T;
 	bool b;
 	T const& value;
 	SerializeBoolAnd( bool b, T const& value ) : b(b), value(value) {}

From 9eeb48c43db3eeddc69a6ec6a9701a539a5e276e Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 29 Jan 2019 08:43:57 -0800
Subject: [PATCH 046/118] Allow to turn on object serializer

This commit includes functionality to turn on
the object serializer for network communication.
This is done the following way:

- On incoming connections, a process will detect
  whether the client supports the object serializer
  and will only serialize responses with it, if it does
- On outgoing connections, the command line flag is used
  to determine whether the object serializer should be used
  to send data.

This way, a cluster can run in mixed mode. To upgrade one
can upgrade one process at a time and set the flag one process
at a time.

This is how this is tested on the simulator:
- The command line flag can take three options: on, off,
  and random.
- For off, the object serializer will never we used.
- For on, the object serializer will be always used.
- For random, the simulator will flip a coin for each
  process it starts up.
---
 fdbclient/NativeAPI.actor.cpp        |   5 +-
 fdbclient/NativeAPI.actor.h          |   3 +-
 fdbclient/vexillographer/fdb.options |   3 +
 fdbrpc/FlowTests.actor.cpp           |   1 +
 fdbrpc/sim2.actor.cpp                |   8 +-
 fdbrpc/simulator.h                   |  13 ++-
 fdbserver/SimulatedCluster.actor.cpp |  62 ++++++----
 fdbserver/SimulatedCluster.h         |   2 +-
 fdbserver/fdbserver.actor.cpp        | 169 ++++++++++++++++-----------
 flow/Net2.actor.cpp                  |  11 +-
 flow/network.h                       |   4 +-
 11 files changed, 172 insertions(+), 109 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4df95c560b..3f5225be1e 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -869,6 +869,9 @@ Future<Void> Cluster::onConnected() {
 void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
 	switch(option) {
 		// SOMEDAY: If the network is already started, should these three throw an error?
+		case FDBNetworkOptions::USE_OBJECT_SERIALIZER:
+			networkOptions.useObjectSerializer = extractIntOption(value) != 0;
+			break;
 		case FDBNetworkOptions::TRACE_ENABLE:
 			networkOptions.traceDirectory = value.present() ? value.get().toString() : "";
 			break;
@@ -1008,7 +1011,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) {
 	if (!networkOptions.logClientInfo.present())
 		networkOptions.logClientInfo = true;
 
-	g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present());
+	g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present(), networkOptions.useObjectSerializer);
 	FlowTransport::createInstance(transportId);
 	Net2FileSystem::newFileSystem();
 
diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h
index 0d1acdfceb..b93094cc71 100644
--- a/fdbclient/NativeAPI.actor.h
+++ b/fdbclient/NativeAPI.actor.h
@@ -56,12 +56,13 @@ struct NetworkOptions {
 	Optional<bool> logClientInfo;
 	Standalone<VectorRef<ClientVersionRef>> supportedVersions;
 	bool slowTaskProfilingEnabled;
+	bool useObjectSerializer;
 
 	// The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h.
 	NetworkOptions()
 	  : localAddress(""), clusterFile(""), traceDirectory(Optional<std::string>()),
 	    traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
-	    traceFormat("xml"), slowTaskProfilingEnabled(false) {}
+	    traceFormat("xml"), slowTaskProfilingEnabled(false), useObjectSerializer(false) {}
 };
 
 class Database {
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index cba06df111..a84ba589dd 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -33,6 +33,9 @@ description is not currently required but encouraged.
     <Option name="local_address"  code="10" 
             paramType="String" paramDescription="IP:PORT" 
             description="Deprecated"/>
+    <Option name="use_object_serializer" code="11"
+            paramType="Int" paramDescription="0 is false, every other value is true"
+            description="enable the object serializer for network communication"/>
     <Option name="cluster_file" code="20"
             paramType="String" paramDescription="path to cluster file"
             description="Deprecated"/>
diff --git a/fdbrpc/FlowTests.actor.cpp b/fdbrpc/FlowTests.actor.cpp
index c6e63d5660..0d1760295e 100644
--- a/fdbrpc/FlowTests.actor.cpp
+++ b/fdbrpc/FlowTests.actor.cpp
@@ -200,6 +200,7 @@ struct YieldMockNetwork : INetwork, ReferenceCounted<YieldMockNetwork> {
 	virtual void run() { return baseNetwork->run(); }
 	virtual void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total)  { return baseNetwork->getDiskBytes(directory,free,total); }
 	virtual bool isAddressOnThisHost(NetworkAddress const& addr) { return baseNetwork->isAddressOnThisHost(addr); }
+	virtual bool useObjectSerializer() const { return baseNetwork->useObjectSerializer(); }
 };
 
 struct NonserializableThing {};
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index be9a0bd4e6..09cff1dc44 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -979,7 +979,7 @@ public:
 	}
 	virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess,
 	                                LocalityData locality, ProcessClass startingClass, const char* dataFolder,
-	                                const char* coordinationFolder) {
+	                                const char* coordinationFolder, bool ObjSerializer) {
 		ASSERT( locality.machineId().present() );
 		MachineInfo& machine = machines[ locality.machineId().get() ];
 		if (!machine.machineId.present())
@@ -1001,7 +1001,7 @@ public:
 		// These files must live on after process kills for sim purposes.
 		if( machine.machineProcess == 0 ) {
 			NetworkAddress machineAddress(ip, 0, false, false);
-			machine.machineProcess = new ProcessInfo("Machine", locality, startingClass, {machineAddress}, this, "", "");
+			machine.machineProcess = new ProcessInfo("Machine", locality, startingClass, {machineAddress}, this, "", "", objSerializer);
 			machine.machineProcess->machine = &machine;
 		}
 
@@ -1011,7 +1011,7 @@ public:
 			addresses.secondaryAddress = NetworkAddress(ip, port+1, true, false);
 		}
 
-		ProcessInfo* m = new ProcessInfo(name, locality, startingClass, addresses, this, dataFolder, coordinationFolder);
+		ProcessInfo* m = new ProcessInfo(name, locality, startingClass, addresses, this, dataFolder, coordinationFolder, objSerializer);
 		for (int processPort = port; processPort < port + listenPerProcess; ++processPort) {
 			NetworkAddress address(ip, processPort, true, false); // SOMEDAY see above about becoming SSL!
 			m->listenerMap[address] = Reference<IListener>( new Sim2Listener(m, address) );
@@ -1572,7 +1572,7 @@ public:
 
 	Sim2() : time(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(-1) {
 		// Not letting currentProcess be NULL eliminates some annoying special cases
-		currentProcess = new ProcessInfo( "NoMachine", LocalityData(Optional<Standalone<StringRef>>(), StringRef(), StringRef(), StringRef()), ProcessClass(), {NetworkAddress()}, this, "", "" );
+		currentProcess = new ProcessInfo("NoMachine", LocalityData(Optional<Standalone<StringRef>>(), StringRef(), StringRef(), StringRef()), ProcessClass(), {NetworkAddress()}, this, "", "", false);
 		g_network = net2 = newNet2(false, true);
 		Net2FileSystem::newFileSystem();
 		check_yield(0);
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index bff9b94efe..02941f0c02 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -53,6 +53,7 @@ public:
 		NetworkAddress address;
 		LocalityData	locality;
 		ProcessClass startingClass;
+		bool useObjectSerializer;
 		TDMetricCollection tdmetrics;
 		std::map<NetworkAddress, Reference<IListener>> listenerMap;
 		bool failed;
@@ -69,10 +70,11 @@ public:
 
 		ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddressList addresses,
 					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder )
-			: name(name), locality(locality), startingClass(startingClass), addresses(addresses), address(addresses.address), dataFolder(dataFolder),
-				network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
-				rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
-				fault_injection_r(0), machine(0), cleared(false) {}
+            : name(name), locality(locality), startingClass(startingClass), useObjectSerializer(useObjectSerializer),
+              addresses(addresses), address(addresses.address), dataFolder(dataFolder),
+              network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
+              rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
+              fault_injection_r(0), machine(0), cleared(false) {}
 
 		Future<KillType> onShutdown() { return shutdownSignal.getFuture(); }
 
@@ -141,7 +143,7 @@ public:
 
 	virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess,
 	                                LocalityData locality, ProcessClass startingClass, const char* dataFolder,
-	                                const char* coordinationFolder) = 0;
+	                                const char* coordinationFolder, bool useObjectSerializer) = 0;
 	virtual void killProcess( ProcessInfo* machine, KillType ) = 0;
 	virtual void rebootProcess(Optional<Standalone<StringRef>> zoneId, bool allProcesses ) = 0;
 	virtual void rebootProcess( ProcessInfo* process, KillType kt ) = 0;
@@ -154,6 +156,7 @@ public:
 	virtual bool isAvailable() const = 0;
 	virtual bool datacenterDead(Optional<Standalone<StringRef>> dcId) const = 0;
 	virtual void displayWorkers() const;
+	virtual bool useObjectSerializer() const { return currentProcess->useObjectSerializer; }
 
 	virtual void addRole(NetworkAddress const& address, std::string const& role) {
 		roleAddresses[address][role] ++;
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index f411b05ba1..95803d46ab 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -121,6 +121,16 @@ static void simInitTLS(Reference<TLSOptions> tlsOptions) {
 	tlsOptions->register_network();
 }
 
+static bool useObjectSerializer(int b) {
+	if (b == 0) {
+		return false;
+	} else if (b == 1) {
+		return true;
+	} else {
+		return g_random->coinflip();
+	}
+}
+
 ACTOR Future<Void> runBackup( Reference<ClusterConnectionFile> connFile ) {
 	state std::vector<Future<Void>> agentFutures;
 
@@ -196,7 +206,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
                                                          LocalityData localities, ProcessClass processClass,
                                                          std::string* dataFolder, std::string* coordFolder,
                                                          std::string baseFolder, ClusterConnectionString connStr,
-                                                         bool useSeedFile, bool runBackupAgents) {
+                                                         bool useSeedFile, bool runBackupAgents, bool objSerializer) {
 	state ISimulator::ProcessInfo *simProcess = g_simulator.getCurrentProcess();
 	state UID randomId = g_nondeterministic_random->randomUniqueID();
 	state int cycles = 0;
@@ -211,8 +221,11 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 
 		wait( delay( waitTime ) );
 
-		state ISimulator::ProcessInfo *process =  g_simulator.newProcess( "Server", ip, port, listenPerProcess, localities, processClass, dataFolder->c_str(), coordFolder->c_str() );
-		wait( g_simulator.onProcess(process, TaskDefaultYield) );	// Now switch execution to the process on which we will run
+		state ISimulator::ProcessInfo* process =
+		    g_simulator.newProcess("Server", ip, port, listenPerProcess, localities, processClass, dataFolder->c_str(),
+		                           coordFolder->c_str(), objSerializer);
+		wait(g_simulator.onProcess(process,
+		                           TaskDefaultYield)); // Now switch execution to the process on which we will run
 		state Future<ISimulator::KillType> onShutdown = process->onShutdown();
 
 		try {
@@ -356,7 +369,7 @@ std::map< Optional<Standalone<StringRef>>, std::vector< std::vector< std::string
 ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr, std::vector<IPAddress> ips, bool sslEnabled,
                                     Reference<TLSOptions> tlsOptions, LocalityData localities,
                                     ProcessClass processClass, std::string baseFolder, bool restarting,
-                                    bool useSeedFile, bool runBackupAgents, bool sslOnly) {
+                                    bool useSeedFile, bool runBackupAgents, bool sslOnly, int objSerializer) {
 	state int bootCount = 0;
 	state std::vector<std::string> myFolders;
 	state std::vector<std::string> coordFolders;
@@ -398,7 +411,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr, std::vector
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
 				Reference<ClusterConnectionFile> clusterFile(useSeedFile ? new ClusterConnectionFile(path, connStr.toString()) : new ClusterConnectionFile(path));
 				const int listenPort = i*listenPerProcess + 1;
-				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, tlsOptions, listenPort, listenPerProcess, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents));
+				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, tlsOptions, listenPort, listenPerProcess, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents, useObjectSerializer));
 				TraceEvent("SimulatedMachineProcess", randomId).detail("Address", NetworkAddress(ips[i], listenPort, true, false)).detail("ZoneId", localities.zoneId()).detail("DataHall", localities.dataHallId()).detail("Folder", myFolders[i]);
 			}
 
@@ -600,9 +613,10 @@ IPAddress makeIPAddressForSim(bool isIPv6, std::array<int, 4> parts) {
 
 #include "fdbclient/MonitorLeader.h"
 
-ACTOR Future<Void> restartSimulatedSystem(
-		vector<Future<Void>> *systemActors, std::string baseFolder, int* pTesterCount,
-		Optional<ClusterConnectionString> *pConnString, Standalone<StringRef> *pStartingConfiguration, Reference<TLSOptions> tlsOptions, int extraDB) {
+ACTOR Future<Void> restartSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFolder, int* pTesterCount,
+                                          Optional<ClusterConnectionString>* pConnString,
+                                          Standalone<StringRef>* pStartingConfiguration,
+                                          Reference<TLSOptions> tlsOptions, int extraDB, int objSerializer) {
 	CSimpleIni ini;
 	ini.SetUnicode();
 	ini.LoadFile(joinPath(baseFolder, "restartInfo.ini").c_str());
@@ -697,9 +711,12 @@ ACTOR Future<Void> restartSimulatedSystem(
 			localities.set(LiteralStringRef("data_hall"), dcUID);
 
 			// SOMEDAY: parse backup agent from test file
-			systemActors->push_back( reportErrors( simulatedMachine(
-				conn, ipAddrs, usingSSL, tlsOptions, localities, processClass, baseFolder, true, i == useSeedForMachine, enableExtraDB, usingSSL && (listenersPerProcess == 1 || processClass == ProcessClass::TesterClass) ),
-				processClass == ProcessClass::TesterClass ? "SimulatedTesterMachine" : "SimulatedMachine") );
+			systemActors->push_back(reportErrors(
+			    simulatedMachine(conn, ipAddrs, usingSSL, tlsOptions, localities, processClass, baseFolder, true,
+			                     i == useSeedForMachine, enableExtraDB,
+			                     usingSSL && (listenersPerProcess == 1 || processClass == ProcessClass::TesterClass),
+			                     objSerializer),
+			    processClass == ProcessClass::TesterClass ? "SimulatedTesterMachine" : "SimulatedMachine"));
 		}
 
 		g_simulator.desiredCoordinators = desiredCoordinators;
@@ -1080,7 +1097,8 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 
 void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFolder, int* pTesterCount,
                           Optional<ClusterConnectionString>* pConnString, Standalone<StringRef>* pStartingConfiguration,
-                          int extraDB, int minimumReplication, int minimumRegions, Reference<TLSOptions> tlsOptions) {
+                          int extraDB, int minimumReplication, int minimumRegions, Reference<TLSOptions> tlsOptions,
+                          int objSerializer) {
 	// SOMEDAY: this does not test multi-interface configurations
 	SimulationConfig simconfig(extraDB, minimumReplication, minimumRegions);
 	StatusObject startingConfigJSON = simconfig.db.toJSON(true);
@@ -1276,7 +1294,7 @@ void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFo
 			LocalityData	localities(Optional<Standalone<StringRef>>(), zoneId, machineId, dcUID);
 			localities.set(LiteralStringRef("data_hall"), dcUID);
 			systemActors->push_back(reportErrors(simulatedMachine(conn, ips, sslEnabled, tlsOptions,
-				localities, processClass, baseFolder, false, machine == useSeedForMachine, true, sslOnly ), "SimulatedMachine"));
+				localities, processClass, baseFolder, false, machine == useSeedForMachine, true, sslOnly, objSerializer), "SimulatedMachine"));
 
 			if (extraDB && g_simulator.extraDB->toString() != conn.toString()) {
 				std::vector<IPAddress> extraIps;
@@ -1290,7 +1308,7 @@ void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFo
 				localities.set(LiteralStringRef("data_hall"), dcUID);
 				systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB, extraIps, sslEnabled, tlsOptions,
 					localities,
-					processClass, baseFolder, false, machine == useSeedForMachine, false, sslOnly ), "SimulatedMachine"));
+					processClass, baseFolder, false, machine == useSeedForMachine, false, sslOnly, objSerializer), "SimulatedMachine"));
 			}
 
 			assignedMachines++;
@@ -1318,7 +1336,7 @@ void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFo
 		systemActors->push_back( reportErrors( simulatedMachine(
 			conn, ips, sslEnabled, tlsOptions,
 			localities, ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource),
-			baseFolder, false, i == useSeedForMachine, false, sslEnabled ),
+			baseFolder, false, i == useSeedForMachine, false, sslEnabled, objSerializer),
 			"SimulatedTesterMachine") );
 	}
 	*pStartingConfiguration = startingConfigString;
@@ -1374,7 +1392,7 @@ void checkExtraDB(const char *testFile, int &extraDB, int &minimumReplication, i
 	ifs.close();
 }
 
-ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool rebooting, Reference<TLSOptions> tlsOptions ) {
+ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool rebooting, Reference<TLSOptions> tlsOptions, int objSerializer ) {
 	state vector<Future<Void>> systemActors;
 	state Optional<ClusterConnectionString> connFile;
 	state Standalone<StringRef> startingConfiguration;
@@ -1391,7 +1409,8 @@ ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool reboot
 	                                        Standalone<StringRef>(g_random->randomUniqueID().toString()),
 	                                        Standalone<StringRef>(g_random->randomUniqueID().toString()),
 	                                        Optional<Standalone<StringRef>>()),
-	                           ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource), "", ""),
+	                           ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource), "", "",
+	                           useObjectSerializer(objSerializer)),
 	    TaskDefaultYield));
 	Sim2FileSystem::newFileSystem();
 	FlowTransport::createInstance(1);
@@ -1404,12 +1423,13 @@ ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool reboot
 	try {
 		//systemActors.push_back( startSystemMonitor(dataFolder) );
 		if (rebooting) {
-			wait( timeoutError( restartSimulatedSystem( &systemActors, dataFolder, &testerCount, &connFile, &startingConfiguration, tlsOptions, extraDB), 100.0 ) );
-		}
-		else {
+			wait(timeoutError(restartSimulatedSystem(&systemActors, dataFolder, &testerCount, &connFile,
+			                                         &startingConfiguration, tlsOptions, extraDB, objSerializer),
+			                  100.0));
+		} else {
 			g_expect_full_pointermap = 1;
 			setupSimulatedSystem(&systemActors, dataFolder, &testerCount, &connFile, &startingConfiguration, extraDB,
-			                     minimumReplication, minimumRegions, tlsOptions);
+			                     minimumReplication, minimumRegions, tlsOptions, objSerializer);
 			wait( delay(1.0) ); // FIXME: WHY!!!  //wait for machines to boot
 		}
 		std::string clusterFileDir = joinPath( dataFolder, g_random->randomUniqueID().toString() );
diff --git a/fdbserver/SimulatedCluster.h b/fdbserver/SimulatedCluster.h
index eb8f325bd3..cb154828f8 100644
--- a/fdbserver/SimulatedCluster.h
+++ b/fdbserver/SimulatedCluster.h
@@ -24,6 +24,6 @@
 #define FDBSERVER_SIMULATEDCLUSTER_H
 #pragma once
 
-void setupAndRun(std::string const& dataFolder, const char* const& testFile, bool const& rebooting, Reference<TLSOptions> const& useSSL);
+void setupAndRun(std::string const& dataFolder, const char* const& testFile, bool const& rebooting, Reference<TLSOptions> const& useSSL, int const& useObjectSerializer);
 
 #endif
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index f25622d7f2..ee69f9e8ad 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -79,82 +79,84 @@
 
 enum {
 	OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_NEWCONSOLE, OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_MACHINEID, OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE, OPT_METRICSPREFIX,
-	OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE, OPT_TRACE_FORMAT };
+	OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE, OPT_TRACE_FORMAT, OPT_USE_OBJECT_SERIALIZER };
 
 CSimpleOpt::SOption g_rgOptions[] = {
-	{ OPT_CONNFILE,             "-C",                          SO_REQ_SEP },
-	{ OPT_CONNFILE,             "--cluster_file",              SO_REQ_SEP },
-	{ OPT_SEEDCONNFILE,         "--seed_cluster_file",         SO_REQ_SEP },
-	{ OPT_SEEDCONNSTRING,       "--seed_connection_string",    SO_REQ_SEP },
-	{ OPT_ROLE,                 "-r",                          SO_REQ_SEP },
-	{ OPT_ROLE,                 "--role",                      SO_REQ_SEP },
-	{ OPT_PUBLICADDR,           "-p",                          SO_REQ_SEP },
-	{ OPT_PUBLICADDR,           "--public_address",            SO_REQ_SEP },
-	{ OPT_LISTEN,               "-l",                          SO_REQ_SEP },
-	{ OPT_LISTEN,               "--listen_address",            SO_REQ_SEP },
+	{ OPT_CONNFILE,              "-C",                          SO_REQ_SEP },
+	{ OPT_CONNFILE,              "--cluster_file",              SO_REQ_SEP },
+	{ OPT_SEEDCONNFILE,          "--seed_cluster_file",         SO_REQ_SEP },
+	{ OPT_SEEDCONNSTRING,        "--seed_connection_string",    SO_REQ_SEP },
+	{ OPT_ROLE,                  "-r",                          SO_REQ_SEP },
+	{ OPT_ROLE,                  "--role",                      SO_REQ_SEP },
+	{ OPT_PUBLICADDR,            "-p",                          SO_REQ_SEP },
+	{ OPT_PUBLICADDR,            "--public_address",            SO_REQ_SEP },
+	{ OPT_LISTEN,                "-l",                          SO_REQ_SEP },
+	{ OPT_LISTEN,                "--listen_address",            SO_REQ_SEP },
 #ifdef __linux__
 	{ OPT_FILESYSTEM,           "--data_filesystem",           SO_REQ_SEP },
 	{ OPT_PROFILER_RSS_SIZE,    "--rsssize",                   SO_REQ_SEP },
 #endif
-	{ OPT_DATAFOLDER,           "-d",                          SO_REQ_SEP },
-	{ OPT_DATAFOLDER,           "--datadir",                   SO_REQ_SEP },
-	{ OPT_LOGFOLDER,            "-L",                          SO_REQ_SEP },
-	{ OPT_LOGFOLDER,            "--logdir",                    SO_REQ_SEP },
-	{ OPT_ROLLSIZE,             "-Rs",                         SO_REQ_SEP },
-	{ OPT_ROLLSIZE,             "--logsize",                   SO_REQ_SEP },
-	{ OPT_MAXLOGS,              "--maxlogs",                   SO_REQ_SEP },
-	{ OPT_MAXLOGSSIZE,          "--maxlogssize",               SO_REQ_SEP },
-	{ OPT_LOGGROUP,             "--loggroup",                  SO_REQ_SEP },
+	{ OPT_DATAFOLDER,            "-d",                          SO_REQ_SEP },
+	{ OPT_DATAFOLDER,            "--datadir",                   SO_REQ_SEP },
+	{ OPT_LOGFOLDER,             "-L",                          SO_REQ_SEP },
+	{ OPT_LOGFOLDER,             "--logdir",                    SO_REQ_SEP },
+	{ OPT_ROLLSIZE,              "-Rs",                         SO_REQ_SEP },
+	{ OPT_ROLLSIZE,              "--logsize",                   SO_REQ_SEP },
+	{ OPT_MAXLOGS,               "--maxlogs",                   SO_REQ_SEP },
+	{ OPT_MAXLOGSSIZE,           "--maxlogssize",               SO_REQ_SEP },
+	{ OPT_LOGGROUP,              "--loggroup",                  SO_REQ_SEP },
 #ifdef _WIN32
-	{ OPT_PARENTPID,            "--parentpid",                 SO_REQ_SEP },
-	{ OPT_NEWCONSOLE,           "-n",                          SO_NONE },
-	{ OPT_NEWCONSOLE,           "--newconsole",                SO_NONE },
-	{ OPT_NOBOX,                "-q",                          SO_NONE },
-	{ OPT_NOBOX,                "--no_dialog",                 SO_NONE },
+	{ OPT_PARENTPID,             "--parentpid",                 SO_REQ_SEP },
+	{ OPT_NEWCONSOLE,            "-n",                          SO_NONE },
+	{ OPT_NEWCONSOLE,            "--newconsole",                SO_NONE },
+	{ OPT_NOBOX,                 "-q",                          SO_NONE },
+	{ OPT_NOBOX,                 "--no_dialog",                 SO_NONE },
 #endif
-	{ OPT_KVFILE,               "--kvfile",                    SO_REQ_SEP },
-	{ OPT_TESTFILE,             "-f",                          SO_REQ_SEP },
-	{ OPT_TESTFILE,             "--testfile",                  SO_REQ_SEP },
-	{ OPT_RESTARTING,           "-R",                          SO_NONE },
-	{ OPT_RESTARTING,           "--restarting",                SO_NONE },
-	{ OPT_RANDOMSEED,           "-s",                          SO_REQ_SEP },
-	{ OPT_RANDOMSEED,           "--seed",                      SO_REQ_SEP },
-	{ OPT_KEY,                  "-k",                          SO_REQ_SEP },
-	{ OPT_KEY,                  "--key",                       SO_REQ_SEP },
-	{ OPT_MEMLIMIT,             "-m",                          SO_REQ_SEP },
-	{ OPT_MEMLIMIT,             "--memory",                    SO_REQ_SEP },
-	{ OPT_STORAGEMEMLIMIT,      "-M",                          SO_REQ_SEP },
-	{ OPT_STORAGEMEMLIMIT,      "--storage_memory",            SO_REQ_SEP },
-	{ OPT_MACHINEID,            "-i",                          SO_REQ_SEP },
-	{ OPT_MACHINEID,            "--machine_id",                SO_REQ_SEP },
-	{ OPT_DCID,                 "-a",                          SO_REQ_SEP },
-	{ OPT_DCID,                 "--datacenter_id",             SO_REQ_SEP },
-	{ OPT_MACHINE_CLASS,        "-c",                          SO_REQ_SEP },
-	{ OPT_MACHINE_CLASS,        "--class",                     SO_REQ_SEP },
-	{ OPT_BUGGIFY,              "-b",                          SO_REQ_SEP },
-	{ OPT_BUGGIFY,              "--buggify",                   SO_REQ_SEP },
-	{ OPT_VERSION,              "-v",                          SO_NONE },
-	{ OPT_VERSION,              "--version",                   SO_NONE },
-	{ OPT_CRASHONERROR,         "--crash",                     SO_NONE },
-	{ OPT_NETWORKIMPL,          "-N",                          SO_REQ_SEP },
-	{ OPT_NETWORKIMPL,          "--network",                   SO_REQ_SEP },
-	{ OPT_NOBUFSTDOUT,          "--unbufferedout",             SO_NONE },
-	{ OPT_BUFSTDOUTERR,         "--bufferedout",               SO_NONE },
-	{ OPT_TRACECLOCK,           "--traceclock",                SO_REQ_SEP },
-	{ OPT_NUMTESTERS,           "--num_testers",               SO_REQ_SEP },
-	{ OPT_HELP,                 "-?",                          SO_NONE },
-	{ OPT_HELP,                 "-h",                          SO_NONE },
-	{ OPT_HELP,                 "--help",                      SO_NONE },
-	{ OPT_DEVHELP,              "--dev-help",                  SO_NONE },
-	{ OPT_KNOB,                 "--knob_",                     SO_REQ_SEP },
-	{ OPT_LOCALITY,             "--locality_",                 SO_REQ_SEP },
-	{ OPT_TESTSERVERS,          "--testservers",               SO_REQ_SEP },
-	{ OPT_TEST_ON_SERVERS,      "--testonservers",             SO_NONE },
-	{ OPT_METRICSCONNFILE,      "--metrics_cluster",           SO_REQ_SEP },
-	{ OPT_METRICSPREFIX,        "--metrics_prefix",            SO_REQ_SEP },
-	{ OPT_IO_TRUST_SECONDS,     "--io_trust_seconds",          SO_REQ_SEP },
-	{ OPT_IO_TRUST_WARN_ONLY,   "--io_trust_warn_only",        SO_NONE },
-	{ OPT_TRACE_FORMAT      ,   "--trace_format",              SO_REQ_SEP },
+	{ OPT_KVFILE,                "--kvfile",                    SO_REQ_SEP },
+	{ OPT_TESTFILE,              "-f",                          SO_REQ_SEP },
+	{ OPT_TESTFILE,              "--testfile",                  SO_REQ_SEP },
+	{ OPT_RESTARTING,            "-R",                          SO_NONE },
+	{ OPT_RESTARTING,            "--restarting",                SO_NONE },
+	{ OPT_RANDOMSEED,            "-s",                          SO_REQ_SEP },
+	{ OPT_RANDOMSEED,            "--seed",                      SO_REQ_SEP },
+	{ OPT_KEY,                   "-k",                          SO_REQ_SEP },
+	{ OPT_KEY,                   "--key",                       SO_REQ_SEP },
+	{ OPT_MEMLIMIT,              "-m",                          SO_REQ_SEP },
+	{ OPT_MEMLIMIT,              "--memory",                    SO_REQ_SEP },
+	{ OPT_STORAGEMEMLIMIT,       "-M",                          SO_REQ_SEP },
+	{ OPT_STORAGEMEMLIMIT,       "--storage_memory",            SO_REQ_SEP },
+	{ OPT_MACHINEID,             "-i",                          SO_REQ_SEP },
+	{ OPT_MACHINEID,             "--machine_id",                SO_REQ_SEP },
+	{ OPT_DCID,                  "-a",                          SO_REQ_SEP },
+	{ OPT_DCID,                  "--datacenter_id",             SO_REQ_SEP },
+	{ OPT_MACHINE_CLASS,         "-c",                          SO_REQ_SEP },
+	{ OPT_MACHINE_CLASS,         "--class",                     SO_REQ_SEP },
+	{ OPT_BUGGIFY,               "-b",                          SO_REQ_SEP },
+	{ OPT_BUGGIFY,               "--buggify",                   SO_REQ_SEP },
+	{ OPT_VERSION,               "-v",                          SO_NONE },
+	{ OPT_VERSION,               "--version",                   SO_NONE },
+	{ OPT_CRASHONERROR,          "--crash",                     SO_NONE },
+	{ OPT_NETWORKIMPL,           "-N",                          SO_REQ_SEP },
+	{ OPT_NETWORKIMPL,           "--network",                   SO_REQ_SEP },
+	{ OPT_NOBUFSTDOUT,           "--unbufferedout",             SO_NONE },
+	{ OPT_BUFSTDOUTERR,          "--bufferedout",               SO_NONE },
+	{ OPT_TRACECLOCK,            "--traceclock",                SO_REQ_SEP },
+	{ OPT_NUMTESTERS,            "--num_testers",               SO_REQ_SEP },
+	{ OPT_HELP,                  "-?",                          SO_NONE },
+	{ OPT_HELP,                  "-h",                          SO_NONE },
+	{ OPT_HELP,                  "--help",                      SO_NONE },
+	{ OPT_DEVHELP,               "--dev-help",                  SO_NONE },
+	{ OPT_KNOB,                  "--knob_",                     SO_REQ_SEP },
+	{ OPT_LOCALITY,              "--locality_",                 SO_REQ_SEP },
+	{ OPT_TESTSERVERS,           "--testservers",               SO_REQ_SEP },
+	{ OPT_TEST_ON_SERVERS,       "--testonservers",             SO_NONE },
+	{ OPT_METRICSCONNFILE,       "--metrics_cluster",           SO_REQ_SEP },
+	{ OPT_METRICSPREFIX,         "--metrics_prefix",            SO_REQ_SEP },
+	{ OPT_IO_TRUST_SECONDS,      "--io_trust_seconds",          SO_REQ_SEP },
+	{ OPT_IO_TRUST_WARN_ONLY,    "--io_trust_warn_only",        SO_NONE },
+	{ OPT_TRACE_FORMAT      ,    "--trace_format",              SO_REQ_SEP },
+	{ OPT_USE_OBJECT_SERIALIZER, "-S",                          SO_REQ_SEP },
+	{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer",         SO_REQ_SEP },
 
 #ifndef TLS_DISABLED
 	TLS_OPTION_FLAGS
@@ -567,6 +569,11 @@ static void printUsage( const char *name, bool devhelp ) {
 		   "                 Machine class (valid options are storage, transaction,\n"
 		   "                 resolution, proxy, master, test, unset, stateless, log, router,\n"
 		   "                 and cluster_controller).\n");
+	printf("  -S %s, --object-serializer %s\n"
+		   "                 Use object serializer for sending messages. The object serializer\n"
+		   "                 is currently a beta feature and it allows fdb processes to talk to\n"
+		   "                 each other even if they don't have the same version\n",
+		   devhelp ? "ON|OFF|RANDOM" : "ON|OFF", devhelp ? "ON|OFF|RANDOM" : "ON|OFF");
 #ifndef TLS_DISABLED
 	printf(TLS_HELP);
 #endif
@@ -924,6 +931,7 @@ int main(int argc, char* argv[]) {
 		double fileIoTimeout = 0.0;
 		bool fileIoWarnOnly = false;
 		uint64_t rsssize = -1;
+		int useObjectSerializer = 0;
 
 		if( argc == 1 ) {
 			printUsage(argv[0], false);
@@ -1260,6 +1268,23 @@ int main(int argc, char* argv[]) {
 						fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg());
 					}
 					break;
+				case OPT_USE_OBJECT_SERIALIZER:
+				{
+					std::string s = args.OptionArg();
+					std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+					if (s == "on" || s == "true" || s == "1") {
+						useObjectSerializer = 1;
+					} else if (s == "off" || s == "false" || s == "0") {
+						useObjectSerializer = 0;
+					} else if (s == "random") {
+						useObjectSerializer = 2;
+					} else {
+						fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str());
+						printHelpTeaser(argv[0]);
+						flushAndExit(FDB_EXIT_ERROR);
+					}
+					break;
+				}
 #ifndef TLS_DISABLED
 				case TLSOptions::OPT_TLS_PLUGIN:
 					args.OptionArg();
@@ -1485,7 +1510,11 @@ int main(int argc, char* argv[]) {
 			startNewSimulator();
 			openTraceFile(NetworkAddress(), rollsize, maxLogsSize, logFolder, "trace", logGroup);
 		} else {
-			g_network = newNet2(useThreadPool, true);
+			if (useObjectSerializer == 2) {
+				fprintf(stderr, "ERROR: 'random' for object serializer is only supported for simulation");
+				flushAndExit(FDB_EXIT_ERROR);
+			}
+			g_network = newNet2(useThreadPool, true, useObjectSerializer == 1);
 			FlowTransport::createInstance(1);
 
 			const bool expectsPublicAddress = (role == FDBD || role == NetworkTestServer || role == Restore);
@@ -1633,7 +1662,7 @@ int main(int argc, char* argv[]) {
 				platform::createDirectory( dataFolder );
 			}
 
-			setupAndRun( dataFolder, testFile, restarting, tlsOptions );
+			setupAndRun( dataFolder, testFile, restarting, tlsOptions, useObjectSerializer );
 			g_simulator.run();
 		} else if (role == FDBD) {
 			ASSERT( connectionFile );
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index 5ee248a33e..6185a60026 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -135,7 +135,7 @@ thread_local INetwork* thread_network = 0;
 class Net2 sealed : public INetwork, public INetworkConnections {
 
 public:
-	Net2(bool useThreadPool, bool useMetrics);
+	Net2(bool useThreadPool, bool useMetrics, bool useObjectSerializer);
 	void run();
 	void initMetrics();
 
@@ -169,9 +169,11 @@ public:
 
 	virtual flowGlobalType global(int id) { return (globals.size() > id) ? globals[id] : NULL; }
 	virtual void setGlobal(size_t id, flowGlobalType v) { globals.resize(std::max(globals.size(),id+1)); globals[id] = v; }
+	virtual bool useObjectSerializer() const { return _useObjectSerializer; }
 	std::vector<flowGlobalType>		globals;
 
 	bool useThreadPool;
+	bool _useObjectSerializer = false;
 //private:
 
 	ASIOReactor reactor;
@@ -501,8 +503,9 @@ struct PromiseTask : public Task, public FastAllocated<PromiseTask> {
 	}
 };
 
-Net2::Net2(bool useThreadPool, bool useMetrics)
+Net2::Net2(bool useThreadPool, bool useMetrics, bool useObjectSerializer)
 	: useThreadPool(useThreadPool),
+	  _useObjectSerializer(useObjectSerializer),
 	  network(this),
 	  reactor(this),
 	  stopped(false),
@@ -1021,9 +1024,9 @@ void ASIOReactor::wake() {
 
 } // namespace net2
 
-INetwork* newNet2(bool useThreadPool, bool useMetrics) {
+INetwork* newNet2(bool useThreadPool, bool useMetrics, bool useObjectSerializer) {
 	try {
-		N2::g_net2 = new N2::Net2(useThreadPool, useMetrics);
+		N2::g_net2 = new N2::Net2(useThreadPool, useMetrics, useObjectSerializer);
 	}
 	catch(boost::system::system_error e) {
 		TraceEvent("Net2InitError").detail("Message", e.what());
diff --git a/flow/network.h b/flow/network.h
index 20124dcf66..a2b3eaa847 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -337,7 +337,7 @@ typedef NetworkAddressList (*NetworkAddressesFuncPtr)();
 
 class INetwork;
 extern INetwork* g_network;
-extern INetwork* newNet2(bool useThreadPool = false, bool useMetrics = false);
+extern INetwork* newNet2(bool useThreadPool = false, bool useMetrics = false, bool useObjectSerializer = false);
 
 class INetwork {
 public:
@@ -399,7 +399,7 @@ public:
 	virtual bool isAddressOnThisHost( NetworkAddress const& addr ) = 0;
 	// Returns true if it is reasonably certain that a connection to the given address would be a fast loopback connection
 
-	virtual bool useObjectSerializer() { return false; }
+	virtual bool useObjectSerializer() const = 0;
 	// Whether or not the object serializer should be used when sending packets
 
 	// Shorthand for transport().getLocalAddress()

From 92bad76479211b6f8d49dea2b19b584f465a4191 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 29 Jan 2019 11:51:37 -0800
Subject: [PATCH 047/118] Wrap ClusterClientInterface into its own type

When a process joins a cluster it fetches the cluster
interface. However, not the whole interface is exposed
to the client. This mechanism relies on the fact that
the serializer keeps the field ordering and doesn't
verify the message before parsing it.

To make this work, we provide a client type with one
member (the ClusterInterface which is exposed to the
client and the server). This client interface has the
same FileIdentifier as the ClusterControllerFullInterface
which has the same first member. This works because
FlatBuffers allows for members to be missing.
---
 fdbclient/ClusterInterface.h            | 18 ++++++++++++++++++
 fdbclient/MonitorLeader.actor.cpp       | 19 +++++++++++++++++++
 fdbclient/MonitorLeader.h               | 25 +++++++++++++++++++++++--
 fdbserver/ClusterRecruitmentInterface.h |  2 ++
 4 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 28ab92be18..c48529fa03 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -29,6 +29,7 @@
 #include "fdbclient/ClientWorkerInterface.h"
 
 struct ClusterInterface {
+    constexpr static FileIdentifier file_identifier = 15888863;
 	RequestStream< struct OpenDatabaseRequest > openDatabase;
 	RequestStream< struct FailureMonitoringRequest > failureMonitoring;
 	RequestStream< struct StatusRequest > databaseStatus;
@@ -56,6 +57,23 @@ struct ClusterInterface {
 	}
 };
 
+struct ClusterControllerClientInterface {
+	constexpr static FileIdentifier file_identifier = 14997695;
+	ClusterInterface clientInterface;
+
+	bool operator==(ClusterControllerClientInterface const& r) const {
+		return clientInterface.id() == r.clientInterface.id();
+	}
+	bool operator!=(ClusterControllerClientInterface const& r) const {
+		return clientInterface.id() != r.clientInterface.id();
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, clientInterface);
+	}
+};
+
 struct ClientVersionRef {
 	StringRef clientVersion;
 	StringRef sourceVersion;
diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 834d2dc090..69934bd52d 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -463,3 +463,22 @@ ACTOR Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> connF
 
 	}
 }
+
+ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> serializedInfo,
+                                                    Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
+	state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
+	    new AsyncVar<Optional<ClusterControllerClientInterface>>{});
+	state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
+	loop {
+		choose {
+			when(wait(deserializer)) { UNSTOPPABLE_ASSERT(false); }
+			when(wait(knownLeader->onChange())) {
+				if (knownLeader->get().present()) {
+					outKnownLeader->set(knownLeader->get().get().clientInterface);
+				} else {
+					outKnownLeader->set(Optional<ClusterInterface>{});
+				}
+			}
+		}
+	}
+}
diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h
index a1ed1c88ad..e635fe428a 100644
--- a/fdbclient/MonitorLeader.h
+++ b/fdbclient/MonitorLeader.h
@@ -24,6 +24,7 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/ClusterInterface.h"
 
 #define CLUSTER_FILE_ENV_VAR_NAME "FDB_CLUSTER_FILE"
 
@@ -39,13 +40,33 @@ Future<Void> monitorLeader( Reference<ClusterConnectionFile> const& connFile, Re
 
 Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Value>> const& outSerializedLeaderInfo, Reference<AsyncVar<int>> const& connectedCoordinatorsNum  );
 
+template <class LeaderInterface>
+struct LeaderDeserializer {
+	Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
+	                        const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
+		return asyncDeserialize(serializedInfo, outKnownLeader);
+	}
+};
+
+Future<Void> asyncDeserializeClusterInterface(const Reference<AsyncVar<Value>>& serializedInfo,
+                                              const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader);
+
+template <>
+struct LeaderDeserializer<ClusterInterface> {
+	Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
+	                        const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader) {
+		return asyncDeserializeClusterInterface(serializedInfo, outKnownLeader);
+	}
+};
+
 template <class LeaderInterface>
 Future<Void> monitorLeader( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader, Reference<AsyncVar<int>> connectedCoordinatorsNum ) {
+	LeaderDeserializer<LeaderInterface> deserializer;
 	Reference<AsyncVar<Value>> serializedInfo( new AsyncVar<Value> );
 	Future<Void> m = monitorLeaderInternal( connFile, serializedInfo, connectedCoordinatorsNum );
-	return m || asyncDeserialize( serializedInfo, outKnownLeader );
+	return m || deserializer( serializedInfo, outKnownLeader );
 }
 
 #pragma endregion
 
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbserver/ClusterRecruitmentInterface.h b/fdbserver/ClusterRecruitmentInterface.h
index caf088c05f..1af8ec9241 100644
--- a/fdbserver/ClusterRecruitmentInterface.h
+++ b/fdbserver/ClusterRecruitmentInterface.h
@@ -35,6 +35,8 @@
 
 // This interface and its serialization depend on slicing, since the client will deserialize only the first part of this structure
 struct ClusterControllerFullInterface {
+    constexpr static FileIdentifier file_identifier =
+        ClusterControllerClientInterface::file_identifier;
 	ClusterInterface clientInterface;
 	RequestStream< struct RecruitFromConfigurationRequest > recruitFromConfiguration;
 	RequestStream< struct RecruitRemoteFromConfigurationRequest > recruitRemoteFromConfiguration;

From 1d5572a07828448cb223c541b059d94cecb80175 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 29 Jan 2019 13:06:41 -0800
Subject: [PATCH 048/118] removed unused knob

---
 flow/Knobs.cpp | 1 -
 flow/Knobs.h   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
index cab8b8f8ec..fd259aad69 100644
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -91,7 +91,6 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
 	init( MAX_PRIOR_MODIFICATION_DELAY,                        1.0 ); if( randomize && BUGGIFY ) MAX_PRIOR_MODIFICATION_DELAY = 10.0;
 
 	//GenericActors
-	init( MAX_DELIVER_DUPLICATE_DELAY,                         1.0 ); if( randomize && BUGGIFY ) MAX_DELIVER_DUPLICATE_DELAY = 10.0;
 	init( BUGGIFY_FLOW_LOCK_RELEASE_DELAY,                     1.0 );
 
 	//IAsyncFile
diff --git a/flow/Knobs.h b/flow/Knobs.h
index 9becb6f052..85cfa4701c 100644
--- a/flow/Knobs.h
+++ b/flow/Knobs.h
@@ -109,7 +109,6 @@ public:
 	double MAX_PRIOR_MODIFICATION_DELAY;
 
 	//GenericActors
-	double MAX_DELIVER_DUPLICATE_DELAY;
 	double BUGGIFY_FLOW_LOCK_RELEASE_DELAY;
 
 	//IAsyncFile

From 6afce017441cf2747bc414a2238a1c24d992381b Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Wed, 30 Jan 2019 13:53:23 -0800
Subject: [PATCH 049/118] Implementation complete (not yet working)

---
 fdbclient/ClientDBInfo.h                |  1 +
 fdbclient/ClientWorkerInterface.h       |  3 +++
 fdbclient/ClusterInterface.h            |  8 ++++++++
 fdbclient/CoordinationInterface.h       |  2 ++
 fdbclient/FDBTypes.h                    |  1 +
 fdbclient/MasterProxyInterface.h        | 11 +++++++++++
 fdbclient/StorageServerInterface.h      | 23 ++++++++++++++++++-----
 fdbrpc/FlowTransport.actor.cpp          | 14 +++++++++++---
 fdbrpc/FlowTransport.h                  |  2 ++
 fdbrpc/Locality.h                       |  1 +
 fdbrpc/PerfMetric.h                     |  1 +
 fdbrpc/ReplicationPolicy.h              |  2 +-
 fdbrpc/fdbrpc.h                         |  3 ++-
 fdbserver/ClusterRecruitmentInterface.h | 11 +++++++++++
 fdbserver/CoordinationInterface.h       |  8 ++++++++
 fdbserver/LeaderElection.actor.cpp      |  6 ++++--
 fdbserver/MasterInterface.h             |  6 ++++++
 fdbserver/NetworkTest.h                 |  3 +++
 fdbserver/ResolverInterface.h           |  7 +++++++
 fdbserver/RestoreInterface.h            |  3 +++
 fdbserver/ServerDBInfo.h                |  1 +
 fdbserver/TLogInterface.h               | 10 ++++++++++
 fdbserver/TesterInterface.actor.h       |  3 +++
 fdbserver/WorkerInterface.actor.h       | 15 +++++++++++++++
 fdbserver/sqlite/btree.h                |  3 +++
 fdbserver/sqlite/sqlite3.amalgamation.c |  3 +++
 fdbserver/workloads/Sideband.actor.cpp  |  2 ++
 flow/Arena.h                            |  3 +++
 flow/Error.h                            |  2 ++
 flow/FileIdentifier.h                   |  4 +---
 flow/IRandom.h                          |  2 ++
 flow/ObjectSerializer.h                 |  2 +-
 flow/Trace.h                            |  1 +
 flow/flow.h                             |  3 +++
 flow/serialize.h                        | 11 ++++++++++-
 tests/TestRunner/TestRunner.py          |  3 +++
 36 files changed, 167 insertions(+), 17 deletions(-)

diff --git a/fdbclient/ClientDBInfo.h b/fdbclient/ClientDBInfo.h
index 3468d0e2fa..cbefda7be0 100644
--- a/fdbclient/ClientDBInfo.h
+++ b/fdbclient/ClientDBInfo.h
@@ -27,6 +27,7 @@
 // ClientDBInfo is all the information needed by a database client to access the database
 // It is returned (and kept up to date) by the OpenDatabaseRequest interface of ClusterInterface
 struct ClientDBInfo {
+	constexpr static FileIdentifier file_identifier = 5355080;
 	UID id;  // Changes each time anything else changes
 	vector< MasterProxyInterface > proxies;
 	double clientTxnInfoSampleRate;
diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h
index 5ea6c499cd..4e3efde268 100644
--- a/fdbclient/ClientWorkerInterface.h
+++ b/fdbclient/ClientWorkerInterface.h
@@ -30,6 +30,7 @@
 // Streams from WorkerInterface that are safe and useful to call from a client.
 // A ClientWorkerInterface is embedded as the first element of a WorkerInterface.
 struct ClientWorkerInterface {
+	constexpr static FileIdentifier file_identifier = 12418152;
 	RequestStream< struct RebootRequest > reboot;
 	RequestStream< struct ProfilerRequest > profiler;
 
@@ -45,6 +46,7 @@ struct ClientWorkerInterface {
 };
 
 struct RebootRequest {
+	constexpr static FileIdentifier file_identifier = 11913957;
 	bool deleteData;
 	bool checkData;
 
@@ -57,6 +59,7 @@ struct RebootRequest {
 };
 
 struct ProfilerRequest {
+	constexpr static FileIdentifier file_identifier = 15437862;
 	ReplyPromise<Void> reply;
 
 	enum class Type : std::int8_t {
diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index c48529fa03..6130739b30 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -131,6 +131,7 @@ struct ClientVersionRef {
 };
 
 struct OpenDatabaseRequest {
+	constexpr static FileIdentifier file_identifier = 2799502;
 	// Sent by the native API to the cluster controller to open a database and track client
 	//   info changes.  Returns immediately if the current client info id is different from
 	//   knownClientInfoID; otherwise returns when it next changes (or perhaps after a long interval)
@@ -152,6 +153,7 @@ struct OpenDatabaseRequest {
 };
 
 struct SystemFailureStatus {
+	constexpr static FileIdentifier file_identifier = 3194108;
 	NetworkAddressList addresses;
 	FailureStatus status;
 
@@ -175,6 +177,7 @@ struct FailureMonitoringRequest {
 	// The failureInformationVersion returned in reply should be passed back to the
 	//   next request to facilitate delta compression of the failure information.
 
+	constexpr static FileIdentifier file_identifier = 5867851;
 	Optional<FailureStatus> senderStatus;
 	Version failureInformationVersion;
 	NetworkAddressList addresses;
@@ -187,6 +190,7 @@ struct FailureMonitoringRequest {
 };
 
 struct FailureMonitoringReply {
+	constexpr static FileIdentifier file_identifier = 6820325;
 	VectorRef< SystemFailureStatus > changes;
 	Version failureInformationVersion;
 	bool allOthersFailed;							// If true, changes are relative to all servers being failed, otherwise to the version given in the request
@@ -201,6 +205,7 @@ struct FailureMonitoringReply {
 };
 
 struct StatusRequest {
+	constexpr static FileIdentifier file_identifier = 14419140;
 	ReplyPromise< struct StatusReply > reply;
 
 	template <class Ar>
@@ -210,6 +215,7 @@ struct StatusRequest {
 };
 
 struct StatusReply {
+	constexpr static FileIdentifier file_identifier = 9980504;
 	StatusObject statusObj;
 	std::string statusStr;
 
@@ -235,6 +241,7 @@ struct StatusReply {
 };
 
 struct GetClientWorkersRequest {
+	constexpr static FileIdentifier file_identifier = 10771791;
 	ReplyPromise<vector<ClientWorkerInterface>> reply;
 
 	GetClientWorkersRequest() {}
@@ -246,6 +253,7 @@ struct GetClientWorkersRequest {
 };
 
 struct ForceRecoveryRequest {
+	constexpr static FileIdentifier file_identifier = 14821350;
 	Key dcId;
 	ReplyPromise<Void> reply;
 
diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h
index b8f91f01ee..7eee9d6aae 100644
--- a/fdbclient/CoordinationInterface.h
+++ b/fdbclient/CoordinationInterface.h
@@ -91,6 +91,7 @@ private:
 };
 
 struct LeaderInfo {
+	constexpr static FileIdentifier file_identifier = 8338794;
 	UID changeID;
 	static const uint64_t mask = ~(127ll << 57);
 	Value serializedInfo;
@@ -126,6 +127,7 @@ struct LeaderInfo {
 };
 
 struct GetLeaderRequest {
+	constexpr static FileIdentifier file_identifier = 214727;
 	Key key;
 	UID knownLeader;
 	ReplyPromise< Optional<LeaderInfo> > reply;
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index eb24e84814..f234ac4f25 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -540,6 +540,7 @@ struct Traceable<RangeResultRef> : std::true_type {
 };
 
 struct KeyValueStoreType {
+	constexpr static FileIdentifier file_identifier = 6560359;
 	// These enumerated values are stored in the database configuration, so can NEVER be changed.  Only add new ones just before END.
 	enum StoreType {
 		SSD_BTREE_V1,
diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h
index 10721871a5..9d51ae1c18 100644
--- a/fdbclient/MasterProxyInterface.h
+++ b/fdbclient/MasterProxyInterface.h
@@ -30,6 +30,7 @@
 #include "flow/Stats.h"
 
 struct MasterProxyInterface {
+	constexpr static FileIdentifier file_identifier = 8954922;
 	enum { LocationAwareLoadBalance = 1 };
 	enum { AlwaysFresh = 1 };
 
@@ -71,6 +72,7 @@ struct MasterProxyInterface {
 };
 
 struct CommitID {
+	constexpr static FileIdentifier file_identifier = 14254927;
 	Version version; 			// returns invalidVersion if transaction conflicts
 	uint16_t txnBatchId;
 	Optional<Value> metadataVersion;
@@ -85,6 +87,7 @@ struct CommitID {
 };
 
 struct CommitTransactionRequest : TimedRequest {
+	constexpr static FileIdentifier file_identifier = 93948;
 	enum { 
 		FLAG_IS_LOCK_AWARE = 0x1,
 		FLAG_FIRST_IN_BATCH = 0x2
@@ -121,6 +124,7 @@ static inline int getBytes( CommitTransactionRequest const& r ) {
 }
 
 struct GetReadVersionReply {
+	constexpr static FileIdentifier file_identifier = 15709388;
 	Version version;
 	bool locked;
 	Optional<Value> metadataVersion;
@@ -132,6 +136,7 @@ struct GetReadVersionReply {
 };
 
 struct GetReadVersionRequest : TimedRequest {
+	constexpr static FileIdentifier file_identifier = 838566;
 	enum { 
 		PRIORITY_SYSTEM_IMMEDIATE = 15 << 24,  // Highest possible priority, always executed even if writes are otherwise blocked
 		PRIORITY_DEFAULT = 8 << 24,
@@ -161,6 +166,7 @@ struct GetReadVersionRequest : TimedRequest {
 };
 
 struct GetKeyServerLocationsReply {
+	constexpr static FileIdentifier file_identifier = 10636023;
 	Arena arena;
 	vector<pair<KeyRangeRef, vector<StorageServerInterface>>> results;
 
@@ -171,6 +177,7 @@ struct GetKeyServerLocationsReply {
 };
 
 struct GetKeyServerLocationsRequest {
+	constexpr static FileIdentifier file_identifier = 9144680;
 	Arena arena;
 	KeyRef begin;
 	Optional<KeyRef> end;
@@ -188,6 +195,7 @@ struct GetKeyServerLocationsRequest {
 };
 
 struct GetRawCommittedVersionRequest {
+	constexpr static FileIdentifier file_identifier = 12954034;
 	Optional<UID> debugID;
 	ReplyPromise<GetReadVersionReply> reply;
 
@@ -200,6 +208,7 @@ struct GetRawCommittedVersionRequest {
 };
 
 struct GetStorageServerRejoinInfoReply {
+	constexpr static FileIdentifier file_identifier = 9469225;
 	Version version;
 	Tag tag;
 	Optional<Tag> newTag;
@@ -213,6 +222,7 @@ struct GetStorageServerRejoinInfoReply {
 };
 
 struct GetStorageServerRejoinInfoRequest {
+	constexpr static FileIdentifier file_identifier = 994279;
 	UID id;
 	Optional<Value> dcId;
 	ReplyPromise< GetStorageServerRejoinInfoReply > reply;
@@ -227,6 +237,7 @@ struct GetStorageServerRejoinInfoRequest {
 };
 
 struct TxnStateRequest {
+	constexpr static FileIdentifier file_identifier = 15250781;
 	Arena arena;
 	VectorRef<KeyValueRef> data;
 	Sequence sequence;
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index d535a81196..d016c2765c 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -30,11 +30,8 @@
 #include "flow/Stats.h"
 
 struct StorageServerInterface {
-	enum { 
-		BUSY_ALLOWED = 0,
-		BUSY_FORCE = 1,
-		BUSY_LOCAL = 2
-	};
+	constexpr static FileIdentifier file_identifier = 15302073;
+	enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 };
 
 	enum { LocationAwareLoadBalance = 1 };
 	enum { AlwaysFresh = 0 };
@@ -114,6 +111,7 @@ struct ServerCacheInfo {
 };
 
 struct GetValueReply : public LoadBalancedReply {
+	constexpr static FileIdentifier file_identifier = 1378929;
 	Optional<Value> value;
 
 	GetValueReply() {}
@@ -126,6 +124,7 @@ struct GetValueReply : public LoadBalancedReply {
 };
 
 struct GetValueRequest : TimedRequest {
+	constexpr static FileIdentifier file_identifier = 8454530;
 	Key key;
 	Version version;
 	Optional<UID> debugID;
@@ -141,6 +140,7 @@ struct GetValueRequest : TimedRequest {
 };
 
 struct WatchValueRequest {
+	constexpr static FileIdentifier file_identifier = 14747733;
 	Key key;
 	Optional<Value> value;
 	Version version;
@@ -157,6 +157,7 @@ struct WatchValueRequest {
 };
 
 struct GetKeyValuesReply : public LoadBalancedReply {
+	constexpr static FileIdentifier file_identifier = 1783066;
 	Arena arena;
 	VectorRef<KeyValueRef> data;
 	Version version; // useful when latestVersion was requested
@@ -169,6 +170,7 @@ struct GetKeyValuesReply : public LoadBalancedReply {
 };
 
 struct GetKeyValuesRequest : TimedRequest {
+	constexpr static FileIdentifier file_identifier = 6795746;
 	Arena arena;
 	KeySelectorRef begin, end;
 	Version version;		// or latestVersion
@@ -185,6 +187,7 @@ struct GetKeyValuesRequest : TimedRequest {
 };
 
 struct GetKeyReply : public LoadBalancedReply {
+	constexpr static FileIdentifier file_identifier = 11226513;
 	KeySelector sel;
 
 	GetKeyReply() {}
@@ -197,6 +200,7 @@ struct GetKeyReply : public LoadBalancedReply {
 };
 
 struct GetKeyRequest : TimedRequest {
+	constexpr static FileIdentifier file_identifier = 10457870;
 	Arena arena;
 	KeySelectorRef sel;
 	Version version;		// or latestVersion
@@ -212,6 +216,7 @@ struct GetKeyRequest : TimedRequest {
 };
 
 struct GetShardStateRequest {
+	constexpr static FileIdentifier file_identifier = 15860168;
 	enum waitMode {
 		NO_WAIT = 0,
 		FETCHING = 1,
@@ -231,6 +236,7 @@ struct GetShardStateRequest {
 };
 
 struct StorageMetrics {
+	constexpr static FileIdentifier file_identifier = 13622226;
 	int64_t bytes;				// total storage
 	int64_t bytesPerKSecond;	// network bandwidth (average over 10s)
 	int64_t iosPerKSecond;
@@ -283,6 +289,7 @@ struct StorageMetrics {
 struct WaitMetricsRequest {
 	// Waits for any of the given minimum or maximum metrics to be exceeded, and then returns the current values
 	// Send a reversed range for min, max to receive an immediate report
+	constexpr static FileIdentifier file_identifier = 1795961;
 	Arena arena;
 	KeyRangeRef keys;
 	StorageMetrics min, max;
@@ -301,6 +308,7 @@ struct WaitMetricsRequest {
 };
 
 struct SplitMetricsReply {
+	constexpr static FileIdentifier file_identifier = 11530792;
 	Standalone<VectorRef<KeyRef>> splits;
 	StorageMetrics used;
 
@@ -311,6 +319,7 @@ struct SplitMetricsReply {
 };
 
 struct SplitMetricsRequest {
+	constexpr static FileIdentifier file_identifier = 10463876;
 	Arena arena;
 	KeyRangeRef keys;
 	StorageMetrics limits;
@@ -329,6 +338,7 @@ struct SplitMetricsRequest {
 };
 
 struct GetPhysicalMetricsReply {
+	constexpr static FileIdentifier file_identifier = 15491478;
 	StorageMetrics load;
 	StorageMetrics free;
 	StorageMetrics capacity;
@@ -340,6 +350,7 @@ struct GetPhysicalMetricsReply {
 };
 
 struct GetPhysicalMetricsRequest {
+	constexpr static FileIdentifier file_identifier = 13290999;
 	ReplyPromise<GetPhysicalMetricsReply> reply;
 
 	template <class Ar>
@@ -350,6 +361,7 @@ struct GetPhysicalMetricsRequest {
 
 struct StorageQueuingMetricsRequest {
 	// SOMEDAY: Send threshold value to avoid polling faster than the information changes?
+	constexpr static FileIdentifier file_identifier = 3978640;
 	ReplyPromise<struct StorageQueuingMetricsReply> reply;
 
 	template <class Ar>
@@ -359,6 +371,7 @@ struct StorageQueuingMetricsRequest {
 };
 
 struct StorageQueuingMetricsReply {
+	constexpr static FileIdentifier file_identifier = 7633366;
 	double localTime;
 	int64_t instanceID;  // changes if bytesDurable and bytesInput reset
 	int64_t bytesDurable, bytesInput;
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 60bff615dd..822a4a0b2b 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -329,7 +329,8 @@ struct Peer : NonCopyable {
 		}
 
 		pkt.connectPacketLength = sizeof(pkt) - sizeof(pkt.connectPacketLength);
-		pkt.protocolVersion = currentProtocolVersion;
+		pkt.protocolVersion =
+		    g_network->useObjectSerializer() ? addObjectSerializerFlag(currentProtocolVersion) : currentProtocolVersion;
 		pkt.connectionId = transport->transportId;
 
 		PacketBuffer* pb_first = new PacketBuffer;
@@ -1013,19 +1014,26 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 		TEST(true); // "Loopback" delivery
 		// SOMEDAY: Would it be better to avoid (de)serialization by doing this check in flow?
 
+		Standalone<StringRef> copy;
+		if (g_network->useObjectSerializer()) {
+			ObjectWriter wr;
+			what.serializeBinaryWriter(wr);
+			copy = wr.toStringRef();
+		} else {
 		BinaryWriter wr( AssumeVersion(currentProtocolVersion) );
 		// we don't need to send using an object writer here. This is a loopback delivery
 		// and therefore it is guaranteed that both versions will have exactly the
 		// same structures - so the backwards compatability capabilities are never needed
 		// here.
 		what.serializeBinaryWriter(wr);
-		Standalone<StringRef> copy = wr.toValue();
+			copy = wr.toStringRef();
+		}
 #if VALGRIND
 		VALGRIND_CHECK_MEM_IS_DEFINED(copy.begin(), copy.size());
 #endif
 
 		deliver(self, destination, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false,
-		        false);
+		        g_network->useObjectSerializer());
 
 		return (PacketID)NULL;
 	} else {
diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h
index ef1f4a186c..63830296ef 100644
--- a/fdbrpc/FlowTransport.h
+++ b/fdbrpc/FlowTransport.h
@@ -24,12 +24,14 @@
 
 #include <algorithm>
 #include "flow/network.h"
+#include "flow/FileIdentifier.h"
 
 #pragma pack(push, 4)
 class Endpoint {
 public:
 	// Endpoint represents a particular service (e.g. a serialized Promise<T> or PromiseStream<T>)
 	// An endpoint is either "local" (used for receiving data) or "remote" (used for sending data)
+	constexpr static FileIdentifier file_identifier = 10618805;
 	typedef UID Token;
 	NetworkAddressList addresses;
 	Token token;
diff --git a/fdbrpc/Locality.h b/fdbrpc/Locality.h
index 1b8b17ec2f..d7cd8da6e9 100644
--- a/fdbrpc/Locality.h
+++ b/fdbrpc/Locality.h
@@ -25,6 +25,7 @@
 #include "flow/flow.h"
 
 struct ProcessClass {
+	constexpr static FileIdentifier file_identifier = 6697257;
 	// This enum is stored in restartInfo.ini for upgrade tests, so be very careful about changing the existing items!
 	enum ClassType { UnsetClass, StorageClass, TransactionClass, ResolutionClass, TesterClass, ProxyClass, MasterClass, StatelessClass, LogClass, ClusterControllerClass, LogRouterClass, DataDistributorClass, CoordinatorClass, RatekeeperClass, InvalidClass = -1 };
 	enum Fitness { BestFit, GoodFit, UnsetFit, OkayFit, WorstFit, ExcludeFit, NeverAssign }; //cannot be larger than 7 because of leader election mask
diff --git a/fdbrpc/PerfMetric.h b/fdbrpc/PerfMetric.h
index 8beea3e1fd..c5e07ff5ea 100644
--- a/fdbrpc/PerfMetric.h
+++ b/fdbrpc/PerfMetric.h
@@ -29,6 +29,7 @@
 using std::vector;
 
 struct PerfMetric {
+	constexpr static FileIdentifier file_identifier = 5980618;
 	PerfMetric() : m_name(""), m_value(0), m_averaged(false), m_format_code( "%.3g" ) {}
 	PerfMetric( std::string name, double value, bool averaged ) : m_name(name), m_value(value), m_averaged(averaged), m_format_code( "%.3g" ) {}
 	PerfMetric( std::string name, double value, bool averaged, std::string format_code ) : m_name(name), m_value(value), m_averaged(averaged), m_format_code(format_code) {}
diff --git a/fdbrpc/ReplicationPolicy.h b/fdbrpc/ReplicationPolicy.h
index 3d6d969144..7963b5b1c1 100644
--- a/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/ReplicationPolicy.h
@@ -392,7 +392,7 @@ void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
 			policy = Reference<IReplicationPolicy>(pointer);
 		}
 		else if(name == LiteralStringRef("And")) {
-			PolicyAnd* pointer = new PolicyAnd({});
+			PolicyAnd* pointer = new PolicyAnd{};
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
 		}
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index 4d466d77ab..26993e5514 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -112,8 +112,9 @@ template <class T>
 class ReplyPromise sealed
 {
 public:
+	constexpr static FileIdentifier file_identifier = (0x2 << 24) | FileIdentifierFor<T>::value;
 	template <class U>
-	void send(U && value) const {
+	void send(U&& value) const {
 		sav->send(std::forward<U>(value));
 	}
 	template <class E>
diff --git a/fdbserver/ClusterRecruitmentInterface.h b/fdbserver/ClusterRecruitmentInterface.h
index 1af8ec9241..a79121cd5d 100644
--- a/fdbserver/ClusterRecruitmentInterface.h
+++ b/fdbserver/ClusterRecruitmentInterface.h
@@ -72,6 +72,7 @@ struct ClusterControllerFullInterface {
 };
 
 struct RecruitFromConfigurationRequest {
+	constexpr static FileIdentifier file_identifier = 2023046;
 	DatabaseConfiguration configuration;
 	bool recruitSeedServers;
 	int maxOldLogRouters;
@@ -88,6 +89,7 @@ struct RecruitFromConfigurationRequest {
 };
 
 struct RecruitFromConfigurationReply {
+	constexpr static FileIdentifier file_identifier = 2224085;
 	vector<WorkerInterface> tLogs;
 	vector<WorkerInterface> satelliteTLogs;
 	vector<WorkerInterface> proxies;
@@ -106,6 +108,7 @@ struct RecruitFromConfigurationReply {
 };
 
 struct RecruitRemoteFromConfigurationRequest {
+	constexpr static FileIdentifier file_identifier = 3235995;
 	DatabaseConfiguration configuration;
 	Optional<Key> dcId;
 	int logRouterCount;
@@ -122,6 +125,7 @@ struct RecruitRemoteFromConfigurationRequest {
 };
 
 struct RecruitRemoteFromConfigurationReply {
+	constexpr static FileIdentifier file_identifier = 9091392;
 	vector<WorkerInterface> remoteTLogs;
 	vector<WorkerInterface> logRouters;
 
@@ -132,6 +136,7 @@ struct RecruitRemoteFromConfigurationReply {
 };
 
 struct RecruitStorageReply {
+	constexpr static FileIdentifier file_identifier = 15877089;
 	WorkerInterface worker;
 	ProcessClass processClass;
 
@@ -142,6 +147,7 @@ struct RecruitStorageReply {
 };
 
 struct RecruitStorageRequest {
+	constexpr static FileIdentifier file_identifier = 905920;
 	std::vector<Optional<Standalone<StringRef>>> excludeMachines;	//< Don't recruit any of these machines
 	std::vector<AddressExclusion> excludeAddresses;		//< Don't recruit any of these addresses
 	std::vector<Optional<Standalone<StringRef>>> includeDCs;
@@ -155,6 +161,7 @@ struct RecruitStorageRequest {
 };
 
 struct RegisterWorkerReply {
+	constexpr static FileIdentifier file_identifier = 16475696;
 	ProcessClass processClass;
 	ClusterControllerPriorityInfo priorityInfo;
 
@@ -168,6 +175,7 @@ struct RegisterWorkerReply {
 };
 
 struct RegisterWorkerRequest {
+	constexpr static FileIdentifier file_identifier = 14332605;
 	WorkerInterface wi;
 	ProcessClass initialClass;
 	ProcessClass processClass;
@@ -189,6 +197,7 @@ struct RegisterWorkerRequest {
 };
 
 struct GetWorkersRequest {
+	constexpr static FileIdentifier file_identifier = 1254174;
 	enum { TESTER_CLASS_ONLY = 0x1, NON_EXCLUDED_PROCESSES_ONLY = 0x2 };
 
 	int flags;
@@ -204,6 +213,7 @@ struct GetWorkersRequest {
 };
 
 struct RegisterMasterRequest {
+	constexpr static FileIdentifier file_identifier = 10773445;
 	UID id;
 	LocalityData mi;
 	LogSystemConfig logSystemConfig;
@@ -231,6 +241,7 @@ struct RegisterMasterRequest {
 };
 
 struct GetServerDBInfoRequest {
+	constexpr static FileIdentifier file_identifier = 9467438;
 	UID knownServerInfoID;
 	Standalone<VectorRef<StringRef>> issues;
 	std::vector<NetworkAddress> incompatiblePeers;
diff --git a/fdbserver/CoordinationInterface.h b/fdbserver/CoordinationInterface.h
index 931fec5500..7d05639594 100644
--- a/fdbserver/CoordinationInterface.h
+++ b/fdbserver/CoordinationInterface.h
@@ -25,6 +25,7 @@
 #include "fdbclient/CoordinationInterface.h"
 
 struct GenerationRegInterface {
+	constexpr static FileIdentifier file_identifier = 16726744;
 	RequestStream< struct GenerationRegReadRequest > read;
 	RequestStream< struct GenerationRegWriteRequest > write;
 
@@ -52,6 +53,7 @@ struct GenerationRegInterface {
 };
 
 struct UniqueGeneration {
+	constexpr static FileIdentifier file_identifier = 16684234;
 	uint64_t generation;
 	UID uid;
 	UniqueGeneration() : generation(0) {}
@@ -71,6 +73,7 @@ struct UniqueGeneration {
 };
 
 struct GenerationRegReadRequest {
+	constexpr static FileIdentifier file_identifier = 8975311;
 	Key key;
 	UniqueGeneration gen;
 	ReplyPromise<struct GenerationRegReadReply> reply;
@@ -83,6 +86,7 @@ struct GenerationRegReadRequest {
 };
 
 struct GenerationRegReadReply {
+	constexpr static FileIdentifier file_identifier = 12623609;
 	Optional<Value> value;
 	UniqueGeneration gen, rgen;
 	GenerationRegReadReply() {}
@@ -94,6 +98,7 @@ struct GenerationRegReadReply {
 };
 
 struct GenerationRegWriteRequest {
+	constexpr static FileIdentifier file_identifier = 3521510;
 	KeyValue kv;
 	UniqueGeneration gen;
 	ReplyPromise< UniqueGeneration > reply;
@@ -116,6 +121,7 @@ struct LeaderElectionRegInterface : ClientLeaderRegInterface {
 };
 
 struct CandidacyRequest {
+	constexpr static FileIdentifier file_identifier = 14473958;
 	Key key;
 	LeaderInfo myInfo;
 	UID knownLeader, prevChangeID;
@@ -131,6 +137,7 @@ struct CandidacyRequest {
 };
 
 struct LeaderHeartbeatRequest {
+	constexpr static FileIdentifier file_identifier = 9495992;
 	Key key;
 	LeaderInfo myInfo;
 	UID prevChangeID;
@@ -146,6 +153,7 @@ struct LeaderHeartbeatRequest {
 };
 
 struct ForwardRequest {
+	constexpr static FileIdentifier file_identifier = 13570359;
 	Key key;
 	Value conn;  // a cluster connection string
 	ReplyPromise<Void> reply;
diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/LeaderElection.actor.cpp
index 473498a445..de5e92cbfc 100644
--- a/fdbserver/LeaderElection.actor.cpp
+++ b/fdbserver/LeaderElection.actor.cpp
@@ -75,8 +75,10 @@ ACTOR Future<Void> changeLeaderCoordinators( ServerCoordinators coordinators, Va
 	return Void();
 }
 
-ACTOR Future<Void> tryBecomeLeaderInternal( ServerCoordinators coordinators, Value proposedSerializedInterface, Reference<AsyncVar<Value>> outSerializedLeader, bool hasConnected, Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo ) {
-	state Reference<AsyncVar<vector<Optional<LeaderInfo>>>> nominees( new AsyncVar<vector<Optional<LeaderInfo>>>() );
+ACTOR Future<Void> tryBecomeLeaderInternal(ServerCoordinators coordinators, Value proposedSerializedInterface,
+                                           Reference<AsyncVar<Value>> outSerializedLeader, bool hasConnected,
+                                           Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo) {
+	state Reference<AsyncVar<vector<Optional<LeaderInfo>>>> nominees(new AsyncVar<vector<Optional<LeaderInfo>>>());
 	state LeaderInfo myInfo;
 	state Future<Void> candidacies;
 	state bool iAmLeader = false;
diff --git a/fdbserver/MasterInterface.h b/fdbserver/MasterInterface.h
index ac53d8ef2e..e6c35e12e3 100644
--- a/fdbserver/MasterInterface.h
+++ b/fdbserver/MasterInterface.h
@@ -31,6 +31,7 @@
 typedef uint64_t DBRecoveryCount;
 
 struct MasterInterface {
+	constexpr static FileIdentifier file_identifier = 5979145;
 	LocalityData locality;
 	RequestStream< ReplyPromise<Void> > waitFailure;
 	RequestStream< struct TLogRejoinRequest > tlogRejoin; // sent by tlog (whether or not rebooted) to communicate with a new master
@@ -54,6 +55,7 @@ struct MasterInterface {
 };
 
 struct TLogRejoinRequest {
+	constexpr static FileIdentifier file_identifier = 15692200;
 	TLogInterface myInterface;
 	ReplyPromise<bool> reply;   // false means someone else registered, so we should re-register.  true means this master is recovered, so don't send again to the same master.
 
@@ -66,6 +68,7 @@ struct TLogRejoinRequest {
 };
 
 struct ChangeCoordinatorsRequest {
+	constexpr static FileIdentifier file_identifier = 13605416;
 	Standalone<StringRef> newConnectionString;
 	ReplyPromise<Void> reply;  // normally throws even on success!
 
@@ -79,6 +82,7 @@ struct ChangeCoordinatorsRequest {
 };
 
 struct ResolverMoveRef {
+	constexpr static FileIdentifier file_identifier = 11945475;
 	KeyRangeRef range;
 	int dest;
 
@@ -104,6 +108,7 @@ struct ResolverMoveRef {
 };
 
 struct GetCommitVersionReply {
+	constexpr static FileIdentifier file_identifier = 3568822;
 	Standalone<VectorRef<ResolverMoveRef>> resolverChanges;
 	Version resolverChangesVersion;
 	Version version;
@@ -120,6 +125,7 @@ struct GetCommitVersionReply {
 };
 
 struct GetCommitVersionRequest {
+	constexpr static FileIdentifier file_identifier = 16683181;
 	uint64_t requestNum;
 	uint64_t mostRecentProcessedRequestNum;
 	UID requestingProxy;
diff --git a/fdbserver/NetworkTest.h b/fdbserver/NetworkTest.h
index 37057bf3a1..94ae95d0fe 100644
--- a/fdbserver/NetworkTest.h
+++ b/fdbserver/NetworkTest.h
@@ -24,6 +24,7 @@
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/fdbrpc.h"
+#include "flow/FileIdentifier.h"
 
 struct NetworkTestInterface {
 	RequestStream< struct NetworkTestRequest > test;
@@ -33,6 +34,7 @@ struct NetworkTestInterface {
 };
 
 struct NetworkTestRequest {
+	constexpr static FileIdentifier file_identifier = 4146513;
 	Key key;
 	uint32_t replySize;
 	ReplyPromise<struct NetworkTestReply> reply;
@@ -45,6 +47,7 @@ struct NetworkTestRequest {
 };
 
 struct NetworkTestReply {
+	constexpr static FileIdentifier file_identifier = 14465374;
 	Value value;
 	NetworkTestReply() {}
 	NetworkTestReply( Value value ) : value(value) {}
diff --git a/fdbserver/ResolverInterface.h b/fdbserver/ResolverInterface.h
index 6bc32ae62a..c011ce02c8 100644
--- a/fdbserver/ResolverInterface.h
+++ b/fdbserver/ResolverInterface.h
@@ -25,6 +25,7 @@
 #include "fdbclient/FDBTypes.h"
 
 struct ResolverInterface {
+	constexpr static FileIdentifier file_identifier = 1755944;
 	enum { LocationAwareLoadBalance = 1 };
 	enum { AlwaysFresh = 1 };
 
@@ -54,6 +55,7 @@ struct ResolverInterface {
 };
 
 struct StateTransactionRef {
+	constexpr static FileIdentifier file_identifier = 6150271;
 	StateTransactionRef() {}
 	StateTransactionRef(const bool committed, VectorRef<MutationRef> const& mutations) : committed(committed), mutations(mutations) {}
 	StateTransactionRef(Arena &p, const StateTransactionRef &toCopy) : committed(toCopy.committed), mutations(p, toCopy.mutations) {}
@@ -70,6 +72,7 @@ struct StateTransactionRef {
 };
 
 struct ResolveTransactionBatchReply {
+	constexpr static FileIdentifier file_identifier = 15472264;
 	Arena arena;
 	VectorRef<uint8_t> committed;
 	Optional<UID> debugID;
@@ -83,6 +86,7 @@ struct ResolveTransactionBatchReply {
 };
 
 struct ResolveTransactionBatchRequest {
+	constexpr static FileIdentifier file_identifier = 16462858;
 	Arena arena;
 
 	Version prevVersion;
@@ -100,6 +104,7 @@ struct ResolveTransactionBatchRequest {
 };
 
 struct ResolutionMetricsRequest {
+	constexpr static FileIdentifier file_identifier = 11663527;
 	ReplyPromise<int64_t> reply;
 
 	template <class Archive>
@@ -109,6 +114,7 @@ struct ResolutionMetricsRequest {
 };
 
 struct ResolutionSplitReply {
+	constexpr static FileIdentifier file_identifier = 12137765;
 	Key key;
 	int64_t used;
 	template <class Archive>
@@ -119,6 +125,7 @@ struct ResolutionSplitReply {
 };
 
 struct ResolutionSplitRequest {
+	constexpr static FileIdentifier file_identifier = 167535;
 	KeyRange range;
 	int64_t offset;
 	bool front;
diff --git a/fdbserver/RestoreInterface.h b/fdbserver/RestoreInterface.h
index cf11686384..fc893c17f5 100644
--- a/fdbserver/RestoreInterface.h
+++ b/fdbserver/RestoreInterface.h
@@ -28,6 +28,7 @@
 #include "fdbrpc/Locality.h"
 
 struct RestoreInterface {
+	constexpr static FileIdentifier file_identifier = 13398189;
 	RequestStream< struct TestRequest > test;
 
 	bool operator == (RestoreInterface const& r) const { return id() == r.id(); }
@@ -46,6 +47,7 @@ struct RestoreInterface {
 };
 
 struct TestRequest {
+	constexpr static FileIdentifier file_identifier = 14404487;
 	int testData;
 	ReplyPromise< struct TestReply > reply;
 
@@ -59,6 +61,7 @@ struct TestRequest {
 };
 
 struct TestReply {
+	constexpr static FileIdentifier file_identifier = 12075719;
 	int replyData;
 
 	TestReply() : replyData(0) {}
diff --git a/fdbserver/ServerDBInfo.h b/fdbserver/ServerDBInfo.h
index 2a5ca96135..fa50cd4d67 100644
--- a/fdbserver/ServerDBInfo.h
+++ b/fdbserver/ServerDBInfo.h
@@ -31,6 +31,7 @@
 #include "fdbserver/LatencyBandConfig.h"
 
 struct ServerDBInfo {
+	constexpr static FileIdentifier file_identifier = 13838807;
 	// This structure contains transient information which is broadcast to all workers for a database,
 	// permitting them to communicate with each other.  It is not available to the client.  This mechanism
 	// (see GetServerDBInfoRequest) is closely parallel to OpenDatabaseRequest for the client.
diff --git a/fdbserver/TLogInterface.h b/fdbserver/TLogInterface.h
index 87d2f92b31..39e9907322 100644
--- a/fdbserver/TLogInterface.h
+++ b/fdbserver/TLogInterface.h
@@ -29,6 +29,7 @@
 #include <iterator>
 
 struct TLogInterface {
+	constexpr static FileIdentifier file_identifier = 16308510;
 	enum { LocationAwareLoadBalance = 1 };
 	enum { AlwaysFresh = 1 };
 
@@ -71,6 +72,7 @@ struct TLogInterface {
 };
 
 struct TLogRecoveryFinishedRequest {
+	constexpr static FileIdentifier file_identifier = 8818668;
 	ReplyPromise<Void> reply;
 
 	TLogRecoveryFinishedRequest() {}
@@ -82,6 +84,7 @@ struct TLogRecoveryFinishedRequest {
 };
 
 struct TLogLockResult {
+	constexpr static FileIdentifier file_identifier = 11822027;
 	Version end;
 	Version knownCommittedVersion;
 
@@ -92,6 +95,7 @@ struct TLogLockResult {
 };
 
 struct TLogConfirmRunningRequest {
+	constexpr static FileIdentifier file_identifier = 10929130;
 	Optional<UID> debugID;
 	ReplyPromise<Void> reply;
 
@@ -136,6 +140,7 @@ struct VerUpdateRef {
 };
 
 struct TLogPeekReply {
+	constexpr static FileIdentifier file_identifier = 11365689;
 	Arena arena;
 	StringRef messages;
 	Version end;
@@ -151,6 +156,7 @@ struct TLogPeekReply {
 };
 
 struct TLogPeekRequest {
+	constexpr static FileIdentifier file_identifier = 11001131;
 	Arena arena;
 	Version begin;
 	Tag tag;
@@ -168,6 +174,7 @@ struct TLogPeekRequest {
 };
 
 struct TLogPopRequest {
+	constexpr static FileIdentifier file_identifier = 5556423;
 	Arena arena;
 	Version to;
 	Version durableKnownCommittedVersion;
@@ -201,6 +208,7 @@ struct TagMessagesRef {
 };
 
 struct TLogCommitRequest {
+	constexpr static FileIdentifier file_identifier = 4022206;
 	Arena arena;
 	Version prevVersion, version, knownCommittedVersion, minKnownCommittedVersion;
 
@@ -219,6 +227,7 @@ struct TLogCommitRequest {
 };
 
 struct TLogQueuingMetricsRequest {
+	constexpr static FileIdentifier file_identifier = 7798476;
 	ReplyPromise<struct TLogQueuingMetricsReply> reply;
 
 	template <class Ar>
@@ -228,6 +237,7 @@ struct TLogQueuingMetricsRequest {
 };
 
 struct TLogQueuingMetricsReply {
+	constexpr static FileIdentifier file_identifier = 12206626;
 	double localTime;
 	int64_t instanceID;  // changes if bytesDurable and bytesInput reset
 	int64_t bytesDurable, bytesInput;
diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h
index e8ee49d342..afdc62be70 100644
--- a/fdbserver/TesterInterface.actor.h
+++ b/fdbserver/TesterInterface.actor.h
@@ -33,6 +33,7 @@
 #include "flow/actorcompiler.h" // has to be last include
 
 struct WorkloadInterface {
+	constexpr static FileIdentifier file_identifier = 4454551;
 	RequestStream<ReplyPromise<Void>> setup;
 	RequestStream<ReplyPromise<Void>> start;
 	RequestStream<ReplyPromise<bool>> check;
@@ -48,6 +49,7 @@ struct WorkloadInterface {
 };
 
 struct WorkloadRequest {
+	constexpr static FileIdentifier file_identifier = 8121024;
 	Arena arena;
 	StringRef title;
 	int timeout;
@@ -79,6 +81,7 @@ struct WorkloadRequest {
 };
 
 struct TesterInterface {
+	constexpr static FileIdentifier file_identifier = 4465210;
 	RequestStream<WorkloadRequest> recruitments;
 
 	UID id() const { return recruitments.getEndpoint().token; }
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index d684109f0e..082dc072cf 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -41,6 +41,7 @@
 #define DUMPTOKEN( name ) TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token)
 
 struct WorkerInterface {
+	constexpr static FileIdentifier file_identifier = 14712718;
 	ClientWorkerInterface clientInterface;
 	LocalityData locality;
 	RequestStream< struct InitializeTLogRequest > tLog;
@@ -89,6 +90,7 @@ struct WorkerDetails {
 };
 
 struct InitializeTLogRequest {
+	constexpr static FileIdentifier file_identifier = 15604392;
 	UID recruitmentID;
 	LogSystemConfig recoverFrom;
 	Version recoverAt;
@@ -116,6 +118,7 @@ struct InitializeTLogRequest {
 };
 
 struct InitializeLogRouterRequest {
+	constexpr static FileIdentifier file_identifier = 2976228;
 	uint64_t recoveryCount;
 	Tag routerTag;
 	Version startVersion;
@@ -132,6 +135,7 @@ struct InitializeLogRouterRequest {
 
 // FIXME: Rename to InitializeMasterRequest, etc
 struct RecruitMasterRequest {
+	constexpr static FileIdentifier file_identifier = 12684574;
 	Arena arena;
 	LifetimeToken lifetime;
 	bool forceRecovery;
@@ -147,6 +151,7 @@ struct RecruitMasterRequest {
 };
 
 struct InitializeMasterProxyRequest {
+	constexpr static FileIdentifier file_identifier = 10344153;
 	MasterInterface master;
 	uint64_t recoveryCount;
 	Version recoveryTransactionVersion;
@@ -184,6 +189,7 @@ struct InitializeRatekeeperRequest {
 };
 
 struct InitializeResolverRequest {
+	constexpr static FileIdentifier file_identifier = 7413317;
 	uint64_t recoveryCount;
 	int proxyCount;
 	int resolverCount;
@@ -196,6 +202,7 @@ struct InitializeResolverRequest {
 };
 
 struct InitializeStorageReply {
+	constexpr static FileIdentifier file_identifier = 10390645;
 	StorageServerInterface interf;
 	Version addedVersion;
 
@@ -206,6 +213,7 @@ struct InitializeStorageReply {
 };
 
 struct InitializeStorageRequest {
+	constexpr static FileIdentifier file_identifier = 16665642;
 	Tag seedTag;									//< If this server will be passed to seedShardServers, this will be a tag, otherwise it is invalidTag
 	UID reqId;
 	UID interfaceId;
@@ -219,6 +227,7 @@ struct InitializeStorageRequest {
 };
 
 struct TraceBatchDumpRequest {
+	constexpr static FileIdentifier file_identifier = 8184121;
 	ReplyPromise<Void> reply;
 
 	template <class Ar>
@@ -228,6 +237,7 @@ struct TraceBatchDumpRequest {
 };
 
 struct LoadedReply {
+	constexpr static FileIdentifier file_identifier = 9956350;
 	Standalone<StringRef> payload;
 	UID id;
 
@@ -238,6 +248,7 @@ struct LoadedReply {
 };
 
 struct LoadedPingRequest {
+	constexpr static FileIdentifier file_identifier = 4590979;
 	UID id;
 	bool loadReply;
 	Standalone<StringRef> payload;
@@ -250,6 +261,7 @@ struct LoadedPingRequest {
 };
 
 struct CoordinationPingMessage {
+	constexpr static FileIdentifier file_identifier = 9982747;
 	UID clusterControllerId;
 	int64_t timeStep;
 
@@ -263,6 +275,7 @@ struct CoordinationPingMessage {
 };
 
 struct SetMetricsLogRateRequest {
+	constexpr static FileIdentifier file_identifier = 4245995;
 	uint32_t metricsLogsPerSecond;
 
 	SetMetricsLogRateRequest() : metricsLogsPerSecond( 1 ) {}
@@ -275,6 +288,7 @@ struct SetMetricsLogRateRequest {
 };
 
 struct EventLogRequest {
+	constexpr static FileIdentifier file_identifier = 122319;
 	bool getLastError;
 	Standalone<StringRef> eventName;
 	ReplyPromise< TraceEventFields > reply;
@@ -309,6 +323,7 @@ struct DebugEntryRef {
 };
 
 struct DiskStoreRequest {
+	constexpr static FileIdentifier file_identifier = 1986262;
 	bool includePartialStores;
 	ReplyPromise<Standalone<VectorRef<UID>>> reply;
 
diff --git a/fdbserver/sqlite/btree.h b/fdbserver/sqlite/btree.h
index b0c748673d..bdf98c14bb 100644
--- a/fdbserver/sqlite/btree.h
+++ b/fdbserver/sqlite/btree.h
@@ -15,6 +15,9 @@
 */
 #ifndef _BTREE_H_
 #define _BTREE_H_
+#ifndef NDEBUG
+#define NDEBUG
+#endif
 
 /* TODO: This definition is just included so other modules compile. It
 ** needs to be revisited.
diff --git a/fdbserver/sqlite/sqlite3.amalgamation.c b/fdbserver/sqlite/sqlite3.amalgamation.c
index 7f1376151e..c9df4ee492 100644
--- a/fdbserver/sqlite/sqlite3.amalgamation.c
+++ b/fdbserver/sqlite/sqlite3.amalgamation.c
@@ -1,3 +1,6 @@
+#ifndef NDEBUG
+#define NDEBUG
+#endif
 #ifndef NDEBUG
     #define SQLITE_DEBUG 1
 #endif
diff --git a/fdbserver/workloads/Sideband.actor.cpp b/fdbserver/workloads/Sideband.actor.cpp
index 3877e64554..f474f7bec6 100644
--- a/fdbserver/workloads/Sideband.actor.cpp
+++ b/fdbserver/workloads/Sideband.actor.cpp
@@ -24,6 +24,7 @@
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
 struct SidebandMessage {
+	constexpr static FileIdentifier file_identifier = 11862046;
 	uint64_t key;
 	Version commitVersion;
 
@@ -37,6 +38,7 @@ struct SidebandMessage {
 };
 
 struct SidebandInterface {
+	constexpr static FileIdentifier file_identifier = 15950544;
 	RequestStream<SidebandMessage> updates;
 
 	UID id() const { return updates.getEndpoint().token; }
diff --git a/flow/Arena.h b/flow/Arena.h
index 4c1e60c6a0..53b096b207 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -27,6 +27,7 @@
 #include "flow/Error.h"
 #include "flow/Trace.h"
 #include "flow/ObjectSerializerTraits.h"
+#include "flow/FileIdentifier.h"
 #include <algorithm>
 #include <stdint.h>
 #include <string>
@@ -512,6 +513,7 @@ extern std::string format(const char* form, ...);
 #pragma pack( push, 4 )
 class StringRef {
 public:
+	constexpr static FileIdentifier file_identifier = 13300811;
 	StringRef() : data(0), length(0) {}
 	StringRef( Arena& p, const StringRef& toCopy ) : data( new (p) uint8_t[toCopy.size()] ), length( toCopy.size() ) {
 		memcpy( (void*)data, toCopy.data, length );
@@ -778,6 +780,7 @@ template <class T>
 class VectorRef {
 public:
 	using value_type = T;
+	static constexpr FileIdentifier file_identifier = (0x8 << 24) | FileIdentifierFor<T>::value;
 
 	// T must be trivially destructible (and copyable)!
 	VectorRef() : data(0), m_size(0), m_capacity(0) {}
diff --git a/flow/Error.h b/flow/Error.h
index 18a3a1c9e6..f298752b08 100644
--- a/flow/Error.h
+++ b/flow/Error.h
@@ -29,6 +29,7 @@
 #include <boost/preprocessor/control/if.hpp>
 #include "flow/Platform.h"
 #include "flow/Knobs.h"
+#include "flow/FileIdentifier.h"
 #include "flow/ObjectSerializerTraits.h"
 
 enum { invalid_error_code = 0xffff };
@@ -41,6 +42,7 @@ public:
 
 class Error {
 public:
+	constexpr static FileIdentifier file_identifier = 14065384;
 	int code() const { return error_code; }
 	const char* name() const;
 	const char* what() const;
diff --git a/flow/FileIdentifier.h b/flow/FileIdentifier.h
index 4bbb1d9d14..394356b55d 100644
--- a/flow/FileIdentifier.h
+++ b/flow/FileIdentifier.h
@@ -25,9 +25,7 @@ using FileIdentifier = uint32_t;
 
 template <class T>
 struct FileIdentifierFor {
-	//constexpr static FileIdentifier value = T::file_identifier;
-	// TODO: use file identifiers for different types
-	constexpr static FileIdentifier value = 0xffffff;
+	constexpr static FileIdentifier value = T::file_identifier;
 };
 
 template <>
diff --git a/flow/IRandom.h b/flow/IRandom.h
index dbf14ac619..ca77984dca 100644
--- a/flow/IRandom.h
+++ b/flow/IRandom.h
@@ -23,6 +23,7 @@
 #pragma once
 
 #include "flow/Platform.h"
+#include "flow/FileIdentifier.h"
 #include "flow/ObjectSerializerTraits.h"
 #include <stdint.h>
 #if (defined(__APPLE__))
@@ -35,6 +36,7 @@
 class UID {
 	uint64_t part[2];
 public:
+	constexpr static FileIdentifier file_identifier = 15597147;
 	UID() { part[0] = part[1] = 0; }
 	UID( uint64_t a, uint64_t b ) { part[0]=a; part[1]=b; }
 	std::string toString() const;
diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h
index b10be8e835..73e1c3c9bc 100644
--- a/flow/ObjectSerializer.h
+++ b/flow/ObjectSerializer.h
@@ -101,7 +101,7 @@ class ObjectWriter {
 public:
 	template <class... Items>
 	void serialize(FileIdentifier file_identifier, Items const&... items) {
-		ASSERT(data = nullptr); // object serializer can only serialize one object
+		ASSERT(data == nullptr); // object serializer can only serialize one object
 		int allocations = 0;
 		auto allocator = [this, &allocations](size_t size_) {
 			++allocations;
diff --git a/flow/Trace.h b/flow/Trace.h
index c09e29374d..b937a9ea0a 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -57,6 +57,7 @@ enum Severity {
 
 class TraceEventFields {
 public:
+	constexpr static FileIdentifier file_identifier = 11262274;
 	typedef std::pair<std::string, std::string> Field;
 	typedef std::vector<Field> FieldContainer;
 	typedef FieldContainer::const_iterator FieldIterator;
diff --git a/flow/flow.h b/flow/flow.h
index 013a6e8ecd..0aac95eb0e 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -44,6 +44,7 @@
 #include "flow/Deque.h"
 #include "flow/ThreadPrimitives.h"
 #include "flow/network.h"
+#include "flow/FileIdentifier.h"
 
 #include <boost/version.hpp>
 
@@ -114,6 +115,7 @@ Standalone<StringRef> concatenate( Iter b, Iter const& e ) {
 
 class Void {
 public:
+	constexpr static FileIdentifier file_identifier = 2010442;
 	template <class Ar>
 	void serialize(Ar&) {}
 };
@@ -123,6 +125,7 @@ class Never {};
 template <class T>
 class ErrorOr {
 public:
+	constexpr static FileIdentifier file_identifier = (0x1 << 24) | FileIdentifierFor<T>::value;
 	ErrorOr() : error(default_error_or()) {}
 	ErrorOr(Error const& error) : error(error) {}
 	ErrorOr(const ErrorOr<T>& o) : error(o.error) {
diff --git a/flow/serialize.h b/flow/serialize.h
index adaf56c560..3d82e761f2 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -715,6 +715,7 @@ private:
 struct ISerializeSource {
 	virtual void serializePacketWriter(PacketWriter&, bool useObjectSerializer) const = 0;
 	virtual void serializeBinaryWriter(BinaryWriter&) const = 0;
+	virtual void serializeBinaryWriter(ObjectWriter&) const = 0;
 };
 
 template <class T, class V>
@@ -737,7 +738,12 @@ struct SerializeSource : MakeSerializeSource<SerializeSource<T>, T> {
 	using value_type = T;
 	T const& value;
 	SerializeSource(T const& value) : value(value) {}
-	template <class Ar> void serialize(Ar& ar) const { ar << value; }
+	virtual void serializeBinaryWriter(ObjectWriter& w) const {
+		w.serialize(value);
+	}
+	template <class Ar> void serialize(Ar& ar) const {
+		ar << value;
+	}
 	virtual T const& get() const { return value; }
 };
 
@@ -748,6 +754,9 @@ struct SerializeBoolAnd : MakeSerializeSource<SerializeBoolAnd<T>, T> {
 	T const& value;
 	SerializeBoolAnd( bool b, T const& value ) : b(b), value(value) {}
 	template <class Ar> void serialize(Ar& ar) const { ar << b << value; }
+	virtual void serializeBinaryWriter(ObjectWriter& w) const {
+		ASSERT(false);
+	}
 	virtual T const& get() const {
 		// This is only used for the streaming serializer
 		ASSERT(false);
diff --git a/tests/TestRunner/TestRunner.py b/tests/TestRunner/TestRunner.py
index 880a3cffa8..5fefc8502c 100755
--- a/tests/TestRunner/TestRunner.py
+++ b/tests/TestRunner/TestRunner.py
@@ -271,6 +271,9 @@ def run_simulation_test(basedir, options):
     if options.testtype == 'test':
         pargs.append('-C')
         pargs.append(os.path.join(args.builddir, 'fdb.cluster'))
+    else:
+        pargs.append('-S')
+        pargs.append('random')
     td = TestDirectory(basedir)
     if options.buggify:
         pargs.append('-b')

From 1df3d8f0c71226150a88b265dfcb66364f0a046c Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Wed, 30 Jan 2019 17:10:25 -0800
Subject: [PATCH 050/118] disable rlimit when ASAN is used

---
 flow/Platform.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/Platform.cpp b/flow/Platform.cpp
index d303e48d80..59efdf0a9c 100644
--- a/flow/Platform.cpp
+++ b/flow/Platform.cpp
@@ -1410,7 +1410,7 @@ void setMemoryQuota( size_t limit ) {
 	}
 	if (!AssignProcessToJobObject( job, GetCurrentProcess() ))
 		TraceEvent(SevWarn, "FailedToSetMemoryLimit").GetLastError();
-#elif defined(__linux__)
+#elif defined(__linux__) && !defined(USE_ASAN)
 	struct rlimit rlim;
 	if (getrlimit(RLIMIT_AS, &rlim)) {
 		TraceEvent(SevError, "GetMemoryLimit").GetLastError();

From b06fe39b475ef30f5ce44d59a8f5977c5a0116fd Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 31 Jan 2019 12:22:48 -0800
Subject: [PATCH 051/118] renamed flatbuffers tests for consistency

---
 flow/flat_buffers.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
index 53739e4b7b..760ad2f8c1 100644
--- a/flow/flat_buffers.cpp
+++ b/flow/flat_buffers.cpp
@@ -424,7 +424,7 @@ struct X {
 	}
 };
 
-TEST_CASE("flow/FlatBuffers/nestedCompat") {
+TEST_CASE("/flow/FlatBuffers/nestedCompat") {
 	X<Y1> x1 = { 1, { 2 }, 3 };
 	X<Y2> x2;
 	Arena arena;
@@ -448,7 +448,7 @@ TEST_CASE("flow/FlatBuffers/nestedCompat") {
 	return Void();
 }
 
-TEST_CASE("flow/FlatBuffers/struct") {
+TEST_CASE("/flow/FlatBuffers/struct") {
 	std::vector<std::tuple<int16_t, bool, int64_t>> x1 = { { 1, true, 2 }, { 3, false, 4 } };
 	decltype(x1) x2;
 	Arena arena;
@@ -462,7 +462,7 @@ TEST_CASE("flow/FlatBuffers/struct") {
 	return Void();
 }
 
-TEST_CASE("flow/FlatBuffers/file_identifier") {
+TEST_CASE("/flow/FlatBuffers/file_identifier") {
 	Arena arena;
 	const uint8_t* out;
 	constexpr FileIdentifier file_identifier{ 1234 };
@@ -472,7 +472,7 @@ TEST_CASE("flow/FlatBuffers/file_identifier") {
 	return Void();
 }
 
-TEST_CASE("flow/FlatBuffers/VectorRef") {
+TEST_CASE("/flow/FlatBuffers/VectorRef") {
 	// this test tests a few weird memory properties of
 	// serialized arenas. This is why it uses weird scoping
 

From 8df7818b17e6536d48873a57c38c2c52c9c9b6bd Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 31 Jan 2019 12:23:12 -0800
Subject: [PATCH 052/118] Added a test to reproduce current failure

Currently, enabling the ObjectSerializer results
in failures. These are hard to debug. Therefore I added
a unit test that reproduces the problem.
---
 fdbclient/MonitorLeader.actor.cpp | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 69934bd52d..6879d77962 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -233,6 +233,37 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/basic") {
 	return Void();
 }
 
+TEST_CASE("/flow/FlatBuffers/LeaderInfo") {
+	LeaderInfo leaderInfo;
+	leaderInfo.forward = g_random->coinflip();
+	leaderInfo.changeID = g_random->randomUniqueID();
+	{
+		std::string rndString(g_random->randomInt(10, 400), 'x');
+		for (auto& c : rndString) {
+			c = g_random->randomAlphaNumeric();
+		}
+		leaderInfo.serializedInfo = rndString;
+	}
+	ErrorOr<Optional<LeaderInfo>> objIn{ Optional<LeaderInfo>{leaderInfo} };
+	ErrorOr<Optional<LeaderInfo>> objOut;
+	Standalone<StringRef> copy;
+	{
+		ObjectWriter writer;
+		writer.serialize(objIn);
+		copy = writer.toStringRef();
+	}
+	ArenaObjectReader reader(copy.arena(), copy);
+	reader.deserialize(objOut);
+
+	ASSERT(!objOut.isError());
+	ASSERT(objOut.get().present());
+	LeaderInfo outLeader = objOut.get().get();
+	ASSERT(outLeader.changeID == leaderInfo.changeID);
+	ASSERT(outLeader.forward == leaderInfo.forward);
+	ASSERT(outLeader.serializedInfo == leaderInfo.serializedInfo);
+	return Void();
+}
+
 TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/fuzz") {
 	// For a static connection string, add in fuzzed comments and whitespace
 	// SOMEDAY: create a series of random connection strings, rather than the one we started with

From 642a96807b1f47254b4cd388fc8a1eeeb5d7fc96 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Wed, 20 Feb 2019 11:52:47 -0800
Subject: [PATCH 053/118] Fixed compilation issues after rebase

---
 fdbrpc/FlowTransport.actor.cpp       |  2 +-
 fdbrpc/FlowTransport.h               | 39 +++++-----------
 fdbrpc/fdbrpc.h                      | 66 +++++++++++-----------------
 fdbrpc/sim2.actor.cpp                |  2 +-
 fdbserver/DataDistributorInterface.h |  2 +
 fdbserver/WorkerInterface.actor.h    |  1 +
 flow/flat_buffers.h                  |  2 +-
 7 files changed, 44 insertions(+), 70 deletions(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 822a4a0b2b..ba2fd5771e 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -553,7 +553,7 @@ ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader
 	auto receiver = self->endpoints.get(destination.token);
 	if (receiver) {
 		try {
-			g_currentDeliveryPeerAddress = destination.address;
+			g_currentDeliveryPeerAddress = destination.addresses;
 			if (useFlatbuffers) {
 				ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll());
 				receiver->receive(objReader);
diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h
index 63830296ef..df3f0b73f5 100644
--- a/fdbrpc/FlowTransport.h
+++ b/fdbrpc/FlowTransport.h
@@ -74,41 +74,26 @@ public:
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061020001LL) {
-			addresses.secondaryAddress = Optional<NetworkAddress>();
-			serializer(ar, addresses.address, token);
-		} else {
+		if constexpr (is_fb_function<Ar>) {
 			serializer(ar, addresses, token);
-			if (ar.isDeserializing) {
+			if constexpr (Ar::isDeserializing) {
 				choosePrimaryAddress();
 			}
+		} else {
+			if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061020001LL) {
+				addresses.resize(1);
+				serializer(ar, addresses[0], token);
+			} else {
+				serializer(ar, addresses, token);
+				if (ar.isDeserializing) {
+					choosePrimaryAddress();
+				}
+			}
 		}
 	}
 };
 #pragma pack(pop)
 
-
-template <>
-struct scalar_traits<Endpoint> : std::true_type {
-	using networkAddress = scalar_traits<NetworkAddress>;
-	using token = scalar_traits<UID>;
-
-	static constexpr size_t size = networkAddress::size + token::size;
-
-	static void save(uint8_t* out, const Endpoint& endpoint) {
-		networkAddress::save(out, endpoint.address);
-		out += networkAddress::size;
-		token::save(out, endpoint.token);
-	}
-
-	template <class C>
-	static void load(const uint8_t* in, Endpoint& endpoint, C& c) {
-		networkAddress::load(in, endpoint.address, c);
-		in += networkAddress::size;
-		token::load(in, endpoint.token, c);
-	}
-};
-
 class ArenaObjectReader;
 class NetworkMessageReceiver {
 public:
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index 26993e5514..54b3ee47d4 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -177,30 +177,22 @@ void load(Ar& ar, ReplyPromise<T>& value) {
 }
 
 template <class T>
-struct scalar_traits<ReplyPromise<T>> : std::true_type {
-	using endpoint = scalar_traits<Endpoint>;
-
-	constexpr static size_t size = endpoint::size;
-
-	static void save(uint8_t* out, const ReplyPromise<T>& reply) {
-		auto const& ep = reply.getEndpoint();
-		endpoint::save(out, ep);
-		ASSERT(!ep.address.isValid() || ep.address.isPublic()); // No re-serializing non-public addresses
-		                                                        // (the reply connection won't be
-		                                                        // available to any other process)
-	}
-
-	template <class C>
-	static void load(const uint8_t* in, ReplyPromise<T>& reply, C& context) {
-		Endpoint ep;
-		endpoint::load(in, ep, context);
-		FlowTransport::transport().loadedEndpoint(ep);
-		reply = ReplyPromise<T>(ep);
-		networkSender(reply.getFuture(), ep);
+struct serializable_traits<ReplyPromise<T>> : std::true_type {
+	template<class Archiver>
+	static void serialize(Archiver& ar, ReplyPromise<T>& p) {
+		if constexpr (Archiver::isDeserializing) {
+			Endpoint endpoint;
+			serializer(ar, endpoint);
+			p = ReplyPromise<T>(endpoint);
+			networkSender(p.getFuture(), endpoint);
+		} else {
+			const auto& ep = p.getEndpoint();
+			serializer(ar, ep);
+			ASSERT(!ep.getPrimaryAddress().isValid() || ep.getPrimaryAddress().isPublic()); // No re-serializing non-public addresses (the reply connection won't be available to any other process)
+		}
 	}
 };
 
-
 template <class Reply>
 ReplyPromise<Reply> const& getReplyPromise(ReplyPromise<Reply> const& p) { return p; }
 
@@ -424,25 +416,19 @@ void load(Ar& ar, RequestStream<T>& value) {
 }
 
 template <class T>
-struct scalar_traits<RequestStream<T>> : std::true_type {
-	using endpointTraits = scalar_traits<Endpoint>;
-
-	static constexpr size_t size = endpointTraits::size;
-
-	static void save(uint8_t* out, const RequestStream<T>& stream) {
-		auto const& ep = stream.getEndpoint();
-		endpointTraits::save(out, ep);
-		UNSTOPPABLE_ASSERT(ep.address.isValid());
-	}
-
-	// Context is an arbitrary type that is plumbed by reference throughout the
-	// load call tree.
-	template <class Context>
-	static void load(const uint8_t* in, RequestStream<T>& stream, Context& context) {
-		Endpoint endpoint;
-		endpointTraits::load(in, endpoint, context);
-		FlowTransport::transport().loadedEndpoint(endpoint);
-		stream = RequestStream<T>(endpoint);
+struct serializable_traits<RequestStream<T>> : std::true_type {
+	template <class Archiver>
+	static void serialize(Archiver& ar, RequestStream<T>& stream) {
+		if constexpr (Archiver::isDeserializing) {
+			Endpoint endpoint;
+			serializer(ar, endpoint);
+			FlowTransport::transport().loadedEndpoint(endpoint);
+			stream = RequestStream<T>(endpoint);
+		} else {
+			const auto& ep = stream.getEndpoint();
+			serializer(ar, ep);
+			UNSTOPPABLE_ASSERT(ep.getPrimaryAddress().isValid());  // No serializing PromiseStreams on a client with no public address
+		}
 	}
 };
 
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 09cff1dc44..4ef1a91494 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -979,7 +979,7 @@ public:
 	}
 	virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess,
 	                                LocalityData locality, ProcessClass startingClass, const char* dataFolder,
-	                                const char* coordinationFolder, bool ObjSerializer) {
+	                                const char* coordinationFolder, bool objSerializer) {
 		ASSERT( locality.machineId().present() );
 		MachineInfo& machine = machines[ locality.machineId().get() ];
 		if (!machine.machineId.present())
diff --git a/fdbserver/DataDistributorInterface.h b/fdbserver/DataDistributorInterface.h
index 403258c01f..d1875cae23 100644
--- a/fdbserver/DataDistributorInterface.h
+++ b/fdbserver/DataDistributorInterface.h
@@ -25,6 +25,7 @@
 #include "fdbrpc/Locality.h"
 
 struct DataDistributorInterface {
+	constexpr static FileIdentifier file_identifier = 12383874;
 	RequestStream<ReplyPromise<Void>> waitFailure;
 	RequestStream<struct HaltDataDistributorRequest> haltDataDistributor;
 	struct LocalityData locality;
@@ -49,6 +50,7 @@ struct DataDistributorInterface {
 };
 
 struct HaltDataDistributorRequest {
+	constexpr static FileIdentifier file_identifier = 1904127;
 	UID requesterID;
 	ReplyPromise<Void> reply;
 
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 082dc072cf..10eea4906e 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -165,6 +165,7 @@ struct InitializeMasterProxyRequest {
 };
 
 struct InitializeDataDistributorRequest {
+	constexpr static FileIdentifier file_identifier = 8858952;
 	UID reqId;
 	ReplyPromise<DataDistributorInterface> reply;
 
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 502997e002..56e3a07d29 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -507,7 +507,7 @@ private:
 };
 
 struct InsertVTableLambda {
-	static constexpr bool isDeserializing = true;
+	static constexpr bool isDeserializing = false;
 	static constexpr bool is_fb_visitor = true;
 	std::set<const VTable*>& vtables;
 	std::set<std::type_index>& known_types;

From 0a3676111ebd29f7558f07cca4d01f114eca3f44 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <anoyes34@gmail.com>
Date: Mon, 11 Feb 2019 16:09:45 -0800
Subject: [PATCH 054/118] Support vector of unions in flatbuffers

---
 flow/flat_buffers.h | 73 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 4 deletions(-)

diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 56e3a07d29..de37bb4a50 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -184,6 +184,9 @@ constexpr bool is_union_like = union_like_traits<T>::value;
 template <class T>
 constexpr bool is_vector_like = vector_like_traits<T>::value;
 
+template <class T>
+constexpr bool is_vector_of_union_like = is_vector_like<T>&& is_union_like<typename vector_like_traits<T>::value_type>;
+
 template <class T>
 constexpr bool is_struct_like = struct_like_traits<T>::value;
 
@@ -415,7 +418,9 @@ constexpr auto fields_helper() {
 	if constexpr (_SizeOf<Member>::size == 0) {
 		return pack<>{};
 	} else if constexpr (is_union_like<Member>) {
-		return pack<uint8_t, uint32_t>{};
+		return pack</*type*/ uint8_t, /*offset*/ uint32_t>{};
+	} else if constexpr (is_vector_of_union_like<Member>) {
+		return pack</*type vector*/ uint32_t, /*offset vector*/ uint32_t>{};
 	} else {
 		return pack<Member>{};
 	}
@@ -480,7 +485,6 @@ struct TraverseMessageTypes {
 	std::enable_if_t<is_vector_like<VectorLike>> operator()(const VectorLike& members) {
 		using VectorTraits = vector_like_traits<VectorLike>;
 		using T = typename VectorTraits::value_type;
-		static_assert(!is_union_like<T>, "vector<union> not yet supported");
 		// we don't need to check for recursion here because the next call
 		// to operator() will do that and we don't generate a vtable for the
 		// vector-like type itself
@@ -643,7 +647,37 @@ struct SaveVisitorLambda {
 		for_each(
 		    [&](const auto& member) {
 			    using Member = std::decay_t<decltype(member)>;
-			    if constexpr (is_union_like<Member>) {
+			    if constexpr (is_vector_of_union_like<Member>) {
+				    using VectorTraits = vector_like_traits<Member>;
+				    using T = typename VectorTraits::value_type;
+				    using UnionTraits = union_like_traits<T>;
+				    uint32_t num_entries = VectorTraits::num_entries(member);
+				    auto typeVectorWriter = writer.getMessageWriter(num_entries); // type tags are one byte
+				    auto offsetVectorWriter = writer.getMessageWriter(num_entries * sizeof(RelativeOffset));
+				    auto iter = VectorTraits::begin(member);
+				    for (int i = 0; i < num_entries; ++i) {
+					    uint8_t type_tag = UnionTraits::index(*iter);
+					    uint8_t fb_type_tag =
+					        UnionTraits::empty(*iter) ? 0 : type_tag + 1; // Flatbuffers indexes from 1.
+					    typeVectorWriter.write(&fb_type_tag, i, sizeof(fb_type_tag));
+					    if (!UnionTraits::empty(*iter)) {
+						    RelativeOffset offset =
+						        (SaveAlternative<Writer, UnionTraits>{ writer, vtableset }).save(type_tag, *iter);
+						    offsetVectorWriter.write(&offset, i * sizeof(offset), sizeof(offset));
+					    }
+					    ++iter;
+				    }
+				    int start = RightAlign(writer.current_buffer_size + num_entries, 4) + 4;
+				    writer.write(&num_entries, start, sizeof(uint32_t));
+				    typeVectorWriter.writeTo(writer, start - sizeof(uint32_t));
+				    auto typeVectorOffset = RelativeOffset{ writer.current_buffer_size };
+				    start = RightAlign(writer.current_buffer_size + num_entries * sizeof(RelativeOffset), 4) + 4;
+				    writer.write(&num_entries, start, sizeof(uint32_t));
+				    offsetVectorWriter.writeTo(writer, start - sizeof(uint32_t));
+				    auto offsetVectorOffset = RelativeOffset{ writer.current_buffer_size };
+				    self.write(&typeVectorOffset, vtable[i++], sizeof(typeVectorOffset));
+				    self.write(&offsetVectorOffset, vtable[i++], sizeof(offsetVectorOffset));
+			    } else if constexpr (is_union_like<Member>) {
 				    using UnionTraits = union_like_traits<Member>;
 				    uint8_t type_tag = UnionTraits::index(member);
 				    uint8_t fb_type_tag = UnionTraits::empty(member) ? 0 : type_tag + 1; // Flatbuffers indexes from 1.
@@ -682,7 +716,38 @@ struct LoadMember {
 	Context& context;
 	template <class Member>
 	void operator()(Member& member) {
-		if constexpr (is_union_like<Member>) {
+		if constexpr (is_vector_of_union_like<Member>) {
+			if (!field_present()) {
+				i += 2;
+				return;
+			}
+			const uint8_t* types_current = &message[vtable[i++]];
+			uint32_t types_current_offset = interpret_as<uint32_t>(types_current);
+			types_current += types_current_offset;
+			types_current += sizeof(uint32_t); // num entries in types vector
+			using VectorTraits = vector_like_traits<Member>;
+			using T = typename Member::value_type;
+			const uint8_t* current = &message[vtable[i++]];
+			uint32_t current_offset = interpret_as<uint32_t>(current);
+			current += current_offset;
+			uint32_t numEntries = interpret_as<uint32_t>(current);
+			current += sizeof(uint32_t);
+			VectorTraits::reserve(member, numEntries, context);
+			auto inserter = VectorTraits::insert(member);
+			for (int i = 0; i < numEntries; ++i) {
+				T value;
+				if (types_current[i] > 0) {
+					uint8_t type_tag = types_current[i] - 1; // Flatbuffers indexes from 1.
+					(LoadAlternative<Context, union_like_traits<T>>{ context, current }).load(type_tag, value);
+				}
+				*inserter = std::move(value);
+				++inserter;
+				current += sizeof(RelativeOffset);
+			}
+			if constexpr (has_deserialization_done<VectorTraits>::value) {
+				VectorTraits::deserialization_done(member);
+			}
+		} else if constexpr (is_union_like<Member>) {
 			if (!field_present()) {
 				i += 2;
 				return;

From 7affe73774dfd04530de6289568348a3c98b8df8 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 31 Jan 2019 14:45:42 -0800
Subject: [PATCH 055/118] Added more low-level tests and implemented enums

---
 fdbclient/MonitorLeader.actor.cpp | 29 ++++++++++--
 flow/ObjectSerializer.h           | 23 ++++++++++
 flow/flow.cpp                     | 76 +++++++++++++++++++++++++++++++
 flow/flow.h                       | 30 +++++++++++-
 4 files changed, 152 insertions(+), 6 deletions(-)

diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 6879d77962..3696edec8b 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -234,6 +234,27 @@ TEST_CASE("/fdbclient/MonitorLeader/parseConnectionString/basic") {
 }
 
 TEST_CASE("/flow/FlatBuffers/LeaderInfo") {
+	{
+		LeaderInfo in;
+		LeaderInfo out;
+		in.forward = g_random->coinflip();
+		in.changeID = g_random->randomUniqueID();
+		{
+			std::string rndString(g_random->randomInt(10, 400), 'x');
+			for (auto& c : rndString) {
+				c = g_random->randomAlphaNumeric();
+			}
+			in.serializedInfo = rndString;
+		}
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(in.forward == out.forward);
+		ASSERT(in.changeID == out.changeID);
+		ASSERT(in.serializedInfo == out.serializedInfo);
+	}
 	LeaderInfo leaderInfo;
 	leaderInfo.forward = g_random->coinflip();
 	leaderInfo.changeID = g_random->randomUniqueID();
@@ -247,11 +268,9 @@ TEST_CASE("/flow/FlatBuffers/LeaderInfo") {
 	ErrorOr<Optional<LeaderInfo>> objIn{ Optional<LeaderInfo>{leaderInfo} };
 	ErrorOr<Optional<LeaderInfo>> objOut;
 	Standalone<StringRef> copy;
-	{
-		ObjectWriter writer;
-		writer.serialize(objIn);
-		copy = writer.toStringRef();
-	}
+	ObjectWriter writer;
+	writer.serialize(objIn);
+	copy = writer.toStringRef();
 	ArenaObjectReader reader(copy.arena(), copy);
 	reader.deserialize(objOut);
 
diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h
index 73e1c3c9bc..8e54111894 100644
--- a/flow/ObjectSerializer.h
+++ b/flow/ObjectSerializer.h
@@ -126,3 +126,26 @@ private:
 	uint8_t* data = nullptr;
 	int size = 0;
 };
+
+// this special case is needed - the code expects
+// Standalone<T> and T to be equivalent for serialization
+namespace detail {
+
+template <class T>
+struct LoadSaveHelper<Standalone<T>> {
+	template <class Context>
+	void load(Standalone<T>& member, const uint8_t* current, Context& context) {
+		helper.load(member.contents(), current, context);
+		context.addArena(member.arena());
+	}
+
+	template <class Writer>
+	RelativeOffset save(const Standalone<T>& member, Writer& writer, const VTableSet* vtables) {
+		return helper.save(member.contents(), writer, vtables);
+	}
+
+private:
+	LoadSaveHelper<T> helper;
+};
+
+} // namespace detail
diff --git a/flow/flow.cpp b/flow/flow.cpp
index aebf8e7c99..93a3a5350b 100644
--- a/flow/flow.cpp
+++ b/flow/flow.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "flow/flow.h"
+#include "flow/UnitTest.h"
 #include <stdarg.h>
 #include <cinttypes>
 
@@ -214,3 +215,78 @@ bool validationIsEnabled() {
 void enableBuggify( bool enabled ) {
 	buggifyActivated = enabled;
 }
+
+TEST_CASE("/flow/FlatBuffers/ErrorOr") {
+	{
+		ErrorOr<int> in(worker_removed());
+		ErrorOr<int> out;
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(out.isError());
+		ASSERT(out.getError().code() == in.getError().code());
+	}
+	{
+		ErrorOr<uint32_t> in(g_random->randomUInt32());
+		ErrorOr<uint32_t> out;
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(!out.isError());
+		ASSERT(out.get() == in.get());
+	}
+	return Void();
+}
+
+TEST_CASE("/flow/FlatBuffers/Optional") {
+	{
+		Optional<int> in;
+		Optional<int> out;
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(!out.present());
+	}
+	{
+		Optional<uint32_t> in(g_random->randomUInt32());
+		Optional<uint32_t> out;
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(out.present());
+		ASSERT(out.get() == in.get());
+	}
+	return Void();
+}
+
+TEST_CASE("/flow/FlatBuffers/Standalone") {
+	{
+		Standalone<StringRef> in(std::string("foobar"));
+		StringRef out;
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(in == out);
+	}
+	{
+		StringRef in = LiteralStringRef("foobar");
+		Standalone<StringRef> out;
+		ObjectWriter writer;
+		writer.serialize(in);
+		Standalone<StringRef> copy = writer.toStringRef();
+		ArenaObjectReader reader(copy.arena(), copy);
+		reader.deserialize(out);
+		ASSERT(in == out);
+	}
+	return Void();
+}
diff --git a/flow/flow.h b/flow/flow.h
index 0aac95eb0e..8ae1a3afa5 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -191,13 +191,41 @@ public:
 
 	bool isError() const { return error.code() != invalid_error_code; }
 	bool isError(int code) const { return error.code() == code; }
-	Error getError() const { ASSERT(isError()); return error; }
+	const Error& getError() const { ASSERT(isError()); return error; }
 
 private:
 	typename std::aligned_storage< sizeof(T), __alignof(T) >::type value;
 	Error error;
 };
 
+template <class T>
+struct union_like_traits<ErrorOr<T>> : std::true_type {
+	using Member = ErrorOr<T>;
+	using alternatives = pack<Error, T>;
+	static uint8_t index(const Member& variant) { return variant.present() ? 1 : 0; }
+	static bool empty(const Member& variant) { return false; }
+
+	template <int i>
+	static const index_t<i, alternatives>& get(const Member& m) {
+		if constexpr (i == 0) {
+			return m.getError();
+		} else {
+			static_assert(i == 1, "ErrorOr only has two members");
+			return m.get();
+		}
+	}
+
+	template <int i, class Alternative>
+	static const void assign(Member& m, const Alternative& a) {
+		if constexpr (i == 0) {
+			m = a;
+		} else {
+			static_assert(i == 1);
+			m = a;
+		}
+	}
+};
+
 template <class T>
 struct Callback {
 	Callback<T> *prev, *next;

From 47fdb787827c924edebe63437a904f3e00441e11 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 31 Jan 2019 15:07:27 -0800
Subject: [PATCH 056/118] use EnsureTable for enums of enums

---
 fdbclient/MonitorLeader.actor.cpp | 8 ++++----
 fdbrpc/fdbrpc.h                   | 4 ++--
 fdbrpc/networksender.actor.h      | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 3696edec8b..fc1110cc5a 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -265,8 +265,8 @@ TEST_CASE("/flow/FlatBuffers/LeaderInfo") {
 		}
 		leaderInfo.serializedInfo = rndString;
 	}
-	ErrorOr<Optional<LeaderInfo>> objIn{ Optional<LeaderInfo>{leaderInfo} };
-	ErrorOr<Optional<LeaderInfo>> objOut;
+	ErrorOr<EnsureTable<Optional<LeaderInfo>>> objIn(leaderInfo);
+	ErrorOr<EnsureTable<Optional<LeaderInfo>>> objOut;
 	Standalone<StringRef> copy;
 	ObjectWriter writer;
 	writer.serialize(objIn);
@@ -275,8 +275,8 @@ TEST_CASE("/flow/FlatBuffers/LeaderInfo") {
 	reader.deserialize(objOut);
 
 	ASSERT(!objOut.isError());
-	ASSERT(objOut.get().present());
-	LeaderInfo outLeader = objOut.get().get();
+	ASSERT(objOut.get().asUnderlyingType().present());
+	LeaderInfo outLeader = objOut.get().asUnderlyingType().get();
 	ASSERT(outLeader.changeID == leaderInfo.changeID);
 	ASSERT(outLeader.forward == leaderInfo.forward);
 	ASSERT(outLeader.serializedInfo == leaderInfo.serializedInfo);
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index 54b3ee47d4..dbe98c3139 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -96,12 +96,12 @@ struct NetSAV : SAV<T>, FlowReceiver, FastAllocated<NetSAV<T>> {
 	virtual void receive(ArenaObjectReader& reader) {
 		if (!SAV<T>::canBeSet()) return;
 		this->addPromiseRef();
-		ErrorOr<T> message;
+		ErrorOr<EnsureTable<T>> message;
 		reader.deserialize(message);
 		if (message.isError()) {
 			SAV<T>::sendErrorAndDelPromiseRef(message.getError());
 		} else {
-			SAV<T>::sendAndDelPromiseRef(message.get());
+			SAV<T>::sendAndDelPromiseRef(message.get().asUnderlyingType());
 		}
 	}
 };
diff --git a/fdbrpc/networksender.actor.h b/fdbrpc/networksender.actor.h
index 629360d188..0d50dac302 100644
--- a/fdbrpc/networksender.actor.h
+++ b/fdbrpc/networksender.actor.h
@@ -35,7 +35,7 @@ void networkSender(Future<T> input, Endpoint endpoint) {
 	try {
 		T value = wait(input);
 		if (g_network->useObjectSerializer()) {
-			FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<T>>(ErrorOr<T>(value)), endpoint);
+			FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<EnsureTable<T>>>(value), endpoint);
 		} else {
 			FlowTransport::transport().sendUnreliable(SerializeBoolAnd<T>(true, value), endpoint, false);
 		}
@@ -43,7 +43,7 @@ void networkSender(Future<T> input, Endpoint endpoint) {
 		// if (err.code() == error_code_broken_promise) return;
 		ASSERT(err.code() != error_code_actor_cancelled);
 		if (g_network->useObjectSerializer()) {
-			FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<T>>(ErrorOr<T>(err)), endpoint);
+			FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<EnsureTable<T>>>(err), endpoint);
 		} else {
 			FlowTransport::transport().sendUnreliable(SerializeBoolAnd<Error>(false, err), endpoint, false);
 		}

From f0d81c6820027b5e0a8c684e744b513374e54a4b Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 21 Feb 2019 12:07:31 -0800
Subject: [PATCH 057/118] make EnsureTable work with streaming serializer

---
 flow/flat_buffers.h | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index de37bb4a50..a91fbf1129 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -1059,11 +1059,15 @@ struct EnsureTable {
 	EnsureTable(const T& t) : t(t) {}
 	template <class Archive>
 	void serialize(Archive& ar) {
-		if constexpr (detail::expect_serialize_member<T>) {
-			if constexpr (serializable_traits<T>::value) {
-				serializable_traits<T>::serialize(ar, t.get());
+		if constexpr (is_fb_function<Archive>) {
+			if constexpr (detail::expect_serialize_member<T>) {
+				if constexpr (serializable_traits<T>::value) {
+					serializable_traits<T>::serialize(ar, t.get());
+				} else {
+					t.get().serialize(ar);
+				}
 			} else {
-				t.get().serialize(ar);
+				serializer(ar, t.get());
 			}
 		} else {
 			serializer(ar, t.get());

From 3bc5ee8c58855cf6d9be1f575e976e714d2c8c3b Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 22 Feb 2019 09:33:54 -0800
Subject: [PATCH 058/118] fix indentation

---
 fdbrpc/FlowTransport.actor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index ba2fd5771e..15440b8066 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -558,9 +558,9 @@ ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader
 				ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll());
 				receiver->receive(objReader);
 			} else {
-                receiver->receive( reader );
-            }
-			g_currentDeliveryPeerAddress = {NetworkAddress()};
+				receiver->receive(reader);
+			}
+			g_currentDeliveryPeerAddress = { NetworkAddress() };
 		} catch (Error& e) {
 			g_currentDeliveryPeerAddress = {NetworkAddress()};
 			TraceEvent(SevError, "ReceiverError").error(e).detail("Token", destination.token.toString()).detail("Peer", destination.getPrimaryAddress());

From 43a7703a1ef05a52cc48e3b434fda3f8f162e279 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 22 Feb 2019 09:34:08 -0800
Subject: [PATCH 059/118] fix addFlag

---
 flow/Net2.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index 6185a60026..b792034183 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -65,7 +65,7 @@ uint64_t removeFlags(uint64_t version) {
 	return version & versionFlagMask;
 }
 uint64_t addObjectSerializerFlag(uint64_t version) {
-	return version | versionFlagMask;
+	return version | objectSerializerFlag;
 }
 bool hasObjectSerializerFlag(uint64_t version) {
 	return (version & objectSerializerFlag) > 0;

From 69fa3d390367103ba43fd0ce31e037b03af68ed8 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 9 Apr 2019 14:29:21 -0700
Subject: [PATCH 060/118] fixed compilation issues after rebase

---
 cmake/ConfigureCompiler.cmake     |   2 +-
 fdbclient/FDBTypes.h              |  19 ---
 fdbclient/MasterProxyInterface.h  |   2 +
 fdbrpc/FlowTransport.actor.cpp    |  14 +-
 fdbrpc/FlowTransport.h            |   6 +-
 fdbrpc/ReplicationPolicy.h        | 218 ++++++++++++++----------------
 fdbrpc/fdbrpc.h                   |   9 +-
 fdbrpc/simulator.h                |  12 +-
 fdbserver/LogSystemConfig.h       |  18 ++-
 fdbserver/RatekeeperInterface.h   |   4 +
 fdbserver/WorkerInterface.actor.h |   2 +
 flow/Arena.h                      |  19 +++
 flow/ObjectSerializerTraits.h     |  20 +++
 flow/network.cpp                  |  36 ++---
 flow/network.h                    |  81 +++++------
 15 files changed, 230 insertions(+), 232 deletions(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index e8f155a019..5258086b1d 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -52,7 +52,7 @@ if(WIN32)
   # see: https://docs.microsoft.com/en-us/windows/desktop/WinProg/using-the-windows-headers
   # this sets the windows target version to Windows 7
   set(WINDOWS_TARGET 0x0601)
-  add_compile_options(/W3 /EHsc /std:c++14 /bigobj $<$<CONFIG:Release>:/Zi> /MP)
+  add_compile_options(/W3 /EHsc /std:c++17 /bigobj $<$<CONFIG:Release>:/Zi> /MP)
   add_compile_definitions(_WIN32_WINNT=${WINDOWS_TARGET} BOOST_ALL_NO_LIB)
 else()
   if(USE_GOLD_LINKER)
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index f234ac4f25..d48324e8fd 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -745,25 +745,6 @@ struct AddressExclusion {
 	}
 };
 
-template <>
-struct scalar_traits<AddressExclusion> : std::true_type {
-	using ip_traits = scalar_traits<uint32_t>;
-	using port_traits = scalar_traits<int>;
-	constexpr static size_t size = ip_traits::size + port_traits::size;
-	static void save(uint8_t* buf, const AddressExclusion& value) {
-		ip_traits::save(buf, value.ip);
-		port_traits::save(buf + ip_traits::size, value.port);
-	}
-
-	// Context is an arbitrary type that is plumbed by reference throughout the
-	// load call tree.
-	template <class Context>
-	static void load(const uint8_t* buf, AddressExclusion& value, Context& context) {
-		ip_traits::load<Context>(buf, value.ip, context);
-		port_traits::load<Context>(buf + ip_traits::size, value.port, context);
-	}
-};
-
 static bool addressExcluded( std::set<AddressExclusion> const& exclusions, NetworkAddress const& addr ) {
 	return exclusions.count( AddressExclusion(addr.ip, addr.port) ) || exclusions.count( AddressExclusion(addr.ip) );
 }
diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h
index 9d51ae1c18..734760c7bf 100644
--- a/fdbclient/MasterProxyInterface.h
+++ b/fdbclient/MasterProxyInterface.h
@@ -252,6 +252,7 @@ struct TxnStateRequest {
 
 struct GetHealthMetricsRequest
 {
+	constexpr static FileIdentifier file_identifier = 11403900;
 	ReplyPromise<struct GetHealthMetricsReply> reply;
 	bool detailed;
 
@@ -266,6 +267,7 @@ struct GetHealthMetricsRequest
 
 struct GetHealthMetricsReply
 {
+	constexpr static FileIdentifier file_identifier = 11544290;
 	Standalone<StringRef> serialized;
 	HealthMetrics healthMetrics;
 
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 15440b8066..0f2608ba8b 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -1020,13 +1020,13 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 			what.serializeBinaryWriter(wr);
 			copy = wr.toStringRef();
 		} else {
-		BinaryWriter wr( AssumeVersion(currentProtocolVersion) );
-		// we don't need to send using an object writer here. This is a loopback delivery
-		// and therefore it is guaranteed that both versions will have exactly the
-		// same structures - so the backwards compatability capabilities are never needed
-		// here.
-		what.serializeBinaryWriter(wr);
-			copy = wr.toStringRef();
+			BinaryWriter wr( AssumeVersion(currentProtocolVersion) );
+			// we don't need to send using an object writer here. This is a loopback delivery
+			// and therefore it is guaranteed that both versions will have exactly the
+			// same structures - so the backwards compatability capabilities are never needed
+			// here.
+			what.serializeBinaryWriter(wr);
+			copy = wr.toValue();
 		}
 #if VALGRIND
 		VALGRIND_CHECK_MEM_IS_DEFINED(copy.begin(), copy.size());
diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h
index df3f0b73f5..5ff872821e 100644
--- a/fdbrpc/FlowTransport.h
+++ b/fdbrpc/FlowTransport.h
@@ -81,8 +81,8 @@ public:
 			}
 		} else {
 			if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061020001LL) {
-				addresses.resize(1);
-				serializer(ar, addresses[0], token);
+				addresses.secondaryAddress = Optional<NetworkAddress>();
+				serializer(ar, addresses.address, token);
 			} else {
 				serializer(ar, addresses, token);
 				if (ar.isDeserializing) {
@@ -170,8 +170,6 @@ public:
 
 	Endpoint loadedEndpoint(const UID& token);
 
-	void loadedEndpoint(Endpoint&);
-
 private:
 	class TransportData* self;
 };
diff --git a/fdbrpc/ReplicationPolicy.h b/fdbrpc/ReplicationPolicy.h
index 7963b5b1c1..d1fe4d8da6 100644
--- a/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/ReplicationPolicy.h
@@ -29,111 +29,99 @@ template <class Ar>
 void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy);
 extern void testReplicationPolicy(int nTests);
 
-
 struct IReplicationPolicy : public ReferenceCounted<IReplicationPolicy> {
-		IReplicationPolicy() {}
-		virtual ~IReplicationPolicy() {}
-		virtual std::string name() const = 0;
-		virtual std::string info() const = 0;
-		virtual void addref() { ReferenceCounted<IReplicationPolicy>::addref(); }
-		virtual void delref() { ReferenceCounted<IReplicationPolicy>::delref(); }
-		virtual int maxResults() const = 0;
-		virtual int depth() const = 0;
-		virtual bool selectReplicas(
-			Reference<LocalitySet> &										fromServers,
-			std::vector<LocalityEntry> const&		alsoServers,
-			std::vector<LocalityEntry>	&				results ) = 0;
-	    virtual void traceLocalityRecords(Reference<LocalitySet> const& fromServers);
-	    virtual void traceOneLocalityRecord(Reference<LocalityRecord> record, Reference<LocalitySet> const& fromServers);
-	    virtual bool validate(
-			std::vector<LocalityEntry>	const&	solutionSet,
-			Reference<LocalitySet> const&								fromServers ) const = 0;
+	IReplicationPolicy() {}
+	virtual ~IReplicationPolicy() {}
+	virtual std::string name() const = 0;
+	virtual std::string info() const = 0;
+	virtual void addref() { ReferenceCounted<IReplicationPolicy>::addref(); }
+	virtual void delref() { ReferenceCounted<IReplicationPolicy>::delref(); }
+	virtual int maxResults() const = 0;
+	virtual int depth() const = 0;
+	virtual bool selectReplicas(Reference<LocalitySet>& fromServers, std::vector<LocalityEntry> const& alsoServers,
+	                            std::vector<LocalityEntry>& results) = 0;
+	virtual void traceLocalityRecords(Reference<LocalitySet> const& fromServers);
+	virtual void traceOneLocalityRecord(Reference<LocalityRecord> record, Reference<LocalitySet> const& fromServers);
+	virtual bool validate(std::vector<LocalityEntry> const& solutionSet,
+	                      Reference<LocalitySet> const& fromServers) const = 0;
 
-		bool operator == ( const IReplicationPolicy& r ) const { return info() == r.info(); }
-		bool operator != ( const IReplicationPolicy& r ) const { return info() != r.info(); }
+	bool operator==(const IReplicationPolicy& r) const { return info() == r.info(); }
+	bool operator!=(const IReplicationPolicy& r) const { return info() != r.info(); }
 
-		template <class Ar>
-		void serialize(Ar& ar) {
-			Reference<IReplicationPolicy>	refThis(this);
-			serializeReplicationPolicy(ar, refThis);
-			refThis->delref_no_destroy();
-		}
-		virtual void deserializationDone() = 0;
+	template <class Ar>
+	void serialize(Ar& ar) {
+		Reference<IReplicationPolicy> refThis(this);
+		serializeReplicationPolicy(ar, refThis);
+		refThis->delref_no_destroy();
+	}
+	virtual void deserializationDone() = 0;
 
-		// Utility functions
-		bool selectReplicas(
-			Reference<LocalitySet> &										fromServers,
-			std::vector<LocalityEntry>	&				results );
-		bool validate(
-			Reference<LocalitySet> const&								solutionSet ) const;
-		bool validateFull(
-			bool																solved,
-			std::vector<LocalityEntry>	const&	solutionSet,
-			std::vector<LocalityEntry> const&		alsoServers,
-			Reference<LocalitySet> const&								fromServers );
+	// Utility functions
+	bool selectReplicas(Reference<LocalitySet>& fromServers, std::vector<LocalityEntry>& results);
+	bool validate(Reference<LocalitySet> const& solutionSet) const;
+	bool validateFull(bool solved, std::vector<LocalityEntry> const& solutionSet,
+	                  std::vector<LocalityEntry> const& alsoServers, Reference<LocalitySet> const& fromServers);
 
-		// Returns a set of the attributes that this policy uses in selection and validation.
-		std::set<std::string> attributeKeys() const
-		{ std::set<std::string> keys; this->attributeKeys(&keys); return keys; }
-		virtual void attributeKeys(std::set<std::string>*) const = 0;
+	// Returns a set of the attributes that this policy uses in selection and validation.
+	std::set<std::string> attributeKeys() const {
+		std::set<std::string> keys;
+		this->attributeKeys(&keys);
+		return keys;
+	}
+	virtual void attributeKeys(std::set<std::string>*) const = 0;
 };
 
 template <class Archive>
-inline void load( Archive& ar, Reference<IReplicationPolicy>& value ) {
+inline void load(Archive& ar, Reference<IReplicationPolicy>& value) {
 	bool present;
 	ar >> present;
 	if (present) {
 		serializeReplicationPolicy(ar, value);
-	}
-	else {
+	} else {
 		value.clear();
 	}
 }
 
 template <class Archive>
-inline void save( Archive& ar, const Reference<IReplicationPolicy>& value ) {
+inline void save(Archive& ar, const Reference<IReplicationPolicy>& value) {
 	bool present = (value.getPtr() != nullptr);
 	ar << present;
 	if (present) {
-		serializeReplicationPolicy(ar, (Reference<IReplicationPolicy>&) value);
+		serializeReplicationPolicy(ar, (Reference<IReplicationPolicy>&)value);
 	}
 }
 
 struct PolicyOne : IReplicationPolicy, public ReferenceCounted<PolicyOne> {
-	PolicyOne() {};
+	PolicyOne(){};
 	explicit PolicyOne(const PolicyOne& o) {}
-	virtual ~PolicyOne() {};
+	virtual ~PolicyOne(){};
 	virtual std::string name() const { return "One"; }
 	virtual std::string info() const { return "1"; }
 	virtual int maxResults() const { return 1; }
 	virtual int depth() const { return 1; }
-	virtual bool validate(
-		std::vector<LocalityEntry>	const&	solutionSet,
-		Reference<LocalitySet> const&				fromServers ) const;
-	virtual bool selectReplicas(
-		Reference<LocalitySet>	&						fromServers,
-		std::vector<LocalityEntry> const&		alsoServers,
-		std::vector<LocalityEntry>	&				results );
+	virtual bool validate(std::vector<LocalityEntry> const& solutionSet,
+	                      Reference<LocalitySet> const& fromServers) const;
+	virtual bool selectReplicas(Reference<LocalitySet>& fromServers, std::vector<LocalityEntry> const& alsoServers,
+	                            std::vector<LocalityEntry>& results);
 	template <class Ar>
-	void serialize(Ar& ar) {
-	}
+	void serialize(Ar& ar) {}
 	virtual void deserializationDone() {}
 	virtual void attributeKeys(std::set<std::string>* set) const override { return; }
 };
 
 struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross> {
 	friend struct serializable_traits<PolicyAcross*>;
-	PolicyAcross(int count, std::string const& attribKey, IRepPolicyRef const policy);
+	PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy);
 	explicit PolicyAcross();
 	explicit PolicyAcross(const PolicyAcross& other) : PolicyAcross(other._count, other._attribKey, other._policy) {}
 	virtual ~PolicyAcross();
 	virtual std::string name() const { return "Across"; }
 	virtual std::string info() const { return format("%s^%d x ", _attribKey.c_str(), _count) + _policy->info(); }
 	virtual int maxResults() const { return _count * _policy->maxResults(); }
-	virtual int depth() const  { return 1 + _policy->depth(); }
-	virtual bool validate(std::vector<LocalityEntry> const& solutionSet, LocalitySetRef const& fromServers) const;
-	virtual bool selectReplicas(LocalitySetRef& fromServers, std::vector<LocalityEntry> const& alsoServers,
-		std::vector<LocalityEntry>	&				results );
+	virtual int depth() const { return 1 + _policy->depth(); }
+	virtual bool validate(std::vector<LocalityEntry> const& solutionSet, Reference<LocalitySet> const& fromServers) const;
+	virtual bool selectReplicas(Reference<LocalitySet>& fromServers, std::vector<LocalityEntry> const& alsoServers,
+	                            std::vector<LocalityEntry>& results);
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -153,22 +141,21 @@ struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross>
 	}
 
 protected:
-	int																_count;
-	std::string												_attribKey;
-	IRepPolicyRef _policy;
+	int _count;
+	std::string _attribKey;
+	Reference<IReplicationPolicy> _policy;
 
 	// Cache temporary members
-	std::vector<AttribValue>					_usedValues;
-	std::vector<LocalityEntry>				_newResults;
-	LocalitySetRef _selected;
-	VectorRef<std::pair<int,int>>			_addedResults;
-	Arena															_arena;
+	std::vector<AttribValue> _usedValues;
+	std::vector<LocalityEntry> _newResults;
+	Reference<LocalitySet> _selected;
+	VectorRef<std::pair<int, int>> _addedResults;
+	Arena _arena;
 };
 
 struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
 	friend struct serializable_traits<PolicyAnd*>;
-	PolicyAnd(std::vector<IRepPolicyRef> policies): _policies(policies), _sortedPolicies(policies)
-	{
+	PolicyAnd(std::vector<Reference<IReplicationPolicy>> policies) : _policies(policies), _sortedPolicies(policies) {
 		// Sort the policy array
 		std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
 	}
@@ -177,9 +164,9 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
 	virtual ~PolicyAnd() {}
 	virtual std::string name() const { return "And"; }
 	virtual std::string info() const {
-		std::string	infoText;
+		std::string infoText;
 		for (auto& policy : _policies) {
-			infoText += ((infoText.length()) ? " & ("  : "(") + policy->info() + ")";
+			infoText += ((infoText.length()) ? " & (" : "(") + policy->info() + ")";
 		}
 		if (_policies.size()) infoText = "(" + infoText + ")";
 		return infoText;
@@ -201,27 +188,26 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
 		}
 		return depthMax;
 	}
-	virtual bool validate(
-		std::vector<LocalityEntry>	const&	solutionSet,
-		Reference<LocalitySet> const&				fromServers ) const;
+	virtual bool validate(std::vector<LocalityEntry> const& solutionSet,
+	                      Reference<LocalitySet> const& fromServers) const;
 
-	virtual bool selectReplicas(
-		Reference<LocalitySet>	&						fromServers,
-		std::vector<LocalityEntry> const&		alsoServers,
-		std::vector<LocalityEntry>	&				results );
+	virtual bool selectReplicas(Reference<LocalitySet>& fromServers, std::vector<LocalityEntry> const& alsoServers,
+	                            std::vector<LocalityEntry>& results);
 
-	static bool comparePolicy(const Reference<IReplicationPolicy>& rhs, const Reference<IReplicationPolicy>& lhs)
-	{ return (lhs->maxResults() < rhs->maxResults()) || (!(rhs->maxResults() < lhs->maxResults()) && (lhs->depth() < rhs->depth())); }
+	static bool comparePolicy(const Reference<IReplicationPolicy>& rhs, const Reference<IReplicationPolicy>& lhs) {
+		return (lhs->maxResults() < rhs->maxResults()) ||
+		       (!(rhs->maxResults() < lhs->maxResults()) && (lhs->depth() < rhs->depth()));
+	}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
 		int count = _policies.size();
 		serializer(ar, count);
 		_policies.resize(count);
-		for(int i = 0; i < count; i++) {
+		for (int i = 0; i < count; i++) {
 			serializeReplicationPolicy(ar, _policies[i]);
 		}
-		if(Ar::isDeserializing) {
+		if (Ar::isDeserializing) {
 			_sortedPolicies = _policies;
 			std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
 		}
@@ -232,12 +218,15 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
 		std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
 	}
 
-	virtual void attributeKeys(std::set<std::string> *set) const override
-	{ for (const Reference<IReplicationPolicy>& r : _policies) { r->attributeKeys(set); } }
+	virtual void attributeKeys(std::set<std::string>* set) const override {
+		for (const Reference<IReplicationPolicy>& r : _policies) {
+			r->attributeKeys(set);
+		}
+	}
 
 protected:
-	std::vector<Reference<IReplicationPolicy>>			_policies;
-	std::vector<Reference<IReplicationPolicy>>			_sortedPolicies;
+	std::vector<Reference<IReplicationPolicy>> _policies;
+	std::vector<Reference<IReplicationPolicy>> _sortedPolicies;
 };
 
 extern int testReplication();
@@ -279,8 +268,8 @@ extern int testReplication();
 	};
 
 template <>
-struct object_construction<IRepPolicyRef> {
-	using type = IRepPolicyRef;
+struct object_construction<Reference<IReplicationPolicy>> {
+	using type = Reference<IReplicationPolicy>;
 	type _impl;
 
 	object_construction() : _impl(new PolicyOne()){};
@@ -296,7 +285,7 @@ POLICY_CONSTRUCTION_WRAPPER(PolicyAcross);
 POLICY_CONSTRUCTION_WRAPPER(PolicyAnd);
 
 template <>
-struct FileIdentifierFor<IRepPolicyRef> {
+struct FileIdentifierFor<Reference<IReplicationPolicy>> {
 	static constexpr FileIdentifier value = 14695621;
 };
 
@@ -323,9 +312,9 @@ struct serializable_traits<PolicyAnd*> : std::true_type {
 };
 
 template <>
-struct serializable_traits<IRepPolicyRef> : std::true_type {
+struct serializable_traits<Reference<IReplicationPolicy>> : std::true_type {
 	template <class Archiver>
-	static void serialize(Archiver& ar, IRepPolicyRef& policy) {
+	static void serialize(Archiver& ar, Reference<IReplicationPolicy>& policy) {
 		::serializer(ar, policy.changePtrUnsafe());
 	}
 };
@@ -374,53 +363,42 @@ struct union_like_traits<IReplicationPolicy*> : std::true_type {
 	}
 };
 
-
 template <class Ar>
 void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
-	if(Ar::isDeserializing) {
+	if (Ar::isDeserializing) {
 		StringRef name;
 		serializer(ar, name);
 
-		if(name == LiteralStringRef("One")) {
+		if (name == LiteralStringRef("One")) {
 			PolicyOne* pointer = new PolicyOne();
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
-		}
-		else if(name == LiteralStringRef("Across")) {
+		} else if (name == LiteralStringRef("Across")) {
 			PolicyAcross* pointer = new PolicyAcross(0, "", Reference<IReplicationPolicy>());
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
-		}
-		else if(name == LiteralStringRef("And")) {
+		} else if (name == LiteralStringRef("And")) {
 			PolicyAnd* pointer = new PolicyAnd{};
 			pointer->serialize(ar);
 			policy = Reference<IReplicationPolicy>(pointer);
-		}
-		else if(name == LiteralStringRef("None")) {
+		} else if (name == LiteralStringRef("None")) {
 			policy = Reference<IReplicationPolicy>();
+		} else {
+			TraceEvent(SevError, "SerializingInvalidPolicyType").detailext("PolicyName", name);
 		}
-		else {
-			TraceEvent(SevError, "SerializingInvalidPolicyType")
-				.detail("PolicyName", name);
-		}
-	}
-	else {
+	} else {
 		std::string name = policy ? policy->name() : "None";
 		Standalone<StringRef> nameRef = StringRef(name);
 		serializer(ar, nameRef);
-		if(name == "One") {
+		if (name == "One") {
 			((PolicyOne*)policy.getPtr())->serialize(ar);
-		}
-		else if(name == "Across") {
+		} else if (name == "Across") {
 			((PolicyAcross*)policy.getPtr())->serialize(ar);
-		}
-		else if(name == "And") {
+		} else if (name == "And") {
 			((PolicyAnd*)policy.getPtr())->serialize(ar);
-		}
-		else if(name == "None") {}
-		else {
-			TraceEvent(SevError, "SerializingInvalidPolicyType")
-				.detail("PolicyName", name);
+		} else if (name == "None") {
+		} else {
+			TraceEvent(SevError, "SerializingInvalidPolicyType").detail("PolicyName", name);
 		}
 	}
 }
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index dbe98c3139..8f3923b82c 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -181,14 +181,14 @@ struct serializable_traits<ReplyPromise<T>> : std::true_type {
 	template<class Archiver>
 	static void serialize(Archiver& ar, ReplyPromise<T>& p) {
 		if constexpr (Archiver::isDeserializing) {
-			Endpoint endpoint;
-			serializer(ar, endpoint);
+			UID token;
+			serializer(ar, token);
+			auto endpoint = FlowTransport::transport().loadedEndpoint(token);
 			p = ReplyPromise<T>(endpoint);
 			networkSender(p.getFuture(), endpoint);
 		} else {
-			const auto& ep = p.getEndpoint();
+			const auto& ep = p.getEndpoint().token;
 			serializer(ar, ep);
-			ASSERT(!ep.getPrimaryAddress().isValid() || ep.getPrimaryAddress().isPublic()); // No re-serializing non-public addresses (the reply connection won't be available to any other process)
 		}
 	}
 };
@@ -422,7 +422,6 @@ struct serializable_traits<RequestStream<T>> : std::true_type {
 		if constexpr (Archiver::isDeserializing) {
 			Endpoint endpoint;
 			serializer(ar, endpoint);
-			FlowTransport::transport().loadedEndpoint(endpoint);
 			stream = RequestStream<T>(endpoint);
 		} else {
 			const auto& ep = stream.getEndpoint();
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index 02941f0c02..6c05a24b4f 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -69,12 +69,12 @@ public:
 		double fault_injection_p1, fault_injection_p2;
 
 		ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddressList addresses,
-					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder )
-            : name(name), locality(locality), startingClass(startingClass), useObjectSerializer(useObjectSerializer),
-              addresses(addresses), address(addresses.address), dataFolder(dataFolder),
-              network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
-              rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
-              fault_injection_r(0), machine(0), cleared(false) {}
+					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder, bool useObjectSerializer )
+			: name(name), locality(locality), startingClass(startingClass), useObjectSerializer(useObjectSerializer),
+			  addresses(addresses), address(addresses.address), dataFolder(dataFolder),
+			  network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
+			  rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
+			  fault_injection_r(0), machine(0), cleared(false) {}
 
 		Future<KillType> onShutdown() { return shutdownSignal.getFuture(); }
 
diff --git a/fdbserver/LogSystemConfig.h b/fdbserver/LogSystemConfig.h
index 2b5277e88a..e66dfbd03e 100644
--- a/fdbserver/LogSystemConfig.h
+++ b/fdbserver/LogSystemConfig.h
@@ -79,6 +79,7 @@ struct serializable_traits<OptionalInterface<Interface>> : std::true_type {
 
 
 struct TLogSet {
+	constexpr static FileIdentifier file_identifier = 6302317;
 	std::vector<OptionalInterface<TLogInterface>> tLogs;
 	std::vector<OptionalInterface<TLogInterface>> logRouters;
 	int32_t tLogWriteAntiQuorum, tLogReplicationFactor;
@@ -136,17 +137,23 @@ struct TLogSet {
 
 	template <class Ar>
 	void serialize( Ar& ar ) {
-		serializer(ar, tLogs, logRouters, tLogWriteAntiQuorum, tLogReplicationFactor, tLogPolicy, tLogLocalities, isLocal, locality, startVersion, satelliteTagLocations);
-		if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061030001LL) {
-			tLogVersion = TLogVersion::V2;
+		if constexpr (is_fb_function<Ar>) {
+			serializer(ar, tLogs, logRouters, tLogWriteAntiQuorum, tLogReplicationFactor, tLogPolicy, tLogLocalities,
+			           isLocal, locality, startVersion, satelliteTagLocations, tLogVersion);
 		} else {
-			serializer(ar, tLogVersion);
+			serializer(ar, tLogs, logRouters, tLogWriteAntiQuorum, tLogReplicationFactor, tLogPolicy, tLogLocalities, isLocal, locality, startVersion, satelliteTagLocations);
+			if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061030001LL) {
+				tLogVersion = TLogVersion::V2;
+			} else {
+				serializer(ar, tLogVersion);
+			}
+			ASSERT(tLogPolicy.getPtr() == nullptr || tLogVersion != TLogVersion::UNSET);
 		}
-		ASSERT(tLogPolicy.getPtr() == nullptr || tLogVersion != TLogVersion::UNSET);
 	}
 };
 
 struct OldTLogConf {
+	constexpr static FileIdentifier file_identifier = 16233772;
 	std::vector<TLogSet> tLogs;
 	Version epochEnd;
 	int32_t logRouterTags;
@@ -188,6 +195,7 @@ enum class LogSystemType {
 BINARY_SERIALIZABLE(LogSystemType);
 
 struct LogSystemConfig {
+	constexpr static FileIdentifier file_identifier = 16360847;
 	LogSystemType logSystemType;
 	std::vector<TLogSet> tLogs;
 	int32_t logRouterTags;
diff --git a/fdbserver/RatekeeperInterface.h b/fdbserver/RatekeeperInterface.h
index 8acec9500e..8d441219c6 100644
--- a/fdbserver/RatekeeperInterface.h
+++ b/fdbserver/RatekeeperInterface.h
@@ -26,6 +26,7 @@
 #include "fdbrpc/Locality.h"
 
 struct RatekeeperInterface {
+	constexpr static FileIdentifier file_identifier = 5983305;
 	RequestStream<ReplyPromise<Void>> waitFailure;
 	RequestStream<struct GetRateInfoRequest> getRateInfo;
 	RequestStream<struct HaltRatekeeperRequest> haltRatekeeper;
@@ -52,6 +53,7 @@ struct RatekeeperInterface {
 };
 
 struct GetRateInfoRequest {
+	constexpr static FileIdentifier file_identifier = 9068521;
 	UID requesterID;
 	int64_t totalReleasedTransactions;
 	int64_t batchReleasedTransactions;
@@ -69,6 +71,7 @@ struct GetRateInfoRequest {
 };
 
 struct GetRateInfoReply {
+	constexpr static FileIdentifier file_identifier = 7845006;
 	double transactionRate;
 	double batchTransactionRate;
 	double leaseDuration;
@@ -81,6 +84,7 @@ struct GetRateInfoReply {
 };
 
 struct HaltRatekeeperRequest {
+	constexpr static FileIdentifier file_identifier = 6997218;
 	UID requesterID;
 	ReplyPromise<Void> reply;
 
diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h
index 10eea4906e..3015e9633c 100644
--- a/fdbserver/WorkerInterface.actor.h
+++ b/fdbserver/WorkerInterface.actor.h
@@ -76,6 +76,7 @@ struct WorkerInterface {
 };
 
 struct WorkerDetails {
+	constexpr static FileIdentifier file_identifier = 9973980;
 	WorkerInterface interf;
 	ProcessClass processClass;
 	bool degraded;
@@ -178,6 +179,7 @@ struct InitializeDataDistributorRequest {
 };
 
 struct InitializeRatekeeperRequest {
+	constexpr static FileIdentifier file_identifier = 6416816;
 	UID reqId;
 	ReplyPromise<RatekeeperInterface> reply;
 
diff --git a/flow/Arena.h b/flow/Arena.h
index 53b096b207..731ff64379 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -349,6 +349,7 @@ inline void save( Archive& ar, const Arena& p ) {
 template <class T>
 class Optional {
 public:
+	static constexpr FileIdentifier file_identifier = (0x10 << 24) | FileIdentifierFor<T>::value;
 	Optional() : valid(false) {}
 	Optional(const Optional<T>& o) : valid(o.valid) {
 		if (valid) new (&value) T(o.get());
@@ -440,6 +441,24 @@ template<class T>
 struct Traceable<Optional<T>> : std::conditional<Traceable<T>::value, std::true_type, std::false_type>::type {
 	static std::string toString(const Optional<T>& value) {
 		return value.present() ? Traceable<T>::toString(value.get()) : "[not set]";
+    }
+};
+
+struct union_like_traits<Optional<T>> : std::true_type {
+	using Member = Optional<T>;
+	using alternatives = pack<T>;
+	static uint8_t index(const Member& variant) { return 0; }
+	static bool empty(const Member& variant) { return !variant.present(); }
+
+	template <int i>
+	static const T& get(const Member& variant) {
+		static_assert(i == 0);
+		return variant.get();
+	}
+
+	template <size_t i>
+	static const void assign(Member& member, const T& t) {
+		member = t;
 	}
 };
 
diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h
index 069146ca9f..af6025bfb8 100644
--- a/flow/ObjectSerializerTraits.h
+++ b/flow/ObjectSerializerTraits.h
@@ -26,6 +26,7 @@
 #include <memory>
 #include <functional>
 #include <vector>
+#include <variant>
 
 template <class T, typename = void>
 struct is_fb_function_t : std::false_type {};
@@ -176,4 +177,23 @@ struct struct_like_traits : std::false_type {
 	static void done(Member&, Context&);
 };
 
+template <class... Alternatives>
+struct union_like_traits<std::variant<Alternatives...>> : std::true_type {
+	using Member = std::variant<Alternatives...>;
+	using alternatives = pack<Alternatives...>;
+	static uint8_t index(const Member& variant) { return variant.index(); }
+	static bool empty(const Member& variant) { return false; }
+
+	template <int i>
+	static const index_t<i, alternatives>& get(const Member& variant) {
+		return std::get<index_t<i, alternatives>>(variant);
+	}
+
+	template <size_t i, class Alternative>
+	static const void assign(Member& member, const Alternative& a) {
+		static_assert(std::is_same_v<index_t<i, alternatives>, Alternative>);
+		member = a;
+	}
+};
+
 
diff --git a/flow/network.cpp b/flow/network.cpp
index d8af7e54d9..222079dd97 100644
--- a/flow/network.cpp
+++ b/flow/network.cpp
@@ -24,20 +24,8 @@
 #include "flow/flow.h"
 #include "flow/UnitTest.h"
 
-IPAddress::IPAddress() : isV6addr(false) {
-	addr.v4 = 0;
-}
-
-IPAddress::IPAddress(const IPAddressStore& v6addr) : isV6addr(true) {
-	addr.v6 = v6addr;
-}
-
-IPAddress::IPAddress(uint32_t v4addr) : isV6addr(false) {
-	addr.v4 = v4addr;
-}
-
 bool IPAddress::operator==(const IPAddress& rhs) const {
-	return isV6addr == rhs.isV6addr && (isV6addr ? addr.v6 == rhs.addr.v6 : addr.v4 == rhs.addr.v4);
+	return addr == rhs.addr;
 }
 
 bool IPAddress::operator!=(const IPAddress& addr) const {
@@ -45,20 +33,15 @@ bool IPAddress::operator!=(const IPAddress& addr) const {
 }
 
 bool IPAddress::operator<(const IPAddress& rhs) const {
-	if(isV6addr != rhs.isV6addr) {
-		return isV6addr < rhs.isV6addr;
-	}
-	if(isV6addr) {
-		return addr.v6 < rhs.addr.v6;
-	}
-	return addr.v4 < rhs.addr.v4;
+	return addr < rhs.addr;
 }
 
 std::string IPAddress::toString() const {
-	if (isV6addr) {
-		return boost::asio::ip::address_v6(addr.v6).to_string();
+	if (isV6()) {
+		return boost::asio::ip::address_v6(std::get<IPAddressStore>(addr)).to_string();
 	} else {
-		return format("%d.%d.%d.%d", (addr.v4 >> 24) & 0xff, (addr.v4 >> 16) & 0xff, (addr.v4 >> 8) & 0xff, addr.v4 & 0xff);
+		auto ip = std::get<uint32_t>(addr);
+		return format("%d.%d.%d.%d", (ip >> 24) & 0xff, (ip >> 16) & 0xff, (ip >> 8) & 0xff, ip & 0xff);
 	}
 }
 
@@ -72,10 +55,11 @@ Optional<IPAddress> IPAddress::parse(std::string str) {
 }
 
 bool IPAddress::isValid() const {
-	if (isV6addr) {
-		return std::any_of(addr.v6.begin(), addr.v6.end(), [](uint8_t part) { return part != 0; });
+	if (isV6()) {
+		const auto& ip = std::get<IPAddressStore>(addr);
+		return std::any_of(ip.begin(), ip.end(), [](uint8_t part) { return part != 0; });
 	}
-	return addr.v4 != 0;
+	return std::get<uint32_t>(addr) != 0;
 }
 
 NetworkAddress NetworkAddress::parse( std::string const& s ) {
diff --git a/flow/network.h b/flow/network.h
index a2b3eaa847..a1b7cc865f 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -25,6 +25,7 @@
 #include <array>
 #include <string>
 #include <stdint.h>
+#include <variant>
 #include "boost/asio.hpp"
 #include "flow/serialize.h"
 #include "flow/IRandom.h"
@@ -91,36 +92,18 @@ struct IPAddress {
 	static_assert(std::is_same<IPAddressStore, std::array<uint8_t, 16>>::value,
 	              "IPAddressStore must be std::array<uint8_t, 16>");
 
-	IPAddress();
-	explicit IPAddress(const IPAddressStore& v6addr);
-	explicit IPAddress(uint32_t v4addr);
+	IPAddress() : addr(uint32_t(0)) {}
+	explicit IPAddress(const IPAddressStore& v6addr) : addr(v6addr) {}
+	explicit IPAddress(uint32_t v4addr) : addr(v4addr) {}
 
-	bool isV6() const { return isV6addr; }
-	bool isV4() const { return !isV6addr; }
+	bool isV6() const { return std::holds_alternative<IPAddressStore>(addr); }
+	bool isV4() const { return !isV6(); }
 	bool isValid() const;
 
-	IPAddress(const IPAddress& rhs) : isV6addr(rhs.isV6addr) {
-		if(isV6addr) {
-			addr.v6 = rhs.addr.v6;
-		} else {
-			addr.v4 = rhs.addr.v4;
-		}
-	}
-
-	IPAddress& operator=(const IPAddress& rhs) {
-		isV6addr = rhs.isV6addr;
-		if(isV6addr) {
-			addr.v6 = rhs.addr.v6;
-		} else {
-			addr.v4 = rhs.addr.v4;
-		}
-		return *this;
-	}
-
 	// Returns raw v4/v6 representation of address. Caller is responsible
 	// to call these functions safely.
-	uint32_t toV4() const { return addr.v4; }
-	const IPAddressStore& toV6() const { return addr.v6; }
+	uint32_t toV4() const { return std::get<uint32_t>(addr); }
+	const IPAddressStore& toV6() const { return std::get<IPAddressStore>(addr); }
 
 	std::string toString() const;
 	static Optional<IPAddress> parse(std::string str);
@@ -131,20 +114,35 @@ struct IPAddress {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		serializer(ar, isV6addr);
-		if(isV6addr) {
-			serializer(ar, addr.v6);
+		if constexpr (is_fb_function<Ar>) {
+			serializer(ar, addr);
 		} else {
-			serializer(ar, addr.v4);
+			bool v6 = isV6();
+			serializer(ar, v6);
+			if (v6) {
+				serializer(ar, std::get<IPAddressStore>(addr));
+			} else {
+				serializer(ar, std::get<uint32_t>(addr));
+			}
 		}
 	}
 
 private:
-	bool isV6addr;
-	union {
-		uint32_t v4;
-		IPAddressStore v6;
-	} addr;
+	std::variant<uint32_t, IPAddressStore> addr;
+};
+
+template<>
+struct vector_like_traits<IPAddress::IPAddressStore> : std::true_type {
+	using type = IPAddress::IPAddressStore;
+	using value_type = uint8_t;
+	using iterator = typename type::const_iterator;
+	using insert_iterator = typename type::iterator;
+
+	static size_t num_entries(const type&) { return 16; }
+	template <class Context>
+	static void reserve(type&, size_t, Context&) {}
+	static insert_iterator insert(type& value) { return value.begin(); }
+	static iterator begin(const type& value) { return value.begin(); }
 };
 
 template<>
@@ -155,6 +153,7 @@ struct Traceable<IPAddress> : std::true_type {
 };
 
 struct NetworkAddress {
+	constexpr static FileIdentifier file_identifier = 14155727;
 	// A NetworkAddress identifies a particular running server (i.e. a TCP endpoint).
 	IPAddress ip;
 	uint16_t port;
@@ -192,12 +191,16 @@ struct NetworkAddress {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
-		if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061030001LL) {
-			uint32_t ipV4;
-			serializer(ar, ipV4, port, flags);
-			ip = IPAddress(ipV4);
-		} else {
+		if constexpr (is_fb_function<Ar>) {
 			serializer(ar, ip, port, flags);
+		} else {
+			if (ar.isDeserializing && ar.protocolVersion() < 0x0FDB00B061030001LL) {
+				uint32_t ipV4;
+				serializer(ar, ipV4, port, flags);
+				ip = IPAddress(ipV4);
+			} else {
+				serializer(ar, ip, port, flags);
+			}
 		}
 	}
 };

From 9c02354255d2e1a4443d3177484d0eef669d5610 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 9 Apr 2019 14:51:32 -0700
Subject: [PATCH 061/118] pass NDEBUG to sqlite to enable debug mode

---
 fdbserver/CMakeLists.txt | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index ccc2667b96..94e166ea83 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -171,7 +171,7 @@ set(FDBSERVER_SRCS
   workloads/WriteBandwidth.actor.cpp
   workloads/WriteDuringRead.actor.cpp)
 
-set(SQLITE_SRCS
+add_library(fdb_sqlite STATIC
   sqlite/btree.h
   sqlite/hash.h
   sqlite/sqlite3.h
@@ -179,9 +179,7 @@ set(SQLITE_SRCS
   sqlite/sqliteInt.h
   sqlite/sqliteLimit.h
   sqlite/sqlite3.amalgamation.c)
-
-add_library(sqlite ${SQLITE_SRCS})
-target_compile_definitions(sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
+target_compile_definitions(fdb_sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
 
 # Suppress warnings in sqlite since it's third party
 if(NOT WIN32)
@@ -200,7 +198,7 @@ add_flow_target(EXECUTABLE NAME fdbserver SRCS ${FDBSERVER_SRCS})
 target_include_directories(fdbserver PRIVATE
   ${CMAKE_CURRENT_BINARY_DIR}/workloads
   ${CMAKE_CURRENT_SOURCE_DIR}/workloads)
-target_link_libraries(fdbserver PRIVATE fdbclient sqlite)
+target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite)
 if(WITH_JAVA_WORKLOAD)
   if(NOT JNI_FOUND)
     message(SEND_ERROR "Trying to build Java workload but couldn't find JNI")

From 44db3450ec5701ead527a30b617d4c6c33503ef8 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Thu, 11 Apr 2019 13:24:00 -0700
Subject: [PATCH 062/118] Several flatbuffers bug fixes

---
 fdbclient/MonitorLeader.actor.cpp |  6 +++---
 fdbclient/MonitorLeader.h         | 12 +++++++-----
 fdbrpc/FlowTransport.actor.cpp    | 25 +++++++++++++++----------
 fdbserver/LeaderElection.h        | 10 +++++++---
 fdbserver/TLogInterface.h         |  4 +++-
 flow/ObjectSerializer.h           | 14 +++++++++++++-
 flow/flat_buffers.h               |  4 +++-
 flow/genericactors.actor.h        | 15 +++++++++++----
 flow/network.h                    |  8 ++++++--
 flow/serialize.h                  |  1 +
 10 files changed, 69 insertions(+), 30 deletions(-)

diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index fc1110cc5a..74337b2791 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -515,10 +515,10 @@ ACTOR Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> connF
 }
 
 ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> serializedInfo,
-                                                    Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
+													Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
 	state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
-	    new AsyncVar<Optional<ClusterControllerClientInterface>>{});
-	state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader);
+		new AsyncVar<Optional<ClusterControllerClientInterface>>{});
+	state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader, g_network->useObjectSerializer());
 	loop {
 		choose {
 			when(wait(deserializer)) { UNSTOPPABLE_ASSERT(false); }
diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h
index e635fe428a..62fcd61427 100644
--- a/fdbclient/MonitorLeader.h
+++ b/fdbclient/MonitorLeader.h
@@ -43,24 +43,26 @@ Future<Void> monitorLeaderInternal( Reference<ClusterConnectionFile> const& conn
 template <class LeaderInterface>
 struct LeaderDeserializer {
 	Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
-	                        const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
-		return asyncDeserialize(serializedInfo, outKnownLeader);
+							const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
+		return asyncDeserialize(serializedInfo, outKnownLeader, g_network->useObjectSerializer());
 	}
 };
 
 Future<Void> asyncDeserializeClusterInterface(const Reference<AsyncVar<Value>>& serializedInfo,
-                                              const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader);
+											  const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader);
 
 template <>
 struct LeaderDeserializer<ClusterInterface> {
 	Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
-	                        const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader) {
+							const Reference<AsyncVar<Optional<ClusterInterface>>>& outKnownLeader) {
 		return asyncDeserializeClusterInterface(serializedInfo, outKnownLeader);
 	}
 };
 
 template <class LeaderInterface>
-Future<Void> monitorLeader( Reference<ClusterConnectionFile> const& connFile, Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader, Reference<AsyncVar<int>> connectedCoordinatorsNum ) {
+Future<Void> monitorLeader(Reference<ClusterConnectionFile> const& connFile,
+						   Reference<AsyncVar<Optional<LeaderInterface>>> const& outKnownLeader,
+						   Reference<AsyncVar<int>> connectedCoordinatorsNum) {
 	LeaderDeserializer<LeaderInterface> deserializer;
 	Reference<AsyncVar<Value>> serializedInfo( new AsyncVar<Value> );
 	Future<Void> m = monitorLeaderInternal( connFile, serializedInfo, connectedCoordinatorsNum );
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 0f2608ba8b..d29d7b01d0 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -541,8 +541,7 @@ TransportData::~TransportData() {
 	}
 }
 
-ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader reader, bool inReadSocket,
-                          bool useFlatbuffers) {
+ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader reader, bool inReadSocket) {
 	int priority = self->endpoints.getPriority(destination.token);
 	if (priority < TaskReadSocket || !inReadSocket) {
 		wait( delay(0, priority) );
@@ -554,7 +553,9 @@ ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader
 	if (receiver) {
 		try {
 			g_currentDeliveryPeerAddress = destination.addresses;
-			if (useFlatbuffers) {
+			if (g_network->useObjectSerializer()) {
+				StringRef data = reader.arenaReadAll();
+				ASSERT(data.size() > 8);
 				ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll());
 				receiver->receive(objReader);
 			} else {
@@ -652,7 +653,7 @@ static void scanPackets(TransportData* transport, uint8_t*& unprocessed_begin, u
 #if VALGRIND
 		VALGRIND_CHECK_MEM_IS_DEFINED(p, packetLen);
 #endif
-		ArenaReader reader(arena, StringRef(p, packetLen), AssumeVersion(removeFlags(peerProtocolVersion)));
+		ArenaReader reader(arena, StringRef(p, packetLen), AssumeVersion(currentProtocolVersion));
 		UID token;
 		reader >> token;
 
@@ -669,7 +670,8 @@ static void scanPackets(TransportData* transport, uint8_t*& unprocessed_begin, u
 				transport->warnAlwaysForLargePacket = false;
 		}
 
-		deliver(transport, Endpoint({ peerAddress }, token), std::move(reader), true, hasObjectSerializerFlag(peerProtocolVersion));
+		ASSERT(!reader.empty());
+		deliver(transport, Endpoint({ peerAddress }, token), std::move(reader), true);
 
 		unprocessed_begin = p = p + packetLen;
 	}
@@ -693,7 +695,7 @@ ACTOR static Future<Void> connectionReader(
 	state bool incompatiblePeerCounted = false;
 	state bool incompatibleProtocolVersionNewer = false;
 	state NetworkAddress peerAddress;
-	state uint64_t peerProtocolVersion = 0;
+	state uint64_t peerProtocolVersion;
 
 	peerAddress = conn->getPeerAddress();
 	if (peer == nullptr) {
@@ -731,7 +733,8 @@ ACTOR static Future<Void> connectionReader(
 						serializer(pktReader, pkt);
 
 						uint64_t connectionId = pkt.connectionId;
-						if( (pkt.protocolVersion & compatibleProtocolVersionMask) != (currentProtocolVersion & compatibleProtocolVersionMask) ) {
+						if(g_network->useObjectSerializer() != hasObjectSerializerFlag(pkt.protocolVersion) ||
+						   (removeFlags(pkt.protocolVersion) & compatibleProtocolVersionMask) != (currentProtocolVersion & compatibleProtocolVersionMask)) {
 							incompatibleProtocolVersionNewer = pkt.protocolVersion > currentProtocolVersion;
 							NetworkAddress addr = pkt.canonicalRemotePort
 							                          ? NetworkAddress(pkt.canonicalRemoteIp(), pkt.canonicalRemotePort)
@@ -778,8 +781,8 @@ ACTOR static Future<Void> connectionReader(
 						unprocessed_begin += connectPacketSize;
 						expectConnectPacket = false;
 
-						peerProtocolVersion = protocolVersion;
 						if (peer != nullptr) {
+							peerProtocolVersion = protocolVersion;
 							// Outgoing connection; port information should be what we expect
 							TraceEvent("ConnectedOutgoing")
 							    .suppressFor(1.0)
@@ -791,7 +794,9 @@ ACTOR static Future<Void> connectionReader(
 								incompatiblePeerCounted = true;
 							}
 							ASSERT( pkt.canonicalRemotePort == peerAddress.port );
+							onConnected.send(peer);
 						} else {
+							peerProtocolVersion = protocolVersion;
 							if (pkt.canonicalRemotePort) {
 								peerAddress = NetworkAddress(pkt.canonicalRemoteIp(), pkt.canonicalRemotePort, true,
 								                             peerAddress.isTLS());
@@ -1032,8 +1037,8 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 		VALGRIND_CHECK_MEM_IS_DEFINED(copy.begin(), copy.size());
 #endif
 
-		deliver(self, destination, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false,
-		        g_network->useObjectSerializer());
+		ASSERT(copy.size() > 0);
+		deliver(self, destination, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false);
 
 		return (PacketID)NULL;
 	} else {
diff --git a/fdbserver/LeaderElection.h b/fdbserver/LeaderElection.h
index 57394a767c..8e90c53034 100644
--- a/fdbserver/LeaderElection.h
+++ b/fdbserver/LeaderElection.h
@@ -24,6 +24,7 @@
 
 #include "fdbrpc/fdbrpc.h"
 #include "fdbrpc/Locality.h"
+#include "fdbclient/FDBTypes.h"
 
 class ServerCoordinators;
 
@@ -57,9 +58,12 @@ Future<Void> tryBecomeLeader( ServerCoordinators const& coordinators,
 							  bool hasConnected,
 							  Reference<AsyncVar<ClusterControllerPriorityInfo>> const& asyncPriorityInfo)
 {
-	Reference<AsyncVar<Value>> serializedInfo( new AsyncVar<Value> );
-	Future<Void> m = tryBecomeLeaderInternal( coordinators, BinaryWriter::toValue(proposedInterface, IncludeVersion()), serializedInfo, hasConnected, asyncPriorityInfo );
-	return m || asyncDeserialize( serializedInfo, outKnownLeader );
+	Reference<AsyncVar<Value>> serializedInfo(new AsyncVar<Value>);
+	Future<Void> m = tryBecomeLeaderInternal(
+		coordinators,
+		g_network->useObjectSerializer() ? ObjectWriter::toValue(proposedInterface) : BinaryWriter::toValue(proposedInterface, IncludeVersion()),
+		serializedInfo, hasConnected, asyncPriorityInfo);
+	return m || asyncDeserialize(serializedInfo, outKnownLeader, g_network->useObjectSerializer());
 }
 
 #pragma endregion
diff --git a/fdbserver/TLogInterface.h b/fdbserver/TLogInterface.h
index 39e9907322..d642de6f44 100644
--- a/fdbserver/TLogInterface.h
+++ b/fdbserver/TLogInterface.h
@@ -65,7 +65,9 @@ struct TLogInterface {
 
 	template <class Ar> 
 	void serialize( Ar& ar ) {
-		ASSERT(ar.isDeserializing || uniqueID != UID());
+		if constexpr (!is_fb_function<Ar>) {
+			ASSERT(ar.isDeserializing || uniqueID != UID());
+		}
 		serializer(ar, uniqueID, sharedTLogID, locality, peekMessages, popMessages
 		  , commit, lock, getQueuingMetrics, confirmRunning, waitFailure, recoveryFinished);
 	}
diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h
index 8e54111894..e048c2aae2 100644
--- a/flow/ObjectSerializer.h
+++ b/flow/ObjectSerializer.h
@@ -106,7 +106,7 @@ public:
 		auto allocator = [this, &allocations](size_t size_) {
 			++allocations;
 			size = size_;
-			data = new uint8_t[size];
+			data = new (arena) uint8_t[size];
 			return data;
 		};
 		auto res = save_members(allocator, file_identifier, items...);
@@ -122,7 +122,19 @@ public:
 		return StringRef(data, size);
 	}
 
+	Standalone<StringRef> toString() const {
+		return Standalone<StringRef>(toStringRef(), arena);
+	}
+
+	template <class Item>
+	static Standalone<StringRef> toValue(Item const& item) {
+		ObjectWriter writer;
+		writer.serialize(item);
+		return writer.toString();
+	}
+
 private:
+	Arena arena;
 	uint8_t* data = nullptr;
 	int size = 0;
 };
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index a91fbf1129..3e0ff4d813 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -881,7 +881,9 @@ struct LoadSaveHelper {
 	auto save(const U& message, Writer& writer, const VTableSet*) {
 		constexpr auto size = scalar_traits<U>::size;
 		std::array<uint8_t, size> result = {};
-		scalar_traits<U>::save(&result[0], message);
+		if constexpr (size > 0) {
+			scalar_traits<U>::save(&result[0], message);
+		}
 		return result;
 	}
 
diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h
index c6e37f58c4..d2e3a57940 100644
--- a/flow/genericactors.actor.h
+++ b/flow/genericactors.actor.h
@@ -736,11 +736,18 @@ private:
 	}
 };
 
-ACTOR template <class T> Future<Void> asyncDeserialize( Reference<AsyncVar<Standalone<StringRef>>> input, Reference<AsyncVar<Optional<T>>> output ) {
+ACTOR template <class T> Future<Void> asyncDeserialize( Reference<AsyncVar<Standalone<StringRef>>> input, Reference<AsyncVar<Optional<T>>> output, bool useObjSerializer ) {
 	loop {
-		if (input->get().size())
-			output->set( BinaryReader::fromStringRef<T>( input->get(), IncludeVersion() ) );
-		else
+		if (input->get().size()) {
+			if (useObjSerializer) {
+				ObjectReader reader(input->get().begin());
+				T res;
+				reader.deserialize(res);
+				output->set(res);
+			} else {
+				output->set( BinaryReader::fromStringRef<T>( input->get(), IncludeVersion() ) );
+			}
+		} else
 			output->set( Optional<T>() );
 		wait( input->onChange() );
 	}
diff --git a/flow/network.h b/flow/network.h
index a1b7cc865f..9307ac56d4 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -120,9 +120,13 @@ struct IPAddress {
 			bool v6 = isV6();
 			serializer(ar, v6);
 			if (v6) {
-				serializer(ar, std::get<IPAddressStore>(addr));
+				IPAddressStore store;
+				serializer(ar, store);
+				addr = store;
 			} else {
-				serializer(ar, std::get<uint32_t>(addr));
+				uint32_t res;
+				serializer(ar, res);
+				addr = res;
 			}
 		}
 	}
diff --git a/flow/serialize.h b/flow/serialize.h
index 3d82e761f2..b7ddb3e6fc 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -725,6 +725,7 @@ struct MakeSerializeSource : ISerializeSource {
 		if (useObjectSerializer) {
 			ObjectWriter writer;
 			writer.serialize(get());
+			w.serializeBytes(writer.toStringRef());
 		} else {
 			static_cast<T const*>(this)->serialize(w);
 		}

From f5fa3a65b4ddd95538bc44365689663cfd443039 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 12 Apr 2019 14:30:46 -0700
Subject: [PATCH 063/118] some more fixes

---
 fdbrpc/FlowTransport.actor.cpp |   4 -
 fdbrpc/ReplicationPolicy.h     | 162 ++++++---------------------------
 flow/Arena.h                   |   1 +
 flow/flat_buffers.h            |  15 +++
 flow/network.h                 |  44 +++++----
 5 files changed, 64 insertions(+), 162 deletions(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index d29d7b01d0..26efa4c1a3 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -1026,10 +1026,6 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 			copy = wr.toStringRef();
 		} else {
 			BinaryWriter wr( AssumeVersion(currentProtocolVersion) );
-			// we don't need to send using an object writer here. This is a loopback delivery
-			// and therefore it is guaranteed that both versions will have exactly the
-			// same structures - so the backwards compatability capabilities are never needed
-			// here.
 			what.serializeBinaryWriter(wr);
 			copy = wr.toValue();
 		}
diff --git a/fdbrpc/ReplicationPolicy.h b/fdbrpc/ReplicationPolicy.h
index d1fe4d8da6..4b9cee1f10 100644
--- a/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/ReplicationPolicy.h
@@ -50,6 +50,7 @@ struct IReplicationPolicy : public ReferenceCounted<IReplicationPolicy> {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
+		static_assert(!is_fb_function<Ar>);
 		Reference<IReplicationPolicy> refThis(this);
 		serializeReplicationPolicy(ar, refThis);
 		refThis->delref_no_destroy();
@@ -104,7 +105,9 @@ struct PolicyOne : IReplicationPolicy, public ReferenceCounted<PolicyOne> {
 	virtual bool selectReplicas(Reference<LocalitySet>& fromServers, std::vector<LocalityEntry> const& alsoServers,
 	                            std::vector<LocalityEntry>& results);
 	template <class Ar>
-	void serialize(Ar& ar) {}
+	void serialize(Ar& ar) {
+		static_assert(!is_fb_function<Ar>);
+	}
 	virtual void deserializationDone() {}
 	virtual void attributeKeys(std::set<std::string>* set) const override { return; }
 };
@@ -125,6 +128,7 @@ struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross>
 
 	template <class Ar>
 	void serialize(Ar& ar) {
+		static_assert(!is_fb_function<Ar>);
 		serializer(ar, _attribKey, _count);
 		serializeReplicationPolicy(ar, _policy);
 	}
@@ -201,6 +205,7 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
 
 	template <class Ar>
 	void serialize(Ar& ar) {
+		static_assert(!is_fb_function<Ar>);
 		int count = _policies.size();
 		serializer(ar, count);
 		_policies.resize(count);
@@ -229,140 +234,6 @@ protected:
 	std::vector<Reference<IReplicationPolicy>> _sortedPolicies;
 };
 
-extern int testReplication();
-
-#define POLICY_CONSTRUCTION_WRAPPER(t)                                                                                 \
-	template <>                                                                                                        \
-	struct object_construction<t*> {                                                                                   \
-		using type = t*;                                                                                               \
-		type obj;                                                                                                      \
-                                                                                                                       \
-		object_construction() : obj(new t()) {}                                                                        \
-		object_construction(object_construction&& other) : obj(other.obj) { other.obj = nullptr; }                     \
-		object_construction(const object_construction& other) : obj() {}                                               \
-                                                                                                                       \
-		object_construction& operator=(object_construction&& other) {                                                  \
-			if (obj != nullptr) {                                                                                      \
-				delete obj;                                                                                            \
-			}                                                                                                          \
-			obj = other.obj;                                                                                           \
-			other.obj = nullptr;                                                                                       \
-			return *this;                                                                                              \
-		}                                                                                                              \
-                                                                                                                       \
-		object_construction& operator=(const object_construction& other) {                                             \
-			if (obj != nullptr) {                                                                                      \
-				delete obj;                                                                                            \
-			}                                                                                                          \
-			obj = new t(*other.obj);                                                                                   \
-			return *this;                                                                                              \
-		}                                                                                                              \
-                                                                                                                       \
-		type& get() { return obj; }                                                                                    \
-		const type& get() const { return obj; }                                                                        \
-		type move() {                                                                                                  \
-			auto res = obj;                                                                                            \
-			obj = nullptr;                                                                                             \
-			return res;                                                                                                \
-		}                                                                                                              \
-	};
-
-template <>
-struct object_construction<Reference<IReplicationPolicy>> {
-	using type = Reference<IReplicationPolicy>;
-	type _impl;
-
-	object_construction() : _impl(new PolicyOne()){};
-
-	type& get() { return _impl; }
-	const type& get() const { return _impl; }
-
-	type move() { return std::move(_impl); }
-};
-
-POLICY_CONSTRUCTION_WRAPPER(PolicyOne);
-POLICY_CONSTRUCTION_WRAPPER(PolicyAcross);
-POLICY_CONSTRUCTION_WRAPPER(PolicyAnd);
-
-template <>
-struct FileIdentifierFor<Reference<IReplicationPolicy>> {
-	static constexpr FileIdentifier value = 14695621;
-};
-
-template <>
-struct serializable_traits<PolicyOne*> : std::true_type {
-	template <class Archiver>
-	static void serialize(Archiver& ar, PolicyOne*& p) {}
-};
-
-template <>
-struct serializable_traits<PolicyAcross*> : std::true_type {
-	template <class Archiver>
-	static void serialize(Archiver& ar, PolicyAcross*& p) {
-		::serializer(ar, p->_count, p->_attribKey, p->_policy);
-	}
-};
-
-template <>
-struct serializable_traits<PolicyAnd*> : std::true_type {
-	template <class Archiver>
-	static void serialize(Archiver& ar, PolicyAnd*& p) {
-		::serializer(ar, p->_policies);
-	}
-};
-
-template <>
-struct serializable_traits<Reference<IReplicationPolicy>> : std::true_type {
-	template <class Archiver>
-	static void serialize(Archiver& ar, Reference<IReplicationPolicy>& policy) {
-		::serializer(ar, policy.changePtrUnsafe());
-	}
-};
-
-template <>
-struct union_like_traits<IReplicationPolicy*> : std::true_type {
-	using Member = IReplicationPolicy*;
-	using alternatives = pack<PolicyOne*, PolicyAcross*, PolicyAnd*>;
-
-	static uint8_t index(const Member& policy) {
-		if (policy->name() == "One") {
-			return 0;
-		} else if (policy->name() == "And") {
-			return 2;
-		} else {
-			return 1;
-		}
-	}
-
-	static bool empty(const Member& policy) { return policy == nullptr; }
-
-	template <int i>
-	static const index_t<i, alternatives>& get(IReplicationPolicy* const& member) {
-		if constexpr (i == 0) {
-			return reinterpret_cast<PolicyOne* const&>(member);
-		} else if constexpr (i == 1) {
-			return reinterpret_cast<PolicyAcross* const&>(member);
-		} else {
-			return reinterpret_cast<PolicyAnd* const&>(member);
-		}
-	}
-
-	template <int i, class Alternative>
-	static const void assign(Member& policy, const Alternative& impl) {
-		if (policy != nullptr) {
-			policy->delref();
-		}
-		policy = impl;
-	}
-
-	template <class Context>
-	static void done(Member& policy, Context&) {
-		if (policy != nullptr) {
-			policy->deserializationDone();
-		}
-	}
-};
-
 template <class Ar>
 void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
 	if (Ar::isDeserializing) {
@@ -403,4 +274,25 @@ void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
 	}
 }
 
+template <>
+struct dynamic_size_traits<Reference<IReplicationPolicy>> : std::true_type {
+	static WriteRawMemory save(const Reference<IReplicationPolicy>& value) {
+		BinaryWriter writer(IncludeVersion());
+		serializeReplicationPolicy(writer, const_cast<Reference<IReplicationPolicy>&>(value));
+		std::unique_ptr<uint8_t[]> memory(new uint8_t[writer.getLength()]);
+		memcpy(memory.get(), writer.getData(), writer.getLength());
+		return std::make_pair<OwnershipErasedPtr<const uint8_t>, size_t>(ownedPtr(const_cast<const uint8_t*>(memory.release())), writer.getLength());
+	}
+
+	// Context is an arbitrary type that is plumbed by reference throughout the
+	// load call tree.
+	template <class Context>
+	static void load(const uint8_t* buf, size_t sz, Reference<IReplicationPolicy>& value, Context&) {
+		StringRef str(buf, sz);
+		BinaryReader reader(str, IncludeVersion());
+		serializeReplicationPolicy(reader, value);
+	}
+};
+
+
 #endif
diff --git a/flow/Arena.h b/flow/Arena.h
index 731ff64379..157fbdaf50 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -514,6 +514,7 @@ public:
 		//T tmp;
 		//ar >> tmp;
 		//*this = tmp;
+		static_assert(!is_fb_function<Archive>);
 		serializer(ar, (*(T*)this), arena());
 	}
 
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 3e0ff4d813..012d55d390 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -123,6 +123,21 @@ struct vector_like_traits<std::vector<T, Alloc>> : std::true_type {
 	static iterator begin(const Vec& v) { return v.begin(); }
 };
 
+template<class T, size_t N>
+struct vector_like_traits<std::array<T, N>> : std::true_type {
+	using Vec = std::array<T, N>;
+	using value_type = typename Vec::value_type;
+	using iterator = typename Vec::const_iterator;
+	using insert_iterator = typename Vec::iterator;
+
+	static size_t num_entries(const Vec& v) { return N; }
+	template <class Context>
+	static void reserve(Vec& v, size_t size, Context&) {
+	}
+	static insert_iterator insert(Vec& v) { return v.begin(); }
+	static iterator begin(const Vec& v) { return v.begin(); }
+};
+
 template <class Key, class T, class Compare, class Allocator>
 struct vector_like_traits<std::map<Key, T, Compare, Allocator>> : std::true_type {
 	using Vec = std::map<Key, T, Compare, Allocator>;
diff --git a/flow/network.h b/flow/network.h
index 9307ac56d4..4b33c9c8c9 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -117,16 +117,28 @@ struct IPAddress {
 		if constexpr (is_fb_function<Ar>) {
 			serializer(ar, addr);
 		} else {
-			bool v6 = isV6();
-			serializer(ar, v6);
-			if (v6) {
-				IPAddressStore store;
-				serializer(ar, store);
-				addr = store;
+			if (Ar::isDeserializing) {
+				bool v6;
+				serializer(ar, v6);
+				if (v6) {
+					IPAddressStore store;
+					serializer(ar, store);
+					addr = store;
+				} else {
+					uint32_t res;
+					serializer(ar, res);
+					addr = res;
+				}
 			} else {
-				uint32_t res;
-				serializer(ar, res);
-				addr = res;
+				bool v6 = isV6();
+				serializer(ar, v6);
+				if (v6) {
+					auto res = toV6();
+					serializer(ar, res);
+				} else {
+					auto res = toV4();
+					serializer(ar, res);
+				}
 			}
 		}
 	}
@@ -135,20 +147,6 @@ private:
 	std::variant<uint32_t, IPAddressStore> addr;
 };
 
-template<>
-struct vector_like_traits<IPAddress::IPAddressStore> : std::true_type {
-	using type = IPAddress::IPAddressStore;
-	using value_type = uint8_t;
-	using iterator = typename type::const_iterator;
-	using insert_iterator = typename type::iterator;
-
-	static size_t num_entries(const type&) { return 16; }
-	template <class Context>
-	static void reserve(type&, size_t, Context&) {}
-	static insert_iterator insert(type& value) { return value.begin(); }
-	static iterator begin(const type& value) { return value.begin(); }
-};
-
 template<>
 struct Traceable<IPAddress> : std::true_type {
 	static std::string toString(const IPAddress& value) {

From 186bff4ae5501801bb2f86de39fb679a0536668e Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 12 Apr 2019 17:19:31 -0700
Subject: [PATCH 064/118] removed unnecessary assert and added test

---
 flow/Arena.h          |  1 -
 flow/flat_buffers.cpp | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/flow/Arena.h b/flow/Arena.h
index 157fbdaf50..731ff64379 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -514,7 +514,6 @@ public:
 		//T tmp;
 		//ar >> tmp;
 		//*this = tmp;
-		static_assert(!is_fb_function<Archive>);
 		serializer(ar, (*(T*)this), arena());
 	}
 
diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
index 760ad2f8c1..81acd94c14 100644
--- a/flow/flat_buffers.cpp
+++ b/flow/flat_buffers.cpp
@@ -507,4 +507,22 @@ TEST_CASE("/flow/FlatBuffers/VectorRef") {
 	return Void();
 }
 
+TEST_CASE("/flow/FlatBuffers/Standalone") {
+	Standalone<VectorRef<StringRef>> vecIn;
+	auto numElements = g_random->randomInt(1, 20);
+	for (int i = 0; i < numElements; ++i) {
+		auto str = g_random->randomAlphaNumeric(g_random->randomInt(0, 30));
+		vecIn.push_back(vecIn.arena(), StringRef(vecIn.arena(), str));
+	}
+	Standalone<StringRef> value = ObjectWriter::toValue(vecIn);
+	ArenaObjectReader reader(value.arena(), value);
+	VectorRef<Standalone<StringRef>> vecOut;
+	reader.deserialize(vecOut);
+	ASSERT(vecOut.size() == vecIn.size());
+	for (int i = 0; i < vecOut.size(); ++i) {
+		ASSERT(vecOut[i] == vecIn[i]);
+	}
+	return Void();
+}
+
 } // namespace unit_tests

From f6fbad506128944dced54045da662786136e06c1 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Sat, 13 Apr 2019 09:51:01 -0700
Subject: [PATCH 065/118] Fix memory bug

---
 fdbserver/tester.actor.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 343eb904e7..1157fcc754 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -757,12 +757,16 @@ ACTOR Future<Void> checkConsistency(Database cx, std::vector< TesterInterface >
 	}
 	
 	Standalone<VectorRef<KeyValueRef>> options;
+	StringRef performQuiescent = LiteralStringRef("false");
+	if (doQuiescentCheck) {
+		performQuiescent = LiteralStringRef("true");
+	}
 	spec.title = LiteralStringRef("ConsistencyCheck");
 	spec.databasePingDelay = databasePingDelay;
 	spec.timeout = 32000;
 	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("testName"), LiteralStringRef("ConsistencyCheck")));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("performQuiescentChecks"), ValueRef(format("%s", LiteralStringRef(doQuiescentCheck ? "true" : "false").toString().c_str()))));
-	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("quiescentWaitTimeout"), ValueRef(format("%f", quiescentWaitTimeout))));
+	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("performQuiescentChecks"), performQuiescent));
+	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("quiescentWaitTimeout"), ValueRef(options.arena(), format("%f", quiescentWaitTimeout))));
 	options.push_back_deep(options.arena(), KeyValueRef(LiteralStringRef("distributed"), LiteralStringRef("false")));
 	spec.options.push_back_deep(spec.options.arena(), options);
 

From 06745568f71e4961d2e36a925fd4530359f43872 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Sat, 13 Apr 2019 09:52:04 -0700
Subject: [PATCH 066/118] Fixed Tag serialization

---
 fdbclient/FDBTypes.h | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index d48324e8fd..837c500771 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -73,20 +73,29 @@ struct Tag {
 template <class Ar> void load( Ar& ar, Tag& tag ) { tag.serialize_unversioned(ar); }
 template <class Ar> void save( Ar& ar, Tag const& tag ) { const_cast<Tag&>(tag).serialize_unversioned(ar); }
 
-template<>
-struct scalar_traits<Tag> : std::true_type {
-	using locality_trait = scalar_traits<decltype(std::declval<Tag>().locality)>;
-	using id_trait = scalar_traits<decltype(std::declval<Tag>().id)>;
-	constexpr static size_t size = locality_trait::size + id_trait::size;
-	static void save(uint8_t* out, const Tag& tag) {
-		locality_trait::save(out, tag.locality);
-		id_trait::save(out + id_trait::size, tag.id);
+template <>
+struct struct_like_traits<Tag> : std::true_type {
+	using Member = Tag;
+	using types = pack<uint16_t, int8_t>;
+
+	template <int i>
+	static const index_t<i, types>& get(const Member& m) {
+		if constexpr (i == 0) {
+			return m.id;
+		} else {
+			static_assert(i == 1);
+			return m.locality;
+		}
 	}
 
-	template <class Context>
-	static void load(const uint8_t* in, Tag& tag, Context& context) {
-		locality_trait::load(in, tag.locality, context);
-		id_trait::load(in + locality_trait::size, tag.id, context);
+	template <int i, class Type>
+	static const void assign(Member& m, const Type& t) {
+		if constexpr (i == 0) {
+			m.id = t;
+		} else {
+			static_assert(i == 1);
+			m.locality = t;
+		}
 	}
 };
 

From 42385c2f81ea8ad191d5f8b521822f223fb05561 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Sat, 13 Apr 2019 10:05:59 -0700
Subject: [PATCH 067/118] Fixed issues introduced during rebase

---
 fdbrpc/ReplicationPolicy.h           | 2 +-
 fdbserver/SimulatedCluster.actor.cpp | 2 +-
 flow/Arena.h                         | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/fdbrpc/ReplicationPolicy.h b/fdbrpc/ReplicationPolicy.h
index 4b9cee1f10..0f4f350cae 100644
--- a/fdbrpc/ReplicationPolicy.h
+++ b/fdbrpc/ReplicationPolicy.h
@@ -255,7 +255,7 @@ void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
 		} else if (name == LiteralStringRef("None")) {
 			policy = Reference<IReplicationPolicy>();
 		} else {
-			TraceEvent(SevError, "SerializingInvalidPolicyType").detailext("PolicyName", name);
+			TraceEvent(SevError, "SerializingInvalidPolicyType").detail("PolicyName", name);
 		}
 	} else {
 		std::string name = policy ? policy->name() : "None";
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 95803d46ab..464fc5585c 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -411,7 +411,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr, std::vector
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
 				Reference<ClusterConnectionFile> clusterFile(useSeedFile ? new ClusterConnectionFile(path, connStr.toString()) : new ClusterConnectionFile(path));
 				const int listenPort = i*listenPerProcess + 1;
-				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, tlsOptions, listenPort, listenPerProcess, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents, useObjectSerializer));
+				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, tlsOptions, listenPort, listenPerProcess, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents, objSerializer));
 				TraceEvent("SimulatedMachineProcess", randomId).detail("Address", NetworkAddress(ips[i], listenPort, true, false)).detail("ZoneId", localities.zoneId()).detail("DataHall", localities.dataHallId()).detail("Folder", myFolders[i]);
 			}
 
diff --git a/flow/Arena.h b/flow/Arena.h
index 731ff64379..37dfcc967b 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -441,9 +441,10 @@ template<class T>
 struct Traceable<Optional<T>> : std::conditional<Traceable<T>::value, std::true_type, std::false_type>::type {
 	static std::string toString(const Optional<T>& value) {
 		return value.present() ? Traceable<T>::toString(value.get()) : "[not set]";
-    }
+	}
 };
 
+template<class T>
 struct union_like_traits<Optional<T>> : std::true_type {
 	using Member = Optional<T>;
 	using alternatives = pack<T>;

From 20c3f7f26427dde45bb9d43b21ffff1bc91a2055 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Sun, 14 Apr 2019 11:11:08 -0700
Subject: [PATCH 068/118] remove mixed-mode support

---
 fdbrpc/sim2.actor.cpp                | 20 ++++++++------
 fdbrpc/simulator.h                   | 11 ++++----
 fdbserver/SimulatedCluster.actor.cpp | 41 ++++++++++------------------
 fdbserver/SimulatedCluster.h         |  2 +-
 fdbserver/fdbserver.actor.cpp        | 25 ++++++-----------
 5 files changed, 41 insertions(+), 58 deletions(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 4ef1a91494..073cf38a79 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -979,7 +979,7 @@ public:
 	}
 	virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess,
 	                                LocalityData locality, ProcessClass startingClass, const char* dataFolder,
-	                                const char* coordinationFolder, bool objSerializer) {
+	                                const char* coordinationFolder) {
 		ASSERT( locality.machineId().present() );
 		MachineInfo& machine = machines[ locality.machineId().get() ];
 		if (!machine.machineId.present())
@@ -1001,7 +1001,7 @@ public:
 		// These files must live on after process kills for sim purposes.
 		if( machine.machineProcess == 0 ) {
 			NetworkAddress machineAddress(ip, 0, false, false);
-			machine.machineProcess = new ProcessInfo("Machine", locality, startingClass, {machineAddress}, this, "", "", objSerializer);
+			machine.machineProcess = new ProcessInfo("Machine", locality, startingClass, {machineAddress}, this, "", "");
 			machine.machineProcess->machine = &machine;
 		}
 
@@ -1011,7 +1011,7 @@ public:
 			addresses.secondaryAddress = NetworkAddress(ip, port+1, true, false);
 		}
 
-		ProcessInfo* m = new ProcessInfo(name, locality, startingClass, addresses, this, dataFolder, coordinationFolder, objSerializer);
+		ProcessInfo* m = new ProcessInfo(name, locality, startingClass, addresses, this, dataFolder, coordinationFolder);
 		for (int processPort = port; processPort < port + listenPerProcess; ++processPort) {
 			NetworkAddress address(ip, processPort, true, false); // SOMEDAY see above about becoming SSL!
 			m->listenerMap[address] = Reference<IListener>( new Sim2Listener(m, address) );
@@ -1078,6 +1078,10 @@ public:
 		return primaryTLogsDead || primaryProcessesDead.validate(storagePolicy);
 	}
 
+	virtual bool useObjectSerializer() const {
+		return net2->useObjectSerializer();
+	}
+
 	// The following function will determine if the specified configuration of available and dead processes can allow the cluster to survive
 	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const
 	{
@@ -1570,10 +1574,10 @@ public:
 		machines.erase(machineId);
 	}
 
-	Sim2() : time(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(-1) {
+	Sim2(bool objSerializer) : time(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(-1) {
 		// Not letting currentProcess be NULL eliminates some annoying special cases
-		currentProcess = new ProcessInfo("NoMachine", LocalityData(Optional<Standalone<StringRef>>(), StringRef(), StringRef(), StringRef()), ProcessClass(), {NetworkAddress()}, this, "", "", false);
-		g_network = net2 = newNet2(false, true);
+		currentProcess = new ProcessInfo("NoMachine", LocalityData(Optional<Standalone<StringRef>>(), StringRef(), StringRef(), StringRef()), ProcessClass(), {NetworkAddress()}, this, "", "");
+		g_network = net2 = newNet2(false, true, objSerializer);
 		Net2FileSystem::newFileSystem();
 		check_yield(0);
 	}
@@ -1678,9 +1682,9 @@ public:
 	int yield_limit;  // how many more times yield may return false before next returning true
 };
 
-void startNewSimulator() {
+void startNewSimulator(bool objSerializer) {
 	ASSERT( !g_network );
-	g_network = g_pSimulator = new Sim2();
+	g_network = g_pSimulator = new Sim2(objSerializer);
 	g_simulator.connectionFailuresDisableDuration = g_random->random01() < 0.5 ? 0 : 1e6;
 }
 
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index 6c05a24b4f..cdd0a7744b 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -53,7 +53,6 @@ public:
 		NetworkAddress address;
 		LocalityData	locality;
 		ProcessClass startingClass;
-		bool useObjectSerializer;
 		TDMetricCollection tdmetrics;
 		std::map<NetworkAddress, Reference<IListener>> listenerMap;
 		bool failed;
@@ -69,8 +68,8 @@ public:
 		double fault_injection_p1, fault_injection_p2;
 
 		ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddressList addresses,
-					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder, bool useObjectSerializer )
-			: name(name), locality(locality), startingClass(startingClass), useObjectSerializer(useObjectSerializer),
+					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder )
+			: name(name), locality(locality), startingClass(startingClass),
 			  addresses(addresses), address(addresses.address), dataFolder(dataFolder),
 			  network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
 			  rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
@@ -143,7 +142,7 @@ public:
 
 	virtual ProcessInfo* newProcess(const char* name, IPAddress ip, uint16_t port, uint16_t listenPerProcess,
 	                                LocalityData locality, ProcessClass startingClass, const char* dataFolder,
-	                                const char* coordinationFolder, bool useObjectSerializer) = 0;
+	                                const char* coordinationFolder) = 0;
 	virtual void killProcess( ProcessInfo* machine, KillType ) = 0;
 	virtual void rebootProcess(Optional<Standalone<StringRef>> zoneId, bool allProcesses ) = 0;
 	virtual void rebootProcess( ProcessInfo* process, KillType kt ) = 0;
@@ -156,7 +155,7 @@ public:
 	virtual bool isAvailable() const = 0;
 	virtual bool datacenterDead(Optional<Standalone<StringRef>> dcId) const = 0;
 	virtual void displayWorkers() const;
-	virtual bool useObjectSerializer() const { return currentProcess->useObjectSerializer; }
+	virtual bool useObjectSerializer() const = 0;
 
 	virtual void addRole(NetworkAddress const& address, std::string const& role) {
 		roleAddresses[address][role] ++;
@@ -328,7 +327,7 @@ private:
 extern ISimulator* g_pSimulator;
 #define g_simulator (*g_pSimulator)
 
-void startNewSimulator();
+void startNewSimulator(bool useObjectSerializer);
 
 //Parameters used to simulate disk performance
 struct DiskParameters : ReferenceCounted<DiskParameters> {
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 464fc5585c..313b060b12 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -121,16 +121,6 @@ static void simInitTLS(Reference<TLSOptions> tlsOptions) {
 	tlsOptions->register_network();
 }
 
-static bool useObjectSerializer(int b) {
-	if (b == 0) {
-		return false;
-	} else if (b == 1) {
-		return true;
-	} else {
-		return g_random->coinflip();
-	}
-}
-
 ACTOR Future<Void> runBackup( Reference<ClusterConnectionFile> connFile ) {
 	state std::vector<Future<Void>> agentFutures;
 
@@ -206,7 +196,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
                                                          LocalityData localities, ProcessClass processClass,
                                                          std::string* dataFolder, std::string* coordFolder,
                                                          std::string baseFolder, ClusterConnectionString connStr,
-                                                         bool useSeedFile, bool runBackupAgents, bool objSerializer) {
+                                                         bool useSeedFile, bool runBackupAgents) {
 	state ISimulator::ProcessInfo *simProcess = g_simulator.getCurrentProcess();
 	state UID randomId = g_nondeterministic_random->randomUniqueID();
 	state int cycles = 0;
@@ -223,7 +213,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 
 		state ISimulator::ProcessInfo* process =
 		    g_simulator.newProcess("Server", ip, port, listenPerProcess, localities, processClass, dataFolder->c_str(),
-		                           coordFolder->c_str(), objSerializer);
+		                           coordFolder->c_str());
 		wait(g_simulator.onProcess(process,
 		                           TaskDefaultYield)); // Now switch execution to the process on which we will run
 		state Future<ISimulator::KillType> onShutdown = process->onShutdown();
@@ -369,7 +359,7 @@ std::map< Optional<Standalone<StringRef>>, std::vector< std::vector< std::string
 ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr, std::vector<IPAddress> ips, bool sslEnabled,
                                     Reference<TLSOptions> tlsOptions, LocalityData localities,
                                     ProcessClass processClass, std::string baseFolder, bool restarting,
-                                    bool useSeedFile, bool runBackupAgents, bool sslOnly, int objSerializer) {
+                                    bool useSeedFile, bool runBackupAgents, bool sslOnly) {
 	state int bootCount = 0;
 	state std::vector<std::string> myFolders;
 	state std::vector<std::string> coordFolders;
@@ -411,7 +401,7 @@ ACTOR Future<Void> simulatedMachine(ClusterConnectionString connStr, std::vector
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
 				Reference<ClusterConnectionFile> clusterFile(useSeedFile ? new ClusterConnectionFile(path, connStr.toString()) : new ClusterConnectionFile(path));
 				const int listenPort = i*listenPerProcess + 1;
-				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, tlsOptions, listenPort, listenPerProcess, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents, objSerializer));
+				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, tlsOptions, listenPort, listenPerProcess, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents));
 				TraceEvent("SimulatedMachineProcess", randomId).detail("Address", NetworkAddress(ips[i], listenPort, true, false)).detail("ZoneId", localities.zoneId()).detail("DataHall", localities.dataHallId()).detail("Folder", myFolders[i]);
 			}
 
@@ -616,7 +606,7 @@ IPAddress makeIPAddressForSim(bool isIPv6, std::array<int, 4> parts) {
 ACTOR Future<Void> restartSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFolder, int* pTesterCount,
                                           Optional<ClusterConnectionString>* pConnString,
                                           Standalone<StringRef>* pStartingConfiguration,
-                                          Reference<TLSOptions> tlsOptions, int extraDB, int objSerializer) {
+                                          Reference<TLSOptions> tlsOptions, int extraDB) {
 	CSimpleIni ini;
 	ini.SetUnicode();
 	ini.LoadFile(joinPath(baseFolder, "restartInfo.ini").c_str());
@@ -714,8 +704,7 @@ ACTOR Future<Void> restartSimulatedSystem(vector<Future<Void>>* systemActors, st
 			systemActors->push_back(reportErrors(
 			    simulatedMachine(conn, ipAddrs, usingSSL, tlsOptions, localities, processClass, baseFolder, true,
 			                     i == useSeedForMachine, enableExtraDB,
-			                     usingSSL && (listenersPerProcess == 1 || processClass == ProcessClass::TesterClass),
-			                     objSerializer),
+			                     usingSSL && (listenersPerProcess == 1 || processClass == ProcessClass::TesterClass)),
 			    processClass == ProcessClass::TesterClass ? "SimulatedTesterMachine" : "SimulatedMachine"));
 		}
 
@@ -1097,8 +1086,7 @@ void SimulationConfig::generateNormalConfig(int minimumReplication, int minimumR
 
 void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFolder, int* pTesterCount,
                           Optional<ClusterConnectionString>* pConnString, Standalone<StringRef>* pStartingConfiguration,
-                          int extraDB, int minimumReplication, int minimumRegions, Reference<TLSOptions> tlsOptions,
-                          int objSerializer) {
+                          int extraDB, int minimumReplication, int minimumRegions, Reference<TLSOptions> tlsOptions) {
 	// SOMEDAY: this does not test multi-interface configurations
 	SimulationConfig simconfig(extraDB, minimumReplication, minimumRegions);
 	StatusObject startingConfigJSON = simconfig.db.toJSON(true);
@@ -1294,7 +1282,7 @@ void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFo
 			LocalityData	localities(Optional<Standalone<StringRef>>(), zoneId, machineId, dcUID);
 			localities.set(LiteralStringRef("data_hall"), dcUID);
 			systemActors->push_back(reportErrors(simulatedMachine(conn, ips, sslEnabled, tlsOptions,
-				localities, processClass, baseFolder, false, machine == useSeedForMachine, true, sslOnly, objSerializer), "SimulatedMachine"));
+				localities, processClass, baseFolder, false, machine == useSeedForMachine, true, sslOnly), "SimulatedMachine"));
 
 			if (extraDB && g_simulator.extraDB->toString() != conn.toString()) {
 				std::vector<IPAddress> extraIps;
@@ -1308,7 +1296,7 @@ void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFo
 				localities.set(LiteralStringRef("data_hall"), dcUID);
 				systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB, extraIps, sslEnabled, tlsOptions,
 					localities,
-					processClass, baseFolder, false, machine == useSeedForMachine, false, sslOnly, objSerializer), "SimulatedMachine"));
+					processClass, baseFolder, false, machine == useSeedForMachine, false, sslOnly), "SimulatedMachine"));
 			}
 
 			assignedMachines++;
@@ -1336,7 +1324,7 @@ void setupSimulatedSystem(vector<Future<Void>>* systemActors, std::string baseFo
 		systemActors->push_back( reportErrors( simulatedMachine(
 			conn, ips, sslEnabled, tlsOptions,
 			localities, ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource),
-			baseFolder, false, i == useSeedForMachine, false, sslEnabled, objSerializer),
+			baseFolder, false, i == useSeedForMachine, false, sslEnabled),
 			"SimulatedTesterMachine") );
 	}
 	*pStartingConfiguration = startingConfigString;
@@ -1392,7 +1380,7 @@ void checkExtraDB(const char *testFile, int &extraDB, int &minimumReplication, i
 	ifs.close();
 }
 
-ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool rebooting, Reference<TLSOptions> tlsOptions, int objSerializer ) {
+ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool rebooting, Reference<TLSOptions> tlsOptions) {
 	state vector<Future<Void>> systemActors;
 	state Optional<ClusterConnectionString> connFile;
 	state Standalone<StringRef> startingConfiguration;
@@ -1409,8 +1397,7 @@ ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool reboot
 	                                        Standalone<StringRef>(g_random->randomUniqueID().toString()),
 	                                        Standalone<StringRef>(g_random->randomUniqueID().toString()),
 	                                        Optional<Standalone<StringRef>>()),
-	                           ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource), "", "",
-	                           useObjectSerializer(objSerializer)),
+	                           ProcessClass(ProcessClass::TesterClass, ProcessClass::CommandLineSource), "", ""),
 	    TaskDefaultYield));
 	Sim2FileSystem::newFileSystem();
 	FlowTransport::createInstance(1);
@@ -1424,12 +1411,12 @@ ACTOR void setupAndRun(std::string dataFolder, const char *testFile, bool reboot
 		//systemActors.push_back( startSystemMonitor(dataFolder) );
 		if (rebooting) {
 			wait(timeoutError(restartSimulatedSystem(&systemActors, dataFolder, &testerCount, &connFile,
-			                                         &startingConfiguration, tlsOptions, extraDB, objSerializer),
+			                                         &startingConfiguration, tlsOptions, extraDB),
 			                  100.0));
 		} else {
 			g_expect_full_pointermap = 1;
 			setupSimulatedSystem(&systemActors, dataFolder, &testerCount, &connFile, &startingConfiguration, extraDB,
-			                     minimumReplication, minimumRegions, tlsOptions, objSerializer);
+			                     minimumReplication, minimumRegions, tlsOptions);
 			wait( delay(1.0) ); // FIXME: WHY!!!  //wait for machines to boot
 		}
 		std::string clusterFileDir = joinPath( dataFolder, g_random->randomUniqueID().toString() );
diff --git a/fdbserver/SimulatedCluster.h b/fdbserver/SimulatedCluster.h
index cb154828f8..eb8f325bd3 100644
--- a/fdbserver/SimulatedCluster.h
+++ b/fdbserver/SimulatedCluster.h
@@ -24,6 +24,6 @@
 #define FDBSERVER_SIMULATEDCLUSTER_H
 #pragma once
 
-void setupAndRun(std::string const& dataFolder, const char* const& testFile, bool const& rebooting, Reference<TLSOptions> const& useSSL, int const& useObjectSerializer);
+void setupAndRun(std::string const& dataFolder, const char* const& testFile, bool const& rebooting, Reference<TLSOptions> const& useSSL);
 
 #endif
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index ee69f9e8ad..515dd35b39 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -569,11 +569,10 @@ static void printUsage( const char *name, bool devhelp ) {
 		   "                 Machine class (valid options are storage, transaction,\n"
 		   "                 resolution, proxy, master, test, unset, stateless, log, router,\n"
 		   "                 and cluster_controller).\n");
-	printf("  -S %s, --object-serializer %s\n"
+	printf("  -S ON|OFF, --object-serializer ON|OFF\n"
 		   "                 Use object serializer for sending messages. The object serializer\n"
 		   "                 is currently a beta feature and it allows fdb processes to talk to\n"
-		   "                 each other even if they don't have the same version\n",
-		   devhelp ? "ON|OFF|RANDOM" : "ON|OFF", devhelp ? "ON|OFF|RANDOM" : "ON|OFF");
+		   "                 each other even if they don't have the same version\n");
 #ifndef TLS_DISABLED
 	printf(TLS_HELP);
 #endif
@@ -931,7 +930,7 @@ int main(int argc, char* argv[]) {
 		double fileIoTimeout = 0.0;
 		bool fileIoWarnOnly = false;
 		uint64_t rsssize = -1;
-		int useObjectSerializer = 0;
+		bool useObjectSerializer = false;
 
 		if( argc == 1 ) {
 			printUsage(argv[0], false);
@@ -1273,12 +1272,10 @@ int main(int argc, char* argv[]) {
 					std::string s = args.OptionArg();
 					std::transform(s.begin(), s.end(), s.begin(), ::tolower);
 					if (s == "on" || s == "true" || s == "1") {
-						useObjectSerializer = 1;
+						useObjectSerializer = true;
 					} else if (s == "off" || s == "false" || s == "0") {
-						useObjectSerializer = 0;
-					} else if (s == "random") {
-						useObjectSerializer = 2;
-					} else {
+						useObjectSerializer = false;
+					}  else {
 						fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str());
 						printHelpTeaser(argv[0]);
 						flushAndExit(FDB_EXIT_ERROR);
@@ -1507,14 +1504,10 @@ int main(int argc, char* argv[]) {
 
 		if (role == Simulation || role == CreateTemplateDatabase) {
 			//startOldSimulator();
-			startNewSimulator();
+			startNewSimulator(useObjectSerializer);
 			openTraceFile(NetworkAddress(), rollsize, maxLogsSize, logFolder, "trace", logGroup);
 		} else {
-			if (useObjectSerializer == 2) {
-				fprintf(stderr, "ERROR: 'random' for object serializer is only supported for simulation");
-				flushAndExit(FDB_EXIT_ERROR);
-			}
-			g_network = newNet2(useThreadPool, true, useObjectSerializer == 1);
+			g_network = newNet2(useThreadPool, true, useObjectSerializer);
 			FlowTransport::createInstance(1);
 
 			const bool expectsPublicAddress = (role == FDBD || role == NetworkTestServer || role == Restore);
@@ -1662,7 +1655,7 @@ int main(int argc, char* argv[]) {
 				platform::createDirectory( dataFolder );
 			}
 
-			setupAndRun( dataFolder, testFile, restarting, tlsOptions, useObjectSerializer );
+			setupAndRun( dataFolder, testFile, restarting, tlsOptions );
 			g_simulator.run();
 		} else if (role == FDBD) {
 			ASSERT( connectionFile );

From 0713e06efc653df89ef72c2a1245f20a6a95c9e7 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 16 Apr 2019 09:30:02 -0700
Subject: [PATCH 069/118] Started to work on Windows

---
 Makefile               |  2 +-
 fdbmonitor/SimpleIni.h |  4 ++--
 fdbrpc/fdbrpc.h        |  3 +--
 flow/Arena.h           |  6 ++---
 flow/FileIdentifier.h  | 52 +++++++++++++++++++++++++++++++++++++++++-
 flow/flat_buffers.h    | 12 +++++-----
 flow/flow.h            |  3 +--
 flow/serialize.h       | 14 ++++++++----
 8 files changed, 74 insertions(+), 22 deletions(-)

diff --git a/Makefile b/Makefile
index 177922a31a..0e31993544 100644
--- a/Makefile
+++ b/Makefile
@@ -43,7 +43,7 @@ ifeq ($(PLATFORM),Linux)
   CC ?= gcc
   CXX ?= g++
 
-  CXXFLAGS += -std=c++0x
+  CXXFLAGS += -std=c++17
 
   BOOST_BASEDIR ?= /opt
   TLS_LIBDIR ?= /usr/local/lib
diff --git a/fdbmonitor/SimpleIni.h b/fdbmonitor/SimpleIni.h
index 5257bc8d92..5b8151d57b 100644
--- a/fdbmonitor/SimpleIni.h
+++ b/fdbmonitor/SimpleIni.h
@@ -324,7 +324,7 @@ public:
 #endif
 
         /** Strict less ordering by name of key only */
-        struct KeyOrder : std::binary_function<Entry, Entry, bool> {
+        struct KeyOrder {
             bool operator()(const Entry & lhs, const Entry & rhs) const {
                 const static SI_STRLESS isLess = SI_STRLESS();
                 return isLess(lhs.pItem, rhs.pItem);
@@ -332,7 +332,7 @@ public:
         };
 
         /** Strict less ordering by order, and then name of key */
-        struct LoadOrder : std::binary_function<Entry, Entry, bool> {
+        struct LoadOrder {
             bool operator()(const Entry & lhs, const Entry & rhs) const {
                 if (lhs.nOrder != rhs.nOrder) {
                     return lhs.nOrder < rhs.nOrder;
diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h
index 8f3923b82c..9853cbe968 100644
--- a/fdbrpc/fdbrpc.h
+++ b/fdbrpc/fdbrpc.h
@@ -109,10 +109,9 @@ struct NetSAV : SAV<T>, FlowReceiver, FastAllocated<NetSAV<T>> {
 
 
 template <class T>
-class ReplyPromise sealed
+class ReplyPromise sealed : public ComposedIdentifier<T, 0x2>
 {
 public:
-	constexpr static FileIdentifier file_identifier = (0x2 << 24) | FileIdentifierFor<T>::value;
 	template <class U>
 	void send(U&& value) const {
 		sav->send(std::forward<U>(value));
diff --git a/flow/Arena.h b/flow/Arena.h
index 37dfcc967b..e2f345835c 100644
--- a/flow/Arena.h
+++ b/flow/Arena.h
@@ -347,9 +347,8 @@ inline void save( Archive& ar, const Arena& p ) {
 }
 
 template <class T>
-class Optional {
+class Optional : public ComposedIdentifier<T, 0x10> {
 public:
-	static constexpr FileIdentifier file_identifier = (0x10 << 24) | FileIdentifierFor<T>::value;
 	Optional() : valid(false) {}
 	Optional(const Optional<T>& o) : valid(o.valid) {
 		if (valid) new (&value) T(o.get());
@@ -797,10 +796,9 @@ template <>
 struct memcpy_able<UID> : std::integral_constant<bool, true> {};
 
 template <class T>
-class VectorRef {
+class VectorRef : public ComposedIdentifier<T, 0x8> {
 public:
 	using value_type = T;
-	static constexpr FileIdentifier file_identifier = (0x8 << 24) | FileIdentifierFor<T>::value;
 
 	// T must be trivially destructible (and copyable)!
 	VectorRef() : data(0), m_size(0), m_capacity(0) {}
diff --git a/flow/FileIdentifier.h b/flow/FileIdentifier.h
index 394356b55d..ca901d4276 100644
--- a/flow/FileIdentifier.h
+++ b/flow/FileIdentifier.h
@@ -20,14 +20,64 @@
 
 #pragma once
 #include <cstdint>
+#include <type_traits>
 
 using FileIdentifier = uint32_t;
 
 template <class T>
-struct FileIdentifierFor {
+struct FileIdentifierOf {
 	constexpr static FileIdentifier value = T::file_identifier;
 };
 
+struct Empty {};
+
+template <typename T, typename = int>
+struct HasFileIdentifierMember : std::false_type {};
+
+template <typename T>
+struct HasFileIdentifierMember<T, decltype((void)T::file_identifier, 0)> : std::true_type {};
+
+template <class T, bool>
+struct FileIdentifierForBase;
+
+template <class T>
+struct FileIdentifierForBase<T, false> {};
+
+template <class T>
+struct FileIdentifierForBase<T, true> {
+	static constexpr FileIdentifier value = T::file_identifier;
+};
+
+template <class T>
+struct FileIdentifierFor : FileIdentifierForBase<T, HasFileIdentifierMember<T>::value> {};
+
+template <typename T, typename = int>
+struct HasFileIdentifier : std::false_type {};
+
+template <typename T>
+struct HasFileIdentifier<T, decltype((void)FileIdentifierFor<T>::value, 0)> : std::true_type {};
+
+template <class T, uint32_t B, bool = HasFileIdentifier<T>::value>
+struct ComposedIdentifier;
+
+template <class T, uint32_t B>
+struct ComposedIdentifier<T, B, true>
+{
+	static constexpr FileIdentifier file_identifier = (B << 24) | FileIdentifierFor<T>::value;
+};
+
+template <class T, uint32_t B>
+struct ComposedIdentifier<T, B, false> {};
+
+template <class T, uint32_t B, bool = HasFileIdentifier<T>::value>
+struct ComposedIdentifierExternal;
+template <class T, uint32_t B>
+struct ComposedIdentifierExternal<T, B, false> {};
+template <class T, uint32_t B>
+struct ComposedIdentifierExternal<T, B, true> {
+	static constexpr FileIdentifier value = ComposedIdentifier<T, B>::file_identifier;
+};
+
 template <>
 struct FileIdentifierFor<int> {
 	constexpr static FileIdentifier value = 1;
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 012d55d390..fc0a8f589a 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -300,8 +300,8 @@ constexpr int fb_align = is_struct_like<T> ? align_helper(typename struct_like_t
 
 template <class T>
 struct _SizeOf {
-	static constexpr int size = fb_size<T>;
-	static constexpr int align = fb_align<T>;
+	static constexpr unsigned int size = fb_size<T>;
+	static constexpr unsigned int align = fb_align<T>;
 };
 
 struct PrecomputeSize {
@@ -351,7 +351,7 @@ struct PrecomputeSize {
 template <class Member, class Context>
 void load_helper(Member&, const uint8_t*, Context&);
 
-class VTableSet;
+struct VTableSet;
 
 template <class T>
 struct is_array : std::false_type {};
@@ -880,7 +880,7 @@ struct LoadSaveHelper {
 		current += sizeof(uint32_t);
 		VectorTraits::reserve(member, numEntries, context);
 		auto inserter = VectorTraits::insert(member);
-		for (int i = 0; i < numEntries; ++i) {
+		for (uint32_t i = 0; i < numEntries; ++i) {
 			T value;
 			load_helper(value, current, context);
 			*inserter = std::move(value);
@@ -945,7 +945,7 @@ struct LoadSaveHelper {
 		uint32_t len = num_entries * size;
 		auto self = writer.getMessageWriter(len);
 		auto iter = VectorTraits::begin(members);
-		for (int i = 0; i < num_entries; ++i) {
+		for (uint32_t i = 0; i < num_entries; ++i) {
 			auto result = save_helper(*iter, writer, vtables);
 			self.write(&result, i * size, size);
 			++iter;
@@ -968,7 +968,7 @@ struct LoadSaveHelper<std::vector<bool, Alloc>> {
 		member.clear();
 		member.resize(length);
 		bool m;
-		for (int i = 0; i < length; ++i) {
+		for (uint32_t i = 0; i < length; ++i) {
 			load_helper(m, current, context);
 			member[i] = m;
 			current += fb_size<bool>;
diff --git a/flow/flow.h b/flow/flow.h
index 8ae1a3afa5..fd39d3dd5e 100644
--- a/flow/flow.h
+++ b/flow/flow.h
@@ -123,9 +123,8 @@ public:
 class Never {};
 
 template <class T>
-class ErrorOr {
+class ErrorOr : public ComposedIdentifier<T, 0x1> {
 public:
-	constexpr static FileIdentifier file_identifier = (0x1 << 24) | FileIdentifierFor<T>::value;
 	ErrorOr() : error(default_error_or()) {}
 	ErrorOr(Error const& error) : error(error) {}
 	ErrorOr(const ErrorOr<T>& o) : error(o.error) {
diff --git a/flow/serialize.h b/flow/serialize.h
index b7ddb3e6fc..ad492208d1 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -130,11 +130,19 @@ public:
 	}
 };
 
+template <class F, class S, bool = HasFileIdentifier<F>::value&& HasFileIdentifier<S>::value>
+struct PairFileIdentifier;
+
 template <class F, class S>
-struct FileIdentifierFor<std::pair<F, S>> {
+struct PairFileIdentifier<F, S, false> {};
+
+template <class F, class S>
+struct PairFileIdentifier<F, S, true> {
 	constexpr static FileIdentifier value = FileIdentifierFor<F>::value ^ FileIdentifierFor<S>::value;
 };
 
+template <class F, class S>
+struct FileIdentifierFor<std::pair<F, S>> : PairFileIdentifier<F, S> {};
 
 template <class Archive, class T1, class T2>
 class Serializer< Archive, std::pair<T1,T2>, void > {
@@ -145,9 +153,7 @@ public:
 };
 
 template <class T, class Allocator>
-struct FileIdentifierFor<std::vector<T, Allocator>> {
-	constexpr static FileIdentifier value = (0x10 << 24) | FileIdentifierFor<T>::value;
-};
+struct FileIdentifierFor<std::vector<T, Allocator>> : ComposedIdentifierExternal<T, 0x10> {};
 
 template <class Archive, class T>
 inline void save( Archive& ar, const std::vector<T>& value ) {

From 96aaa31a6c081226843c9d857a29f571c8d2f4cf Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 16 Apr 2019 15:00:12 -0700
Subject: [PATCH 070/118] Compiling on clang again

---
 fdbclient/ClusterInterface.h            | 50 +++++++++---------
 fdbclient/MasterProxyInterface.h        | 30 +++++------
 fdbclient/StorageServerInterface.h      | 22 ++++----
 fdbserver/ClusterRecruitmentInterface.h | 69 ++++++++++---------------
 fdbserver/CoordinationInterface.h       | 24 ++++-----
 fdbserver/NetworkTest.h                 | 22 ++++----
 fdbserver/RatekeeperInterface.h         | 26 +++++-----
 fdbserver/RestoreInterface.h            | 26 +++++-----
 fdbserver/ServerDBInfo.h                | 13 +++++
 fdbserver/TLogInterface.h               | 20 +++----
 flow/FileIdentifier.h                   |  6 ---
 flow/flat_buffers.h                     |  1 +
 12 files changed, 152 insertions(+), 157 deletions(-)

diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 6130739b30..95641ea8f0 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -166,6 +166,21 @@ struct SystemFailureStatus {
 	}
 };
 
+struct FailureMonitoringReply {
+	constexpr static FileIdentifier file_identifier = 6820325;
+	VectorRef< SystemFailureStatus > changes;
+	Version failureInformationVersion;
+	bool allOthersFailed;							// If true, changes are relative to all servers being failed, otherwise to the version given in the request
+	int clientRequestIntervalMS,        // after this many milliseconds, send another request
+		considerServerFailedTimeoutMS;  // after this many additional milliseconds, consider the ClusterController itself to be failed
+	Arena arena;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena);
+	}
+};
+
 struct FailureMonitoringRequest {
 	// Sent by all participants to the cluster controller reply.clientRequestIntervalMS
 	//   ms after receiving the previous reply.
@@ -189,31 +204,6 @@ struct FailureMonitoringRequest {
 	}
 };
 
-struct FailureMonitoringReply {
-	constexpr static FileIdentifier file_identifier = 6820325;
-	VectorRef< SystemFailureStatus > changes;
-	Version failureInformationVersion;
-	bool allOthersFailed;							// If true, changes are relative to all servers being failed, otherwise to the version given in the request
-	int clientRequestIntervalMS,        // after this many milliseconds, send another request
-		considerServerFailedTimeoutMS;  // after this many additional milliseconds, consider the ClusterController itself to be failed
-	Arena arena;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, changes, failureInformationVersion, allOthersFailed, clientRequestIntervalMS, considerServerFailedTimeoutMS, arena);
-	}
-};
-
-struct StatusRequest {
-	constexpr static FileIdentifier file_identifier = 14419140;
-	ReplyPromise< struct StatusReply > reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
 struct StatusReply {
 	constexpr static FileIdentifier file_identifier = 9980504;
 	StatusObject statusObj;
@@ -240,6 +230,16 @@ struct StatusReply {
 	}
 };
 
+struct StatusRequest {
+	constexpr static FileIdentifier file_identifier = 14419140;
+	ReplyPromise< struct StatusReply > reply;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
 struct GetClientWorkersRequest {
 	constexpr static FileIdentifier file_identifier = 10771791;
 	ReplyPromise<vector<ClientWorkerInterface>> reply;
diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h
index 734760c7bf..8fedd2ed1f 100644
--- a/fdbclient/MasterProxyInterface.h
+++ b/fdbclient/MasterProxyInterface.h
@@ -250,21 +250,6 @@ struct TxnStateRequest {
 	}
 };
 
-struct GetHealthMetricsRequest
-{
-	constexpr static FileIdentifier file_identifier = 11403900;
-	ReplyPromise<struct GetHealthMetricsReply> reply;
-	bool detailed;
-
-	explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
-
-	template <class Ar>
-	void serialize(Ar& ar)
-	{
-		serializer(ar, reply, detailed);
-	}
-};
-
 struct GetHealthMetricsReply
 {
 	constexpr static FileIdentifier file_identifier = 11544290;
@@ -295,4 +280,19 @@ struct GetHealthMetricsReply
 	}
 };
 
+struct GetHealthMetricsRequest
+{
+	constexpr static FileIdentifier file_identifier = 11403900;
+	ReplyPromise<struct GetHealthMetricsReply> reply;
+	bool detailed;
+
+	explicit GetHealthMetricsRequest(bool detailed = false) : detailed(detailed) {}
+
+	template <class Ar>
+	void serialize(Ar& ar)
+	{
+		serializer(ar, reply, detailed);
+	}
+};
+
 #endif
diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h
index d016c2765c..19573dda63 100644
--- a/fdbclient/StorageServerInterface.h
+++ b/fdbclient/StorageServerInterface.h
@@ -359,17 +359,6 @@ struct GetPhysicalMetricsRequest {
 	}
 };
 
-struct StorageQueuingMetricsRequest {
-	// SOMEDAY: Send threshold value to avoid polling faster than the information changes?
-	constexpr static FileIdentifier file_identifier = 3978640;
-	ReplyPromise<struct StorageQueuingMetricsReply> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
 struct StorageQueuingMetricsReply {
 	constexpr static FileIdentifier file_identifier = 7633366;
 	double localTime;
@@ -387,4 +376,15 @@ struct StorageQueuingMetricsReply {
 	}
 };
 
+struct StorageQueuingMetricsRequest {
+	// SOMEDAY: Send threshold value to avoid polling faster than the information changes?
+	constexpr static FileIdentifier file_identifier = 3978640;
+	ReplyPromise<struct StorageQueuingMetricsReply> reply;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
 #endif
diff --git a/fdbserver/ClusterRecruitmentInterface.h b/fdbserver/ClusterRecruitmentInterface.h
index a79121cd5d..4ae7a95af7 100644
--- a/fdbserver/ClusterRecruitmentInterface.h
+++ b/fdbserver/ClusterRecruitmentInterface.h
@@ -71,23 +71,6 @@ struct ClusterControllerFullInterface {
 	}
 };
 
-struct RecruitFromConfigurationRequest {
-	constexpr static FileIdentifier file_identifier = 2023046;
-	DatabaseConfiguration configuration;
-	bool recruitSeedServers;
-	int maxOldLogRouters;
-	ReplyPromise< struct RecruitFromConfigurationReply > reply;
-
-	RecruitFromConfigurationRequest() {}
-	explicit RecruitFromConfigurationRequest(DatabaseConfiguration const& configuration, bool recruitSeedServers, int maxOldLogRouters)
-		: configuration(configuration), recruitSeedServers(recruitSeedServers), maxOldLogRouters(maxOldLogRouters) {}
-
-	template <class Ar>
-	void serialize( Ar& ar ) {
-		serializer(ar, configuration, recruitSeedServers, maxOldLogRouters, reply);
-	}
-};
-
 struct RecruitFromConfigurationReply {
 	constexpr static FileIdentifier file_identifier = 2224085;
 	vector<WorkerInterface> tLogs;
@@ -107,6 +90,34 @@ struct RecruitFromConfigurationReply {
 	}
 };
 
+struct RecruitFromConfigurationRequest {
+	constexpr static FileIdentifier file_identifier = 2023046;
+	DatabaseConfiguration configuration;
+	bool recruitSeedServers;
+	int maxOldLogRouters;
+	ReplyPromise< struct RecruitFromConfigurationReply > reply;
+
+	RecruitFromConfigurationRequest() {}
+	explicit RecruitFromConfigurationRequest(DatabaseConfiguration const& configuration, bool recruitSeedServers, int maxOldLogRouters)
+		: configuration(configuration), recruitSeedServers(recruitSeedServers), maxOldLogRouters(maxOldLogRouters) {}
+
+	template <class Ar>
+	void serialize( Ar& ar ) {
+		serializer(ar, configuration, recruitSeedServers, maxOldLogRouters, reply);
+	}
+};
+
+struct RecruitRemoteFromConfigurationReply {
+	constexpr static FileIdentifier file_identifier = 9091392;
+	vector<WorkerInterface> remoteTLogs;
+	vector<WorkerInterface> logRouters;
+
+	template <class Ar>
+	void serialize( Ar& ar ) {
+		serializer(ar, remoteTLogs, logRouters);
+	}
+};
+
 struct RecruitRemoteFromConfigurationRequest {
 	constexpr static FileIdentifier file_identifier = 3235995;
 	DatabaseConfiguration configuration;
@@ -124,17 +135,6 @@ struct RecruitRemoteFromConfigurationRequest {
 	}
 };
 
-struct RecruitRemoteFromConfigurationReply {
-	constexpr static FileIdentifier file_identifier = 9091392;
-	vector<WorkerInterface> remoteTLogs;
-	vector<WorkerInterface> logRouters;
-
-	template <class Ar>
-	void serialize( Ar& ar ) {
-		serializer(ar, remoteTLogs, logRouters);
-	}
-};
-
 struct RecruitStorageReply {
 	constexpr static FileIdentifier file_identifier = 15877089;
 	WorkerInterface worker;
@@ -240,19 +240,6 @@ struct RegisterMasterRequest {
 	}
 };
 
-struct GetServerDBInfoRequest {
-	constexpr static FileIdentifier file_identifier = 9467438;
-	UID knownServerInfoID;
-	Standalone<VectorRef<StringRef>> issues;
-	std::vector<NetworkAddress> incompatiblePeers;
-	ReplyPromise< struct ServerDBInfo > reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, knownServerInfoID, issues, incompatiblePeers, reply);
-	}
-};
-
 #include "fdbserver/ServerDBInfo.h" // include order hack
 
 #endif
diff --git a/fdbserver/CoordinationInterface.h b/fdbserver/CoordinationInterface.h
index 7d05639594..b943618ab3 100644
--- a/fdbserver/CoordinationInterface.h
+++ b/fdbserver/CoordinationInterface.h
@@ -72,6 +72,18 @@ struct UniqueGeneration {
 	}
 };
 
+struct GenerationRegReadReply {
+	constexpr static FileIdentifier file_identifier = 12623609;
+	Optional<Value> value;
+	UniqueGeneration gen, rgen;
+	GenerationRegReadReply() {}
+	GenerationRegReadReply( Optional<Value> value, UniqueGeneration gen, UniqueGeneration rgen ) : value(value), gen(gen), rgen(rgen) {}
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, value, gen, rgen);
+	}
+};
+
 struct GenerationRegReadRequest {
 	constexpr static FileIdentifier file_identifier = 8975311;
 	Key key;
@@ -85,18 +97,6 @@ struct GenerationRegReadRequest {
 	}
 };
 
-struct GenerationRegReadReply {
-	constexpr static FileIdentifier file_identifier = 12623609;
-	Optional<Value> value;
-	UniqueGeneration gen, rgen;
-	GenerationRegReadReply() {}
-	GenerationRegReadReply( Optional<Value> value, UniqueGeneration gen, UniqueGeneration rgen ) : value(value), gen(gen), rgen(rgen) {}
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, value, gen, rgen);
-	}
-};
-
 struct GenerationRegWriteRequest {
 	constexpr static FileIdentifier file_identifier = 3521510;
 	KeyValue kv;
diff --git a/fdbserver/NetworkTest.h b/fdbserver/NetworkTest.h
index 94ae95d0fe..b68c795215 100644
--- a/fdbserver/NetworkTest.h
+++ b/fdbserver/NetworkTest.h
@@ -33,6 +33,17 @@ struct NetworkTestInterface {
 	NetworkTestInterface( INetwork* local );
 };
 
+struct NetworkTestReply {
+	constexpr static FileIdentifier file_identifier = 14465374;
+	Value value;
+	NetworkTestReply() {}
+	NetworkTestReply( Value value ) : value(value) {}
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, value);
+	}
+};
+
 struct NetworkTestRequest {
 	constexpr static FileIdentifier file_identifier = 4146513;
 	Key key;
@@ -46,17 +57,6 @@ struct NetworkTestRequest {
 	}
 };
 
-struct NetworkTestReply {
-	constexpr static FileIdentifier file_identifier = 14465374;
-	Value value;
-	NetworkTestReply() {}
-	NetworkTestReply( Value value ) : value(value) {}
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, value);
-	}
-};
-
 Future<Void> networkTestServer();
 
 Future<Void> networkTestClient( std:: string const& testServers );
diff --git a/fdbserver/RatekeeperInterface.h b/fdbserver/RatekeeperInterface.h
index 8d441219c6..f3cdd49574 100644
--- a/fdbserver/RatekeeperInterface.h
+++ b/fdbserver/RatekeeperInterface.h
@@ -52,6 +52,19 @@ struct RatekeeperInterface {
 	}
 };
 
+struct GetRateInfoReply {
+	constexpr static FileIdentifier file_identifier = 7845006;
+	double transactionRate;
+	double batchTransactionRate;
+	double leaseDuration;
+	HealthMetrics healthMetrics;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, transactionRate, batchTransactionRate, leaseDuration, healthMetrics);
+	}
+};
+
 struct GetRateInfoRequest {
 	constexpr static FileIdentifier file_identifier = 9068521;
 	UID requesterID;
@@ -70,19 +83,6 @@ struct GetRateInfoRequest {
 	}
 };
 
-struct GetRateInfoReply {
-	constexpr static FileIdentifier file_identifier = 7845006;
-	double transactionRate;
-	double batchTransactionRate;
-	double leaseDuration;
-	HealthMetrics healthMetrics;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, transactionRate, batchTransactionRate, leaseDuration, healthMetrics);
-	}
-};
-
 struct HaltRatekeeperRequest {
 	constexpr static FileIdentifier file_identifier = 6997218;
 	UID requesterID;
diff --git a/fdbserver/RestoreInterface.h b/fdbserver/RestoreInterface.h
index fc893c17f5..847e940d9b 100644
--- a/fdbserver/RestoreInterface.h
+++ b/fdbserver/RestoreInterface.h
@@ -46,6 +46,19 @@ struct RestoreInterface {
 	}
 };
 
+struct TestReply {
+	constexpr static FileIdentifier file_identifier = 12075719;
+	int replyData;
+
+	TestReply() : replyData(0) {}
+	explicit TestReply(int replyData) : replyData(replyData) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, replyData);
+	}
+};
+
 struct TestRequest {
 	constexpr static FileIdentifier file_identifier = 14404487;
 	int testData;
@@ -60,19 +73,6 @@ struct TestRequest {
 	}
 };
 
-struct TestReply {
-	constexpr static FileIdentifier file_identifier = 12075719;
-	int replyData;
-
-	TestReply() : replyData(0) {}
-	explicit TestReply(int replyData) : replyData(replyData) {}
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, replyData);
-	}
-};
-
 Future<Void> restoreWorker(Reference<ClusterConnectionFile> const& ccf, LocalityData const& locality);
 
 #endif
diff --git a/fdbserver/ServerDBInfo.h b/fdbserver/ServerDBInfo.h
index fa50cd4d67..595507b1dd 100644
--- a/fdbserver/ServerDBInfo.h
+++ b/fdbserver/ServerDBInfo.h
@@ -62,4 +62,17 @@ struct ServerDBInfo {
 	}
 };
 
+struct GetServerDBInfoRequest {
+	constexpr static FileIdentifier file_identifier = 9467438;
+	UID knownServerInfoID;
+	Standalone<VectorRef<StringRef>> issues;
+	std::vector<NetworkAddress> incompatiblePeers;
+	ReplyPromise< struct ServerDBInfo > reply;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, knownServerInfoID, issues, incompatiblePeers, reply);
+	}
+};
+
 #endif
diff --git a/fdbserver/TLogInterface.h b/fdbserver/TLogInterface.h
index d642de6f44..585caa23a3 100644
--- a/fdbserver/TLogInterface.h
+++ b/fdbserver/TLogInterface.h
@@ -228,16 +228,6 @@ struct TLogCommitRequest {
 	}
 };
 
-struct TLogQueuingMetricsRequest {
-	constexpr static FileIdentifier file_identifier = 7798476;
-	ReplyPromise<struct TLogQueuingMetricsReply> reply;
-
-	template <class Ar>
-	void serialize(Ar& ar) {
-		serializer(ar, reply);
-	}
-};
-
 struct TLogQueuingMetricsReply {
 	constexpr static FileIdentifier file_identifier = 12206626;
 	double localTime;
@@ -252,4 +242,14 @@ struct TLogQueuingMetricsReply {
 	}
 };
 
+struct TLogQueuingMetricsRequest {
+	constexpr static FileIdentifier file_identifier = 7798476;
+	ReplyPromise<struct TLogQueuingMetricsReply> reply;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
 #endif
diff --git a/flow/FileIdentifier.h b/flow/FileIdentifier.h
index ca901d4276..1ae1d59374 100644
--- a/flow/FileIdentifier.h
+++ b/flow/FileIdentifier.h
@@ -24,11 +24,6 @@
 
 using FileIdentifier = uint32_t;
 
-template <class T>
-struct FileIdentifierOf {
-	constexpr static FileIdentifier value = T::file_identifier;
-};
-
 struct Empty {};
 
 template <typename T, typename = int>
@@ -142,4 +137,3 @@ template <>
 struct FileIdentifierFor<double> {
 	constexpr static FileIdentifier value = 9348150;
 };
-
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index fc0a8f589a..1599449b39 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -1070,6 +1070,7 @@ inline FileIdentifier read_file_identifier(const uint8_t* in) {
 // introduce the indirection only when necessary.
 template <class T>
 struct EnsureTable {
+	static_assert(HasFileIdentifier<T>::value);
 	constexpr static FileIdentifier file_identifier = FileIdentifierFor<T>::value;
 	EnsureTable() = default;
 	EnsureTable(const object_construction<T>& t) : t(t) {}

From 57912b33a540ab40c27ea3fb8fe2653c72e18930 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Tue, 16 Apr 2019 17:43:46 -0700
Subject: [PATCH 071/118] fixed merge error

---
 fdbserver/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt
index 94e166ea83..bb211ff9d2 100644
--- a/fdbserver/CMakeLists.txt
+++ b/fdbserver/CMakeLists.txt
@@ -179,11 +179,11 @@ add_library(fdb_sqlite STATIC
   sqlite/sqliteInt.h
   sqlite/sqliteLimit.h
   sqlite/sqlite3.amalgamation.c)
-target_compile_definitions(fdb_sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
 
 # Suppress warnings in sqlite since it's third party
 if(NOT WIN32)
-  target_compile_options(sqlite BEFORE PRIVATE -w) # disable warnings for third party
+  target_compile_definitions(fdb_sqlite PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
+  target_compile_options(fdb_sqlite BEFORE PRIVATE -w) # disable warnings for third party
 endif()
 
 set(java_workload_docstring "Build the Java workloads (makes fdbserver link against JNI)")

From b4fa13c33ba5cc48b5d432c7cd2a7848b79c49c1 Mon Sep 17 00:00:00 2001
From: Markus Pilman <markus@pilman.ch>
Date: Wed, 17 Apr 2019 10:33:31 -0700
Subject: [PATCH 072/118] Remove obsolete function

---
 flow/FastRef.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flow/FastRef.h b/flow/FastRef.h
index 380ad0ecda..3ddcdd55e9 100644
--- a/flow/FastRef.h
+++ b/flow/FastRef.h
@@ -156,7 +156,6 @@ public:
 
 	bool isValid() const { return ptr != NULL; }
 	explicit operator bool() const { return ptr != NULL; }
-	P*& changePtrUnsafe() { return ptr; }
 
 private:
 	P *ptr;

From a92233a37197ce5979b158da0ddef702dc277b91 Mon Sep 17 00:00:00 2001
From: Andrew Noyes <anoyes34@gmail.com>
Date: Wed, 17 Apr 2019 10:34:28 -0700
Subject: [PATCH 073/118] Added TODO to optimize packet serialization

Co-Authored-By: mpilman <markus@pilman.ch>
---
 flow/serialize.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/serialize.h b/flow/serialize.h
index ad492208d1..53c1266c4c 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -731,7 +731,7 @@ struct MakeSerializeSource : ISerializeSource {
 		if (useObjectSerializer) {
 			ObjectWriter writer;
 			writer.serialize(get());
-			w.serializeBytes(writer.toStringRef());
+			w.serializeBytes(writer.toStringRef()); // TODO(atn34) Eliminate unnecessary memcpy
 		} else {
 			static_cast<T const*>(this)->serialize(w);
 		}

From 46e7a0ca56e3848dd18cd87a724c76df4eb18679 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Wed, 17 Apr 2019 10:49:19 -0700
Subject: [PATCH 074/118] address reviews and make compile with
 `-Wunused-variable`

---
 fdbclient/ClusterInterface.h               | 5 +++--
 fdbrpc/FlowTransport.actor.cpp             | 2 +-
 fdbserver/workloads/JavaWorkload.actor.cpp | 1 +
 flow/ObjectSerializer.h                    | 2 +-
 flow/Platform.cpp                          | 2 +-
 flow/flat_buffers.h                        | 1 -
 flow/serialize.h                           | 6 +++---
 7 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 95641ea8f0..0b9083d740 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -146,9 +146,10 @@ struct OpenDatabaseRequest {
 	template <class Ar>
 	void serialize(Ar& ar) {
 		if constexpr (!is_fb_function<Ar>) {
-                ASSERT( ar.protocolVersion() >= 0x0FDB00A400040001LL );
+			ASSERT(ar.protocolVersion() >= 0x0FDB00A400040001LL);
 		}
-        serializer(ar, issues, supportedVersions, connectedCoordinatorsNum, traceLogGroup, knownClientInfoID, reply, arena);
+		serializer(ar, issues, supportedVersions, connectedCoordinatorsNum, traceLogGroup, knownClientInfoID, reply,
+				   arena);
 	}
 };
 
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 26efa4c1a3..02101b81f3 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -1022,7 +1022,7 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 		Standalone<StringRef> copy;
 		if (g_network->useObjectSerializer()) {
 			ObjectWriter wr;
-			what.serializeBinaryWriter(wr);
+			what.serializeObjectWriter(wr);
 			copy = wr.toStringRef();
 		} else {
 			BinaryWriter wr( AssumeVersion(currentProtocolVersion) );
diff --git a/fdbserver/workloads/JavaWorkload.actor.cpp b/fdbserver/workloads/JavaWorkload.actor.cpp
index 3d15e2eedc..8c77c361a7 100644
--- a/fdbserver/workloads/JavaWorkload.actor.cpp
+++ b/fdbserver/workloads/JavaWorkload.actor.cpp
@@ -203,6 +203,7 @@ struct JVMContext {
 	bool checkException() {
 		auto flag = env->ExceptionCheck();
 		if (flag) {
+			env->ExceptionOccurred();
 			TraceEvent(SevError, "JavaException");
 			env->ExceptionDescribe();
 			env->ExceptionClear();
diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h
index e048c2aae2..6ebc9ad586 100644
--- a/flow/ObjectSerializer.h
+++ b/flow/ObjectSerializer.h
@@ -109,7 +109,7 @@ public:
 			data = new (arena) uint8_t[size];
 			return data;
 		};
-		auto res = save_members(allocator, file_identifier, items...);
+		save_members(allocator, file_identifier, items...);
 		ASSERT(allocations == 1);
 	}
 
diff --git a/flow/Platform.cpp b/flow/Platform.cpp
index 59efdf0a9c..d303e48d80 100644
--- a/flow/Platform.cpp
+++ b/flow/Platform.cpp
@@ -1410,7 +1410,7 @@ void setMemoryQuota( size_t limit ) {
 	}
 	if (!AssignProcessToJobObject( job, GetCurrentProcess() ))
 		TraceEvent(SevWarn, "FailedToSetMemoryLimit").GetLastError();
-#elif defined(__linux__) && !defined(USE_ASAN)
+#elif defined(__linux__)
 	struct rlimit rlim;
 	if (getrlimit(RLIMIT_AS, &rlim)) {
 		TraceEvent(SevError, "GetMemoryLimit").GetLastError();
diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 1599449b39..3a0c7d6a0b 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -816,7 +816,6 @@ struct LoadSaveHelper {
 	std::enable_if_t<is_struct_like<U>> load(U& member, const uint8_t* current, Context& context) {
 		using StructTraits = struct_like_traits<U>;
 		using types = typename StructTraits::types;
-		constexpr auto size = struct_size(types{});
 		for_each_i<pack_size(types{})>([&](auto i_type) {
 			constexpr int i = decltype(i_type)::value;
 			using type = index_t<i, types>;
diff --git a/flow/serialize.h b/flow/serialize.h
index 53c1266c4c..d47da029d8 100644
--- a/flow/serialize.h
+++ b/flow/serialize.h
@@ -721,7 +721,7 @@ private:
 struct ISerializeSource {
 	virtual void serializePacketWriter(PacketWriter&, bool useObjectSerializer) const = 0;
 	virtual void serializeBinaryWriter(BinaryWriter&) const = 0;
-	virtual void serializeBinaryWriter(ObjectWriter&) const = 0;
+	virtual void serializeObjectWriter(ObjectWriter&) const = 0;
 };
 
 template <class T, class V>
@@ -745,7 +745,7 @@ struct SerializeSource : MakeSerializeSource<SerializeSource<T>, T> {
 	using value_type = T;
 	T const& value;
 	SerializeSource(T const& value) : value(value) {}
-	virtual void serializeBinaryWriter(ObjectWriter& w) const {
+	virtual void serializeObjectWriter(ObjectWriter& w) const {
 		w.serialize(value);
 	}
 	template <class Ar> void serialize(Ar& ar) const {
@@ -761,7 +761,7 @@ struct SerializeBoolAnd : MakeSerializeSource<SerializeBoolAnd<T>, T> {
 	T const& value;
 	SerializeBoolAnd( bool b, T const& value ) : b(b), value(value) {}
 	template <class Ar> void serialize(Ar& ar) const { ar << b << value; }
-	virtual void serializeBinaryWriter(ObjectWriter& w) const {
+	virtual void serializeObjectWriter(ObjectWriter& w) const {
 		ASSERT(false);
 	}
 	virtual T const& get() const {

From 5a139159604a11b44c21518f411b49c795873487 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 10 May 2019 13:14:17 -0700
Subject: [PATCH 075/118] implemented std::set serializer for flatbuffers

---
 flow/flat_buffers.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/flow/flat_buffers.h b/flow/flat_buffers.h
index 3a0c7d6a0b..a7ff261358 100644
--- a/flow/flat_buffers.h
+++ b/flow/flat_buffers.h
@@ -153,6 +153,21 @@ struct vector_like_traits<std::map<Key, T, Compare, Allocator>> : std::true_type
 	static iterator begin(const Vec& v) { return v.begin(); }
 };
 
+template<class Key, class Compare, class Allocator>
+struct vector_like_traits<std::set<Key, Compare, Allocator>> : std::true_type {
+	using Vec = std::set<Key, Compare, Allocator>;
+	using value_type = Key;
+	using iterator = typename Vec::const_iterator;
+	using insert_iterator = std::insert_iterator<Vec>;
+
+	static size_t num_entries(const Vec& v) { return v.size(); }
+	template <class Context>
+	static void reserve(Vec&, size_t, Context&) {}
+
+	static insert_iterator insert(Vec& v) { return std::inserter(v, v.end()); }
+	static iterator begin(const Vec& v) { return v.begin(); }
+};
+
 template <>
 struct dynamic_size_traits<std::string> : std::true_type {
 private:

From b601fb2df1978480ba1b929c7497e163719984af Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Mon, 13 May 2019 14:08:30 -0700
Subject: [PATCH 076/118] Added flat_buffers source files to Windows project in
 order to support building via make

---
 flow/flow.vcxproj | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj
index b6b647db4e..caa74e1311 100644
--- a/flow/flow.vcxproj
+++ b/flow/flow.vcxproj
@@ -30,6 +30,7 @@
     <ClCompile Include="IThreadPool.cpp" />
     <ClCompile Include="network.cpp" />
     <ClCompile Include="flow.cpp" />
+		<ClCompile Include="flat_buffers.cpp" />
     <ActorCompiler Include="genericactors.actor.cpp" />
     <ClCompile Include="Hash3.c" />
     <ClCompile Include="IndexedSet.cpp" />
@@ -63,6 +64,7 @@
     <ClInclude Include="FaultInjection.h" />
     <ClInclude Include="network.h" />
     <ClInclude Include="flow.h" />
+		<ClInclude Include="flat_buffers.h" />
     <ActorCompiler Include="EventTypes.actor.h">
       <EnableCompile>false</EnableCompile>
       <EnableCompile>false</EnableCompile>

From 7d38a1e273eaecb94113319ebd228aa1bf6050b2 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 10 May 2019 13:17:46 -0700
Subject: [PATCH 077/118] Use std::atomics in thread-safe queue.

---
 flow/ThreadSafeQueue.h | 96 +++++++++++++++++++++++++++++-------------
 1 file changed, 66 insertions(+), 30 deletions(-)

diff --git a/flow/ThreadSafeQueue.h b/flow/ThreadSafeQueue.h
index 360ac35ef7..b362ed1ba3 100644
--- a/flow/ThreadSafeQueue.h
+++ b/flow/ThreadSafeQueue.h
@@ -27,63 +27,88 @@ THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED WA
 
 The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of Dmitry Vyukov.*/
 
+#include <atomic>
+
 template <class T>
 class ThreadSafeQueue : NonCopyable {
 	struct BaseNode {
-		BaseNode* volatile  next;
+		std::atomic<BaseNode*> next;
+		BaseNode() : next(nullptr) {}
 	};
 
-	struct Node : BaseNode, FastAllocated<Node>
-	{
+	struct Node : BaseNode, FastAllocated<Node> {
 		T data;
 		Node( T const& data ) : data(data) {}
 	};
-	BaseNode* volatile  head;
-	BaseNode*           tail;
+	std::atomic<BaseNode*> head;
+	BaseNode* tail;
 	BaseNode stub, sleeping;
 	bool sleepy;
 
 	BaseNode* popNode() {
 		BaseNode* tail = this->tail;
-		BaseNode* next = tail->next;
+		BaseNode* next = tail->next.load();
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_BEFORE(&tail->next);
+#endif
 		if (tail == &this->stub) {
-			if (0 == next)
-				return 0;
+			if (!next) {
+				return nullptr;
+			}
 			this->tail = next;
 			tail = next;
-			next = next->next;
+			next = next->next.load();
+#ifdef VALGRIND
+			ANNOTATE_HAPPENS_BEFORE(&next->next);
+#endif
 		}
-		if (next)
-		{
+		if (next) {
 			this->tail = next;
 			return tail;
 		}
-		BaseNode* head = this->head;
-		if (tail != head)
-			return 0;
-		pushNode( &this->stub );
-		next = tail->next;
-		if (next)
-		{
+		BaseNode* head = this->head.load();
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_BEFORE(&this->head);
+#endif
+		if (tail != head) {
+			return nullptr;
+		}
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_AFTER(&this->stub.next);
+#endif
+		this->stub.next.store(nullptr);
+		pushNode(&this->stub);
+		next = tail->next.load();
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_BEFORE(&tail->next);
+#endif
+		if (next) {
 			this->tail = next;
 			return tail;
 		}
-		return 0;
+		return nullptr;
 	}
 
 	// Pushes n at the end of the queue and returns the node immediately before it
 	BaseNode* pushNode( BaseNode* n ) {
-		n->next = 0;
-		BaseNode* prev = interlockedExchangePtr( &head, n );
-		prev->next = n;
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_AFTER(&head);
+#endif
+		BaseNode* prev = head.exchange(n);
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_BEFORE(&head);
+		ANNOTATE_HAPPENS_AFTER(&prev->next);
+#endif
+		prev->next.store(n);
 		return prev;
 	}
 public:
 	ThreadSafeQueue() {
-		this->head = &this->stub;
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_AFTER(&this->head);
+#endif
+		this->head.store(&this->stub);
 		this->tail = &this->stub;
-		this->stub.next = 0;
-		this->sleeping.next = 0;
 		this->sleepy = false;
 	}
 	~ThreadSafeQueue() {
@@ -101,10 +126,21 @@ public:
 
 	// If canSleep returns true, then the queue is empty and the next push() will return true
 	bool canSleep() {
-		if (sleepy) return false;  // We already have sleeping in the queue from a previous call to canSleep.  Pop it and then maybe you can sleep!
-		if (this->tail != &stub || this->tail->next != 0) return false;  // There is definitely something in the queue.  This is a rejection test not needed for correctness, but avoids calls to pushNode
+		if (sleepy) {
+			return false;  // We already have sleeping in the queue from a previous call to canSleep.  Pop it and then maybe you can sleep!
+		}
+		if (this->tail != &stub || this->tail->next.load()) {
+#ifdef VALGRIND
+			ANNOTATE_HAPPENS_BEFORE(&this->tail->next);
+#endif
+			return false;  // There is definitely something in the queue.  This is a rejection test not needed for correctness, but avoids calls to pushNode
+		}
+#ifdef VALGRIND
+		ANNOTATE_HAPPENS_BEFORE(&this->tail->next);
+#endif
 		// sleeping is definitely not in the queue, so we can safely try...
-		bool ok = pushNode( &sleeping ) == &stub;
+		sleeping.next.store(nullptr);
+		bool ok = pushNode(&sleeping) == &stub;
 		sleepy = true; // sleeping is in the queue, regardless of whether it's the first thing; we need to pop it before sleeping again
 		return ok;
 	}
@@ -112,8 +148,8 @@ public:
 	Optional<T> pop() {
 		BaseNode* b = popNode();
 		if (b == &sleeping) { sleepy = false; b = popNode(); }
-		if ( b == &sleeping ) ASSERT(false);
-		if ( b == &stub ) ASSERT(false);
+		if (b == &sleeping) ASSERT(false);
+		if (b == &stub) ASSERT(false);
 		if (!b) return Optional<T>();
 
 		Node* n = (Node*)b;

From 436645e9568f1e184b558d14874b06e1f13a6729 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@users.noreply.github.com>
Date: Fri, 10 May 2019 15:51:44 -0700
Subject: [PATCH 078/118] Update flow/ThreadSafeQueue.h

Co-Authored-By: Alex Miller <35046903+alexmiller-apple@users.noreply.github.com>
---
 flow/ThreadSafeQueue.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/flow/ThreadSafeQueue.h b/flow/ThreadSafeQueue.h
index b362ed1ba3..75d3ea6744 100644
--- a/flow/ThreadSafeQueue.h
+++ b/flow/ThreadSafeQueue.h
@@ -29,6 +29,10 @@ The views and conclusions contained in the software and documentation are those
 
 #include <atomic>
 
+#if VALGRIND
+#include <drd.h>
+#endif
+  
 template <class T>
 class ThreadSafeQueue : NonCopyable {
 	struct BaseNode {
@@ -157,4 +161,4 @@ public:
 		delete n;
 		return Optional<T>( std::move(data) );
 	}
-};
\ No newline at end of file
+};

From 60ce790a2b1911d75c7acf5ffe533691dc748247 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 10 May 2019 15:55:06 -0700
Subject: [PATCH 079/118] Consistent use of #if VALGRIND

---
 flow/ThreadSafeQueue.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/flow/ThreadSafeQueue.h b/flow/ThreadSafeQueue.h
index 75d3ea6744..1822b636af 100644
--- a/flow/ThreadSafeQueue.h
+++ b/flow/ThreadSafeQueue.h
@@ -52,7 +52,7 @@ class ThreadSafeQueue : NonCopyable {
 	BaseNode* popNode() {
 		BaseNode* tail = this->tail;
 		BaseNode* next = tail->next.load();
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_BEFORE(&tail->next);
 #endif
 		if (tail == &this->stub) {
@@ -62,7 +62,7 @@ class ThreadSafeQueue : NonCopyable {
 			this->tail = next;
 			tail = next;
 			next = next->next.load();
-#ifdef VALGRIND
+#if VALGRIND
 			ANNOTATE_HAPPENS_BEFORE(&next->next);
 #endif
 		}
@@ -71,19 +71,19 @@ class ThreadSafeQueue : NonCopyable {
 			return tail;
 		}
 		BaseNode* head = this->head.load();
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_BEFORE(&this->head);
 #endif
 		if (tail != head) {
 			return nullptr;
 		}
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_AFTER(&this->stub.next);
 #endif
 		this->stub.next.store(nullptr);
 		pushNode(&this->stub);
 		next = tail->next.load();
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_BEFORE(&tail->next);
 #endif
 		if (next) {
@@ -95,11 +95,11 @@ class ThreadSafeQueue : NonCopyable {
 
 	// Pushes n at the end of the queue and returns the node immediately before it
 	BaseNode* pushNode( BaseNode* n ) {
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_AFTER(&head);
 #endif
 		BaseNode* prev = head.exchange(n);
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_BEFORE(&head);
 		ANNOTATE_HAPPENS_AFTER(&prev->next);
 #endif
@@ -108,7 +108,7 @@ class ThreadSafeQueue : NonCopyable {
 	}
 public:
 	ThreadSafeQueue() {
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_AFTER(&this->head);
 #endif
 		this->head.store(&this->stub);
@@ -134,12 +134,12 @@ public:
 			return false;  // We already have sleeping in the queue from a previous call to canSleep.  Pop it and then maybe you can sleep!
 		}
 		if (this->tail != &stub || this->tail->next.load()) {
-#ifdef VALGRIND
+#if VALGRIND
 			ANNOTATE_HAPPENS_BEFORE(&this->tail->next);
 #endif
 			return false;  // There is definitely something in the queue.  This is a rejection test not needed for correctness, but avoids calls to pushNode
 		}
-#ifdef VALGRIND
+#if VALGRIND
 		ANNOTATE_HAPPENS_BEFORE(&this->tail->next);
 #endif
 		// sleeping is definitely not in the queue, so we can safely try...

From 7e4da69343d6c0849d78c775b17ee892113850c5 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Mon, 13 May 2019 23:16:58 -0700
Subject: [PATCH 080/118] Added stdc++ library to link libraries

---
 bindings/c/local.mk | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/bindings/c/local.mk b/bindings/c/local.mk
index eebd6233c1..f4ede256e8 100644
--- a/bindings/c/local.mk
+++ b/bindings/c/local.mk
@@ -24,7 +24,7 @@ fdb_c_CFLAGS := $(fdbclient_CFLAGS)
 fdb_c_LDFLAGS := $(fdbrpc_LDFLAGS)
 fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(FDB_TLS_LIB)
 fdb_c_STATIC_LIBS := $(TLS_LIBS)
-fdb_c_tests_LIBS := -Llib -lfdb_c
+fdb_c_tests_LIBS := -Llib -lfdb_c -lstdc++
 fdb_c_tests_HEADERS := -Ibindings/c
 
 CLEAN_TARGETS += fdb_c_tests_clean
@@ -86,11 +86,12 @@ bindings/c/foundationdb/fdb_c_options.g.h: bin/vexillographer.exe fdbclient/vexi
 
 bin/fdb_c_performance_test: bindings/c/test/performance_test.c bindings/c/test/test.h fdb_c
 	@echo "Compiling      fdb_c_performance_test"
-	@$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c $(fdb_c_tests_LIBS) 
+	@echo "$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c $(fdb_c_tests_LIBS)"
+	@$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c $(fdb_c_tests_LIBS)
 
 bin/fdb_c_ryw_benchmark: bindings/c/test/ryw_benchmark.c bindings/c/test/test.h fdb_c
 	@echo "Compiling      fdb_c_ryw_benchmark"
-	@$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/ryw_benchmark.c $(fdb_c_tests_LIBS) 
+	@$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/ryw_benchmark.c $(fdb_c_tests_LIBS)
 
 packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz: bin/fdb_c_performance_test bin/fdb_c_ryw_benchmark
 	@echo "Packaging      $@"

From b0362a654e4c746e00299df56750e7da95483259 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Mon, 13 May 2019 23:17:35 -0700
Subject: [PATCH 081/118] Removed the debug compilation command display

---
 bindings/c/local.mk | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bindings/c/local.mk b/bindings/c/local.mk
index f4ede256e8..5bdbe750e0 100644
--- a/bindings/c/local.mk
+++ b/bindings/c/local.mk
@@ -86,7 +86,6 @@ bindings/c/foundationdb/fdb_c_options.g.h: bin/vexillographer.exe fdbclient/vexi
 
 bin/fdb_c_performance_test: bindings/c/test/performance_test.c bindings/c/test/test.h fdb_c
 	@echo "Compiling      fdb_c_performance_test"
-	@echo "$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c $(fdb_c_tests_LIBS)"
 	@$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c $(fdb_c_tests_LIBS)
 
 bin/fdb_c_ryw_benchmark: bindings/c/test/ryw_benchmark.c bindings/c/test/test.h fdb_c

From 2ed1176d6aade0ea2aa4ee51fefec0c9f3144cd2 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 02:37:16 -0700
Subject: [PATCH 082/118] Add support for statically linking stdc++ to
 libfdbc_c on Centos while using custom hack for Ubuntu Set compilationn flag
 to c++17

---
 Makefile                      | 13 ++++++-------
 bindings/c/local.mk           | 10 ++++++++--
 bindings/flow/tester/local.mk |  2 +-
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 78b83afe80..1f63242c44 100644
--- a/Makefile
+++ b/Makefile
@@ -1,14 +1,13 @@
 export
 PLATFORM := $(shell uname)
 ARCH := $(shell uname -m)
-ifeq ("$(wildcard /etc/centos-release)", "")
-	LIBSTDCPP_HACK = 1
-else
-	LIBSTDCPP_HACK = 0
-endif
-
 TOPDIR := $(shell pwd)
 
+# Allow custom libc++ hack for Ubuntu
+ifeq ("$(wildcard /etc/centos-release)", "")
+  LIBSTDCPP_HACK ?= 1
+endif
+
 ifeq ($(ARCH),x86_64)
   ARCH := x64
 else
@@ -43,7 +42,7 @@ ifeq ($(PLATFORM),Linux)
   CC ?= gcc
   CXX ?= g++
 
-  CXXFLAGS += -std=c++0x
+  CXXFLAGS += -std=c++17
 
   BOOST_BASEDIR ?= /opt
   TLS_LIBDIR ?= /usr/local/lib
diff --git a/bindings/c/local.mk b/bindings/c/local.mk
index 5bdbe750e0..df2859c6a3 100644
--- a/bindings/c/local.mk
+++ b/bindings/c/local.mk
@@ -31,8 +31,14 @@ CLEAN_TARGETS += fdb_c_tests_clean
 
 ifeq ($(PLATFORM),linux)
   fdb_c_LDFLAGS += -Wl,--version-script=bindings/c/fdb_c.map -static-libgcc -Wl,-z,nodelete -lm -lpthread -lrt -ldl
-  ifeq ($(LIBSTDCPP_HACK),1)
-    fdb_c_LIBS += lib/libstdc++.a
+	# Link our custom libstdc++ statically in Ubuntu, if hacking
+  ifeq ("$(wildcard /etc/centos-release)", "")
+    ifeq ($(LIBSTDCPP_HACK),1)
+      fdb_c_LIBS += lib/libstdc++.a
+    endif
+	# Link stdc++ statically in Centos, if not hacking
+  else
+    fdb_c_STATIC_LIBS += -static-libstdc++
   endif
   fdb_c_tests_LIBS += -lpthread
 endif
diff --git a/bindings/flow/tester/local.mk b/bindings/flow/tester/local.mk
index 6e59625c5f..83444774bd 100644
--- a/bindings/flow/tester/local.mk
+++ b/bindings/flow/tester/local.mk
@@ -35,7 +35,7 @@ _fdb_flow_tester_clean:
 	@rm -rf bindings/flow/bin
 
 ifeq ($(PLATFORM),linux)
-  fdb_flow_tester_LDFLAGS += -static-libstdc++ -static-libgcc -ldl -lpthread -lrt
+  fdb_flow_tester_LDFLAGS += -static-libstdc++ -static-libgcc -ldl -lpthread -lrt -lm
 else ifeq ($(PLATFORM),osx)
   fdb_flow_tester_LDFLAGS += -lc++
 endif

From 7421d7808d56893fbccd360d6b85401149e1edd2 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 02:38:16 -0700
Subject: [PATCH 083/118] Set MacOS compilation flag to c++17

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1f63242c44..7e2386cc85 100644
--- a/Makefile
+++ b/Makefile
@@ -56,7 +56,7 @@ else ifeq ($(PLATFORM),Darwin)
   CXX := /usr/bin/clang
 
   CFLAGS += -mmacosx-version-min=10.7 -stdlib=libc++
-  CXXFLAGS += -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
+  CXXFLAGS += -mmacosx-version-min=10.7 -std=c++17 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
 
   .LIBPATTERNS := lib%.dylib lib%.a
 

From b78632a00e93a4b530e78c2ac4ea7b6df07e7e6d Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 09:08:24 -0700
Subject: [PATCH 084/118] Changed min macos version to support due to use of
 std::get requirement

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 7e2386cc85..b61e8e0590 100644
--- a/Makefile
+++ b/Makefile
@@ -55,8 +55,8 @@ else ifeq ($(PLATFORM),Darwin)
   CC := /usr/bin/clang
   CXX := /usr/bin/clang
 
-  CFLAGS += -mmacosx-version-min=10.7 -stdlib=libc++
-  CXXFLAGS += -mmacosx-version-min=10.7 -std=c++17 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
+  CFLAGS += -mmacosx-version-min=10.14 -stdlib=libc++
+  CXXFLAGS += -mmacosx-version-min=10.14 -std=c++17 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
 
   .LIBPATTERNS := lib%.dylib lib%.a
 

From f3897238f865a4595b178cffdf87d019e670ad04 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 15 May 2019 10:13:38 -0700
Subject: [PATCH 085/118] added the ability to add a read conflict range on the
 metadata version key without the READ_SYSTEM_KEYS option

---
 fdbclient/ReadYourWrites.actor.cpp | 2 +-
 fdbclient/SystemData.cpp           | 1 +
 fdbclient/SystemData.h             | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp
index 2b5c60f4fb..0811167273 100644
--- a/fdbclient/ReadYourWrites.actor.cpp
+++ b/fdbclient/ReadYourWrites.actor.cpp
@@ -1344,7 +1344,7 @@ void ReadYourWritesTransaction::addReadConflictRange( KeyRangeRef const& keys )
 	}
 	
 	if (tr.apiVersionAtLeast(300)) {
-		if (keys.begin > getMaxReadKey() || keys.end > getMaxReadKey()) {
+		if ((keys.begin > getMaxReadKey() || keys.end > getMaxReadKey()) && (keys.begin != metadataVersionKey || keys.end != metadataVersionKeyEnd)) {
 			throw key_outside_legal_range();
 		}
 	}
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index a8cf78f009..8c6552a109 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -587,6 +587,7 @@ const KeyRef maxUIDKey = LiteralStringRef("\xff\xff\xff\xff\xff\xff\xff\xff\xff\
 
 const KeyRef databaseLockedKey = LiteralStringRef("\xff/dbLocked");
 const KeyRef metadataVersionKey = LiteralStringRef("\xff/metadataVersion");
+const KeyRef metadataVersionKeyEnd = LiteralStringRef("\xff/metadataVersion\x00");
 const KeyRef metadataVersionRequiredValue = LiteralStringRef("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00");
 const KeyRef mustContainSystemMutationsKey = LiteralStringRef("\xff/mustContainSystemMutations");
 
diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h
index f2b1debdec..a4e15f76f9 100644
--- a/fdbclient/SystemData.h
+++ b/fdbclient/SystemData.h
@@ -266,6 +266,7 @@ extern const KeyRef maxUIDKey;
 
 extern const KeyRef databaseLockedKey;
 extern const KeyRef metadataVersionKey;
+extern const KeyRef metadataVersionKeyEnd;
 extern const KeyRef metadataVersionRequiredValue;
 extern const KeyRef mustContainSystemMutationsKey;
 

From c93ebe1e8ec0536659445800464e9953276dac77 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 10:46:52 -0700
Subject: [PATCH 086/118] Created new build docker with valgrind and default
 compilation flags

---
 build/Dockerfile          | 8 +++++---
 build/docker-compose.yaml | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index 3a0f61908a..32e7a3f2f5 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -1,6 +1,6 @@
 FROM centos:6
-LABEL version=0.1.3
-ENV DOCKER_IMAGEVER=0.1.3
+LABEL version=0.1.4
+ENV DOCKER_IMAGEVER=0.1.4
 
 # Install dependencies for developer tools, bindings,\
 # documentation, actorcompiler, and packaging tools\
@@ -9,7 +9,7 @@ RUN yum install -y yum-utils &&\
 	yum -y install centos-release-scl epel-release &&\
 	yum -y install devtoolset-8 java-1.8.0-openjdk-devel \
 		rh-python36-python-devel mono-core rh-ruby24 golang python27 \
-		rpm-build debbuild python-pip npm dos2unix &&\
+		rpm-build debbuild python-pip npm dos2unix valgrind &&\
 	pip install boto3==1.1.1
 
 USER root
@@ -41,4 +41,6 @@ RUN curl -L https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz >
     rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz
 
 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
+ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
+ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++
 CMD scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash
diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml
index 28dea93408..a663b31770 100644
--- a/build/docker-compose.yaml
+++ b/build/docker-compose.yaml
@@ -2,7 +2,7 @@ version: "3"
 
 services:
   common: &common
-    image: foundationdb/foundationdb-build:0.1.3
+    image: foundationdb/foundationdb-build:0.1.4
 
   build-setup: &build-setup
     <<: *common

From 6c51b10c84e8d7634aa446b63b30dd62211f614e Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 11:08:06 -0700
Subject: [PATCH 087/118] Removed default compilation flag environmental
 variables

---
 build/Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index 32e7a3f2f5..7a53d0ceb9 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -41,6 +41,4 @@ RUN curl -L https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz >
     rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz
 
 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
-ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
-ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++
 CMD scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash

From ee1e3f620a0ff1bd2a66a5cf69ead12887070be1 Mon Sep 17 00:00:00 2001
From: Austin Seipp <aseipp@pobox.com>
Date: Sun, 12 May 2019 12:41:04 -0500
Subject: [PATCH 088/118] cmake: add support for choosing 'ld' via -DUSE_LD

This introduces a new USE_LD cached variable which can be set to one of
three values: LD, GOLD, or LLD. In the latter two cases, we propagate
the -fuse-ld={gold,lld} option to the compiler. The default case
defaults to whatever the ordinary system linker is, i.e. it does
nothing.

Signed-off-by: Austin Seipp <aseipp@pobox.com>
---
 cmake/ConfigureCompiler.cmake | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 5258086b1d..dfaaa02fdd 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -1,10 +1,11 @@
 set(USE_GPERFTOOLS OFF CACHE BOOL "Use gperfools for profiling")
 set(PORTABLE_BINARY OFF CACHE BOOL "Create a binary that runs on older OS versions")
 set(USE_VALGRIND OFF CACHE BOOL "Compile for valgrind usage")
-set(USE_GOLD_LINKER OFF CACHE BOOL "Use gold linker")
 set(ALLOC_INSTRUMENTATION OFF CACHE BOOL "Instrument alloc")
 set(WITH_UNDODB OFF CACHE BOOL "Use rr or undodb")
+set(USE_ASAN OFF CACHE BOOL "Compile with address sanitizer")
 set(FDB_RELEASE OFF CACHE BOOL "This is a building of a final release")
+set(USE_LD "LD" CACHE STRING "The linker to use for building: can be LD (system default, default choice), GOLD, or LLD")
 
 if(USE_GPERFTOOLS)
   find_package(Gperftools REQUIRED)
@@ -47,7 +48,6 @@ include(CheckFunctionExists)
 set(CMAKE_REQUIRED_INCLUDES stdlib.h malloc.h)
 set(CMAKE_REQUIRED_LIBRARIES c)
 
-
 if(WIN32)
   # see: https://docs.microsoft.com/en-us/windows/desktop/WinProg/using-the-windows-headers
   # this sets the windows target version to Windows 7
@@ -55,10 +55,6 @@ if(WIN32)
   add_compile_options(/W3 /EHsc /std:c++17 /bigobj $<$<CONFIG:Release>:/Zi> /MP)
   add_compile_definitions(_WIN32_WINNT=${WINDOWS_TARGET} BOOST_ALL_NO_LIB)
 else()
-  if(USE_GOLD_LINKER)
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold -Wl,--disable-new-dtags")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold -Wl,--disable-new-dtags")
-  endif()
 
   set(GCC NO)
   set(CLANG NO)
@@ -70,10 +66,29 @@ else()
     set(GCC YES)
   endif()
 
+  # check linker flags.
+  if ((NOT (USE_LD STREQUAL "LD")) AND (NOT (USE_LD STREQUAL "GOLD")) AND (NOT (USE_LD STREQUAL "LLD")))
+    message (FATAL_ERROR "USE_LD must be set to LD, GOLD, or LLD!")
+  endif()
+
+  # if USE_LD=LD, then we don't do anything, defaulting to whatever system
+  # linker is available (e.g. binutils doesn't normally exist on macOS, so this
+  # implies the default xcode linker, and other distros may choose others by
+  # default).
+
+  if(USE_LD STREQUAL "GOLD")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold -Wl,--disable-new-dtags")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold -Wl,--disable-new-dtags")
+  endif()
+
+  if(USE_LD STREQUAL "LLD")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld -Wl,--disable-new-dtags")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld -Wl,--disable-new-dtags")
+  endif()
+
   # we always compile with debug symbols. CPack will strip them out
   # and create a debuginfo rpm
   add_compile_options(-ggdb)
-  set(USE_ASAN OFF CACHE BOOL "Compile with address sanitizer")
   if(USE_ASAN)
     add_compile_options(
       -fno-omit-frame-pointer -fsanitize=address

From 28e7255f4b9fd5d46bb37828265638a7e31c1bc4 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 13:23:12 -0700
Subject: [PATCH 089/118] Added valgrind development package for Valgrind
 builds

---
 build/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index 7a53d0ceb9..9c10df46cc 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -9,7 +9,7 @@ RUN yum install -y yum-utils &&\
 	yum -y install centos-release-scl epel-release &&\
 	yum -y install devtoolset-8 java-1.8.0-openjdk-devel \
 		rh-python36-python-devel mono-core rh-ruby24 golang python27 \
-		rpm-build debbuild python-pip npm dos2unix valgrind &&\
+		rpm-build debbuild python-pip npm dos2unix valgrind-devel &&\
 	pip install boto3==1.1.1
 
 USER root

From 43f142ac139ee86700aa2ba8e61d2ea0026a568a Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 13:27:08 -0700
Subject: [PATCH 090/118] Upgraded devtoolset package from 7 to 8

---
 build/docker-compose.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml
index a663b31770..1833b62dfa 100644
--- a/build/docker-compose.yaml
+++ b/build/docker-compose.yaml
@@ -29,16 +29,16 @@ services:
     volumes:
       - ..:/foundationdb
     working_dir: /foundationdb
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage'
 
 
   release-packages: &release-packages
     <<: *release-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
 
   snapshot-packages: &snapshot-packages
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages'
 
   prb-packages:
     <<: *snapshot-packages
@@ -46,11 +46,11 @@ services:
 
   release-bindings: &release-bindings
     <<: *release-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
 
   snapshot-bindings: &snapshot-bindings
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings'
 
   prb-bindings:
     <<: *snapshot-bindings
@@ -58,7 +58,7 @@ services:
 
   snapshot-cmake: &snapshot-cmake
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" packages preinstall && cpack'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" packages preinstall && cpack'
 
   prb-cmake:
     <<: *snapshot-cmake
@@ -66,7 +66,7 @@ services:
 
   snapshot-ctest: &snapshot-ctest
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure'
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure'
 
   prb-ctest:
     <<: *snapshot-ctest
@@ -74,7 +74,7 @@ services:
 
   snapshot-correctness: &snapshot-correctness
     <<: *build-setup
-    command: scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure
+    command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && make -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure
 
   prb-correctness:
     <<: *snapshot-correctness

From 6a55949e03e727c183071e4c36b46ed53a4fd7d3 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 14:32:21 -0700
Subject: [PATCH 091/118] Addded in developer package for valgrind

---
 build/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/build/Dockerfile b/build/Dockerfile
index 9c10df46cc..890066e1e9 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -8,8 +8,9 @@ RUN yum install -y yum-utils &&\
 	yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
 	yum -y install centos-release-scl epel-release &&\
 	yum -y install devtoolset-8 java-1.8.0-openjdk-devel \
-		rh-python36-python-devel mono-core rh-ruby24 golang python27 \
-		rpm-build debbuild python-pip npm dos2unix valgrind-devel &&\
+		rh-python36-python-devel devtoolset-8-valgrind-devel \
+		mono-core rh-ruby24 golang python27 rpm-build debbuild \
+		python-pip npm dos2unix valgrind-devel &&\
 	pip install boto3==1.1.1
 
 USER root

From 148122800f75618848f33fb9a5712530e94acd86 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 22:32:16 -0700
Subject: [PATCH 092/118] Removed dead project from Windows solution

---
 foundationdb.sln | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/foundationdb.sln b/foundationdb.sln
index f0162c7b60..b02682b919 100644
--- a/foundationdb.sln
+++ b/foundationdb.sln
@@ -144,10 +144,6 @@ Global
 		{E22D4EF8-E75D-4281-93F9-A9F73936DE54}.Debug|X64.Build.0 = Debug|x64
 		{E22D4EF8-E75D-4281-93F9-A9F73936DE54}.Release|X64.ActiveCfg = Release|x64
 		{E22D4EF8-E75D-4281-93F9-A9F73936DE54}.Release|X64.Build.0 = Release|x64
-		{9463CB25-DCA0-9D45-C46E-0A8E68EE7FAE}.Debug|X64.ActiveCfg = Debug|x64
-		{9463CB25-DCA0-9D45-C46E-0A8E68EE7FAE}.Debug|X64.Build.0 = Debug|x64
-		{9463CB25-DCA0-9D45-C46E-0A8E68EE7FAE}.Release|X64.ActiveCfg = Release|x64
-		{9463CB25-DCA0-9D45-C46E-0A8E68EE7FAE}.Release|X64.Build.0 = Release|x64
 		{2BA0A5E2-EB4C-4A32-948C-CBAABD77AF87}.Debug|X64.ActiveCfg = Debug|X64
 		{2BA0A5E2-EB4C-4A32-948C-CBAABD77AF87}.Debug|X64.Build.0 = Debug|X64
 		{2BA0A5E2-EB4C-4A32-948C-CBAABD77AF87}.Release|X64.ActiveCfg = Release|X64

From 8f6af3ceff4da73d1e754b50f3a9d475093b82ee Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Wed, 15 May 2019 15:33:58 -1000
Subject: [PATCH 093/118] procossed -> processed

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6d46b458f0..6f949245e5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -196,7 +196,7 @@ if (CMAKE_EXPORT_COMPILE_COMMANDS)
 	DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
 	COMMENT "Build compile commands for IDE"
 	)
-  add_custom_target(procossed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json)
+  add_custom_target(processed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json)
 endif()
 
 ################################################################################

From f271d8816aee10392c4b5a077411bf95f29efc5f Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 15 May 2019 23:41:19 -0700
Subject: [PATCH 094/118] Removed dead project id from solution file

---
 foundationdb.sln | 1 -
 1 file changed, 1 deletion(-)

diff --git a/foundationdb.sln b/foundationdb.sln
index b02682b919..35a08f9918 100644
--- a/foundationdb.sln
+++ b/foundationdb.sln
@@ -54,7 +54,6 @@ EndProject
 Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MSIInstaller", "packaging\msi\MSIInstaller.wixproj", "{C797E922-C07D-489F-B3CF-5AC35B709F4C}"
 	ProjectSection(ProjectDependencies) = postProject
 		{0C938D09-39F6-48C0-9C6A-45D3ADBDCFB7} = {0C938D09-39F6-48C0-9C6A-45D3ADBDCFB7}
-		{9463CB25-DCA0-9D45-C46E-0A8E68EE7FAE} = {9463CB25-DCA0-9D45-C46E-0A8E68EE7FAE}
 		{9617584C-22E8-4272-934F-733F378BF6AE} = {9617584C-22E8-4272-934F-733F378BF6AE}
 		{CACB2C8E-3E55-4309-A411-2A9C56C6C1CB} = {CACB2C8E-3E55-4309-A411-2A9C56C6C1CB}
 		{4631CC93-52A3-4537-9BE9-6B237A3AC6B2} = {4631CC93-52A3-4537-9BE9-6B237A3AC6B2}

From 8077a428c02eb58e225fa51aed2e90c11537ac1d Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Thu, 16 May 2019 09:30:34 -0700
Subject: [PATCH 095/118] Fix invalid trace suppression

---
 FDBLibTLS/FDBLibTLSSession.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp
index 352e4ae00b..fff6ac8d1d 100644
--- a/FDBLibTLS/FDBLibTLSSession.cpp
+++ b/FDBLibTLS/FDBLibTLSSession.cpp
@@ -345,7 +345,7 @@ bool FDBLibTLSSession::verify_peer() {
 	if (!rc) {
 		// log the various failure reasons
 		for (std::string reason : verify_failure_reasons) {
-			TraceEvent("FDBLibTLSVerifyFailure", uid).detail("Reason", reason).suppressFor(1.0);
+			TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason);
 		}
 	}
 

From 6389a129317bdbba2f9eb732591e3f5ed24eae84 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Thu, 16 May 2019 10:37:33 -0700
Subject: [PATCH 096/118] Switched variant to use boost variant for source
 consistency

---
 flow/ObjectSerializerTraits.h | 4 +---
 flow/network.h                | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h
index af6025bfb8..ea08e127bb 100644
--- a/flow/ObjectSerializerTraits.h
+++ b/flow/ObjectSerializerTraits.h
@@ -26,7 +26,7 @@
 #include <memory>
 #include <functional>
 #include <vector>
-#include <variant>
+#include <boost/variant.hpp>
 
 template <class T, typename = void>
 struct is_fb_function_t : std::false_type {};
@@ -195,5 +195,3 @@ struct union_like_traits<std::variant<Alternatives...>> : std::true_type {
 		member = a;
 	}
 };
-
-
diff --git a/flow/network.h b/flow/network.h
index 4b33c9c8c9..6751f081d2 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -25,7 +25,7 @@
 #include <array>
 #include <string>
 #include <stdint.h>
-#include <variant>
+#include <boost/variant.hpp>
 #include "boost/asio.hpp"
 #include "flow/serialize.h"
 #include "flow/IRandom.h"

From 17a2983943e1e1c61da5858ce2161b7beb6a79cb Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Thu, 16 May 2019 13:05:49 -0700
Subject: [PATCH 097/118] Switch boost variant to C++17 variant

---
 flow/ObjectSerializerTraits.h |  2 +-
 flow/flat_buffers.cpp         | 35 ++++++++---------------------------
 flow/network.h                |  2 +-
 3 files changed, 10 insertions(+), 29 deletions(-)

diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h
index ea08e127bb..3301214e76 100644
--- a/flow/ObjectSerializerTraits.h
+++ b/flow/ObjectSerializerTraits.h
@@ -26,7 +26,7 @@
 #include <memory>
 #include <functional>
 #include <vector>
-#include <boost/variant.hpp>
+#include <variant>
 
 template <class T, typename = void>
 struct is_fb_function_t : std::false_type {};
diff --git a/flow/flat_buffers.cpp b/flow/flat_buffers.cpp
index 81acd94c14..00546098d7 100644
--- a/flow/flat_buffers.cpp
+++ b/flow/flat_buffers.cpp
@@ -26,7 +26,7 @@
 
 #include <algorithm>
 #include <iomanip>
-#include <boost/variant.hpp>
+#include <variant>
 
 namespace detail {
 
@@ -283,29 +283,10 @@ TEST_CASE("flow/FlatBuffers/serializeDeserializeMembers") {
 
 } // namespace unit_tests
 
-template <class... Alternatives>
-struct union_like_traits<boost::variant<Alternatives...>> : std::true_type {
-	using Member = boost::variant<Alternatives...>;
-	using alternatives = pack<Alternatives...>;
-	static uint8_t index(const Member& variant) { return variant.which(); }
-	static bool empty(const Member& variant) { return false; }
-
-	template <int i>
-	static const index_t<i, alternatives>& get(const Member& variant) {
-		return boost::get<index_t<i, alternatives>>(variant);
-	}
-
-	template <size_t i, class Alternative>
-	static const void assign(Member& member, const Alternative& a) {
-		static_assert(std::is_same_v<index_t<i, alternatives>, Alternative>);
-		member = a;
-	}
-};
-
 namespace unit_tests {
 
 TEST_CASE("flow/FlatBuffers/variant") {
-	using V = boost::variant<int, double, Nested2>;
+	using V = std::variant<int, double, Nested2>;
 	V v1;
 	V v2;
 	Arena arena;
@@ -316,21 +297,21 @@ TEST_CASE("flow/FlatBuffers/variant") {
 	out = save_members(arena, FileIdentifier{}, v1);
 	// print_buffer(out, arena.get_size(out));
 	load_members(out, context, v2);
-	ASSERT(boost::get<int>(v1) == boost::get<int>(v2));
+	ASSERT(std::get<int>(v1) == std::get<int>(v2));
 
 	v1 = 1.0;
 	out = save_members(arena, FileIdentifier{}, v1);
 	// print_buffer(out, arena.get_size(out));
 	load_members(out, context, v2);
-	ASSERT(boost::get<double>(v1) == boost::get<double>(v2));
+	ASSERT(std::get<double>(v1) == std::get<double>(v2));
 
 	v1 = Nested2{ 1, { "abc", "def" }, 2 };
 	out = save_members(arena, FileIdentifier{}, v1);
 	// print_buffer(out, arena.get_size(out));
 	load_members(out, context, v2);
-	ASSERT(boost::get<Nested2>(v1).a == boost::get<Nested2>(v2).a);
-	ASSERT(boost::get<Nested2>(v1).b == boost::get<Nested2>(v2).b);
-	ASSERT(boost::get<Nested2>(v1).c == boost::get<Nested2>(v2).c);
+	ASSERT(std::get<Nested2>(v1).a == std::get<Nested2>(v2).a);
+	ASSERT(std::get<Nested2>(v1).b == std::get<Nested2>(v2).b);
+	ASSERT(std::get<Nested2>(v1).c == std::get<Nested2>(v2).c);
 	return Void();
 }
 
@@ -404,7 +385,7 @@ struct Y1 {
 
 struct Y2 {
 	int a;
-	boost::variant<int> b;
+	std::variant<int> b;
 
 	template <class Archiver>
 	void serialize(Archiver& ar) {
diff --git a/flow/network.h b/flow/network.h
index 6751f081d2..4b33c9c8c9 100644
--- a/flow/network.h
+++ b/flow/network.h
@@ -25,7 +25,7 @@
 #include <array>
 #include <string>
 #include <stdint.h>
-#include <boost/variant.hpp>
+#include <variant>
 #include "boost/asio.hpp"
 #include "flow/serialize.h"
 #include "flow/IRandom.h"

From 94aed513c7bbccecdd70bf6766a3f31800590981 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Thu, 16 May 2019 15:05:11 -0700
Subject: [PATCH 098/118] Switched Windows tools within projects to 2017

---
 FDBLibTLS/FDBLibTLS.vcxproj                  | 6 +++---
 bindings/c/fdb_c.vcxproj                     | 6 +++---
 bindings/flow/fdb_flow.vcxproj               | 6 +++---
 bindings/flow/tester/fdb_flow_tester.vcxproj | 6 +++---
 bindings/java/fdb_java.vcxproj               | 6 +++---
 fdbbackup/fdbbackup.vcxproj                  | 6 +++---
 fdbcli/fdbcli.vcxproj                        | 6 +++---
 fdbclient/fdbclient.vcxproj                  | 6 +++---
 fdbmonitor/fdbmonitor.vcxproj                | 6 +++---
 fdbrpc/fdbrpc.vcxproj                        | 6 +++---
 fdbserver/fdbserver.vcxproj                  | 6 +++---
 fdbservice/fdbservice.vcxproj                | 6 +++---
 flow/flow.vcxproj                            | 6 +++---
 13 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/FDBLibTLS/FDBLibTLS.vcxproj b/FDBLibTLS/FDBLibTLS.vcxproj
index 579e03297f..2ebdc603f7 100644
--- a/FDBLibTLS/FDBLibTLS.vcxproj
+++ b/FDBLibTLS/FDBLibTLS.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|X64">
       <Configuration>Debug</Configuration>
@@ -23,11 +23,11 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
 		<ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
 		<ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
 </Project>
diff --git a/bindings/c/fdb_c.vcxproj b/bindings/c/fdb_c.vcxproj
index fd646eb75e..ba2f721633 100644
--- a/bindings/c/fdb_c.vcxproj
+++ b/bindings/c/fdb_c.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup Condition="'$(Release)' != 'true' ">
   </PropertyGroup>
   <PropertyGroup Condition="'$(Release)' == 'true' ">
@@ -38,14 +38,14 @@
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/bindings/flow/fdb_flow.vcxproj b/bindings/flow/fdb_flow.vcxproj
index d7b0de94c1..56348258c9 100755
--- a/bindings/flow/fdb_flow.vcxproj
+++ b/bindings/flow/fdb_flow.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup Condition="'$(Release)' != 'true' ">
   </PropertyGroup>
   <PropertyGroup Condition="'$(Release)' == 'true' ">
@@ -63,12 +63,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/bindings/flow/tester/fdb_flow_tester.vcxproj b/bindings/flow/tester/fdb_flow_tester.vcxproj
index 621f16afc3..b1fa3bcd6b 100644
--- a/bindings/flow/tester/fdb_flow_tester.vcxproj
+++ b/bindings/flow/tester/fdb_flow_tester.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -35,14 +35,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>false</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/bindings/java/fdb_java.vcxproj b/bindings/java/fdb_java.vcxproj
index 8196ac9962..7117fd3bdd 100644
--- a/bindings/java/fdb_java.vcxproj
+++ b/bindings/java/fdb_java.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -28,14 +28,14 @@
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbbackup/fdbbackup.vcxproj b/fdbbackup/fdbbackup.vcxproj
index fc12aed414..b2c058426b 100644
--- a/fdbbackup/fdbbackup.vcxproj
+++ b/fdbbackup/fdbbackup.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -37,12 +37,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbcli/fdbcli.vcxproj b/fdbcli/fdbcli.vcxproj
index 0cd91518b3..fc15760cf1 100644
--- a/fdbcli/fdbcli.vcxproj
+++ b/fdbcli/fdbcli.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -39,14 +39,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>false</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj
index 610bdfc77f..a1fef6502f 100644
--- a/fdbclient/fdbclient.vcxproj
+++ b/fdbclient/fdbclient.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -149,12 +149,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbmonitor/fdbmonitor.vcxproj b/fdbmonitor/fdbmonitor.vcxproj
index 80ac3835a0..a2673c8bc7 100644
--- a/fdbmonitor/fdbmonitor.vcxproj
+++ b/fdbmonitor/fdbmonitor.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|x64">
       <Configuration>Debug</Configuration>
@@ -19,14 +19,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbrpc/fdbrpc.vcxproj b/fdbrpc/fdbrpc.vcxproj
index 553bd7a422..e170700322 100644
--- a/fdbrpc/fdbrpc.vcxproj
+++ b/fdbrpc/fdbrpc.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|X64">
       <Configuration>Debug</Configuration>
@@ -138,12 +138,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbserver/fdbserver.vcxproj b/fdbserver/fdbserver.vcxproj
index 27c3c48353..deec23685e 100644
--- a/fdbserver/fdbserver.vcxproj
+++ b/fdbserver/fdbserver.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -240,12 +240,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/fdbservice/fdbservice.vcxproj b/fdbservice/fdbservice.vcxproj
index 1b6b528147..6875a0275e 100644
--- a/fdbservice/fdbservice.vcxproj
+++ b/fdbservice/fdbservice.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(SolutionDir)versions.target" />
   <PropertyGroup Condition="'$(Release)' != 'true' ">
     <PreReleaseDecoration>-PRERELEASE</PreReleaseDecoration>
@@ -28,14 +28,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj
index caa74e1311..63ebd03d61 100644
--- a/flow/flow.vcxproj
+++ b/flow/flow.vcxproj
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="14.1" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|X64">
       <Configuration>Debug</Configuration>
@@ -118,12 +118,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v140_xp</PlatformToolset>
+    <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">

From 7b8e1407880e692372485f4446539ae8c3c45fa0 Mon Sep 17 00:00:00 2001
From: Stephen Atherton <stevea@apple.com>
Date: Thu, 16 May 2019 17:23:33 -0700
Subject: [PATCH 099/118] Added UID field to fdbbackup status --json output.

---
 fdbclient/FileBackupAgent.actor.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 57e61d661e..26c8b3c6a5 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3928,6 +3928,8 @@ public:
 				doc.setKey("Tag", tag.tagName);
 
 				if(uidAndAbortedFlag.present()) {
+					doc.setKey("UID", uidAndAbortedFlag.get().first.toString().c_str());
+
 					state BackupConfig config(uidAndAbortedFlag.get().first);
 
 					state EBackupState backupState = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN));

From 53b97fe506029c38b792dadd9307f3a937c3d2c0 Mon Sep 17 00:00:00 2001
From: "Paul J. Davis" <paul.joseph.davis@gmail.com>
Date: Thu, 25 Apr 2019 14:49:14 -0500
Subject: [PATCH 100/118] Extend support for parentpid

This adds support for the `--parentpid` option to non-Windows platforms.
This option is intended for testing layer implementations. When running
higher level CI chains its useful to ensure that any ephemeral instances
of fdbserver are automatically reaped.
---
 fdbserver/fdbserver.actor.cpp | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index 515dd35b39..9a0904db09 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -105,8 +105,8 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_MAXLOGS,               "--maxlogs",                   SO_REQ_SEP },
 	{ OPT_MAXLOGSSIZE,           "--maxlogssize",               SO_REQ_SEP },
 	{ OPT_LOGGROUP,              "--loggroup",                  SO_REQ_SEP },
-#ifdef _WIN32
 	{ OPT_PARENTPID,             "--parentpid",                 SO_REQ_SEP },
+#ifdef _WIN32
 	{ OPT_NEWCONSOLE,            "-n",                          SO_NONE },
 	{ OPT_NEWCONSOLE,            "--newconsole",                SO_NONE },
 	{ OPT_NOBOX,                 "-q",                          SO_NONE },
@@ -504,6 +504,15 @@ void parentWatcher(void *parentHandle) {
 		criticalError( FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited" );
 	TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError();
 }
+#else
+void* parentWatcher(void *arg) {
+	int *parent_pid = (int*) arg;
+	while(1) {
+		sleep(1);
+		if(getppid() != *parent_pid)
+			criticalError( FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited" );
+	}
+}
 #endif
 
 static void printVersion() {
@@ -1166,6 +1175,14 @@ int main(int argc, char* argv[]) {
 				case OPT_NOBOX:
 					SetErrorMode(SetErrorMode(0) | SEM_NOGPFAULTERRORBOX);
 					break;
+	#else
+				case OPT_PARENTPID: {
+					auto pid_str = args.OptionArg();
+					int *parent_pid = new(int);
+					*parent_pid = atoi(pid_str);
+					startThread(&parentWatcher, parent_pid);
+					break;
+				}
 	#endif
 				case OPT_TESTFILE:
 					testFile = args.OptionArg();

From e495afdc6eead90192cebf48307a1e7f2bca0952 Mon Sep 17 00:00:00 2001
From: Stephen Atherton <stevea@apple.com>
Date: Thu, 16 May 2019 17:26:37 -0700
Subject: [PATCH 101/118] Removed unnecessary c_str().

---
 fdbclient/FileBackupAgent.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index 26c8b3c6a5..ca25ca83fa 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -3928,7 +3928,7 @@ public:
 				doc.setKey("Tag", tag.tagName);
 
 				if(uidAndAbortedFlag.present()) {
-					doc.setKey("UID", uidAndAbortedFlag.get().first.toString().c_str());
+					doc.setKey("UID", uidAndAbortedFlag.get().first.toString());
 
 					state BackupConfig config(uidAndAbortedFlag.get().first);
 

From a745a8094ecd8ddcd1f331ff0f6637c6914357f2 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Thu, 16 May 2019 17:26:48 -0700
Subject: [PATCH 102/118] do not close a connection due to a missed ping if the
 process is still receiving data from the connection

---
 fdbrpc/FlowTransport.actor.cpp | 35 +++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 5319078f37..c0908d26ed 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -282,16 +282,18 @@ struct Peer : NonCopyable {
 	ReliablePacketList reliable;
 	AsyncTrigger dataToSend;  // Triggered when unsent.empty() becomes false
 	Future<Void> connect;
-	AsyncTrigger incompatibleDataRead;
+	AsyncTrigger resetPing;
 	bool compatible;
 	bool outgoingConnectionIdle;  // We don't actually have a connection open and aren't trying to open one because we don't have anything to send
 	double lastConnectTime;
 	double reconnectionDelay;
 	int peerReferences;
 	bool incompatibleProtocolVersionNewer;
+	int64_t bytesReceived;
 
 	explicit Peer( TransportData* transport, NetworkAddress const& destination )
-		: transport(transport), destination(destination), outgoingConnectionIdle(false), lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), compatible(true), incompatibleProtocolVersionNewer(false), peerReferences(-1)
+		: transport(transport), destination(destination), outgoingConnectionIdle(false), lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), 
+		  compatible(true), incompatibleProtocolVersionNewer(false), peerReferences(-1), bytesReceived(0)
 	{
 		connect = connectionKeeper(this);
 	}
@@ -389,11 +391,23 @@ struct Peer : NonCopyable {
 
 			state ReplyPromise<Void> reply;
 			FlowTransport::transport().sendUnreliable( SerializeSource<ReplyPromise<Void>>(reply), remotePing.getEndpoint() );
-
-			choose {
-				when (wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) { TraceEvent("ConnectionTimeout").suppressFor(1.0).detail("WithAddr", peer->destination); throw connection_failed(); }
-				when (wait( reply.getFuture() )) {}
-				when (wait( peer->incompatibleDataRead.onTrigger())) {}
+			state int64_t startingBytes = peer->bytesReceived;
+			loop {
+				choose {
+					when (wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) {
+						if(startingBytes == peer->bytesReceived) {
+							TraceEvent("ConnectionTimeout").suppressFor(1.0).detail("WithAddr", peer->destination);
+							throw connection_failed();
+						}
+						startingBytes = peer->bytesReceived;
+					}
+					when (wait( reply.getFuture() )) {
+						break;
+					}
+					when (wait( peer->resetPing.onTrigger())) {
+						break;
+					}
+				}
 			}
 		}
 	}
@@ -697,6 +711,9 @@ ACTOR static Future<Void> connectionReader(
 				}
 
 				int readBytes = conn->read( unprocessed_end, buffer_end );
+				if(peer) {
+					peer->bytesReceived += readBytes;
+				}
 				if (!readBytes) break;
 				state bool readWillBlock = readBytes != readAllBytes;
 				unprocessed_end += readBytes;
@@ -792,7 +809,7 @@ ACTOR static Future<Void> connectionReader(
 				}
 				else if(!expectConnectPacket) {
 					unprocessed_begin = unprocessed_end;
-					peer->incompatibleDataRead.trigger();
+					peer->resetPing.trigger();
 				}
 
 				if (readWillBlock)
@@ -959,7 +976,7 @@ void FlowTransport::removePeerReference( const Endpoint& endpoint, NetworkMessag
 				.detail("Token", endpoint.token);
 		}
 		if(peer->peerReferences == 0 && peer->reliable.empty() && peer->unsent.empty()) {
-			peer->incompatibleDataRead.trigger();
+			peer->resetPing.trigger();
 		}
 	}
 }

From 3acaa7343e16d1d3ac0f5dc1d2958c2b0007cd42 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Thu, 16 May 2019 17:44:13 -0700
Subject: [PATCH 103/118] Enabled C++17 for all Windows projects Set Visual
 Studio version to 2017 (first version to support C++17)

---
 FDBLibTLS/FDBLibTLS.vcxproj                  | 4 ++--
 bindings/c/fdb_c.vcxproj                     | 6 ++++--
 bindings/flow/fdb_flow.vcxproj               | 6 ++++--
 bindings/flow/tester/fdb_flow_tester.vcxproj | 7 +++++--
 bindings/java/fdb_java.vcxproj               | 6 ++++--
 fdbbackup/fdbbackup.vcxproj                  | 7 +++++--
 fdbcli/fdbcli.vcxproj                        | 7 +++++--
 fdbclient/fdbclient.vcxproj                  | 7 +++++--
 fdbmonitor/fdbmonitor.vcxproj                | 6 ++++--
 fdbrpc/fdbrpc.vcxproj                        | 6 ++++--
 fdbserver/fdbserver.vcxproj                  | 7 +++++--
 fdbservice/fdbservice.vcxproj                | 7 +++++--
 flow/flow.vcxproj                            | 6 ++++--
 packaging/msi/MSIInstaller.wixproj           | 2 +-
 14 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/FDBLibTLS/FDBLibTLS.vcxproj b/FDBLibTLS/FDBLibTLS.vcxproj
index 2ebdc603f7..315ea7a37a 100644
--- a/FDBLibTLS/FDBLibTLS.vcxproj
+++ b/FDBLibTLS/FDBLibTLS.vcxproj
@@ -23,11 +23,11 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
 		<ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
 		<ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
 </Project>
diff --git a/bindings/c/fdb_c.vcxproj b/bindings/c/fdb_c.vcxproj
index ba2f721633..61322f8c23 100644
--- a/bindings/c/fdb_c.vcxproj
+++ b/bindings/c/fdb_c.vcxproj
@@ -38,14 +38,14 @@
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -83,6 +83,7 @@ FOR /F "tokens=1" %%i in ('hg.exe id') do copy /Y "$(TargetPath)" "$(TargetPath)
       <PreprocessorDefinitions>TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
       <AdditionalOptions>/bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
@@ -98,6 +99,7 @@ FOR /F "tokens=1" %%i in ('hg.exe id') do copy /Y "$(TargetPath)" "$(TargetPath)
       <PreprocessorDefinitions>TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <AdditionalOptions>/bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
diff --git a/bindings/flow/fdb_flow.vcxproj b/bindings/flow/fdb_flow.vcxproj
index 56348258c9..d6a71427c0 100755
--- a/bindings/flow/fdb_flow.vcxproj
+++ b/bindings/flow/fdb_flow.vcxproj
@@ -63,12 +63,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -99,6 +99,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);..\c</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <AdditionalOptions>/bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -126,6 +127,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <MinimalRebuild>false</MinimalRebuild>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/bindings/flow/tester/fdb_flow_tester.vcxproj b/bindings/flow/tester/fdb_flow_tester.vcxproj
index b1fa3bcd6b..ef8405f60a 100644
--- a/bindings/flow/tester/fdb_flow_tester.vcxproj
+++ b/bindings/flow/tester/fdb_flow_tester.vcxproj
@@ -35,14 +35,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>false</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -69,6 +69,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">
@@ -82,6 +83,7 @@
       <MinimalRebuild>false</MinimalRebuild>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
       <AdditionalOptions> @../../../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -107,6 +109,7 @@
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
       <AdditionalOptions> @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/bindings/java/fdb_java.vcxproj b/bindings/java/fdb_java.vcxproj
index 7117fd3bdd..d5cba49bc6 100644
--- a/bindings/java/fdb_java.vcxproj
+++ b/bindings/java/fdb_java.vcxproj
@@ -28,14 +28,14 @@
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>DynamicLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -63,6 +63,7 @@
       <PreprocessorDefinitions>TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
       <AdditionalOptions>/bigobj "@$(SolutionDir)flow/no_intellisense.opt" %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -78,6 +79,7 @@
       <PreprocessorDefinitions>TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <AdditionalOptions>/bigobj "@$(SolutionDir)flow/no_intellisense.opt" %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
diff --git a/fdbbackup/fdbbackup.vcxproj b/fdbbackup/fdbbackup.vcxproj
index b2c058426b..1ffa3ec495 100644
--- a/fdbbackup/fdbbackup.vcxproj
+++ b/fdbbackup/fdbbackup.vcxproj
@@ -37,12 +37,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -66,6 +66,7 @@
     </Lib>
     <ClCompile>
       <PreprocessorDefinitions>FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">
@@ -82,6 +83,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <AdditionalOptions>/bigobj @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -107,6 +109,7 @@
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <MinimalRebuild>false</MinimalRebuild>
+      <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/fdbcli/fdbcli.vcxproj b/fdbcli/fdbcli.vcxproj
index fc15760cf1..2c22697ee6 100644
--- a/fdbcli/fdbcli.vcxproj
+++ b/fdbcli/fdbcli.vcxproj
@@ -39,14 +39,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>false</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -73,6 +73,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">
@@ -87,6 +88,7 @@
       <MinimalRebuild>false</MinimalRebuild>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
       <AdditionalOptions> @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -113,6 +115,7 @@
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
       <AdditionalOptions> @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj
index a1fef6502f..94ab1d8024 100644
--- a/fdbclient/fdbclient.vcxproj
+++ b/fdbclient/fdbclient.vcxproj
@@ -149,12 +149,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -174,6 +174,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">
@@ -190,6 +191,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <AdditionalOptions>/bigobj @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -217,6 +219,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <MinimalRebuild>false</MinimalRebuild>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/fdbmonitor/fdbmonitor.vcxproj b/fdbmonitor/fdbmonitor.vcxproj
index a2673c8bc7..41097a6d73 100644
--- a/fdbmonitor/fdbmonitor.vcxproj
+++ b/fdbmonitor/fdbmonitor.vcxproj
@@ -19,14 +19,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -48,6 +48,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
@@ -59,6 +60,7 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
diff --git a/fdbrpc/fdbrpc.vcxproj b/fdbrpc/fdbrpc.vcxproj
index e170700322..4276f05271 100644
--- a/fdbrpc/fdbrpc.vcxproj
+++ b/fdbrpc/fdbrpc.vcxproj
@@ -138,12 +138,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -181,6 +181,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <AdditionalOptions>/bigobj @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -209,6 +210,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <MinimalRebuild>false</MinimalRebuild>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/fdbserver/fdbserver.vcxproj b/fdbserver/fdbserver.vcxproj
index deec23685e..5563e2511e 100644
--- a/fdbserver/fdbserver.vcxproj
+++ b/fdbserver/fdbserver.vcxproj
@@ -240,12 +240,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -269,6 +269,7 @@
     </Lib>
     <ClCompile>
       <PreprocessorDefinitions>FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">
@@ -286,6 +287,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <AdditionalOptions>/bigobj @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
       <PreprocessToFile>false</PreprocessToFile>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -311,6 +313,7 @@
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <MinimalRebuild>false</MinimalRebuild>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/fdbservice/fdbservice.vcxproj b/fdbservice/fdbservice.vcxproj
index 6875a0275e..18dc8084e8 100644
--- a/fdbservice/fdbservice.vcxproj
+++ b/fdbservice/fdbservice.vcxproj
@@ -28,14 +28,14 @@
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -52,6 +52,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -60,6 +61,7 @@
       <Optimization>Disabled</Optimization>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <PreprocessorDefinitions>_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
@@ -74,6 +76,7 @@
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <PreprocessorDefinitions>_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+			<LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj
index 63ebd03d61..fc0fa2a412 100644
--- a/flow/flow.vcxproj
+++ b/flow/flow.vcxproj
@@ -118,12 +118,12 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|X64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <CharacterSet>MultiByte</CharacterSet>
-    <PlatformToolset>v141_xp</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">
@@ -160,6 +160,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <AdditionalOptions>/bigobj @../flow/no_intellisense.opt %(AdditionalOptions)</AdditionalOptions>
+      <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -187,6 +188,7 @@
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
       <MinimalRebuild>false</MinimalRebuild>
+      <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
diff --git a/packaging/msi/MSIInstaller.wixproj b/packaging/msi/MSIInstaller.wixproj
index 8339e8d2f7..8a3b49be55 100644
--- a/packaging/msi/MSIInstaller.wixproj
+++ b/packaging/msi/MSIInstaller.wixproj
@@ -76,4 +76,4 @@
     </Content>
   </ItemGroup>
   <Import Project="$(WixTargetsPath)" />
-</Project>
\ No newline at end of file
+</Project>

From 4166ecdfe7420af09691665a8a1520ed80bb17e0 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Thu, 16 May 2019 18:03:16 -0700
Subject: [PATCH 104/118] Increase the configure timeout to 60 seconds, to
 avoid spurious configuration failures

---
 fdbclient/ManagementAPI.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index e96e2cd704..9d1c76bc5a 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -302,7 +302,7 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
 		}
 	}
 
-	state Future<Void> tooLong = delay(4.5);
+	state Future<Void> tooLong = delay(60);
 	state Key versionKey = BinaryWriter::toValue(g_random->randomUniqueID(),Unversioned());
 	state bool oldReplicationUsesDcId = false;
 	loop {

From d68e31e711c13287e2daeef5501def7015199af5 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Thu, 16 May 2019 18:09:12 -0700
Subject: [PATCH 105/118] Updated release notes for 6.1.7

---
 documentation/sphinx/source/release-notes.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 79720284ae..7bb5248425 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -2,7 +2,7 @@
 Release Notes
 #############
 
-6.1.6
+6.1.7
 =====
 
 Features
@@ -50,6 +50,7 @@ Performance
 * Increase the rate that deleted pages are made available for reuse in the SQLite storage engine. Rename and add knobs to provide more control over this process. [6.1.3] `(PR #1485) <https://github.com/apple/foundationdb/pull/1485>`_
 * SQLite page files now grow and shrink in chunks based on a knob which defaults to an effective chunk size of 100MB. [6.1.4] `(PR #1482) <https://github.com/apple/foundationdb/pull/1482>`_ `(PR #1499) <https://github.com/apple/foundationdb/pull/1499>`_
 * Reduced the rate at which data is moved between servers, to reduce the impact a failure has on cluster performance. [6.1.4] `(PR #1499) <https://github.com/apple/foundationdb/pull/1499>`_
+* Avoid closing saturated network connections which have not received ping packets. [6.1.7] `(PR #1601) <https://github.com/apple/foundationdb/pull/1601>`_
 
 Fixes
 -----
@@ -127,6 +128,7 @@ Fixes only impacting 6.1.0+
 * Memory tracking trace events could cause the program to crash when called from inside a trace event. [6.1.5] `(PR #1541) <https://github.com/apple/foundationdb/pull/1541>`_
 * TLogs will replace a large file with an empty file rather than doing a large truncate operation. [6.1.5] `(PR #1545) <https://github.com/apple/foundationdb/pull/1545>`_
 * Fix PR #1545 to work on Windows and Linux. [6.1.6] `(PR #1556) <https://github.com/apple/foundationdb/pull/1556>`_
+* Adding a read conflict range for the metadata version key. [6.1.6] `(PR #1556) <https://github.com/apple/foundationdb/pull/1556>`_
 
 Earlier release notes
 ---------------------

From 20e93c67ea933f4605d8d63cba7419b2ad8b34a1 Mon Sep 17 00:00:00 2001
From: Trevor Clinkenbeard <trevorclinkenbeard@gmail.com>
Date: Fri, 17 May 2019 01:03:44 -0700
Subject: [PATCH 106/118] Allow sqlite pages to be checked for CRC32 checksum

Future versions of FDB will write sqlite pages with CRC32 checksums. In
order to roll back to this version from a version that writes CRC32
checksums, this version must be able to verify those checksums.
---
 fdbserver/KeyValueStoreSQLite.actor.cpp | 57 +++++++++++++++----------
 1 file changed, 35 insertions(+), 22 deletions(-)

diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 9134914bfe..b9c02be40d 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -20,6 +20,7 @@
 
 
 #define SQLITE_THREADSAFE 0  // also in sqlite3.amalgamation.c!
+#include "fdbrpc/crc32c.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/CoroFlow.h"
 #include "fdbserver/Knobs.h"
@@ -90,32 +91,44 @@ struct PageChecksumCodec {
 
 		char *pData = (char *)data;
 		int dataLen = pageLen - sizeof(SumType);
-		SumType sum;
 		SumType *pSumInPage = (SumType *)(pData + dataLen);
 
-		// Write sum directly to page or to sum variable based on mode
-		SumType *sumOut = write ? pSumInPage : &sum;
-		sumOut->part1 = pageNumber; //DO NOT CHANGE
-		sumOut->part2 = 0x5ca1ab1e;
-		hashlittle2(pData, dataLen, &sumOut->part1, &sumOut->part2);
-
-		// Verify if not in write mode
-		if(!write && sum != *pSumInPage) {
-			if(!silent)
-				TraceEvent (SevError, "SQLitePageChecksumFailure")
-					.error(checksum_failed())
-					.detail("CodecPageSize", pageSize)
-					.detail("CodecReserveSize", reserveSize)
-					.detail("Filename", filename)
-					.detail("PageNumber", pageNumber)
-					.detail("PageSize", pageLen)
-					.detail("ChecksumInPage", pSumInPage->toString())
-					.detail("ChecksumCalculated", sum.toString());
-
-			return false;
+		if (write) {
+			// Always write a hashlittle2 checksum for new pages
+			pSumInPage->part1 = pageNumber; // DO NOT CHANGE
+			pSumInPage->part2 = 0x5ca1ab1e;
+			hashlittle2(pData, dataLen, &pSumInPage->part1, &pSumInPage->part2);
+			return true;
 		}
 
-		return true;
+		SumType sum;
+		if (pSumInPage->part1 == 0) {
+			// part1 being 0 indicates with high probability that a CRC32 checksum
+			// was used, so check that first. If this checksum fails, there is still
+			// some chance the page was written with hashlittle2, so fall back to checking
+			// hashlittle2
+			sum.part1 = 0;
+			sum.part2 = crc32c_append(0xfdbeefdb, static_cast<uint8_t*>(data), dataLen);
+			if (sum == *pSumInPage) return true;
+		}
+
+		SumType hashLittle2Sum;
+		hashLittle2Sum.part1 = pageNumber; // DO NOT CHANGE
+		hashLittle2Sum.part2 = 0x5ca1ab1e;
+		hashlittle2(pData, dataLen, &hashLittle2Sum.part1, &hashLittle2Sum.part2);
+		if (hashLittle2Sum == *pSumInPage) return true;
+
+		TraceEvent tr(SevError, "SQLitePageChecksumFailure");
+		tr.error(checksum_failed())
+		    .detail("CodecPageSize", pageSize)
+		    .detail("CodecReserveSize", reserveSize)
+		    .detail("Filename", filename)
+		    .detail("PageNumber", pageNumber)
+		    .detail("PageSize", pageLen)
+		    .detail("ChecksumInPage", pSumInPage->toString())
+		    .detail("ChecksumCalculatedHL2", hashLittle2Sum.toString());
+		if (pSumInPage->part1 == 0) tr.detail("ChecksumCalculatedCRC", sum.toString());
+		return false;
 	}
 
 	static void * codec(void *vpSelf, void *data, Pgno pageNumber, int op) {

From cea7ab3a25d3cfd2f574635801c954975d91ee52 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <36455792+etschannen@users.noreply.github.com>
Date: Fri, 17 May 2019 10:41:21 -0700
Subject: [PATCH 107/118] Update documentation/sphinx/source/release-notes.rst

Co-Authored-By: A.J. Beamon <ajbeamon@users.noreply.github.com>
---
 documentation/sphinx/source/release-notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 7bb5248425..94492be883 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -128,7 +128,7 @@ Fixes only impacting 6.1.0+
 * Memory tracking trace events could cause the program to crash when called from inside a trace event. [6.1.5] `(PR #1541) <https://github.com/apple/foundationdb/pull/1541>`_
 * TLogs will replace a large file with an empty file rather than doing a large truncate operation. [6.1.5] `(PR #1545) <https://github.com/apple/foundationdb/pull/1545>`_
 * Fix PR #1545 to work on Windows and Linux. [6.1.6] `(PR #1556) <https://github.com/apple/foundationdb/pull/1556>`_
-* Adding a read conflict range for the metadata version key. [6.1.6] `(PR #1556) <https://github.com/apple/foundationdb/pull/1556>`_
+* Adding a read conflict range for the metadata version key no longer requires read access to the system keys. [6.1.6] `(PR #1556) <https://github.com/apple/foundationdb/pull/1556>`_
 
 Earlier release notes
 ---------------------

From 3fac380b90f5db238d28ebfc1ae508b1a7119cfe Mon Sep 17 00:00:00 2001
From: Trevor Clinkenbeard <trevorclinkenbeard@gmail.com>
Date: Fri, 17 May 2019 10:43:28 -0700
Subject: [PATCH 108/118] Avoid tracing in PageChecksumCodec::checksum if
 silent flag is set

---
 fdbserver/KeyValueStoreSQLite.actor.cpp | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index b9c02be40d..a053e39b0d 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -118,16 +118,18 @@ struct PageChecksumCodec {
 		hashlittle2(pData, dataLen, &hashLittle2Sum.part1, &hashLittle2Sum.part2);
 		if (hashLittle2Sum == *pSumInPage) return true;
 
-		TraceEvent tr(SevError, "SQLitePageChecksumFailure");
-		tr.error(checksum_failed())
-		    .detail("CodecPageSize", pageSize)
-		    .detail("CodecReserveSize", reserveSize)
-		    .detail("Filename", filename)
-		    .detail("PageNumber", pageNumber)
-		    .detail("PageSize", pageLen)
-		    .detail("ChecksumInPage", pSumInPage->toString())
-		    .detail("ChecksumCalculatedHL2", hashLittle2Sum.toString());
-		if (pSumInPage->part1 == 0) tr.detail("ChecksumCalculatedCRC", sum.toString());
+		if (!silent) {
+			TraceEvent trEvent(SevError, "SQLitePageChecksumFailure");
+			trEvent.error(checksum_failed())
+			    .detail("CodecPageSize", pageSize)
+			    .detail("CodecReserveSize", reserveSize)
+			    .detail("Filename", filename)
+			    .detail("PageNumber", pageNumber)
+			    .detail("PageSize", pageLen)
+			    .detail("ChecksumInPage", pSumInPage->toString())
+			    .detail("ChecksumCalculatedHL2", hashLittle2Sum.toString());
+			if (pSumInPage->part1 == 0) trEvent.detail("ChecksumCalculatedCRC", sum.toString());
+		}
 		return false;
 	}
 

From b8e7fc1b840846e90054a0c0d2bcfb3f4850aa24 Mon Sep 17 00:00:00 2001
From: Jingyu Zhou <jingyuzhou@gmail.com>
Date: Thu, 16 May 2019 13:54:06 -0700
Subject: [PATCH 109/118] Refactor: add std:: qualifier and use emplace_back

---
 fdbclient/DatabaseContext.h                   |   2 +-
 fdbclient/FDBTypes.h                          |   9 +-
 fdbclient/MasterProxyInterface.h              |   7 +-
 fdbclient/NativeAPI.actor.cpp                 |   1 +
 fdbserver/ConflictSet.h                       |  15 +-
 fdbserver/LogSystem.h                         |  20 +--
 fdbserver/SkipList.cpp                        |  84 +++++------
 fdbserver/TagPartitionedLogSystem.actor.cpp   |   2 +-
 fdbserver/pubsub.actor.cpp                    |   6 +-
 fdbserver/storageserver.actor.cpp             |  27 ++--
 fdbserver/workloads/AsyncFileRead.actor.cpp   |  40 ++---
 fdbserver/workloads/BulkSetup.actor.h         |  36 +++--
 .../workloads/ConsistencyCheck.actor.cpp      |  10 +-
 fdbserver/workloads/QueuePush.actor.cpp       |  29 ++--
 fdbserver/workloads/ReadWrite.actor.cpp       | 140 +++++++++---------
 fdbserver/workloads/WriteBandwidth.actor.cpp  |  40 ++---
 16 files changed, 244 insertions(+), 224 deletions(-)

diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h
index 926cc77c66..b59fce81a5 100644
--- a/fdbclient/DatabaseContext.h
+++ b/fdbclient/DatabaseContext.h
@@ -60,7 +60,7 @@ public:
 
 	Database clone() const { return Database(new DatabaseContext( cluster, clientInfo, clientInfoMonitor, dbId, taskID, clientLocality, enableLocalityLoadBalance, lockAware, apiVersion )); }
 
-	pair<KeyRange,Reference<LocationInfo>> getCachedLocation( const KeyRef&, bool isBackward = false );
+	std::pair<KeyRange,Reference<LocationInfo>> getCachedLocation( const KeyRef&, bool isBackward = false );
 	bool getCachedLocations( const KeyRangeRef&, vector<std::pair<KeyRange,Reference<LocationInfo>>>&, int limit, bool reverse );
 	Reference<LocationInfo> setCachedLocation( const KeyRangeRef&, const vector<struct StorageServerInterface>& );
 	void invalidateCache( const KeyRef&, bool isBackward = false );
diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h
index 837c500771..edb83f5f92 100644
--- a/fdbclient/FDBTypes.h
+++ b/fdbclient/FDBTypes.h
@@ -21,11 +21,14 @@
 #ifndef FDBCLIENT_FDBTYPES_H
 #define FDBCLIENT_FDBTYPES_H
 
+#include <algorithm>
+#include <set>
+#include <string>
+#include <vector>
+
 #include "flow/flow.h"
 #include "fdbclient/Knobs.h"
 
-using std::vector;
-using std::pair;
 typedef int64_t Version;
 typedef uint64_t LogEpoch;
 typedef uint64_t Sequence;
@@ -761,7 +764,7 @@ static bool addressExcluded( std::set<AddressExclusion> const& exclusions, Netwo
 struct ClusterControllerPriorityInfo {
 	enum DCFitness { FitnessPrimary, FitnessRemote, FitnessPreferred, FitnessUnknown, FitnessBad }; //cannot be larger than 7 because of leader election mask
 
-	static DCFitness calculateDCFitness(Optional<Key> const& dcId, vector<Optional<Key>> const& dcPriority) {
+	static DCFitness calculateDCFitness(Optional<Key> const& dcId, std::vector<Optional<Key>> const& dcPriority) {
 		if(!dcPriority.size()) {
 			return FitnessUnknown;
 		} else if(dcPriority.size() == 1) {
diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h
index 8fedd2ed1f..17e43d5930 100644
--- a/fdbclient/MasterProxyInterface.h
+++ b/fdbclient/MasterProxyInterface.h
@@ -23,6 +23,9 @@
 #define FDBCLIENT_MASTERPROXYINTERFACE_H
 #pragma once
 
+#include <utility>
+#include <vector>
+
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/StorageServerInterface.h"
 #include "fdbclient/CommitTransaction.h"
@@ -168,7 +171,7 @@ struct GetReadVersionRequest : TimedRequest {
 struct GetKeyServerLocationsReply {
 	constexpr static FileIdentifier file_identifier = 10636023;
 	Arena arena;
-	vector<pair<KeyRangeRef, vector<StorageServerInterface>>> results;
+	std::vector<std::pair<KeyRangeRef, vector<StorageServerInterface>>> results;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -213,7 +216,7 @@ struct GetStorageServerRejoinInfoReply {
 	Tag tag;
 	Optional<Tag> newTag;
 	bool newLocality;
-	vector<pair<Version, Tag>> history;
+	std::vector<std::pair<Version, Tag>> history;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 3f5225be1e..9b6d7399e4 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -62,6 +62,7 @@ extern const char* getHGVersion();
 using std::make_pair;
 using std::max;
 using std::min;
+using std::pair;
 
 NetworkOptions networkOptions;
 Reference<TLSOptions> tlsOptions;
diff --git a/fdbserver/ConflictSet.h b/fdbserver/ConflictSet.h
index 06e2ca3094..5c219fc5d9 100644
--- a/fdbserver/ConflictSet.h
+++ b/fdbserver/ConflictSet.h
@@ -22,6 +22,9 @@
 #define CONFLICTSET_H
 #pragma once
 
+#include <utility>
+#include <vector>
+
 #include "fdbclient/CommitTransaction.h"
 
 struct ConflictSet;
@@ -40,23 +43,23 @@ struct ConflictBatch {
 	};
 
 	void addTransaction( const CommitTransactionRef& transaction );
-	void detectConflicts(Version now, Version newOldestVersion, vector<int>& nonConflicting, vector<int>* tooOldTransactions = NULL);
-	void GetTooOldTransactions(vector<int>& tooOldTransactions);
+	void detectConflicts(Version now, Version newOldestVersion, std::vector<int>& nonConflicting, std::vector<int>* tooOldTransactions = NULL);
+	void GetTooOldTransactions(std::vector<int>& tooOldTransactions);
 
 private:
 	ConflictSet* cs;
 	Standalone< VectorRef< struct TransactionInfo* > > transactionInfo;
-	vector<struct KeyInfo> points;
+	std::vector<struct KeyInfo> points;
 	int transactionCount;
-	vector< pair<StringRef,StringRef> > combinedWriteConflictRanges;
-	vector< struct ReadConflictRange > combinedReadConflictRanges;
+	std::vector< std::pair<StringRef,StringRef> > combinedWriteConflictRanges;
+	std::vector< struct ReadConflictRange > combinedReadConflictRanges;
 	bool* transactionConflictStatus;
 
 	void checkIntraBatchConflicts();
 	void combineWriteConflictRanges();
 	void checkReadConflictRanges();
 	void mergeWriteConflictRanges(Version now);
-	void addConflictRanges(Version now, vector< pair<StringRef,StringRef> >::iterator begin, vector< pair<StringRef,StringRef> >::iterator end, class SkipList* part);
+	void addConflictRanges(Version now, std::vector< std::pair<StringRef,StringRef> >::iterator begin, std::vector< std::pair<StringRef,StringRef> >::iterator end, class SkipList* part);
 };
 
 #endif
\ No newline at end of file
diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index fb9c5506fa..824ff310f2 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -204,7 +204,7 @@ public:
 		return tag.id % logServers.size();
 	}
 
-	void updateLocalitySet( vector<LocalityData> const& localities ) {
+	void updateLocalitySet( std::vector<LocalityData> const& localities ) {
 		LocalityMap<int>* logServerMap;
 
 		logServerSet = Reference<LocalitySet>(new LocalityMap<int>());
@@ -418,7 +418,7 @@ struct ILogSystem {
 
 	struct MergedPeekCursor : IPeekCursor, ReferenceCounted<MergedPeekCursor> {
 		Reference<LogSet> logSet;
-		vector< Reference<IPeekCursor> > serverCursors;
+		std::vector< Reference<IPeekCursor> > serverCursors;
 		std::vector<LocalityEntry> locations;
 		std::vector< std::pair<LogMessageVersion, int> > sortedVersions;
 		Tag tag;
@@ -429,9 +429,9 @@ struct ILogSystem {
 		UID randomID;
 		int tLogReplicationFactor;
 
-		MergedPeekCursor( vector< Reference<ILogSystem::IPeekCursor> > const& serverCursors, Version begin );
+		MergedPeekCursor( std::vector< Reference<ILogSystem::IPeekCursor> > const& serverCursors, Version begin );
 		MergedPeekCursor( std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers, int bestServer, int readQuorum, Tag tag, Version begin, Version end, bool parallelGetMore, std::vector<LocalityData> const& tLogLocalities, Reference<IReplicationPolicy> const tLogPolicy, int tLogReplicationFactor );
-		MergedPeekCursor( vector< Reference<IPeekCursor> > const& serverCursors, LogMessageVersion const& messageVersion, int bestServer, int readQuorum, Optional<LogMessageVersion> nextVersion, Reference<LogSet> logSet, int tLogReplicationFactor );
+		MergedPeekCursor( std::vector< Reference<IPeekCursor> > const& serverCursors, LogMessageVersion const& messageVersion, int bestServer, int readQuorum, Optional<LogMessageVersion> nextVersion, Reference<LogSet> logSet, int tLogReplicationFactor );
 
 		virtual Reference<IPeekCursor> cloneNoMore();
 		virtual void setProtocolVersion( uint64_t version );
@@ -636,7 +636,7 @@ struct ILogSystem {
 	virtual Reference<IPeekCursor> peek( UID dbgid, Version begin, Optional<Version> end, std::vector<Tag> tags, bool parallelGetMore = false ) = 0;
 		// Same contract as peek(), but for a set of tags
 
-	virtual Reference<IPeekCursor> peekSingle( UID dbgid, Version begin, Tag tag, vector<pair<Version,Tag>> history = vector<pair<Version,Tag>>() ) = 0;
+	virtual Reference<IPeekCursor> peekSingle( UID dbgid, Version begin, Tag tag, std::vector<std::pair<Version,Tag>> history = std::vector<std::pair<Version,Tag>>() ) = 0;
 		// Same contract as peek(), but blocks until the preferred log server(s) for the given tag are available (and is correspondingly less expensive)
 
 	virtual Reference<IPeekCursor> peekLogRouter( UID dbgid, Version begin, Tag tag ) = 0;
@@ -690,7 +690,7 @@ struct ILogSystem {
 	virtual Future<Void> onLogSystemConfigChange() = 0;
 		// Returns when the log system configuration has changed due to a tlog rejoin.
 
-	virtual void getPushLocations( std::vector<Tag> const& tags, vector<int>& locations ) = 0;
+	virtual void getPushLocations( std::vector<Tag> const& tags, std::vector<int>& locations ) = 0;
 
 	virtual bool hasRemoteLogs() = 0;
 
@@ -807,10 +807,10 @@ struct LogPushData : NonCopyable {
 
 private:
 	Reference<ILogSystem> logSystem;
-	vector<Tag> next_message_tags;
-	vector<Tag> prev_tags;
-	vector<BinaryWriter> messagesWriter;
-	vector<int> msg_locations;
+	std::vector<Tag> next_message_tags;
+	std::vector<Tag> prev_tags;
+	std::vector<BinaryWriter> messagesWriter;
+	std::vector<int> msg_locations;
 	uint32_t subsequence;
 };
 
diff --git a/fdbserver/SkipList.cpp b/fdbserver/SkipList.cpp
index 58c03b5b99..0104d117ba 100644
--- a/fdbserver/SkipList.cpp
+++ b/fdbserver/SkipList.cpp
@@ -24,6 +24,7 @@
 #include <algorithm>
 #include <numeric>
 #include <string>
+#include <vector>
 
 /*
 #ifdef __GNUG__
@@ -43,9 +44,8 @@
 
 using std::min;
 using std::max;
-using std::make_pair;
 
-static vector<PerfDoubleCounter*> skc;
+static std::vector<PerfDoubleCounter*> skc;
 
 static thread_local uint32_t g_seed = 0;
 
@@ -199,7 +199,7 @@ bool operator == (const KeyInfo& lhs, const KeyInfo& rhs ) {
 	return !(lhs<rhs || rhs<lhs);
 }
 
-void swapSort(vector<KeyInfo>& points, int a, int b){
+void swapSort(std::vector<KeyInfo>& points, int a, int b){
 	if (points[b] < points[a]){
 		KeyInfo temp;
 		temp = points[a];
@@ -208,7 +208,7 @@ void swapSort(vector<KeyInfo>& points, int a, int b){
 	}
 }
 
-void smallSort(vector<KeyInfo>& points, int start, int N){
+void smallSort(std::vector<KeyInfo>& points, int start, int N){
 	for (int i=1;i<N;i++)
 		for (int j=i;j>0;j-=2)
 			swapSort(points, start+j-1, start+j);
@@ -224,12 +224,12 @@ struct SortTask {
 	SortTask(int begin, int size, int character) : begin(begin), size(size), character(character) {}
 };
 
-void sortPoints(vector<KeyInfo>& points){
-	vector<SortTask> tasks;
-	vector<KeyInfo> newPoints;
-	vector<int> counts;
+void sortPoints(std::vector<KeyInfo>& points){
+	std::vector<SortTask> tasks;
+	std::vector<KeyInfo> newPoints;
+	std::vector<int> counts;
 
-	tasks.push_back( SortTask(0, points.size(), 0) );
+	tasks.emplace_back(0, points.size(), 0);
 
 	while (tasks.size()){
 		SortTask st = tasks.back();
@@ -259,7 +259,7 @@ void sortPoints(vector<KeyInfo>& points){
 		for(int i=0;i<counts.size();i++){
 			int temp = counts[i];
 			if (temp > 1)
-				tasks.push_back(SortTask(st.begin+total, temp, st.character+1));
+				tasks.emplace_back(st.begin+total, temp, st.character+1);
 			counts[i] = total;
 			total += temp;
 		}
@@ -569,7 +569,7 @@ public:
 	}
 
 	void concatenate( SkipList* input, int count ) {
-		vector<Finger> ends( count-1 );
+		std::vector<Finger> ends( count-1 );
 		for(int i=0; i<ends.size(); i++)
 			input[i].getEnd( ends[i] );
 
@@ -948,9 +948,9 @@ struct ConflictSet {
 	SkipList versionHistory;
 	Key removalKey;
 	Version oldestVersion;
-	vector<PAction> worker_nextAction;
-	vector<Event*> worker_ready;
-	vector<Event*> worker_finished;
+	std::vector<PAction> worker_nextAction;
+	std::vector<Event*> worker_ready;
+	std::vector<Event*> worker_finished;
 };
 
 ConflictSet* newConflictSet() { return new ConflictSet; }
@@ -989,18 +989,18 @@ void ConflictBatch::addTransaction( const CommitTransactionRef& tr ) {
 		info->readRanges.resize( arena, tr.read_conflict_ranges.size() );
 		info->writeRanges.resize( arena, tr.write_conflict_ranges.size() );
 
-		vector<KeyInfo> &points = this->points;
+		std::vector<KeyInfo>& points = this->points;
 		for(int r=0; r<tr.read_conflict_ranges.size(); r++) {
 			const KeyRangeRef& range = tr.read_conflict_ranges[r];
-			points.push_back( KeyInfo( range.begin, false, true, false, t, &info->readRanges[r].first ) );
+			points.emplace_back(range.begin, false, true, false, t, &info->readRanges[r].first);
 			//points.back().keyEnd = StringRef(buf,range.second);
-			points.push_back( KeyInfo( range.end, false, false, false, t, &info->readRanges[r].second ) );
-			combinedReadConflictRanges.push_back( ReadConflictRange( range.begin, range.end, tr.read_snapshot, t ) );
+			points.emplace_back(range.end, false, false, false, t, &info->readRanges[r].second);
+			combinedReadConflictRanges.emplace_back(range.begin, range.end, tr.read_snapshot, t);
 		}
 		for(int r=0; r<tr.write_conflict_ranges.size(); r++) {
 			const KeyRangeRef& range = tr.write_conflict_ranges[r];
-			points.push_back( KeyInfo( range.begin, false, true, true, t, &info->writeRanges[r].first ) );
-			points.push_back( KeyInfo( range.end, false, false, true, t, &info->writeRanges[r].second ) );
+			points.emplace_back(range.begin, false, true, true, t, &info->writeRanges[r].first);
+			points.emplace_back(range.end, false, false, true, t, &info->writeRanges[r].second);
 		}
 	}
 
@@ -1008,7 +1008,7 @@ void ConflictBatch::addTransaction( const CommitTransactionRef& tr ) {
 }
 
 class MiniConflictSet2 : NonCopyable {
-	vector<bool> values;
+	std::vector<bool> values;
 public:
 	explicit MiniConflictSet2( int size ) {
 		values.assign( size, false );
@@ -1028,21 +1028,21 @@ public:
 class MiniConflictSet : NonCopyable {
 	typedef uint64_t wordType;
 	enum { bucketShift = 6, bucketMask=sizeof(wordType)*8-1 };
-	vector<wordType> values; // undefined when andValues is true for a range of values
-	vector<wordType> orValues;
-	vector<wordType> andValues;
+	std::vector<wordType> values; // undefined when andValues is true for a range of values
+	std::vector<wordType> orValues;
+	std::vector<wordType> andValues;
 	MiniConflictSet2 debug;		// SOMEDAY: Test on big ranges, eliminate this
 
 	uint64_t bitMask(unsigned int bit){ // computes results for bit%word
 		return (((wordType)1) << ( bit & bucketMask )); // '&' unnecesary?
 	}
-	void setNthBit(vector<wordType> &v, const unsigned int bit){
+	void setNthBit(std::vector<wordType>& v, const unsigned int bit){
 		v[bit>>bucketShift] |= bitMask(bit);
 	}
-	void clearNthBit(vector<wordType> &v, const unsigned int bit){
+	void clearNthBit(std::vector<wordType>& v, const unsigned int bit){
 		v[bit>>bucketShift] &= ~(bitMask(bit));
 	}
-	bool getNthBit(const vector<wordType> &v, const unsigned int bit){
+	bool getNthBit(const std::vector<wordType>& v, const unsigned int bit){
 		return (v[bit>>bucketShift] & bitMask(bit)) != 0;
 	}
 	int wordsForNBits(unsigned int bits){
@@ -1060,7 +1060,7 @@ class MiniConflictSet : NonCopyable {
 		return (b&bucketMask) ? lowBits(b) : -1;
 	}
 
-	void setBits(vector<wordType> &v, int bitBegin, int bitEnd, bool fillMiddle){
+	void setBits(std::vector<wordType>& v, int bitBegin, int bitEnd, bool fillMiddle){
 		if (bitBegin >= bitEnd)	return;
 		int beginWord = bitBegin>>bucketShift;
 		int lastWord = ((bitEnd+bucketMask) >> bucketShift) - 1;
@@ -1075,7 +1075,7 @@ class MiniConflictSet : NonCopyable {
 		}
 	}
 
-	bool orBits(vector<wordType> &v, int bitBegin, int bitEnd, bool getMiddle) {
+	bool orBits(std::vector<wordType>& v, int bitBegin, int bitEnd, bool getMiddle) {
 		if (bitBegin >= bitEnd) return false;
 		int beginWord = bitBegin >> bucketShift;
 		int lastWord = ((bitEnd+bucketMask) >> bucketShift) - 1;
@@ -1152,7 +1152,7 @@ void ConflictBatch::checkIntraBatchConflicts() {
 	}
 }
 
-void ConflictBatch::GetTooOldTransactions(vector<int>& tooOldTransactions) {
+void ConflictBatch::GetTooOldTransactions(std::vector<int>& tooOldTransactions) {
 	for (int i = 0; i<transactionInfo.size(); i++) {
 		if (transactionInfo[i]->tooOld) {
 			tooOldTransactions.push_back(i);
@@ -1160,7 +1160,7 @@ void ConflictBatch::GetTooOldTransactions(vector<int>& tooOldTransactions) {
 	}
 }
 
-void ConflictBatch::detectConflicts(Version now, Version newOldestVersion, vector<int>& nonConflicting, vector<int>* tooOldTransactions) {
+void ConflictBatch::detectConflicts(Version now, Version newOldestVersion, std::vector<int>& nonConflicting, std::vector<int>* tooOldTransactions) {
 	double t = timer();
 	sortPoints( points );
 	//std::sort( combinedReadConflictRanges.begin(), combinedReadConflictRanges.end() );
@@ -1232,7 +1232,7 @@ DISABLE_ZERO_DIVISION_FLAG
 	}
 }
 
-void ConflictBatch::addConflictRanges(Version now, vector< pair<StringRef,StringRef> >::iterator begin, vector< pair<StringRef,StringRef> >::iterator end,SkipList* part) {
+void ConflictBatch::addConflictRanges(Version now, std::vector< std::pair<StringRef,StringRef> >::iterator begin, std::vector< std::pair<StringRef,StringRef> >::iterator end,SkipList* part) {
 	int count = end-begin;
 #if 0
 	//for(auto w = begin; w != end; ++w)
@@ -1262,16 +1262,16 @@ void ConflictBatch::mergeWriteConflictRanges(Version now) {
 		return;
 
 	if (PARALLEL_THREAD_COUNT) {
-		vector<SkipList> parts;
+		std::vector<SkipList> parts;
 		for (int i = 0; i < PARALLEL_THREAD_COUNT; i++)
-			parts.push_back(SkipList());
+			parts.emplace_back();
 
-		vector<StringRef> splits( parts.size()-1 );
+		std::vector<StringRef> splits( parts.size()-1 );
 		for(int s=0; s<splits.size(); s++)
 			splits[s] = combinedWriteConflictRanges[ (s+1)*combinedWriteConflictRanges.size()/parts.size() ].first;
 
 		cs->versionHistory.partition( splits.size() ? &splits[0] : NULL, splits.size(), &parts[0] );
-		vector<double> tstart(PARALLEL_THREAD_COUNT), tend(PARALLEL_THREAD_COUNT);
+		std::vector<double> tstart(PARALLEL_THREAD_COUNT), tend(PARALLEL_THREAD_COUNT);
 		Event done[PARALLEL_THREAD_COUNT ? PARALLEL_THREAD_COUNT : 1];
 		double before = timer();
 		for(int t=0; t<parts.size(); t++) {
@@ -1325,8 +1325,8 @@ void ConflictBatch::combineWriteConflictRanges()
 		if (point.write && !transactionConflictStatus[ point.transaction ]) {
 			if (point.begin) {
  				activeWriteCount++;
-  				if (activeWriteCount == 1)
-  					combinedWriteConflictRanges.push_back( make_pair( point.key, KeyRef() ) );
+				if (activeWriteCount == 1)
+					combinedWriteConflictRanges.emplace_back(point.key, KeyRef());
 			} else /*if (point.end)*/ {
 				activeWriteCount--;
 				if (activeWriteCount == 0)
@@ -1431,8 +1431,8 @@ void skipListTest() {
 	Arena testDataArena;
 	VectorRef< VectorRef<KeyRangeRef> > testData;
 	testData.resize(testDataArena, 500);
-	vector<vector<uint8_t>> success( testData.size() );
-	vector<vector<uint8_t>> success2( testData.size() );
+	std::vector<std::vector<uint8_t>> success( testData.size() );
+	std::vector<std::vector<uint8_t>> success2( testData.size() );
 	for(int i=0; i<testData.size(); i++) {
 		testData[i].resize(testDataArena, 5000);
 		success[i].assign( testData[i].size(), false );
@@ -1454,10 +1454,10 @@ void skipListTest() {
 	int cranges = 0, tcount = 0;
 
 	start = timer();
-	vector<vector<int>> nonConflict( testData.size() );
+	std::vector<std::vector<int>> nonConflict( testData.size() );
 	for(int i=0; i<testData.size(); i++) {
 		Arena buf;
-		vector<CommitTransactionRef> trs;
+		std::vector<CommitTransactionRef> trs;
 		double t = timer();
 		for(int j=0; j+readCount+writeCount<=testData[i].size(); j+=readCount+writeCount) {
 			CommitTransactionRef tr;
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 2a3beea47a..001fa25939 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -767,7 +767,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
 		}
 	}
 
-	virtual Reference<IPeekCursor> peekSingle( UID dbgid, Version begin, Tag tag, vector<pair<Version,Tag>> history ) {
+	virtual Reference<IPeekCursor> peekSingle( UID dbgid, Version begin, Tag tag, std::vector<std::pair<Version,Tag>> history ) {
 		while(history.size() && begin >= history.back().first) {
 			history.pop_back();
 		}
diff --git a/fdbserver/pubsub.actor.cpp b/fdbserver/pubsub.actor.cpp
index c6bff2a365..3b1ec58bd7 100644
--- a/fdbserver/pubsub.actor.cpp
+++ b/fdbserver/pubsub.actor.cpp
@@ -429,12 +429,12 @@ ACTOR Future<std::vector<Message>> _listInboxMessages(Database cx, uint64_t inbo
 				//printf(" -> cached message %016llx from feed %016llx\n", messageId, feed);
 				if(messageId >= cursor) {
 					//printf(" -> entering message %016llx from feed %016llx\n", messageId, feed);
-					feedLatest.insert(pair<MessageId, Feed>(messageId, feed));
+					feedLatest.emplace(messageId, feed);
 				} else {
 					// replace this with the first message older than the cursor
 					MessageId mId = wait(getFeedLatestAtOrAfter(&tr, feed, cursor));
 					if(mId) {
-						feedLatest.insert(pair<MessageId, Feed>(mId, feed));
+						feedLatest.emplace(mId, feed);
 					}
 				}
 			}
@@ -465,7 +465,7 @@ ACTOR Future<std::vector<Message>> _listInboxMessages(Database cx, uint64_t inbo
 
 				MessageId nextMessage = wait(getFeedLatestAtOrAfter(&tr, f, id + 1));
 				if(nextMessage) {
-					feedLatest.insert(pair<MessageId, Feed>(nextMessage, f));
+					feedLatest.emplace(nextMessage, f);
 				}
 			}
 
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index dde7c9ed9f..b5d387484c 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -52,6 +52,7 @@
 #include "flow/TDMetric.actor.h"
 #include "flow/actorcompiler.h"  // This must be the last #include.
 
+using std::pair;
 using std::make_pair;
 
 #pragma region Data Structures
@@ -199,11 +200,11 @@ private:
 };
 
 struct UpdateEagerReadInfo {
-	vector<KeyRef> keyBegin;
-	vector<Key> keyEnd; // these are for ClearRange
+	std::vector<KeyRef> keyBegin;
+	std::vector<Key> keyEnd; // these are for ClearRange
 
-	vector<pair<KeyRef, int>> keys;
-	vector<Optional<Value>> value;
+	std::vector<std::pair<KeyRef, int>> keys;
+	std::vector<Optional<Value>> value;
 
 	Arena arena;
 
@@ -223,13 +224,13 @@ struct UpdateEagerReadInfo {
 				// CompareAndClear is likely to be used after another atomic operation on same key.
 				keys.back().second = std::max(keys.back().second, m.param2.size() + 1);
 			} else {
-				keys.push_back(pair<KeyRef, int>(m.param1, m.param2.size() + 1));
+				keys.emplace_back(m.param1, m.param2.size() + 1);
 			}
 		} else if ((m.type == MutationRef::AppendIfFits) || (m.type == MutationRef::ByteMin) ||
 		           (m.type == MutationRef::ByteMax))
-			keys.push_back(pair<KeyRef, int>(m.param1, CLIENT_KNOBS->VALUE_SIZE_LIMIT));
+			keys.emplace_back(m.param1, CLIENT_KNOBS->VALUE_SIZE_LIMIT);
 		else if (isAtomicOp((MutationRef::Type) m.type))
-			keys.push_back(pair<KeyRef, int>(m.param1, m.param2.size()));
+			keys.emplace_back(m.param1, m.param2.size());
 	}
 
 	void finishKeyBegin() {
@@ -2239,8 +2240,8 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss
 	// version.  The latter depends on data->newestAvailableVersion, so loop over the ranges of that.
 	// SOMEDAY: Could this just use shards?  Then we could explicitly do the removeDataRange here when an adding/transferred shard is cancelled
 	auto vr = data->newestAvailableVersion.intersectingRanges(keys);
-	vector<std::pair<KeyRange,Version>> changeNewestAvailable;
-	vector<KeyRange> removeRanges;
+	std::vector<std::pair<KeyRange,Version>> changeNewestAvailable;
+	std::vector<KeyRange> removeRanges;
 	for (auto r = vr.begin(); r != vr.end(); ++r) {
 		KeyRangeRef range = keys & r->range();
 		bool dataAvailable = r->value()==latestVersion || r->value() >= version;
@@ -2255,7 +2256,7 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss
 			if (dataAvailable) {
 				ASSERT( r->value() == latestVersion);  // Not that we care, but this used to be checked instead of dataAvailable
 				ASSERT( data->mutableData().getLatestVersion() > version || context == CSK_RESTORE );
-				changeNewestAvailable.push_back(make_pair(range, version));
+				changeNewestAvailable.emplace_back(range, version);
 				removeRanges.push_back( range );
 			}
 			data->addShard( ShardInfo::newNotAssigned(range) );
@@ -2263,7 +2264,7 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss
 		} else if (!dataAvailable) {
 			// SOMEDAY: Avoid restarting adding/transferred shards
 			if (version==0){ // bypass fetchkeys; shard is known empty at version 0
-				changeNewestAvailable.push_back(make_pair(range, latestVersion));
+				changeNewestAvailable.emplace_back(range, latestVersion);
 				data->addShard( ShardInfo::newReadWrite(range, data) );
 				setAvailableStatus(data, range, true);
 			} else {
@@ -2272,7 +2273,7 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss
 					data->addShard( ShardInfo::newAdding(data, range) );
 			}
 		} else {
-			changeNewestAvailable.push_back(make_pair(range, latestVersion));
+			changeNewestAvailable.emplace_back(range, latestVersion);
 			data->addShard( ShardInfo::newReadWrite(range, data) );
 		}
 	}
@@ -2438,7 +2439,7 @@ private:
 				rollback( data, rollbackVersion, currentVersion );
 			}
 
-			data->recoveryVersionSkips.push_back(std::make_pair(rollbackVersion, currentVersion - rollbackVersion));
+			data->recoveryVersionSkips.emplace_back(rollbackVersion, currentVersion - rollbackVersion);
 		} else if (m.type == MutationRef::SetValue && m.param1 == killStoragePrivateKey) {
 			throw worker_removed();
 		} else if ((m.type == MutationRef::SetValue || m.type == MutationRef::ClearRange) && m.param1.substr(1).startsWith(serverTagPrefix)) {
diff --git a/fdbserver/workloads/AsyncFileRead.actor.cpp b/fdbserver/workloads/AsyncFileRead.actor.cpp
index 5f60e611b0..89dc69d56c 100644
--- a/fdbserver/workloads/AsyncFileRead.actor.cpp
+++ b/fdbserver/workloads/AsyncFileRead.actor.cpp
@@ -17,6 +17,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <utility>
+#include <vector>
 
 #include "fdbserver/workloads/workloads.actor.h"
 #include "flow/ActorCollection.h"
@@ -75,18 +77,18 @@ struct IOLog {
 	ProcessLog issueR, completionR, durationR;
 	ProcessLog issueW, completionW, durationW;
 
-	vector<pair<std::string, ProcessLog*> > logs;
+	std::vector<std::pair<std::string, ProcessLog*> > logs;
 
 	IOLog(){
-		logs.push_back(std::make_pair("issue", &issue));
-		logs.push_back(std::make_pair("completion", &completion));
-		logs.push_back(std::make_pair("duration", &duration));
-		logs.push_back(std::make_pair("issueR", &issueR));
-		logs.push_back(std::make_pair("completionR", &completionR));
-		logs.push_back(std::make_pair("durationR", &durationR));
-		logs.push_back(std::make_pair("issueW", &issueW));
-		logs.push_back(std::make_pair("completionW", &completionW));
-		logs.push_back(std::make_pair("durationW", &durationW));
+		logs.emplace_back("issue", &issue);
+		logs.emplace_back("completion", &completion);
+		logs.emplace_back("duration", &duration);
+		logs.emplace_back("issueR", &issueR);
+		logs.emplace_back("completionR", &completionR);
+		logs.emplace_back("durationR", &durationR);
+		logs.emplace_back("issueW", &issueW);
+		logs.emplace_back("completionW", &completionW);
+		logs.emplace_back("durationW", &durationW);
 
 		duration.logLatency = true;
 		durationR.logLatency = true;
@@ -138,10 +140,10 @@ struct IOLog {
 struct AsyncFileReadWorkload : public AsyncFileWorkload
 {
 	//Buffers used to store what is being read or written
-	vector<Reference<AsyncFileBuffer> > readBuffers;
+	std::vector<Reference<AsyncFileBuffer> > readBuffers;
 
 	//The futures for the asynchronous read operations
-	vector<Future<int> > readFutures;
+	std::vector<Future<int> > readFutures;
 
 	//Number of reads to perform in parallel.  Read tests are performed only if this is greater than zero
 	int numParallelReads;
@@ -280,7 +282,7 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload
 		if (self->unbatched) {
 			self->ioLog = new IOLog();
 
-			vector<Future<Void>> readers;
+			std::vector<Future<Void>> readers;
 
 			for(int i=0; i<self->numParallelReads; i++)
 				readers.push_back( readLoop(self, i, self->fixedRate / self->numParallelReads) );
@@ -333,14 +335,12 @@ struct AsyncFileReadWorkload : public AsyncFileWorkload
 		}
 	}
 
-	virtual void getMetrics(vector<PerfMetric>& m)
-	{
-		if(enabled)
-		{
-			m.push_back(PerfMetric("Bytes read/sec", bytesRead.getValue() / testDuration, false));
-			m.push_back(PerfMetric("Average CPU Utilization (Percentage)", averageCpuUtilization * 100, false));
+	virtual void getMetrics(std::vector<PerfMetric>& m) {
+		if (enabled) {
+			m.emplace_back("Bytes read/sec", bytesRead.getValue() / testDuration, false);
+			m.emplace_back("Average CPU Utilization (Percentage)", averageCpuUtilization * 100, false);
 		}
 	}
- };
+};
 
 WorkloadFactory<AsyncFileReadWorkload> AsyncFileReadWorkloadFactory("AsyncFileRead");
diff --git a/fdbserver/workloads/BulkSetup.actor.h b/fdbserver/workloads/BulkSetup.actor.h
index c660b7b205..c67b3905b1 100644
--- a/fdbserver/workloads/BulkSetup.actor.h
+++ b/fdbserver/workloads/BulkSetup.actor.h
@@ -29,6 +29,9 @@
 		#define FDBSERVER_BULK_SETUP_ACTOR_H
 
 #include <string>
+#include <utility>
+#include <vector>
+
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
 #include "fdbserver/ServerDBInfo.h"
@@ -102,14 +105,14 @@ Future<uint64_t> setupRange( Database cx, T* workload, uint64_t begin, uint64_t
 }
 
 ACTOR template<class T>
-Future<uint64_t> setupRangeWorker( Database cx, T* workload, vector<std::pair<uint64_t,uint64_t>>* jobs, double maxKeyInsertRate, int keySaveIncrement, int actorId) {
+Future<uint64_t> setupRangeWorker( Database cx, T* workload, std::vector<std::pair<uint64_t,uint64_t>>* jobs, double maxKeyInsertRate, int keySaveIncrement, int actorId) {
 	state double nextStart;
 	state uint64_t loadedRanges = 0;
 	state int lastStoredKeysLoaded = 0;
 	state uint64_t keysLoaded = 0;
 	state uint64_t bytesStored = 0;
 	while (jobs->size()){
-		state pair<uint64_t, uint64_t> job = jobs->back();
+		state std::pair<uint64_t, uint64_t> job = jobs->back();
 		jobs->pop_back();
 		nextStart = now() + (job.second-job.first)/maxKeyInsertRate;
 		uint64_t numBytes = wait( setupRange(cx, workload, job.first, job.second) );
@@ -152,7 +155,7 @@ Future<uint64_t> setupRangeWorker( Database cx, T* workload, vector<std::pair<ui
 //to reach that mark.  Returns a vector of times (in seconds) corresponding to the counts in the countsOfInterest vector.
 
 //Expects countsOfInterest to be sorted in ascending order
-ACTOR static Future<vector<pair<uint64_t, double> > > trackInsertionCount(Database cx, vector<uint64_t> countsOfInterest, double checkInterval)
+ACTOR static Future<std::vector<std::pair<uint64_t, double> > > trackInsertionCount(Database cx, std::vector<uint64_t> countsOfInterest, double checkInterval)
 {
 	state KeyRange keyPrefix = KeyRangeRef(std::string("keycount"), std::string("keycount") + char(255));
 	state KeyRange bytesPrefix = KeyRangeRef(std::string("bytesstored"), std::string("bytesstored") + char(255));
@@ -160,7 +163,7 @@ ACTOR static Future<vector<pair<uint64_t, double> > > trackInsertionCount(Databa
 	state uint64_t lastInsertionCount = 0;
 	state int currentCountIndex = 0;
 
-	state vector<pair<uint64_t, double> > countInsertionRates;
+	state std::vector<std::pair<uint64_t, double> > countInsertionRates;
 
 	state double startTime = now();
 
@@ -184,7 +187,7 @@ ACTOR static Future<vector<pair<uint64_t, double> > > trackInsertionCount(Databa
 				bytesInserted += *(uint64_t*)bytes[i].value.begin();
 
 			while(currentCountIndex < countsOfInterest.size() && countsOfInterest[currentCountIndex] > lastInsertionCount && countsOfInterest[currentCountIndex] <= numInserted)
-				countInsertionRates.push_back(pair<uint64_t, double>(countsOfInterest[currentCountIndex++], bytesInserted / (now() - startTime)));
+				countInsertionRates.emplace_back(countsOfInterest[currentCountIndex++], bytesInserted / (now() - startTime));
 
 			lastInsertionCount = numInserted;
 			wait(delay(checkInterval));
@@ -198,13 +201,16 @@ ACTOR static Future<vector<pair<uint64_t, double> > > trackInsertionCount(Databa
 	return countInsertionRates;
 }
 
-ACTOR template<class T>
-Future<Void> bulkSetup( Database cx, T* workload, uint64_t nodeCount, Promise<double> setupTime,
-		bool valuesInconsequential = false, double postSetupWarming = 0.0, double maxKeyInsertRate = 1e12,
-		vector<uint64_t> insertionCountsToMeasure = vector<uint64_t>(), Promise<vector<pair<uint64_t, double> > > ratesAtKeyCounts = Promise<vector<pair<uint64_t, double> > >(),
-		int keySaveIncrement = 0, double keyCheckInterval = 0.1 ) {
+ACTOR template <class T>
+Future<Void> bulkSetup(Database cx, T* workload, uint64_t nodeCount, Promise<double> setupTime,
+                       bool valuesInconsequential = false, double postSetupWarming = 0.0,
+                       double maxKeyInsertRate = 1e12,
+                       std::vector<uint64_t> insertionCountsToMeasure = std::vector<uint64_t>(),
+                       Promise<std::vector<std::pair<uint64_t, double>>> ratesAtKeyCounts =
+                           Promise<std::vector<std::pair<uint64_t, double>>>(),
+                       int keySaveIncrement = 0, double keyCheckInterval = 0.1) {
 
-	state vector<pair<uint64_t,uint64_t>> jobs;
+	state std::vector<std::pair<uint64_t,uint64_t>> jobs;
 	state uint64_t startNode = (nodeCount * workload->clientId) / workload->clientCount;
 	state uint64_t endNode = (nodeCount * (workload->clientId+1)) / workload->clientCount;
 
@@ -226,7 +232,7 @@ Future<Void> bulkSetup( Database cx, T* workload, uint64_t nodeCount, Promise<do
 				.detail("End", endNode)
 				.detail("CheckMethod", "SimpleValueSize");
 			setupTime.send(0.0);
-			ratesAtKeyCounts.send(vector<pair<uint64_t, double> >());
+			ratesAtKeyCounts.send(std::vector<std::pair<uint64_t, double> >());
 			return Void();
 		} else {
 			TraceEvent("BulkRangeNotFound")
@@ -257,14 +263,14 @@ Future<Void> bulkSetup( Database cx, T* workload, uint64_t nodeCount, Promise<do
 
 	// create a random vector of range-create jobs
 	for(uint64_t n=startNode; n<endNode; n+=BULK_SETUP_RANGE_SIZE)
-		jobs.push_back( std::make_pair( n, std::min(endNode, n+BULK_SETUP_RANGE_SIZE) ) );
+		jobs.emplace_back(n, std::min(endNode, n+BULK_SETUP_RANGE_SIZE));
 	g_random->randomShuffle(jobs);
 
 	// fire up the workers and wait for them to eat all the jobs
 	double maxWorkerInsertRate = maxKeyInsertRate / BULK_SETUP_WORKERS / workload->clientCount;
-	state vector<Future<uint64_t>> fs;
+	state std::vector<Future<uint64_t>> fs;
 
-	state Future<vector<pair<uint64_t, double> > > insertionTimes = vector<pair<uint64_t, double> >();
+	state Future<std::vector<std::pair<uint64_t, double> > > insertionTimes = std::vector<std::pair<uint64_t, double>>();
 
 	if(insertionCountsToMeasure.size() > 0)
 	{
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index f344562668..2de59bb0bc 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -268,11 +268,11 @@ struct ConsistencyCheckWorkload : TestWorkload
 				}
 
 				//Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
-				state Promise<vector<pair<KeyRange, vector<StorageServerInterface>>>> keyServerPromise;
+				state Promise<std::vector<std::pair<KeyRange, std::vector<StorageServerInterface>>>> keyServerPromise;
 				bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise));
 				if(keyServerResult)
 				{
-					state vector<pair<KeyRange, vector<StorageServerInterface>>> keyServers = keyServerPromise.getFuture().get();
+					state std::vector<std::pair<KeyRange, vector<StorageServerInterface>>> keyServers = keyServerPromise.getFuture().get();
 
 					//Get the locations of all the shards in the database
 					state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
@@ -323,9 +323,9 @@ struct ConsistencyCheckWorkload : TestWorkload
 	//Get a list of storage servers from the master and compares them with the TLogs.
 	//If this is a quiescent check, then each master proxy needs to respond, otherwise only one needs to respond.
 	//Returns false if there is a failure (in this case, keyServersPromise will never be set)
-	ACTOR Future<bool> getKeyServers(Database cx, ConsistencyCheckWorkload *self, Promise<vector<pair<KeyRange, vector<StorageServerInterface>>>> keyServersPromise)
+	ACTOR Future<bool> getKeyServers(Database cx, ConsistencyCheckWorkload *self, Promise<std::vector<std::pair<KeyRange, vector<StorageServerInterface>>>> keyServersPromise)
 	{
-		state vector<pair<KeyRange, vector<StorageServerInterface>>> keyServers;
+		state std::vector<std::pair<KeyRange, vector<StorageServerInterface>>> keyServers;
 
 		//Try getting key server locations from the master proxies
 		state vector<Future<ErrorOr<GetKeyServerLocationsReply>>> keyServerLocationFutures;
@@ -382,7 +382,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 
 	//Retrieves the locations of all shards in the database
 	//Returns false if there is a failure (in this case, keyLocationPromise will never be set)
-	ACTOR Future<bool> getKeyLocations(Database cx, vector<pair<KeyRange, vector<StorageServerInterface>>> shards, ConsistencyCheckWorkload *self, Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise)
+	ACTOR Future<bool> getKeyLocations(Database cx, std::vector<std::pair<KeyRange, vector<StorageServerInterface>>> shards, ConsistencyCheckWorkload *self, Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise)
 	{
 		state Standalone<VectorRef<KeyValueRef>> keyLocations;
 		state Key beginKey = allKeys.begin.withPrefix(keyServersPrefix);
diff --git a/fdbserver/workloads/QueuePush.actor.cpp b/fdbserver/workloads/QueuePush.actor.cpp
index 98e1d92d02..a3226166c1 100644
--- a/fdbserver/workloads/QueuePush.actor.cpp
+++ b/fdbserver/workloads/QueuePush.actor.cpp
@@ -17,6 +17,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <vector>
 
 #include "fdbrpc/ContinuousSample.h"
 #include "fdbclient/NativeAPI.actor.h"
@@ -33,7 +34,7 @@ struct QueuePushWorkload : TestWorkload {
 	std::string valueString;
 	Key endingKey, startingKey;
 
-	vector<Future<Void>> clients;
+	std::vector<Future<Void>> clients;
 	PerfIntCounter transactions, retries;
 	ContinuousSample<double> commitLatencies, GRVLatencies;
 
@@ -58,25 +59,25 @@ struct QueuePushWorkload : TestWorkload {
 
 	virtual Future<bool> check( Database const& cx ) { return true; }
 
-	virtual void getMetrics( vector<PerfMetric>& m ) {
+	virtual void getMetrics(std::vector<PerfMetric>& m ) {
 		double duration = testDuration;
 		int writes = transactions.getValue();
-		m.push_back( PerfMetric( "Measured Duration", duration, true ) );
-		m.push_back( PerfMetric( "Operations/sec", writes / duration, false ) );
+		m.emplace_back("Measured Duration", duration, true);
+		m.emplace_back("Operations/sec", writes / duration, false);
 		m.push_back( transactions.getMetric() );
 		m.push_back( retries.getMetric() );
 
-		m.push_back( PerfMetric( "Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), true ) );
-		m.push_back( PerfMetric( "Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), true ) );
-		m.push_back( PerfMetric( "90% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.90 ), true ) );
-		m.push_back( PerfMetric( "98% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.98 ), true ) );
+		m.emplace_back("Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), true);
+		m.emplace_back("Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), true);
+		m.emplace_back("90% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.90 ), true);
+		m.emplace_back("98% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.98 ), true);
 
-		m.push_back( PerfMetric( "Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), true ) );
-		m.push_back( PerfMetric( "Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), true ) );
-		m.push_back( PerfMetric( "90% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.90 ), true ) );
-		m.push_back( PerfMetric( "98% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.98 ), true ) );
+		m.emplace_back("Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), true);
+		m.emplace_back("Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), true);
+		m.emplace_back("90% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.90 ), true);
+		m.emplace_back("98% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.98 ), true);
 
-		m.push_back( PerfMetric( "Bytes written/sec", (writes * (keyBytes + valueBytes)) / duration, false ) );
+		m.emplace_back("Bytes written/sec", (writes * (keyBytes + valueBytes)) / duration, false);
 	}
 
 	static Key keyForIndex( int base, int offset ) { return StringRef( format( "%08x%08x", base, offset ) ); }
@@ -127,7 +128,7 @@ struct QueuePushWorkload : TestWorkload {
 							lastKey = self->endingKey;
 					}
 					
-					pair<int, int> unpacked = valuesForKey( lastKey );
+					std::pair<int, int> unpacked = valuesForKey( lastKey );
 
 					if( self->forward )
 						tr.set( keyForIndex( unpacked.first + unpacked.second, g_random->randomInt(1, 1000) ), 
diff --git a/fdbserver/workloads/ReadWrite.actor.cpp b/fdbserver/workloads/ReadWrite.actor.cpp
index 9614f6e0a6..8de4258a6d 100644
--- a/fdbserver/workloads/ReadWrite.actor.cpp
+++ b/fdbserver/workloads/ReadWrite.actor.cpp
@@ -19,6 +19,8 @@
  */
 
 #include <boost/lexical_cast.hpp>
+#include <utility>
+#include <vector>
 
 #include "fdbrpc/ContinuousSample.h"
 #include "fdbclient/NativeAPI.actor.h"
@@ -106,15 +108,15 @@ struct ReadWriteWorkload : KVWorkload {
 	EventMetricHandle<TransactionFailureMetric> transactionFailureMetric;
 	EventMetricHandle<ReadMetric> readMetric;
 
-	vector<Future<Void>> clients;
+	std::vector<Future<Void>> clients;
 	PerfIntCounter aTransactions, bTransactions, retries;
 	ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, fullReadLatencies;
 	double readLatencyTotal; int readLatencyCount;
 
-	vector<uint64_t> insertionCountsToMeasure;
-	vector<pair<uint64_t, double> > ratesAtKeyCounts;
+	std::vector<uint64_t> insertionCountsToMeasure;
+	std::vector<std::pair<uint64_t, double> > ratesAtKeyCounts;
 
-	vector<PerfMetric> periodicMetrics;
+	std::vector<PerfMetric> periodicMetrics;
 
 	bool doSetup;
 
@@ -193,9 +195,9 @@ struct ReadWriteWorkload : KVWorkload {
 			ASSERT((keyForIndex(a, false) <= keyForIndex(b, false)));
 		}
 
-		vector<std::string> insertionCountsToMeasureString = getOption(options, LiteralStringRef("insertionCountsToMeasure"), vector<std::string>());
-		for(int i = 0; i < insertionCountsToMeasureString.size(); i++)
-		{
+		std::vector<std::string> insertionCountsToMeasureString =
+		    getOption(options, LiteralStringRef("insertionCountsToMeasure"), std::vector<std::string>());
+		for (int i = 0; i < insertionCountsToMeasureString.size(); i++) {
 			try
 			{
 				uint64_t count = boost::lexical_cast<uint64_t>(insertionCountsToMeasureString[i]);
@@ -225,7 +227,7 @@ struct ReadWriteWorkload : KVWorkload {
 	ACTOR static Future<bool> traceDumpWorkers( Reference<AsyncVar<ServerDBInfo>> db ) {
 		try {
 			loop {
-				ErrorOr<vector<WorkerDetails>> workerList = wait( db->get().clusterInterface.getWorkers.tryGetReply( GetWorkersRequest() ) );
+				ErrorOr<std::vector<WorkerDetails>> workerList = wait( db->get().clusterInterface.getWorkers.tryGetReply( GetWorkersRequest() ) );
 				if( workerList.present() ) {
 					std::vector<Future<ErrorOr<Void>>> dumpRequests;
 					for( int i = 0; i < workerList.get().size(); i++)
@@ -254,53 +256,53 @@ struct ReadWriteWorkload : KVWorkload {
 			return true;
 	}
 
-	virtual void getMetrics( vector<PerfMetric>& m ) {
+	virtual void getMetrics(std::vector<PerfMetric>& m) {
 		double duration = metricsDuration;
 		int reads = (aTransactions.getValue() * readsPerTransactionA) + (bTransactions.getValue() * readsPerTransactionB);
 		int writes = (aTransactions.getValue() * writesPerTransactionA) + (bTransactions.getValue() * writesPerTransactionB);
-		m.push_back( PerfMetric( "Measured Duration", duration, true ) );
-		m.push_back( PerfMetric( "Transactions/sec", (aTransactions.getValue() + bTransactions.getValue()) / duration, false ) );
-		m.push_back( PerfMetric( "Operations/sec", ( ( reads + writes ) / duration ), false ) );
+		m.emplace_back("Measured Duration", duration, true);
+		m.emplace_back("Transactions/sec", (aTransactions.getValue() + bTransactions.getValue()) / duration, false);
+		m.emplace_back("Operations/sec", ( ( reads + writes ) / duration ), false);
 		m.push_back( aTransactions.getMetric() );
 		m.push_back( bTransactions.getMetric() );
 		m.push_back( retries.getMetric() );
-		m.push_back( PerfMetric( "Mean load time (seconds)", loadTime, true ) );
-		m.push_back( PerfMetric( "Read rows", reads, false ) );
-		m.push_back( PerfMetric( "Write rows", writes, false ) );
+		m.emplace_back("Mean load time (seconds)", loadTime, true);
+		m.emplace_back("Read rows", reads, false);
+		m.emplace_back("Write rows", writes, false);
 
 		if(!rampUpLoad) {
-			m.push_back(PerfMetric("Mean Latency (ms)", 1000 * latencies.mean(), true));
-			m.push_back(PerfMetric("Median Latency (ms, averaged)", 1000 * latencies.median(), true));
-			m.push_back(PerfMetric("90% Latency (ms, averaged)", 1000 * latencies.percentile(0.90), true));
-			m.push_back(PerfMetric("98% Latency (ms, averaged)", 1000 * latencies.percentile(0.98), true));
-			m.push_back(PerfMetric("Max Latency (ms, averaged)", 1000 * latencies.max(), true));
+			m.emplace_back("Mean Latency (ms)", 1000 * latencies.mean(), true);
+			m.emplace_back("Median Latency (ms, averaged)", 1000 * latencies.median(), true);
+			m.emplace_back("90% Latency (ms, averaged)", 1000 * latencies.percentile(0.90), true);
+			m.emplace_back("98% Latency (ms, averaged)", 1000 * latencies.percentile(0.98), true);
+			m.emplace_back("Max Latency (ms, averaged)", 1000 * latencies.max(), true);
 
-			m.push_back(PerfMetric("Mean Row Read Latency (ms)", 1000 * readLatencies.mean(), true));
-			m.push_back(PerfMetric("Median Row Read Latency (ms, averaged)", 1000 * readLatencies.median(), true));
-			m.push_back(PerfMetric("Max Row Read Latency (ms, averaged)", 1000 * readLatencies.max(), true));
+			m.emplace_back("Mean Row Read Latency (ms)", 1000 * readLatencies.mean(), true);
+			m.emplace_back("Median Row Read Latency (ms, averaged)", 1000 * readLatencies.median(), true);
+			m.emplace_back("Max Row Read Latency (ms, averaged)", 1000 * readLatencies.max(), true);
 
-			m.push_back(PerfMetric("Mean Total Read Latency (ms)", 1000 * fullReadLatencies.mean(), true));
-			m.push_back(PerfMetric("Median Total Read Latency (ms, averaged)", 1000 * fullReadLatencies.median(), true));
-			m.push_back(PerfMetric("Max Total Latency (ms, averaged)", 1000 * fullReadLatencies.max(), true));
+			m.emplace_back("Mean Total Read Latency (ms)", 1000 * fullReadLatencies.mean(), true);
+			m.emplace_back("Median Total Read Latency (ms, averaged)", 1000 * fullReadLatencies.median(), true);
+			m.emplace_back("Max Total Latency (ms, averaged)", 1000 * fullReadLatencies.max(), true);
 
-			m.push_back(PerfMetric("Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), true));
-			m.push_back(PerfMetric("Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), true));
-			m.push_back(PerfMetric("Max GRV Latency (ms, averaged)", 1000 * GRVLatencies.max(), true));
+			m.emplace_back("Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), true);
+			m.emplace_back("Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), true);
+			m.emplace_back("Max GRV Latency (ms, averaged)", 1000 * GRVLatencies.max(), true);
 
-			m.push_back(PerfMetric("Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), true));
-			m.push_back(PerfMetric("Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), true));
-			m.push_back(PerfMetric("Max Commit Latency (ms, averaged)", 1000 * commitLatencies.max(), true));
+			m.emplace_back("Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), true);
+			m.emplace_back("Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), true);
+			m.emplace_back("Max Commit Latency (ms, averaged)", 1000 * commitLatencies.max(), true);
 		}
 
-		m.push_back( PerfMetric( "Read rows/sec", reads / duration, false ) );
-		m.push_back( PerfMetric( "Write rows/sec", writes / duration, false ) );
-		m.push_back( PerfMetric( "Bytes read/sec", (reads * (keyBytes + (minValueBytes+maxValueBytes)*0.5)) / duration, false ) );
-		m.push_back( PerfMetric( "Bytes written/sec", (writes * (keyBytes + (minValueBytes+maxValueBytes)*0.5)) / duration, false ) );
+		m.emplace_back("Read rows/sec", reads / duration, false);
+		m.emplace_back("Write rows/sec", writes / duration, false);
+		m.emplace_back("Bytes read/sec", (reads * (keyBytes + (minValueBytes+maxValueBytes)*0.5)) / duration, false);
+		m.emplace_back("Bytes written/sec", (writes * (keyBytes + (minValueBytes+maxValueBytes)*0.5)) / duration, false);
 		m.insert(m.end(), periodicMetrics.begin(), periodicMetrics.end());
 
-		vector<pair<uint64_t, double> >::iterator ratesItr = ratesAtKeyCounts.begin();
+		std::vector<std::pair<uint64_t, double> >::iterator ratesItr = ratesAtKeyCounts.begin();
 		for(; ratesItr != ratesAtKeyCounts.end(); ratesItr++)
-			m.push_back(PerfMetric(format("%ld keys imported bytes/sec", ratesItr->first), ratesItr->second, false));
+			m.emplace_back(format("%ld keys imported bytes/sec", ratesItr->first), ratesItr->second, false);
 	}
 
 	Value randomValue() { return StringRef( (uint8_t*)valueString.c_str(), g_random->randomInt(minValueBytes, maxValueBytes+1) );	}
@@ -336,40 +338,40 @@ struct ReadWriteWorkload : KVWorkload {
 			bool recordEnd   = self->shouldRecord( now() );
 			if( recordBegin && recordEnd ) {
 				std::string ts = format("T=%04.0fs:", elapsed);
-				self->periodicMetrics.push_back( PerfMetric( ts + "Operations/sec", (ops-last_ops)/self->periodicLoggingInterval, false ) );
+				self->periodicMetrics.emplace_back(ts + "Operations/sec", (ops-last_ops)/self->periodicLoggingInterval, false);
 
 				//if(self->rampUpLoad) {
-					self->periodicMetrics.push_back(PerfMetric(ts + "Mean Latency (ms)", 1000 * self->latencies.mean(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "Median Latency (ms, averaged)", 1000 * self->latencies.median(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "5% Latency (ms, averaged)", 1000 * self->latencies.percentile(.05), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "95% Latency (ms, averaged)", 1000 * self->latencies.percentile(.95), true));
+					self->periodicMetrics.emplace_back(ts + "Mean Latency (ms)", 1000 * self->latencies.mean(), true);
+					self->periodicMetrics.emplace_back(ts + "Median Latency (ms, averaged)", 1000 * self->latencies.median(), true);
+					self->periodicMetrics.emplace_back(ts + "5% Latency (ms, averaged)", 1000 * self->latencies.percentile(.05), true);
+					self->periodicMetrics.emplace_back(ts + "95% Latency (ms, averaged)", 1000 * self->latencies.percentile(.95), true);
 
-					self->periodicMetrics.push_back(PerfMetric(ts + "Mean Row Read Latency (ms)", 1000 * self->readLatencies.mean(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "Median Row Read Latency (ms, averaged)", 1000 * self->readLatencies.median(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "5% Row Read Latency (ms, averaged)", 1000 * self->readLatencies.percentile(.05), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "95% Row Read Latency (ms, averaged)", 1000 * self->readLatencies.percentile(.95), true));
+					self->periodicMetrics.emplace_back(ts + "Mean Row Read Latency (ms)", 1000 * self->readLatencies.mean(), true);
+					self->periodicMetrics.emplace_back(ts + "Median Row Read Latency (ms, averaged)", 1000 * self->readLatencies.median(), true);
+					self->periodicMetrics.emplace_back(ts + "5% Row Read Latency (ms, averaged)", 1000 * self->readLatencies.percentile(.05), true);
+					self->periodicMetrics.emplace_back(ts + "95% Row Read Latency (ms, averaged)", 1000 * self->readLatencies.percentile(.95), true);
 
-					self->periodicMetrics.push_back(PerfMetric(ts + "Mean Total Read Latency (ms)", 1000 * self->fullReadLatencies.mean(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "Median Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.median(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "5% Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.percentile(.05), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "95% Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.percentile(.95), true));
+					self->periodicMetrics.emplace_back(ts + "Mean Total Read Latency (ms)", 1000 * self->fullReadLatencies.mean(), true);
+					self->periodicMetrics.emplace_back(ts + "Median Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.median(), true);
+					self->periodicMetrics.emplace_back(ts + "5% Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.percentile(.05), true);
+					self->periodicMetrics.emplace_back(ts + "95% Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.percentile(.95), true);
 
-					self->periodicMetrics.push_back(PerfMetric(ts + "Mean GRV Latency (ms)", 1000 * self->GRVLatencies.mean(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "Median GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.median(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "5% GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.percentile(.05), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "95% GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.percentile(.95), true));
+					self->periodicMetrics.emplace_back(ts + "Mean GRV Latency (ms)", 1000 * self->GRVLatencies.mean(), true);
+					self->periodicMetrics.emplace_back(ts + "Median GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.median(), true);
+					self->periodicMetrics.emplace_back(ts + "5% GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.percentile(.05), true);
+					self->periodicMetrics.emplace_back(ts + "95% GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.percentile(.95), true);
 
-					self->periodicMetrics.push_back(PerfMetric(ts + "Mean Commit Latency (ms)", 1000 * self->commitLatencies.mean(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "Median Commit Latency (ms, averaged)", 1000 * self->commitLatencies.median(), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "5% Commit Latency (ms, averaged)", 1000 * self->commitLatencies.percentile(.05), true));
-					self->periodicMetrics.push_back(PerfMetric(ts + "95% Commit Latency (ms, averaged)", 1000 * self->commitLatencies.percentile(.95), true));
+					self->periodicMetrics.emplace_back(ts + "Mean Commit Latency (ms)", 1000 * self->commitLatencies.mean(), true);
+					self->periodicMetrics.emplace_back(ts + "Median Commit Latency (ms, averaged)", 1000 * self->commitLatencies.median(), true);
+					self->periodicMetrics.emplace_back(ts + "5% Commit Latency (ms, averaged)", 1000 * self->commitLatencies.percentile(.05), true);
+					self->periodicMetrics.emplace_back(ts + "95% Commit Latency (ms, averaged)", 1000 * self->commitLatencies.percentile(.95), true);
 				//}
 
-				self->periodicMetrics.push_back(PerfMetric(ts + "Max Latency (ms, averaged)", 1000 * self->latencies.max(), true));
-				self->periodicMetrics.push_back(PerfMetric(ts + "Max Row Read Latency (ms, averaged)", 1000 * self->readLatencies.max(), true));
-				self->periodicMetrics.push_back(PerfMetric(ts + "Max Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.max(), true));
-				self->periodicMetrics.push_back(PerfMetric(ts + "Max GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.max(), true));
-				self->periodicMetrics.push_back(PerfMetric(ts + "Max Commit Latency (ms, averaged)", 1000 * self->commitLatencies.max(), true));
+				self->periodicMetrics.emplace_back(ts + "Max Latency (ms, averaged)", 1000 * self->latencies.max(), true);
+				self->periodicMetrics.emplace_back(ts + "Max Row Read Latency (ms, averaged)", 1000 * self->readLatencies.max(), true);
+				self->periodicMetrics.emplace_back(ts + "Max Total Read Latency (ms, averaged)", 1000 * self->fullReadLatencies.max(), true);
+				self->periodicMetrics.emplace_back(ts + "Max GRV Latency (ms, averaged)", 1000 * self->GRVLatencies.max(), true);
+				self->periodicMetrics.emplace_back(ts + "Max Commit Latency (ms, averaged)", 1000 * self->commitLatencies.max(), true);
 			}
 			last_ops = ops;
 
@@ -450,7 +452,7 @@ struct ReadWriteWorkload : KVWorkload {
 			return Void();
 
 		state Promise<double> loadTime;
-		state Promise<vector<pair<uint64_t, double> > > ratesAtKeyCounts;
+		state Promise<std::vector<std::pair<uint64_t, double> > > ratesAtKeyCounts;
 
 		wait( bulkSetup( cx, self, self->nodeCount, loadTime, self->insertionCountsToMeasure.empty(), self->warmingDelay, self->maxInsertRate, 
 								  self->insertionCountsToMeasure, ratesAtKeyCounts ) );
@@ -481,7 +483,7 @@ struct ReadWriteWorkload : KVWorkload {
 
 		wait( delay( std::max(0.1, 1.0 - (now() - startTime) ) ) );
 
-		vector<Future<Void>> clients;
+		std::vector<Future<Void>> clients;
 		if(self->enableReadLatencyLogging)
 			clients.push_back(tracePeriodically(self));
 
@@ -554,9 +556,9 @@ struct ReadWriteWorkload : KVWorkload {
 				state double tstart = now();
 				state bool aTransaction = g_random->random01() > (self->rampTransactionType ? self->sweepAlpha(startTime) : self->alpha);
 
-				state vector<int64_t> keys;
-				state vector<Value> values;
-				state vector<KeyRange> extra_ranges;
+				state std::vector<int64_t> keys;
+				state std::vector<Value> values;
+				state std::vector<KeyRange> extra_ranges;
 				int reads = aTransaction ? self->readsPerTransactionA : self->readsPerTransactionB;
 				state int writes = aTransaction ? self->writesPerTransactionA : self->writesPerTransactionB;
 				state int extra_read_conflict_ranges = writes ? self->extraReadConflictRangesPerTransaction : 0;
diff --git a/fdbserver/workloads/WriteBandwidth.actor.cpp b/fdbserver/workloads/WriteBandwidth.actor.cpp
index d9cc8a4db2..74286c23ae 100644
--- a/fdbserver/workloads/WriteBandwidth.actor.cpp
+++ b/fdbserver/workloads/WriteBandwidth.actor.cpp
@@ -33,7 +33,7 @@ struct WriteBandwidthWorkload : KVWorkload {
 	double testDuration, warmingDelay, loadTime, maxInsertRate;
 	std::string valueString;
 
-	vector<Future<Void>> clients;
+	std::vector<Future<Void>> clients;
 	PerfIntCounter transactions, retries;
 	ContinuousSample<double> commitLatencies, GRVLatencies;
 
@@ -56,29 +56,29 @@ struct WriteBandwidthWorkload : KVWorkload {
 
 	virtual Future<bool> check( Database const& cx ) { return true; }
 
-	virtual void getMetrics( vector<PerfMetric>& m ) {
+	virtual void getMetrics( std::vector<PerfMetric>& m ) {
 		double duration = testDuration;
 		int writes = transactions.getValue() * keysPerTransaction;
-		m.push_back( PerfMetric( "Measured Duration", duration, true ) );
-		m.push_back( PerfMetric( "Transactions/sec", transactions.getValue() / duration, false ) );
-		m.push_back( PerfMetric( "Operations/sec", writes / duration, false ) );
-		m.push_back( transactions.getMetric() );
-		m.push_back( retries.getMetric() );
-		m.push_back( PerfMetric( "Mean load time (seconds)", loadTime, true ) );
-		m.push_back( PerfMetric( "Write rows", writes, false ) );
+		m.emplace_back("Measured Duration", duration, true);
+		m.emplace_back("Transactions/sec", transactions.getValue() / duration, false);
+		m.emplace_back("Operations/sec", writes / duration, false);
+		m.push_back(transactions.getMetric());
+		m.push_back(retries.getMetric());
+		m.emplace_back("Mean load time (seconds)", loadTime, true);
+		m.emplace_back("Write rows", writes, false);
 
-		m.push_back( PerfMetric( "Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), true ) );
-		m.push_back( PerfMetric( "Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), true ) );
-		m.push_back( PerfMetric( "90% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.90 ), true ) );
-		m.push_back( PerfMetric( "98% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.98 ), true ) );
+		m.emplace_back("Mean GRV Latency (ms)", 1000 * GRVLatencies.mean(), true);
+		m.emplace_back("Median GRV Latency (ms, averaged)", 1000 * GRVLatencies.median(), true);
+		m.emplace_back("90% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.90 ), true);
+		m.emplace_back("98% GRV Latency (ms, averaged)", 1000 * GRVLatencies.percentile( 0.98 ), true);
 
-		m.push_back( PerfMetric( "Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), true ) );
-		m.push_back( PerfMetric( "Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), true ) );
-		m.push_back( PerfMetric( "90% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.90 ), true ) );
-		m.push_back( PerfMetric( "98% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.98 ), true ) );
+		m.emplace_back("Mean Commit Latency (ms)", 1000 * commitLatencies.mean(), true);
+		m.emplace_back("Median Commit Latency (ms, averaged)", 1000 * commitLatencies.median(), true);
+		m.emplace_back("90% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.90 ), true);
+		m.emplace_back("98% Commit Latency (ms, averaged)", 1000 * commitLatencies.percentile( 0.98 ), true);
 
-		m.push_back( PerfMetric( "Write rows/sec", writes / duration, false ) );
-		m.push_back( PerfMetric( "Bytes written/sec", (writes * (keyBytes + (minValueBytes+maxValueBytes)*0.5)) / duration, false ) );
+		m.emplace_back("Write rows/sec", writes / duration, false);
+		m.emplace_back("Bytes written/sec", (writes * (keyBytes + (minValueBytes+maxValueBytes)*0.5)) / duration, false);
 	}
 
 	Value randomValue() { return StringRef( (uint8_t*)valueString.c_str(), g_random->randomInt(minValueBytes, maxValueBytes+1) );	}
@@ -89,7 +89,7 @@ struct WriteBandwidthWorkload : KVWorkload {
 
 	ACTOR Future<Void> _setup( Database cx, WriteBandwidthWorkload *self ) {
 		state Promise<double> loadTime;
-		state Promise<vector<pair<uint64_t, double> > > ratesAtKeyCounts;
+		state Promise<std::vector<std::pair<uint64_t, double> > > ratesAtKeyCounts;
 
 		wait( bulkSetup( cx, self, self->nodeCount, loadTime, true, self->warmingDelay, self->maxInsertRate ) );
 		self->loadTime = loadTime.getFuture().get();

From 97051a23e47d56cd965e2e0ddde4417d4dcd3950 Mon Sep 17 00:00:00 2001
From: Bhaskar Muppana <mbhaskar@apple.com>
Date: Thu, 16 May 2019 13:34:55 -0700
Subject: [PATCH 110/118] Fixing typo in tls.rst

---
 documentation/sphinx/source/tls.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/tls.rst b/documentation/sphinx/source/tls.rst
index f5b5c94852..f2c23fc279 100644
--- a/documentation/sphinx/source/tls.rst
+++ b/documentation/sphinx/source/tls.rst
@@ -351,4 +351,4 @@ A verification string of::
 Would pass, and:
 
 * Require that the Subject has a Subject Alternative Name extension, which has one or more members of type DNS that begins with the value ``prod.``.
-* Require that the Subject has a Subject Alternative Name extension, which has one or more members of type DNS that ends with the value ``.com``.
+* Require that the Subject has a Subject Alternative Name extension, which has one or more members of type DNS that ends with the value ``.org``.

From a0fa81aa1c57ed10ad47b394ea094710667cf983 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 17 May 2019 14:12:44 -0700
Subject: [PATCH 111/118] remove memory-hungry benchmark

This test was used as a benchmark to prove that the
new `IsTraceable` feature improves performance. However
it will cause smaller machines to OOM and it doesn't
help for correctness testing. Therefore this test should
likely be removed
---
 flow/ActorCollection.actor.cpp | 71 ----------------------------------
 1 file changed, 71 deletions(-)

diff --git a/flow/ActorCollection.actor.cpp b/flow/ActorCollection.actor.cpp
index 0a013f8e01..a15142cea8 100644
--- a/flow/ActorCollection.actor.cpp
+++ b/flow/ActorCollection.actor.cpp
@@ -76,74 +76,3 @@ struct Traceable<std::pair<T, U>> {
 		return result;
 	}
 };
-
-
-TEST_CASE("/flow/TraceEvent") {
-	state unsigned i;
-	state double startTime;
-	state std::vector<std::string> strings;
-	state std::vector<int> keyIdx;
-	state std::vector<int> pairRnd;
-	state std::vector<int> num;
-	state std::vector<double> doub;
-	state std::vector<int> strIdx;
-	strings.reserve(10000);
-	keyIdx.reserve(1e6);
-	pairRnd.reserve(1e6);
-	num.reserve(1e6);
-	doub.reserve(1e6);
-	strIdx.reserve(1e6);
-	for (i = 0; i < 100; ++i) {
-		for (int j = 0; j < 100; ++j) {
-			strings.emplace_back(g_random->randomAlphaNumeric(g_random->randomInt(1, 30)));
-		}
-		wait(delay(0));
-	}
-	for (i = 0; i < 1e6; ++i) {
-		keyIdx.emplace_back(g_random->randomInt(0, strings.size()));
-		pairRnd.emplace_back(g_random->randomInt(-1000, 1000));
-		num.emplace_back(g_random->randomInt(0, 1000));
-		doub.emplace_back(g_random->random01());
-		strIdx.emplace_back(g_random->randomInt(0, strings.size()));
-	}
-	TraceEvent("PairsFilled")
-		.detail("MemoryUsage", getMemoryUsage());
-	printf("Sleeping for 20 seconds - attach perf now to PID %d\n", getpid());
-	wait(delay(20));
-	printf("Done sleeping\n");
-	startTime = g_network->now();
-	for (i = 0; i < 100000; ++i) {
-		for (unsigned j = 0; j < 100; ++j) {
-			int idx = (i+1)*j % keyIdx.size();
-			StringRef key(strings[keyIdx[idx]]);
-			auto p = std::make_pair(key, pairRnd[idx]);
-			TraceEvent("TestTraceLineNoDebug")
-				.detail("Num", num[idx])
-				.detail("Double", doub[idx])
-				.detail("Str", strings[strIdx[idx]])
-				.detail("Pair", p);
-		}
-		wait(delay(0));
-	}
-	TraceEvent("TraceDuration")
-		.detail("Duration", g_network->now() - startTime);
-	startTime = g_network->now();
-	for (i = 0; i < 1000000; ++i) {
-		for (unsigned j = 0; j < 100; ++j) {
-			int idx = (i+1)*j % keyIdx.size();
-			StringRef key(strings[keyIdx[idx]]);
-			auto p = std::make_pair(key, pairRnd[idx]);
-			TraceEvent(SevDebug, "TestTraceLineDebug")
-				.detail("Num", num[idx])
-				.detail("Double", doub[idx])
-				.detail("Str", strings[strIdx[idx]])
-				.detail("Pair", p);
-		}
-		wait(delay(0));
-	}
-	TraceEvent("TraceDuration")
-		.detail("Duration", g_network->now() - startTime);
-	printf("benchmark done\n");
-	wait(delay(10));
-	return Void();
-}

From f673be456bcc6176a2c96371ccb4262fe609f2c9 Mon Sep 17 00:00:00 2001
From: mpilman <markus.pilman@snowflake.com>
Date: Fri, 17 May 2019 14:06:08 -0700
Subject: [PATCH 112/118] Added jenkins file for Snowflake

We (snowflake) want to run build and simulation tests against PRs.
For this we need a jenkins file in the repository. This should not
impact anything else
---
 tests/SnowflakeJenkins | 107 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 tests/SnowflakeJenkins

diff --git a/tests/SnowflakeJenkins b/tests/SnowflakeJenkins
new file mode 100644
index 0000000000..4584a87bbd
--- /dev/null
+++ b/tests/SnowflakeJenkins
@@ -0,0 +1,107 @@
+def buildScmInfo
+stage("Build") {
+    node('test-dynamic-slave') {
+        cleanWs()
+
+        sfScmInfo = checkout([$class: 'GitSCM',
+            branches: [[name: '*']],
+            doGenerateSubmoduleConfigurations: false,
+            extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'snowflake']],
+            submoduleCfg: [],
+            userRemoteConfigs: [[credentialsId: 'a0395839-84c7-4ceb-90e2-bcf66b2d6885', url: 'ssh://bitbucket-internal.int.snowflakecomputing.com:7999/opfdb/fdb_snowflake.git']]
+            ])
+        println("$sfScmInfo")
+
+        buildScmInfo = checkout([
+            $class: 'GitSCM',
+            branches: scm.branches,
+            doGenerateSubmoduleConfigurations: scm.doGenerateSubmoduleConfigurations,
+            extensions: scm.extensions,
+            userRemoteConfigs: scm.userRemoteConfigs,
+            extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'snowflake/jenkins/foundationdb']]
+            ])
+        println("$buildScmInfo")
+
+        sh """
+            |export GIT_SPECIFIER=${buildScmInfo.GIT_COMMIT}
+            |rm -rf venv
+            |virtualenv-3.4 venv
+            |source venv/bin/activate
+            |pip3 install docker-compose
+            |docker-compose --version
+            |git config --global user.name jenkins
+            |git config --global user.email fdb-devs@snowflake.net
+            |cd snowflake/jenkins
+            |./build.sh check_uploaded package sql sql_upload upload
+          """.stripMargin()
+    }
+}
+
+def makeTestStep(iteration) {
+    return {
+        node("test-dynamic-slave") {
+            cleanWs()
+            sfScmInfo = checkout([$class: 'GitSCM',
+                branches: [[name: '*']],
+                doGenerateSubmoduleConfigurations: false,
+                extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'snowflake']],
+                submoduleCfg: [],
+                userRemoteConfigs: [[credentialsId: 'a0395839-84c7-4ceb-90e2-bcf66b2d6885', url: 'ssh://bitbucket-internal.int.snowflakecomputing.com:7999/opfdb/fdb_snowflake.git']]
+                ])
+            println("$sfScmInfo")
+
+            scmInfo = checkout([
+                $class: 'GitSCM',
+                branches: scm.branches,
+                doGenerateSubmoduleConfigurations: scm.doGenerateSubmoduleConfigurations,
+                extensions: scm.extensions,
+                userRemoteConfigs: scm.userRemoteConfigs,
+                extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'snowflake/jenkins/foundationdb']]
+                ])
+            println("$scmInfo")
+            sh """
+                |# Clean up the jenkins output; gets messy with too many iterations
+                |set +x
+                |exec 3>&1
+                |exec 1> \$WORKSPACE/setup_${iteration}.log
+                |exec 2>&1
+                |
+                |export GIT_SPECIFIER=${scmInfo.GIT_COMMIT}
+                |rm -rf venv
+                |virtualenv-3.4 venv
+                |source venv/bin/activate
+                |pip3 install docker-compose
+                |docker-compose --version
+                |git config --global user.name jenkins
+                |git config --global user.email fdb-devs@snowflake.net
+                |
+                |cd snowflake/jenkins
+                |echo Iteration ${iteration} building >&3
+                |./build.sh configure download test sql sql_upload > \$WORKSPACE/iteration_${iteration}.log 2>&1
+                |rc=\$?
+                |seed=\$(find . -name traces.json -exec grep -m 1 CMakeSEED {} \\; | awk '{print \$2}'  | head -1 | tr -d '"}')
+                |echo Iteration ${iteration} completed with \$rc - seed \$seed >&3
+                |mv \$WORKSPACE/iteration_${iteration}.log \$WORKSPACE/iteration_${iteration}_\${seed}.log
+                |find . -name traces.json -exec gzip -c {} > \$WORKSPACE/traces_${iteration}_\${seed}.json.gz \\;
+                |#cat \$WORKSPACE/iteration_${iteration}.log
+              """.stripMargin()
+              archiveArtifacts 'setup_*log,iteration_*log,traces_*.json.gz'
+        }
+    }
+}
+
+stage("Test") {
+    def testSteps = [:]
+    for (int i = 0; i < 4; i++) {
+        testSteps["Iteration ${i}"] = makeTestStep(i)
+    }
+    println(testSteps)
+
+    parallel testSteps
+    build job: "NotifyGitHub",
+        parameters: [
+            string(name: 'pr_branch', value: buildScmInfo.GIT_BRANCH),
+            string(name: 'publish_url', value: "https://foo.bar/stuff")
+        ],
+        propagate: false
+}

From 2b8b7954a9f51c70600472cc1a3b4457dfde99a3 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 17 May 2019 15:05:32 -0700
Subject: [PATCH 113/118] in simulation, prevent data from being received over
 a connection 1 second after the connection is closed on the other end

---
 fdbrpc/sim2.actor.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 5bc89266f9..9df572b01b 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -175,7 +175,7 @@ SimClogging g_clogging;
 
 struct Sim2Conn : IConnection, ReferenceCounted<Sim2Conn> {
 	Sim2Conn( ISimulator::ProcessInfo* process )
-		: process(process), dbgid( g_random->randomUniqueID() ), opened(false), closedByCaller(false)
+		: process(process), dbgid( g_random->randomUniqueID() ), opened(false), closedByCaller(false), stopReceive(Never())
 	{
 		pipes = sender(this) && receiver(this);
 	}
@@ -210,6 +210,7 @@ struct Sim2Conn : IConnection, ReferenceCounted<Sim2Conn> {
 
 	void peerClosed() {
 		leakedConnectionTracker = trackLeakedConnection(this);
+		stopReceive = delay(1.0);
 	}
 
 	// Reads as many bytes as possible from the read buffer into [begin,end) and returns the number of bytes read (might be 0)
@@ -286,6 +287,7 @@ private:
 	Future<Void> leakedConnectionTracker;
 
 	Future<Void> pipes;
+	Future<Void> stopReceive;
 
 	int availableSendBufferForPeer() const { return sendBufSize - (writtenBytes.get() - receivedBytes.get()); }  // SOMEDAY: acknowledgedBytes instead of receivedBytes
 
@@ -318,6 +320,9 @@ private:
 			ASSERT( g_simulator.getCurrentProcess() == self->process );
 			wait( delay( g_clogging.getRecvDelay( self->process->address, self->peerProcess->address ) ) );
 			ASSERT( g_simulator.getCurrentProcess() == self->process );
+			if(self->stopReceive.isReady()) {
+				wait(Future<Void>(Never()));
+			}
 			self->receivedBytes.set( pos );
 			wait( Future<Void>(Void()) );  // Prior notification can delete self and cancel this actor
 			ASSERT( g_simulator.getCurrentProcess() == self->process );

From 26ab2aee1cabd10472843fdb6ca61f36afb062fa Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 17 May 2019 16:15:44 -0700
Subject: [PATCH 114/118] updated documentation for 6.1.7

---
 documentation/sphinx/source/downloads.rst | 24 +++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst
index 63a25896f6..fbd5ca91ca 100644
--- a/documentation/sphinx/source/downloads.rst
+++ b/documentation/sphinx/source/downloads.rst
@@ -10,38 +10,38 @@ macOS
 
 The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
 
-* `FoundationDB-6.1.6.pkg <https://www.foundationdb.org/downloads/6.1.6/macOS/installers/FoundationDB-6.1.6.pkg>`_
+* `FoundationDB-6.1.7.pkg <https://www.foundationdb.org/downloads/6.1.7/macOS/installers/FoundationDB-6.1.7.pkg>`_
 
 Ubuntu
 ------
 
 The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
 
-* `foundationdb-clients-6.1.6-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.6/ubuntu/installers/foundationdb-clients_6.1.6-1_amd64.deb>`_
-* `foundationdb-server-6.1.6-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.6/ubuntu/installers/foundationdb-server_6.1.6-1_amd64.deb>`_ (depends on the clients package)
+* `foundationdb-clients-6.1.7-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.7/ubuntu/installers/foundationdb-clients_6.1.7-1_amd64.deb>`_
+* `foundationdb-server-6.1.7-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.7/ubuntu/installers/foundationdb-server_6.1.7-1_amd64.deb>`_ (depends on the clients package)
 
 RHEL/CentOS EL6
 ---------------
 
 The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
 
-* `foundationdb-clients-6.1.6-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel6/installers/foundationdb-clients-6.1.6-1.el6.x86_64.rpm>`_
-* `foundationdb-server-6.1.6-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel6/installers/foundationdb-server-6.1.6-1.el6.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.1.7-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.7/rhel6/installers/foundationdb-clients-6.1.7-1.el6.x86_64.rpm>`_
+* `foundationdb-server-6.1.7-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.7/rhel6/installers/foundationdb-server-6.1.7-1.el6.x86_64.rpm>`_ (depends on the clients package)
 
 RHEL/CentOS EL7
 ---------------
 
 The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
 
-* `foundationdb-clients-6.1.6-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel7/installers/foundationdb-clients-6.1.6-1.el7.x86_64.rpm>`_
-* `foundationdb-server-6.1.6-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.6/rhel7/installers/foundationdb-server-6.1.6-1.el7.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.1.7-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.7/rhel7/installers/foundationdb-clients-6.1.7-1.el7.x86_64.rpm>`_
+* `foundationdb-server-6.1.7-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.1.7/rhel7/installers/foundationdb-server-6.1.7-1.el7.x86_64.rpm>`_ (depends on the clients package)
 
 Windows
 -------
 
 The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
 
-* `foundationdb-6.1.6-x64.msi <https://www.foundationdb.org/downloads/6.1.6/windows/installers/foundationdb-6.1.6-x64.msi>`_
+* `foundationdb-6.1.7-x64.msi <https://www.foundationdb.org/downloads/6.1.7/windows/installers/foundationdb-6.1.7-x64.msi>`_
 
 API Language Bindings
 =====================
@@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
 
 If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package:
 
-* `foundationdb-6.1.6.tar.gz <https://www.foundationdb.org/downloads/6.1.6/bindings/python/foundationdb-6.1.6.tar.gz>`_
+* `foundationdb-6.1.7.tar.gz <https://www.foundationdb.org/downloads/6.1.7/bindings/python/foundationdb-6.1.7.tar.gz>`_
 
 Ruby 1.9.3/2.0.0+
 -----------------
 
-* `fdb-6.1.6.gem <https://www.foundationdb.org/downloads/6.1.6/bindings/ruby/fdb-6.1.6.gem>`_
+* `fdb-6.1.7.gem <https://www.foundationdb.org/downloads/6.1.7/bindings/ruby/fdb-6.1.7.gem>`_
 
 Java 8+
 -------
 
-* `fdb-java-6.1.6.jar <https://www.foundationdb.org/downloads/6.1.6/bindings/java/fdb-java-6.1.6.jar>`_
-* `fdb-java-6.1.6-javadoc.jar <https://www.foundationdb.org/downloads/6.1.6/bindings/java/fdb-java-6.1.6-javadoc.jar>`_
+* `fdb-java-6.1.7.jar <https://www.foundationdb.org/downloads/6.1.7/bindings/java/fdb-java-6.1.7.jar>`_
+* `fdb-java-6.1.7-javadoc.jar <https://www.foundationdb.org/downloads/6.1.7/bindings/java/fdb-java-6.1.7-javadoc.jar>`_
 
 Go 1.1+
 -------

From d34b9e572fb52c135753724b138d5eecaaedf883 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 17 May 2019 16:18:27 -0700
Subject: [PATCH 115/118] update installer WIX GUID following release

---
 packaging/msi/FDBInstaller.wxs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
index 9947b6138a..bff3b25911 100644
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{92B71EC3-E4A1-4473-8D72-7A4D61435C8F}'
+           Id='{515105D6-BCF8-4205-9D2D-E1693AEBF234}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'

From 2763062727a7c53b5d5432ad51d8e4d97e0bd067 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 19 May 2019 10:22:54 -0700
Subject: [PATCH 116/118] update versions target to 6.1.8

---
 versions.target | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.target b/versions.target
index 872c948e7b..9c2cfae58e 100644
--- a/versions.target
+++ b/versions.target
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
-    <Version>6.1.7</Version>
+    <Version>6.1.8</Version>
     <PackageName>6.1</PackageName>
   </PropertyGroup>
 </Project>

From 263baf64c2f9e9072ab852fc2ec54c2311a393d3 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Sun, 19 May 2019 10:22:54 -0700
Subject: [PATCH 117/118] update installer WIX GUID following release

---
 packaging/msi/FDBInstaller.wxs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
index bff3b25911..c3adf2db47 100644
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{515105D6-BCF8-4205-9D2D-E1693AEBF234}'
+           Id='{B440CAB2-F9C8-4185-9863-E8A7E7587FED}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'

From 76333ad87ab8798b43d91be44e1b4778c29cb7ee Mon Sep 17 00:00:00 2001
From: Alec Grieser <agrieser@apple.com>
Date: Mon, 20 May 2019 13:49:22 -0700
Subject: [PATCH 118/118] Resolves #1612: Go bindings: Use single line comments
 in CGO

This converts the multi-line comments for C imports to single line comments, which at the least is more idiomatic.
---
 bindings/go/src/fdb/cluster.go     |  6 ++----
 bindings/go/src/fdb/database.go    |  6 ++----
 bindings/go/src/fdb/errors.go      |  6 ++----
 bindings/go/src/fdb/fdb.go         |  8 +++-----
 bindings/go/src/fdb/futures.go     | 30 ++++++++++++++----------------
 bindings/go/src/fdb/generated.go   | 21 ++++++++++++++-------
 bindings/go/src/fdb/range.go       |  6 ++----
 bindings/go/src/fdb/transaction.go |  6 ++----
 8 files changed, 41 insertions(+), 48 deletions(-)

diff --git a/bindings/go/src/fdb/cluster.go b/bindings/go/src/fdb/cluster.go
index 379589685e..d6342170ab 100644
--- a/bindings/go/src/fdb/cluster.go
+++ b/bindings/go/src/fdb/cluster.go
@@ -22,10 +22,8 @@
 
 package fdb
 
-/*
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
-*/
+// #define FDB_API_VERSION 610
+// #include <foundationdb/fdb_c.h>
 import "C"
 
 // Deprecated: Use OpenDatabase or OpenDefault to obtain a database handle directly
diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go
index 34b90ee926..92ae56300d 100644
--- a/bindings/go/src/fdb/database.go
+++ b/bindings/go/src/fdb/database.go
@@ -22,10 +22,8 @@
 
 package fdb
 
-/*
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
-*/
+// #define FDB_API_VERSION 610
+// #include <foundationdb/fdb_c.h>
 import "C"
 
 import (
diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go
index 5b39724d7b..b828dd4c26 100644
--- a/bindings/go/src/fdb/errors.go
+++ b/bindings/go/src/fdb/errors.go
@@ -22,10 +22,8 @@
 
 package fdb
 
-/*
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
-*/
+// #define FDB_API_VERSION 610
+// #include <foundationdb/fdb_c.h>
 import "C"
 
 import (
diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go
index 3566c7f2ce..6e983d2ddc 100644
--- a/bindings/go/src/fdb/fdb.go
+++ b/bindings/go/src/fdb/fdb.go
@@ -22,11 +22,9 @@
 
 package fdb
 
-/*
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
- #include <stdlib.h>
-*/
+// #define FDB_API_VERSION 610
+// #include <foundationdb/fdb_c.h>
+// #include <stdlib.h>
 import "C"
 
 import (
diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go
index 747a1135e7..6f20d9d807 100644
--- a/bindings/go/src/fdb/futures.go
+++ b/bindings/go/src/fdb/futures.go
@@ -22,22 +22,20 @@
 
 package fdb
 
-/*
- #cgo LDFLAGS: -lfdb_c -lm
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
- #include <string.h>
-
- extern void unlockMutex(void*);
-
- void go_callback(FDBFuture* f, void* m) {
-     unlockMutex(m);
- }
-
- void go_set_callback(void* f, void* m) {
-     fdb_future_set_callback(f, (FDBCallback)&go_callback, m);
- }
-*/
+//  #cgo LDFLAGS: -lfdb_c -lm
+//  #define FDB_API_VERSION 610
+//  #include <foundationdb/fdb_c.h>
+//  #include <string.h>
+//
+//  extern void unlockMutex(void*);
+//
+//  void go_callback(FDBFuture* f, void* m) {
+//      unlockMutex(m);
+//  }
+//
+//  void go_set_callback(void* f, void* m) {
+//      fdb_future_set_callback(f, (FDBCallback)&go_callback, m);
+//  }
 import "C"
 
 import (
diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index 6842072df8..e059e7da63 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -46,6 +46,13 @@ func (o NetworkOptions) SetLocalAddress(param string) error {
 	return o.setOpt(10, []byte(param))
 }
 
+// enable the object serializer for network communication
+//
+// Parameter: 0 is false, every other value is true
+func (o NetworkOptions) SetUseObjectSerializer(param int64) error {
+	return o.setOpt(11, int64ToBytes(param))
+}
+
 // Deprecated
 //
 // Parameter: path to cluster file
@@ -444,7 +451,7 @@ const (
 	// Infrequently used. The client has passed a specific row limit and wants
 	// that many rows delivered in a single batch. Because of iterator operation
 	// in client drivers make request batches transparent to the user, consider
-	// ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this
+	// “WANT_ALL“ StreamingMode instead. A row limit must be specified if this
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
@@ -561,15 +568,15 @@ type ErrorPredicate int
 
 const (
 
-	// Returns ``true`` if the error indicates the operations in the
-	// transactions should be retried because of transient error.
+	// Returns “true“ if the error indicates the operations in the transactions
+	// should be retried because of transient error.
 	ErrorPredicateRetryable ErrorPredicate = 50000
 
-	// Returns ``true`` if the error indicates the transaction may have
-	// succeeded, though not in a way the system can verify.
+	// Returns “true“ if the error indicates the transaction may have succeeded,
+	// though not in a way the system can verify.
 	ErrorPredicateMaybeCommitted ErrorPredicate = 50001
 
-	// Returns ``true`` if the error indicates the transaction has not
-	// committed, though in a way that can be retried.
+	// Returns “true“ if the error indicates the transaction has not committed,
+	// though in a way that can be retried.
 	ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002
 )
diff --git a/bindings/go/src/fdb/range.go b/bindings/go/src/fdb/range.go
index a4383e0770..78b95f35fd 100644
--- a/bindings/go/src/fdb/range.go
+++ b/bindings/go/src/fdb/range.go
@@ -22,10 +22,8 @@
 
 package fdb
 
-/*
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
-*/
+// #define FDB_API_VERSION 610
+// #include <foundationdb/fdb_c.h>
 import "C"
 
 import (
diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go
index 4cd0186404..adfb5b62ec 100644
--- a/bindings/go/src/fdb/transaction.go
+++ b/bindings/go/src/fdb/transaction.go
@@ -22,10 +22,8 @@
 
 package fdb
 
-/*
- #define FDB_API_VERSION 610
- #include <foundationdb/fdb_c.h>
-*/
+// #define FDB_API_VERSION 610
+// #include <foundationdb/fdb_c.h>
 import "C"
 
 // A ReadTransaction can asynchronously read from a FoundationDB