diff --git a/Makefile b/Makefile
index de59197f71..cf8fcb228c 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,7 @@ else ifeq ($(PLATFORM),Darwin)
   CXX := /usr/bin/clang
 
   CFLAGS += -mmacosx-version-min=10.7 -stdlib=libc++
-  CXXFLAGS += -std=c++11 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
+  CXXFLAGS += -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
 
   .LIBPATTERNS := lib%.dylib lib%.a
 
diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py
index d00f2faf55..a8d2c6b9a1 100644
--- a/bindings/bindingtester/known_testers.py
+++ b/bindings/bindingtester/known_testers.py
@@ -63,6 +63,6 @@ testers = {
     'java_async' :             Tester('java', _java_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION),
     'java_completable' :       Tester('java', _java_completable_cmd + 'StackTester', 2040, 500, MAX_API_VERSION),
     'java_completable_async' : Tester('java', _java_completable_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION),
-    'go' :                     Tester('go', _absolute_path('go/bin/_stacktester'), 63, 200, MAX_API_VERSION),
+    'go' :                     Tester('go', _absolute_path('go/build/bin/_stacktester'), 63, 200, MAX_API_VERSION),
     'flow' :                   Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION),
 }
diff --git a/bindings/bindingtester/run_binding_tester.sh b/bindings/bindingtester/run_binding_tester.sh
old mode 100755
new mode 100644
diff --git a/bindings/c/local.mk b/bindings/c/local.mk
index 4df783d5c1..baa757c16a 100644
--- a/bindings/c/local.mk
+++ b/bindings/c/local.mk
@@ -23,14 +23,18 @@
 fdb_c_CFLAGS := $(fdbclient_CFLAGS)
 fdb_c_LDFLAGS := $(fdbrpc_LDFLAGS)
 fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a
+fdb_c_tests_LIBS := -Llib -lfdb_c
+fdb_c_tests_HEADERS := -Ibindings/c
 
 ifeq ($(PLATFORM),linux)
   fdb_c_LIBS += lib/libstdc++.a -lm -lpthread -lrt -ldl
   fdb_c_LDFLAGS += -Wl,--version-script=bindings/c/fdb_c.map -static-libgcc -Wl,-z,nodelete
+  fdb_c_tests_LIBS += -lpthread
 endif
 
 ifeq ($(PLATFORM),osx)
   fdb_c_LDFLAGS += -lc++ -Xlinker -exported_symbols_list -Xlinker bindings/c/fdb_c.symbols
+  fdb_c_tests_LIBS += -lpthread
 
   lib/libfdb_c.dylib: bindings/c/fdb_c.symbols
 
@@ -74,3 +78,24 @@ fdb_c_BUILD_SOURCES += bindings/c/fdb_c.g.S
 bindings/c/foundationdb/fdb_c_options.g.h: bin/vexillographer.exe fdbclient/vexillographer/fdb.options $(ALL_MAKEFILES)
 	@echo "Building       $@"
 	@$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options c $@
+
+bin/fdb_c_performance_test: bindings/c/test/performance_test.c bindings/c/test/test.h fdb_c
+	@echo "Compiling      fdb_c_performance_test"
+	@$(CC) $(CFLAGS) $(fdb_c_tests_LIBS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c
+
+bin/fdb_c_ryw_benchmark: bindings/c/test/ryw_benchmark.c bindings/c/test/test.h fdb_c
+	@echo "Compiling      fdb_c_ryw_benchmark"
+	@$(CC) $(CFLAGS) $(fdb_c_tests_LIBS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/ryw_benchmark.c
+
+packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz: bin/fdb_c_performance_test bin/fdb_c_ryw_benchmark
+	@echo "Packaging      $@"
+	@rm -rf packages/fdb-c-tests-$(VERSION)-$(PLATFORM)
+	@mkdir -p packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin
+	@cp bin/fdb_c_performance_test packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin
+	@cp bin/fdb_c_ryw_benchmark packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin
+	@tar -C packages -czvf $@ fdb-c-tests-$(VERSION)-$(PLATFORM) > /dev/null
+	@rm -rf packages/fdb-c-tests-$(VERSION)-$(PLATFORM)
+
+fdb_c_tests: packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz
+
+packages: fdb_c_tests
diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c
new file mode 100644
index 0000000000..aef1926f77
--- /dev/null
+++ b/bindings/c/test/performance_test.c
@@ -0,0 +1,623 @@
+/*
+ * performance_test.c
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test.h"
+#include <foundationdb/fdb_c.h>
+#include <foundationdb/fdb_c_options.g.h>
+
+#include <stdio.h>
+#include <pthread.h>
+
+pthread_t netThread;
+
+int numKeys = 1000000;
+int keySize = 16;
+uint8_t** keys = NULL;
+int valueSize = 100;
+uint8_t *valueStr = NULL;
+
+fdb_error_t waitError(FDBFuture *f) {
+	fdb_error_t blockError = fdb_future_block_until_ready(f);
+	if(!blockError) {
+		return fdb_future_get_error(f);
+	} else {
+		return blockError;
+	}
+}
+
+struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
+	FDBTransaction *tr = NULL;
+	checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
+	fdb_error_t e = fdb_database_create_transaction(db, &tr);
+	checkError(e, "create transaction", rs);
+
+	while(1) {
+		struct RunResult r = func(rs, tr);
+		e = r.e;
+		if(!e) {
+			FDBFuture *f = fdb_transaction_commit(tr);
+			e = waitError(f);
+			fdb_future_destroy(f);
+		}
+
+		if(e) {
+			FDBFuture *f = fdb_transaction_on_error(tr, e);
+			fdb_error_t retryE = waitError(f);
+			fdb_future_destroy(f);
+			if (retryE) {
+				return (struct RunResult) {0, retryE};
+			}
+		} else {
+			return r;
+		}
+	}
+
+	return RES(0, 4100); // internal_error ; we should never get here
+}
+
+int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
+	int numRuns = 25;
+	int *results = malloc(sizeof(int)*numRuns);
+	int i = 0;
+	for(; i < numRuns; ++i) {
+		struct RunResult res = run(rs, db, testFxn);
+		if(res.e) {
+			logError(res.e, kpiName, rs);
+			free(results);
+			return 0;
+		}
+		results[i] = res.res;
+		if(results[i] < 0) {
+			free(results);
+			return -1;
+		}
+	}
+
+	int result = median(results, numRuns);
+	free(results);
+
+	addKpi(rs, kpiName, result, "keys/s");
+
+	return result;
+}
+
+int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
+	int numRuns = 25;
+	int *results = malloc(sizeof(int)*numRuns);
+	int i = 0;
+	for(; i < numRuns; ++i) {
+		struct RunResult res = testFxn(rs, db);
+		if(res.e) {
+			logError(res.e, kpiName, rs);
+			free(results);
+			return 0;
+		}
+		results[i] = res.res;
+		if(results[i] < 0) {
+			free(results);
+			return -1;
+		}
+	}
+
+	int result = median(results, numRuns);
+	free(results);
+
+	addKpi(rs, kpiName, result, "keys/s");
+
+	return result;
+}
+
+
+struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
+	return RES(0, 0);
+}
+
+uint32_t start = 0;
+uint32_t stop = 0;
+struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) {
+	int i;
+	for(i = start; i < stop; i++) {
+		fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize);
+	}
+	return RES(0, 0);
+}
+
+void insertData(struct ResultSet *rs, FDBDatabase *db) {
+	checkError(run(rs, db, &clearAll).e, "clearing database", rs);
+
+	// TODO: Do this asynchronously.
+	start = 0;
+	while(start < numKeys) {
+		stop = start + 1000;
+		if(stop > numKeys) stop = numKeys;
+		checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
+		start = stop;
+	}
+}
+
+fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) {
+	return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t));
+}
+
+uint32_t FUTURE_LATENCY_COUNT = 100000;
+const char *FUTURE_LATENCY_KPI = "C future throughput (local client)";
+struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	FDBFuture *f = fdb_transaction_get_read_version(tr);
+	e = waitError(f);
+	fdb_future_destroy(f);
+	maybeLogError(e, "getting initial read version", rs);
+	if(e) return RES(0, e);
+
+	double start = getTime();
+	int i;
+	for(i = 0; i < FUTURE_LATENCY_COUNT; i++) {
+		FDBFuture *f = fdb_transaction_get_read_version(tr);
+		e = waitError(f);
+		fdb_future_destroy(f);
+		maybeLogError(e, "getting read version", rs);
+		if(e) return RES(0, e);
+	}
+	double end = getTime();
+
+	return RES(FUTURE_LATENCY_COUNT/(end - start), 0);
+}
+
+uint32_t CLEAR_COUNT = 100000;
+const char *CLEAR_KPI = "C clear throughput (local client)";
+struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) {
+	double start = getTime();
+	int i;
+	for(i = 0; i < CLEAR_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		fdb_transaction_clear(tr, keys[k], keySize);
+	}
+	double end = getTime();
+
+	fdb_transaction_reset(tr); // Don't actually clear things.
+	return RES(CLEAR_COUNT/(end - start), 0);
+}
+
+uint32_t CLEAR_RANGE_COUNT = 100000;
+const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)";
+struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) {
+	double start = getTime();
+	int i;
+	for(i = 0; i < CLEAR_RANGE_COUNT; i++) {
+		int k = ((uint64_t)rand()) % (numKeys - 1);
+		fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize);
+	}
+	double end = getTime();
+
+	fdb_transaction_reset(tr); // Don't actually clear things.
+	return RES(CLEAR_RANGE_COUNT/(end - start), 0);
+}
+
+uint32_t SET_COUNT = 100000;
+const char *SET_KPI = "C set throughput (local client)";
+struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) {
+	double start = getTime();
+	int i;
+	for(i = 0; i < SET_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
+	}
+	double end = getTime();
+
+	fdb_transaction_reset(tr); // Don't actually set things.
+	return RES(SET_COUNT/(end - start), 0);
+}
+
+uint32_t PARALLEL_GET_COUNT = 10000;
+const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)";
+struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
+
+	double start = getTime();
+
+	int i;
+	for(i = 0; i < PARALLEL_GET_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
+	}
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	for(i = 0; i < PARALLEL_GET_COUNT; i++) {
+		e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		fdb_future_destroy(futures[i]);
+	}
+
+	double end = getTime();
+
+	free(futures);
+	return RES(PARALLEL_GET_COUNT/(end - start), 0);
+}
+
+uint32_t ALTERNATING_GET_SET_COUNT = 2000;
+const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
+struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
+
+	double start = getTime();
+
+	int i;
+	for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
+		futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
+	}
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
+		e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		fdb_future_destroy(futures[i]);
+	}
+
+	double end = getTime();
+
+	free(futures);
+	return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0);
+}
+
+uint32_t SERIAL_GET_COUNT = 2000;
+const char *SERIAL_GET_KPI = "C serial get throughput (local client)";
+struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	int i;
+	uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
+
+	if(SERIAL_GET_COUNT > numKeys/2) {
+		for(i = 0; i < SERIAL_GET_COUNT; i++) {
+			keyIndices[i] = ((uint64_t)rand()) % numKeys;
+		}
+	} else {
+		for(i = 0; i < SERIAL_GET_COUNT; i++) {
+			while(1) {
+				// Yes, this is a linear scan. This happens outside
+				// the part we are measuring.
+				uint32_t index = ((uint64_t)rand()) % numKeys;
+				int j;
+				fdb_bool_t found = 0;
+				for(j = 0; j < i; j++) {
+					if(keyIndices[j] == index) {
+						found = 1;
+						break;
+					}
+				}
+
+				if(!found) {
+					keyIndices[i] = index;
+					break;
+				}
+			}
+		}
+	}
+
+	double start = getTime();
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	for(i = 0; i < SERIAL_GET_COUNT; i++) {
+		FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
+		fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs);
+		if(e) {
+			free(keyIndices);
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
+		fdb_future_destroy(f);
+		if(e) {
+			free(keyIndices);
+			return RES(0, e);
+		}
+	}
+
+	double end = getTime();
+
+	free(keyIndices);
+	return RES(SERIAL_GET_COUNT/(end - start), 0);
+}
+
+uint32_t GET_RANGE_COUNT = 100000;
+const char *GET_RANGE_KPI = "C get range throughput (local client)";
+struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
+
+	double start = getTime();
+
+	const FDBKeyValue *outKv;
+	int outCount;
+	fdb_bool_t outMore = 1;
+	int totalOut = 0;
+	int iteration = 0;
+
+	FDBFuture *f = fdb_transaction_get_range(tr,
+		keys[startKey], keySize, 1, 0,
+		keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
+		0, 0,
+		FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
+
+	while(outMore) {
+		e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		totalOut += outCount;
+
+		if(outMore) {
+			FDBFuture *f2 = fdb_transaction_get_range(tr,
+				outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1,
+				keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
+				0, 0,
+				FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
+			fdb_future_destroy(f);
+			f = f2;
+		}
+	}
+
+	if(totalOut != GET_RANGE_COUNT) {
+		char *msg = (char*)malloc((sizeof(char)) * 200);
+		sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
+		logError(4100, msg, rs);
+		free(msg);
+		fdb_future_destroy(f);
+		return RES(0, 4100);
+	}
+	if(outMore) {
+		logError(4100, "verifying no more in range", rs);
+		fdb_future_destroy(f);
+		return RES(0, 4100);
+	}
+	fdb_future_destroy(f);
+
+	double end = getTime();
+
+	return RES(GET_RANGE_COUNT/(end - start), 0);
+}
+
+uint32_t GET_KEY_COUNT = 2000;
+const char *GET_KEY_KPI = "C get key throughput (local client)";
+struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	double start = getTime();
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	int i;
+	for(i = 0; i < GET_KEY_COUNT; i++) {
+		int key = ((uint64_t)rand()) % numKeys;
+		int offset = (((uint64_t)rand()) % 21) - 10;
+		FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
+
+		e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
+		fdb_future_destroy(f);
+		if(e) {
+			return RES(0, e);
+		}
+	}
+
+	double end = getTime();
+
+	return RES(GET_KEY_COUNT/(end - start), 0);
+}
+
+uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
+const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
+struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	double start = getTime();
+
+	const FDBKeyValue *outKv;
+	int outCount;
+	fdb_bool_t outMore;
+
+	int i;
+	for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
+		int key = ((uint64_t)rand()) % (numKeys - 1);
+		FDBFuture *f = fdb_transaction_get_range(tr,
+			keys[key], keySize, 1, 0,
+			keys[key + 1], keySize, 1, 0,
+			0, 0,
+			FDB_STREAMING_MODE_WANT_ALL, 1, 0, 0);
+
+		e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		if(outCount != 1) {
+			logError(4100, "non-1 number of keys returned in single key range read", rs);
+			fdb_future_destroy(f);
+			return RES(0, 4100);
+		}
+		if(outMore) {
+			logError(4100, "more keys to read in single key range read", rs);
+			fdb_future_destroy(f);
+			return RES(0, 4100);
+		}
+
+		fdb_future_destroy(f);
+	}
+
+	double end = getTime();
+
+	return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0);
+}
+
+struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) {
+	int k = ((uint64_t)rand()) % numKeys;
+	fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
+	return RES(0, 0);
+}
+
+uint32_t WRITE_TRANSACTION_COUNT = 1000;
+const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
+struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
+	double start = getTime();
+
+	int i;
+	for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
+		struct RunResult res = run(rs, db, &singleKey);
+		if(res.e) return res;
+	}
+
+	double end = getTime();
+
+	return RES(WRITE_TRANSACTION_COUNT/(end - start), 0);
+}
+
+void runTests(struct ResultSet *rs) {
+	FDBDatabase *db = openDatabase(rs, &netThread);
+
+	printf("Loading database...\n");
+	insertData(rs, db);
+
+	printf("future_latency\n");
+	runTest(&futureLatency, db, rs, FUTURE_LATENCY_KPI);
+
+	printf("clear\n");
+	runTest(&clear, db, rs, CLEAR_KPI);
+
+	printf("clear_range\n");
+	runTest(&clearRange, db, rs, CLEAR_RANGE_KPI);
+
+	printf("set\n");
+	runTest(&set, db, rs, SET_KPI);
+
+	printf("parallel_get\n");
+	runTest(&parallelGet, db, rs, PARALLEL_GET_KPI);
+
+	printf("alternating_get_set\n");
+	runTest(&alternatingGetSet, db, rs, ALTERNATING_GET_SET_KPI);
+
+	printf("serial_get\n");
+	runTest(&serialGet, db, rs, SERIAL_GET_KPI);
+
+	printf("get_range\n");
+	runTest(&getRange, db, rs, GET_RANGE_KPI);
+
+	printf("get_key\n");
+	runTest(&getKey, db, rs, GET_KEY_KPI);
+
+	printf("get_single_key_range\n");
+	runTest(&getSingleKeyRange, db, rs, GET_SINGLE_KEY_RANGE_KPI);
+
+	printf("write_transaction\n");
+	runTestDb(&writeTransaction, db, rs, WRITE_TRANSACTION_KPI);
+
+	fdb_database_destroy(db);
+	fdb_stop_network();
+}
+
+int main(int argc, char **argv) {
+	srand(time(NULL));
+	struct ResultSet *rs = newResultSet();
+	checkError(fdb_select_api_version(500), "select API version", rs);
+	printf("Running performance test at client version: %s\n", fdb_get_client_version());
+
+	valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize);
+	int i;
+	for(i = 0; i < valueSize; i++) {
+		valueStr[i] = (uint8_t)'x';
+	}
+
+	keys = generateKeys(numKeys, keySize);
+	runTests(rs);
+	writeResultSet(rs);
+
+	free(valueStr);
+	freeResultSet(rs);
+	freeKeys(keys, numKeys);
+
+	return 0;
+}
diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c
new file mode 100644
index 0000000000..1a780bf1b3
--- /dev/null
+++ b/bindings/c/test/ryw_benchmark.c
@@ -0,0 +1,257 @@
+/*
+ * ryw_benchmark.c
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test.h"
+#include <foundationdb/fdb_c.h>
+#include <foundationdb/fdb_c_options.g.h>
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+
+pthread_t netThread;
+
+int numKeys = 10000;
+int keySize = 16;
+uint8_t** keys;
+
+void insertData(FDBTransaction *tr) {
+	fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
+
+	uint8_t *v = (uint8_t*)"foo";
+	uint32_t i;
+	for(i = 0; i <= numKeys; ++i) {
+		fdb_transaction_set(tr, keys[i], keySize, v, 3);
+	}
+}
+
+int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) {
+	int numRuns = 25;
+	int *results = malloc(sizeof(int)*numRuns);
+	int i = 0;
+	for(; i < numRuns; ++i) {
+		results[i] = testFxn(tr, rs);
+		if(results[i] < 0) {
+			free(results);
+			return -1;
+		}
+	}
+
+	int result = median(results, numRuns);
+	free(results);
+
+	addKpi(rs, kpiName, result, "keys/s");
+
+	return result;
+}
+
+int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
+	int present;
+	uint8_t const *value;
+	int length;
+	int i;
+
+	double start = getTime();
+	for(i = 0; i < numKeys; ++i) {
+		FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0);
+		if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1;
+		if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1;
+		fdb_future_destroy(f);
+	}
+	double end = getTime();
+
+	return numKeys / (end - start);
+}
+
+int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
+	int present;
+	uint8_t const *value;
+	int length;
+	int i;
+
+	double start = getTime();
+	for(i = 0; i < numKeys; ++i) {
+		FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0);
+		if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1;
+		if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1;
+		fdb_future_destroy(f);
+	}
+	double end = getTime();
+
+	return numKeys / (end - start);
+}
+
+int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
+	int count;
+	const FDBKeyValue *kvs;
+	int more;
+	int i;
+
+	double start = getTime();
+	for(i = 0; i < 100; ++i) {
+		FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
+
+		if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1;
+		if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1;
+
+		if(count != numKeys) {
+			fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
+			addError(rs, "GetRangeBasic bad count");
+			return -1;
+		}
+	}
+	double end = getTime();
+
+	return 100 * numKeys / (end - start);
+}
+
+int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
+	int count;
+	const FDBKeyValue *kvs;
+	int more;
+	int i;
+
+	for(i = 0; i < numKeys; i+=2) {
+		fdb_transaction_clear(tr, keys[i], keySize);
+	}
+
+	double start = getTime();
+	for(i = 0; i < 100; ++i) {
+		FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
+
+		if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1;
+		if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1;
+
+		fdb_future_destroy(f);
+
+		if(count != numKeys/2) {
+			fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
+			addError(rs, "SingleClearGetRange bad count");
+			return -1;
+		}
+	}
+	double end = getTime();
+
+	insertData(tr);
+	return 100 * numKeys / 2 / (end - start);
+}
+
+int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
+	int count;
+	const FDBKeyValue *kvs;
+	int more;
+	int i;
+
+	for(i = 0; i < numKeys; i+=4) {
+		fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize);
+	}
+
+	double start = getTime();
+	for(i = 0; i < 100; ++i) {
+		FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
+
+		if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1;
+		if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1;
+
+		fdb_future_destroy(f);
+
+		if(count != numKeys*3/4) {
+			fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4);
+			addError(rs, "ClearRangeGetRange bad count");
+			return -1;
+		}
+	}
+	double end = getTime();
+
+	insertData(tr);
+	return 100 * numKeys * 3 / 4 / (end - start);
+}
+
+int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
+	int present;
+	uint8_t const *value;
+	int length;
+	int i;
+
+	uint8_t *k = (uint8_t*)"foo";
+	uint8_t v[10];
+	int num = 1;
+
+	double start = getTime();
+	sprintf((char*)v, "%d", num);
+	fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
+
+	for(i = 0; i < 10000; ++i) {
+		FDBFuture *f = fdb_transaction_get(tr, k, 3, 0);
+		if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1;
+		if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1;
+		fdb_future_destroy(f);
+
+		sprintf((char*)v, "%d", ++num);
+		fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
+	}
+	double end = getTime();
+
+	return 10000 / (end - start);
+}
+
+void runTests(struct ResultSet *rs) {
+	FDBDatabase *db = openDatabase(rs, &netThread);
+
+	FDBTransaction *tr;
+	checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
+
+	FDBFuture *f = fdb_transaction_get_read_version(tr);
+	checkError(fdb_future_block_until_ready(f), "block for read version", rs);
+
+	int64_t version;
+	checkError(fdb_future_get_version(f, &version), "get version", rs);
+	fdb_future_destroy(f);
+
+	insertData(tr);
+
+	runTest(&getSingle, tr, rs, "C: get single cached value throughput");
+	runTest(&getManySequential, tr, rs, "C: get sequential cached values throughput");
+	runTest(&getRangeBasic, tr, rs, "C: get range cached values throughput");
+	runTest(&singleClearGetRange, tr, rs, "C: get range cached values with clears throughput");
+	runTest(&clearRangeGetRange, tr, rs, "C: get range cached values with clear ranges throughput");
+	runTest(&interleavedSetsGets, tr, rs, "C: interleaved sets and gets on a single key throughput");
+
+	fdb_database_destroy(db);
+	fdb_stop_network();
+}
+
+int main(int argc, char **argv) {
+	srand(time(NULL));
+	struct ResultSet *rs = newResultSet();
+	checkError(fdb_select_api_version(500), "select API version", rs);
+	printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
+
+	keys = generateKeys(numKeys, keySize);
+	runTests(rs);
+	writeResultSet(rs);
+	freeResultSet(rs);
+	freeKeys(keys, numKeys);
+
+	return 0;
+}
+
diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h
new file mode 100644
index 0000000000..9cb1d9f2ae
--- /dev/null
+++ b/bindings/c/test/test.h
@@ -0,0 +1,257 @@
+/*
+ * test.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sys/time.h>
+#include <arpa/inet.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+#ifndef FDB_API_VERSION
+#define FDB_API_VERSION 500
+#endif
+
+#include <foundationdb/fdb_c.h>
+#include <foundationdb/fdb_c_options.g.h>
+
+double getTime() {
+	static struct timeval tv;
+	gettimeofday(&tv, NULL);
+	return tv.tv_usec/1000000.0 + tv.tv_sec;
+}
+
+void writeKey(uint8_t **dest, int key, int keySize) {
+	*dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize);
+	sprintf((char*)*dest, "%0*d", keySize, key);
+}
+
+uint8_t **generateKeys(int numKeys, int keySize) {
+	uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1));
+
+	uint32_t i;
+	for(i = 0; i <= numKeys; ++i) {
+		writeKey(keys + i, i, keySize);
+	}
+
+	return keys;
+}
+void freeKeys(uint8_t **keys, int numKeys) {
+	uint32_t i;
+	for(i = 0; i < numKeys; i++) {
+		free(keys[i]);
+	}
+	free(keys);
+}
+
+int cmpfunc(const void* a, const void* b) {
+	return (*(int*)a - *(int*)b);
+}
+
+int median(int *values, int length) {
+	qsort(values, length, sizeof(int), cmpfunc);
+	return values[length/2];
+}
+
+struct RunResult {
+	int res;
+	fdb_error_t e;
+};
+#define RES(x, y) (struct RunResult) { x, y }
+
+struct Kpi {
+	const char *name;
+	int value;
+	const char *units;
+
+	struct Kpi *next;
+};
+
+struct Error {
+	char *message;
+
+	struct Error *next;
+};
+
+struct ResultSet {
+	struct Kpi *kpis;
+	struct Error *errors;
+};
+
+struct ResultSet* newResultSet() {
+	struct ResultSet *rs = malloc(sizeof(struct ResultSet));
+
+	rs->kpis = NULL;
+	rs->errors = NULL;
+
+	return rs;
+}
+
+void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) {
+	struct Kpi *k = malloc(sizeof(struct Kpi));
+	k->name = name;
+	k->value = value;
+	k->units = units;
+	k->next = rs->kpis;
+	rs->kpis = k;
+}
+
+void addError(struct ResultSet *rs, const char *message) {
+	struct Error *e = malloc(sizeof(struct Error));
+	e->message = (char*)malloc(strlen(message)+1);
+	strcpy(e->message, message);
+	e->next = rs->errors;
+	rs->errors = e;
+}
+
+void writeResultSet(struct ResultSet *rs) {
+	uint64_t id = ((uint64_t)rand() << 32) + rand();
+	char name[100];
+	sprintf(name, "fdb-c_result-%llu.json", id);
+	FILE *fp = fopen(name, "w");
+	if(!fp) {
+		fprintf(stderr, "Could not open results file %s\n", name);
+		exit(1);
+	}
+
+	fprintf(fp, "{\n");
+	fprintf(fp, "\t\"kpis\": {\n");
+
+	struct Kpi *k = rs->kpis;
+	while(k != NULL) {
+		fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value);
+		if(k->next != NULL) {
+			fprintf(fp, ",");
+		}
+		fprintf(fp, "\n");
+		k = k->next;
+	}
+
+	fprintf(fp, "\t},\n");
+	fprintf(fp, "\t\"errors\": [\n");
+
+	struct Error *e = rs->errors;
+	while(e != NULL) {
+		fprintf(fp, "\t\t\"%s\"", e->message);
+		if(e->next != NULL) {
+			fprintf(fp, ",");
+		}
+		fprintf(fp, "\n");
+		e = e->next;
+	}
+
+	fprintf(fp, "\t]\n");
+	fprintf(fp, "}\n");
+
+	fclose(fp);
+}
+
+void freeResultSet(struct ResultSet *rs) {
+	struct Kpi *k = rs->kpis;
+	while(k != NULL) {
+		struct Kpi *next = k->next;
+		free(k);
+		k = next;
+	}
+
+	struct Error *e = rs->errors;
+	while(e != NULL) {
+		struct Error *next = e->next;
+		free(e->message);
+		free(e);
+		e = next;
+	}
+
+	free(rs);
+}
+
+fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	if(err) {
+		char *msg = (char*)malloc(strlen(context) + 100);
+		sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
+		fprintf(stderr, "%s\n", msg);
+		if(rs != NULL) {
+			addError(rs, msg);
+		}
+
+		free(msg);
+	}
+
+	return err;
+}
+
+void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	if(getError(err, context, rs)) {
+		if(rs != NULL) {
+			writeResultSet(rs);
+			freeResultSet(rs);
+		}
+		exit(1);
+	}
+}
+
+fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	char *msg = (char*)malloc(strlen(context) + 100);
+	sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
+	fprintf(stderr, "%s\n", msg);
+	if(rs != NULL) {
+		addError(rs, msg);
+	}
+
+	free(msg);
+	return err;
+}
+
+fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) {
+		return logError(err, context, rs);
+	}
+	return err;
+}
+
+void* runNetwork() {
+	checkError(fdb_run_network(), "run network", NULL);
+	return NULL;
+}
+
+FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
+	checkError(fdb_setup_network(), "setup network", rs);
+	pthread_create(netThread, NULL, &runNetwork, NULL);
+
+	FDBFuture *f = fdb_create_cluster(NULL);
+	checkError(fdb_future_block_until_ready(f), "block for cluster", rs);
+
+	FDBCluster *cluster;
+	checkError(fdb_future_get_cluster(f, &cluster), "get cluster", rs);
+
+	fdb_future_destroy(f);
+
+	f = fdb_cluster_create_database(cluster, (uint8_t*)"DB", 2);
+	checkError(fdb_future_block_until_ready(f), "block for database", rs);
+
+	FDBDatabase *db;
+	checkError(fdb_future_get_database(f, &db), "get database", rs);
+
+	fdb_future_destroy(f);
+	fdb_cluster_destroy(cluster);
+
+	return db;
+}
diff --git a/bindings/flow/local.mk b/bindings/flow/local.mk
index 95814f970e..1f2260e4c1 100644
--- a/bindings/flow/local.mk
+++ b/bindings/flow/local.mk
@@ -22,5 +22,23 @@
 
 fdb_flow_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS)
 fdb_flow_LDFLAGS := -Llib -lfdb_c $(fdbrpc_LDFLAGS)
-fdb_flow_LIBS := lib/libfdbrpc.a
+fdb_flow_LIBS :=
 
+packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz: fdb_flow
+	@echo "Packaging      fdb_flow"
+	@rm -rf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+	@mkdir -p packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/flow packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/c/foundationdb
+	@cp lib/libfdb_flow.a packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib
+	@find bindings/flow -name '*.h' -not -name 'bindings/flow/tester/*' -exec cp {} packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/flow \;
+	@find bindings/c/foundationdb -name '*.h' -exec cp {} packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/c/foundationdb \;
+	@tar czf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz -C packages fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+	@rm -rf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+
+FDB_FLOW: packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz
+
+FDB_FLOW_clean:
+	@echo "Cleaning       fdb_flow package"
+	@rm -rf packages/fdb-flow-*.tar.gz
+
+packages: FDB_FLOW
+packages_clean: FDB_FLOW_clean
diff --git a/bindings/flow/tester/local.mk b/bindings/flow/tester/local.mk
index 570ad9c0bf..695e98bea9 100644
--- a/bindings/flow/tester/local.mk
+++ b/bindings/flow/tester/local.mk
@@ -22,7 +22,7 @@
 
 fdb_flow_tester_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS)
 fdb_flow_tester_LDFLAGS := -Llib $(fdbrpc_LDFLAGS) -lfdb_c
-fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libfdbrpc.a lib/libflow.a lib/libfdb_c.$(DLEXT)
+fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libflow.a lib/libfdb_c.$(DLEXT)
 
 fdb_flow_tester: lib/libfdb_c.$(DLEXT)
 	@mkdir -p bindings/flow/bin
diff --git a/bindings/go/Gopkg.lock b/bindings/go/Gopkg.lock
new file mode 100644
index 0000000000..bef2d0092e
--- /dev/null
+++ b/bindings/go/Gopkg.lock
@@ -0,0 +1,9 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7"
+  solver-name = "gps-cdcl"
+  solver-version = 1
diff --git a/bindings/go/Gopkg.toml b/bindings/go/Gopkg.toml
new file mode 100644
index 0000000000..525774ebad
--- /dev/null
+++ b/bindings/go/Gopkg.toml
@@ -0,0 +1,2 @@
+# The FoundationDB go bindings currently have no external golang dependencies outside of
+# the go standard library.
diff --git a/bindings/go/README.md b/bindings/go/README.md
index 89f93976b2..13e8f07940 100644
--- a/bindings/go/README.md
+++ b/bindings/go/README.md
@@ -10,10 +10,20 @@ This package requires:
 
 Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-500.
 
-To install this package, in the top level of this repository run:
+To build this package, in the top level of this repository run:
 
     make fdb_go
 
+This will create binary packages for the appropriate platform within the "build" subdirectory of this folder.
+
+To install this package, you can run the "fdb-go-install.sh" script:
+
+    ./fdb-go-install.sh install 
+
+The "install" command of this script does not depend on the presence of the repo in general and will download the repository into
+your local go path. Running "localinstall" instead of "install" will use the local copy here (with a symlink) instead
+of downloading from the remote repository.
+
 Documentation
 -------------
 
diff --git a/bindings/go/fdb-go-install.sh b/bindings/go/fdb-go-install.sh
new file mode 100755
index 0000000000..c8122b94cf
--- /dev/null
+++ b/bindings/go/fdb-go-install.sh
@@ -0,0 +1,304 @@
+#!/bin/bash -eu
+#
+# fdb-go-install.sh
+#
+# Installs the FoundationDB Go bindings for a client. This will download
+# the repository from the remote repo either into the go directory
+# with the appropriate semantic version. It will then build a few
+# generated files that need to be present for the go build to work.
+# At the end, it has some advice for flags to modify within your
+# go environment so that other packages may successfully use this
+# library.
+#
+
+DESTDIR="${DESTDIR:-}"
+FDBVER="${FDBVER:-5.0.1}"
+REMOTE="${REMOTE:-github.com}"
+FDBREPO="${FDBREPO:-apple/foundationdb}"
+
+status=0
+
+platform=$(uname)
+if [[ "${platform}" == "Darwin" ]] ; then
+    FDBLIBDIR="${FDBLIBDIR:-/usr/local/lib}"
+    libfdbc="libfdb_c.dylib"
+elif [[ "${platform}" == "Linux" ]] ; then
+    FDBLIBDIR="${FDBLIBDIR:-/usr/lib}"
+    libfdbc="libfdb_c.so"
+else
+    echo "Unsupported platform ${platform}".
+    echo "At the moment, only macOS and Linux are supported by this script."
+    let status="${status} + 1"
+fi
+
+filedir=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
+destdir=""
+
+function printUsage() {
+    echo "Usage: fdb-go-install.sh <cmd>"
+    echo
+    echo "cmd: One of the commands to run. The options are:"
+    echo "     install         Download the FDB go bindings and install them"
+    echo "     localinstall    Install a into the go path a local copy of the repo"
+    echo "     download        Download but do not prepare the FoundationDB bindings"
+    echo "     help            Print this help message and then quit"
+    echo
+    echo "Command Line Options:"
+    echo "     --fdbver <version>    FoundationDB semantic version (default is ${FDBVER})"
+    echo "     -d/--dest-dir <dest>  Local location for the repo (default is to place in go path)"
+    echo
+    echo "Environment Variable Options:"
+    echo "     REMOTE          Remote repository to download from (currently ${REMOTE})"
+    echo "     FDBREPO         Repository of FoundationDB library to download (currently ${FDBREPO})"
+    echo "     FDBLIBDIR       Directory within which should be the FoundationDB c library (currently ${FDBLIBDIR})"
+}
+
+function parseArgs() {
+    local status=0
+
+    if [[ "${#}" -lt 0 ]] ; then
+        printUsage
+        let status="${status} + 1"
+    else
+        operation="${1}"
+        shift
+        if [[ "${operation}" != "install" ]] && [[ "${operation}" != "localinstall" ]] && [[ "${operation}" != "download" ]] && [[ "${operation}" != "help" ]] ; then
+            echo "Unknown command: ${operation}"
+            printUsage
+            let status="${status} + 1"
+        fi
+    fi
+
+    while [[ "${#}" -gt 0 ]] && [[ "${status}" -eq 0 ]] ; do
+        local key="${1}"
+        case "${key}" in
+            --fdbver)
+            if [[ "${#}" -lt 2 ]] ; then
+                echo "No version specified with --fdbver flag"
+                printUsage
+                let status="${status} + 1"
+            else
+                FDBVER="${2}"
+            fi
+            shift
+            ;;
+
+            -d|--dest-dir)
+            if [[ "${#}" -lt 2 ]] ; then
+                echo "No destination specified with ${key} flag"
+                printUsage
+                let status="${status} + 1"
+            else
+                destdir="${2}"
+            fi
+            shift
+            ;;
+
+            *)
+            echo "Unrecognized argument ${key}"
+            printUsage
+            let status="${status} + 1"
+        esac
+        shift
+    done
+
+    return "${status}"
+}
+
+function checkBin() {
+    if [[ "${#}" -lt 1 ]] ; then
+        echo "Usage: checkBin <binary>"
+        return 1
+    else
+        if [[ -n $(which "${1}") ]] ; then
+            return 0
+        else
+            return 1
+        fi
+    fi
+}
+
+if [[ "${status}" -gt 0 ]] ; then
+    # We have already failed.
+    :
+elif [[ "${#}" -lt 1 ]] ; then
+    printUsage
+else
+    required_bins=( 'go' 'git' 'make' 'mono' )
+
+    missing_bins=()
+    for bin in "${required_bins[@]}" ; do
+        if ! checkBin "${bin}" ; then
+            missing_bins+=("${bin}")
+            let status="${status} + 1"
+        fi
+    done
+
+    if [[ "${status}" -gt 0 ]] ; then
+        echo "Missing binaries: ${missing_bins[*]}"
+    elif ! parseArgs ${@} ; then
+        let status="${status} + 1"
+    elif [[ "${operation}" == "help" ]] ; then
+        printUsage
+    else
+        # Add go-specific environment variables.
+        eval $(go env)
+
+        golibdir=$(dirname "${GOPATH}/src/${REMOTE}/${FDBREPO}")
+        if [[ -z "${destdir}" ]] ; then
+            if [[ "${operation}" == "localinstall" ]] ; then
+                # Assume its the local directory.
+                destdir=$(cd "${filedir}/../../.." && pwd)
+            else
+                destdir="${golibdir}"
+            fi
+        fi
+
+        if [[ ! -d "${destdir}" ]] ; then
+            cmd=( 'mkdir' '-p' "${destdir}" )
+            echo "${cmd[*]}"
+            if ! "${cmd[@]}" ; then
+                let status="${status} + 1"
+                echo "Could not create destination directory ${destdir}."
+            fi
+        fi
+
+        # Step 1: Make sure repository is present.
+
+        if [[ "${status}" -eq 0 ]] ; then
+            destdir=$( cd "${destdir}" && pwd ) # Get absolute path of destination dir.
+            fdbdir="${destdir}/foundation"
+
+            if [[ ! -d "${destdir}" ]] ; then
+                cmd=("mkdir" "-p" "${destdir}")
+                echo "${cmd[*]}"
+                if ! "${cmd[@]}" ; then
+                    echo "Could not create destination directory ${destdir}."
+                    let status="${status} + 1"
+                fi
+            fi
+        fi
+
+        if [[ "${operation}" == "localinstall" ]] ; then
+            # No download occurs in this case.
+            :
+        else
+            if [[ -d "${fdbdir}" ]] ; then
+                echo "Directory ${fdbdir} already exists ; checking out appropriate tag"
+                cmd1=( 'git' '-C' "${fdbdir}" 'fetch' 'origin' )
+                cmd2=( 'git' '-C' "${fdbdir}" 'checkout' "release-${FDBVER}" )
+
+                if ! echo "${cmd1[*]}" || ! "${cmd1[@]}" ; then
+                    let status="${status} + 1"
+                    echo "Could not pull latest changes from origin"
+                elif ! echo "${cmd2[*]}" ||  ! "${cmd2[@]}" ; then
+                    let status="${status} + 1"
+                    echo "Could not checkout tag release-${FDBVER}."
+                fi
+            else
+                echo "Downloading foundation repository into ${destdir}:"
+                cmd=( 'git' '-C' "${destdir}" 'clone' '--branch' "release-${FDBVER}" "git@${REMOTE}:${FDBREPO}.git" )
+
+                echo "${cmd[*]}"
+                if ! "${cmd[@]}" ; then
+                    let status="${status} + 1"
+                    echo "Could not download repository."
+                fi
+            fi
+        fi
+
+        # Step 2: Build generated things.
+
+        if [[ "${operation}" == "download" ]] ; then
+            # The generated files are not created under a strict download.
+            :
+        elif [[ "${status}" -eq 0 ]] ; then
+            echo "Building generated files."
+            cmd=( 'make' '-C' "${fdbdir}" 'bindings/c/foundationdb/fdb_c_options.g.h' )
+
+            echo "${cmd[*]}"
+            if ! "${cmd[@]}" ; then
+                let status="${status} + 1"
+                echo "Could not generate required c header"
+            else
+                infile="${fdbdir}/fdbclient/vexillographer/fdb.options"
+                outfile="${fdbdir}/bindings/go/src/fdb/generated.go"
+                cmd=( 'go' 'run' "${fdbdir}/bindings/go/src/_util/translate_fdb_options.go" )
+                echo "${cmd[*]} < ${infile} > ${outfile}"
+                if ! "${cmd[@]}" < "${infile}" > "${outfile}" ; then
+                    let status="${status} + 1"
+                    echo "Could not generate generated go file."
+                fi
+            fi
+        fi
+
+        # Step 3: Add to go path.
+
+        if [[ "${operation}" == "download" ]] ; then
+            # The files are not moved under a strict download.
+            :
+        elif [[ "${status}" -eq 0 ]] ; then
+            linkpath="${GOPATH}/src/${REMOTE}/${FDBREPO}"
+            if [[ "${linkpath}" == "${fdbdir}" ]] ; then
+                # Downloaded directly into go path. Skip making the link.
+                :
+            elif [[ -e "${linkpath}" ]] ; then
+                echo "Warning: link path (${linkpath}) already exists. Leaving in place."
+            else
+                dirpath=$(dirname "${linkpath}")
+                if [[ ! -d "${dirpath}" ]] ; then
+                    cmd=( 'mkdir' '-p' "${dirpath}" )
+                    echo "${cmd[*]}"
+                    if ! "${cmd[@]}" ; then
+                        let status="${status} + 1"
+                        echo "Could not create directory for link."
+                    fi
+                fi
+
+                if [[ "${status}" -eq 0 ]] ; then
+                    cmd=( 'ln' '-s' "${fdbdir}" "${linkpath}" )
+                    echo "${cmd[*]}"
+                    if ! "${cmd[@]}" ; then
+                        let status="${status} + 1"
+                        echo "Could not create link within go path."
+                    fi
+                fi
+            fi
+        fi
+
+        # Step 4: Build the binaries.
+
+        if [[ "${operation}" == "download" ]] ; then
+            # Do not install if only downloading
+            :
+        elif [[ "${status}" -eq 0 ]] ; then
+            cgo_cflags="-g -O2 -I${linkpath}/bindings/c"
+            cgo_ldflags="-g -O2 -L${FDBLIBDIR}"
+            fdb_go_path="${REMOTE}/${FDBREPO}/bindings/go/src"
+
+            if [[ ! -e "${FDBLIBDIR}/${libfdbc}" ]] ; then
+                # Just a warning. Don't fail script.
+                echo
+                echo "WARNING: The FoundationDB C library was not found within ${FDBLIBDIR}."
+                echo "Your installation may be incomplete."
+                echo
+            elif ! CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" go install "${fdb_go_path}/fdb" "${fdb_go_path}/fdb/tuple" "${fdb_go_path}/fdb/subspace" "${fdb_go_path}/fdb/directory" ; then
+                let status="${status} + 1"
+                echo "Could not build FoundationDB go libraries."
+            fi
+        fi
+
+        # Step 5: Explain CGO flags.
+
+        if [[ "${status}" -eq 0 && ("${operation}" == "localinstall" || "${operation}" == "install" ) ]] ; then
+            echo
+            echo "The FoundationDB go bindings were successfully installed."
+            echo "To build packages which use the go bindings, you will need to"
+            echo "set the following environment variables:"
+            echo "   CGO_CFLAGS=\"${cgo_cflags}\""
+            echo "   CGO_LDFLAGS=\"${cgo_ldflags}\""
+        fi
+    fi
+fi
+
+exit "${status}"
diff --git a/bindings/go/include.mk b/bindings/go/include.mk
index 4a5ea38433..bfaa2a0334 100644
--- a/bindings/go/include.mk
+++ b/bindings/go/include.mk
@@ -21,7 +21,11 @@
 TARGETS += fdb_go fdb_go_tester
 CLEAN_TARGETS += fdb_go_clean fdb_go_tester_clean
 
-GOPATH := $(CURDIR)/bindings/go
+GOPATH := $(CURDIR)/bindings/go/build
+GO_IMPORT_PATH := github.com/apple/foundationdb/bindings/go/src
+GO_DEST := $(GOPATH)/src/$(GO_IMPORT_PATH)
+
+.PHONY: fdb_go fdb_go_path fdb_go_tester fdb_go_tester_clean godoc godoc_clean
 
 # We only override if the environment didn't set it (this is used by
 # the fdbwebsite documentation build process)
@@ -38,18 +42,23 @@ else
   $(error Not prepared to compile on platform $(PLATFORM))
 endif
 
-GO_PACKAGE_OUTDIR := $(GOPATH)/pkg/$(GOPLATFORM)
+GO_PACKAGE_OUTDIR := $(GOPATH)/pkg/$(GOPLATFORM)/$(GO_IMPORT_PATH)
 
 GO_PACKAGES := fdb fdb/tuple fdb/subspace fdb/directory
 GO_PACKAGE_OBJECTS := $(addprefix $(GO_PACKAGE_OUTDIR)/,$(GO_PACKAGES:=.a))
 
-GO_SRC := $(shell find $(GOPATH)/src -name '*.go')
+GO_SRC := $(shell find $(CURDIR)/bindings/go/src -name '*.go')
 
 fdb_go: $(GO_PACKAGE_OBJECTS) $(GO_SRC)
 
+fdb_go_path: $(GO_SRC)
+	@echo "Creating       fdb_go_path"
+	@mkdir -p $(GO_DEST)
+	@cp -r bindings/go/src/* $(GO_DEST)
+
 fdb_go_clean:
 	@echo "Cleaning       fdb_go"
-	@rm -rf $(GO_PACKAGE_OUTDIR)
+	@rm -rf $(GOPATH)
 
 fdb_go_tester: $(GOPATH)/bin/_stacktester
 
@@ -57,40 +66,40 @@ fdb_go_tester_clean:
 	@echo "Cleaning       fdb_go_tester"
 	@rm -rf $(GOPATH)/bin
 
-$(GOPATH)/bin/_stacktester: $(GO_SRC) $(GO_PACKAGE_OBJECTS) bindings/go/src/fdb/generated.go
+$(GOPATH)/bin/_stacktester: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OBJECTS) $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      $(basename $(notdir $@))"
-	@go install _stacktester
+	@go install $(GO_IMPORT_PATH)/_stacktester
 
-$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb/tuple"
-	@go install fdb/tuple
+	@go install $(GO_IMPORT_PATH)/fdb/tuple
 
-$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb/subspace"
-	@go install fdb/subspace
+	@go install $(GO_IMPORT_PATH)/fdb/subspace
 
-$(GO_PACKAGE_OUTDIR)/fdb/directory.a: $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb/directory.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb/directory"
-	@go install fdb/directory
+	@go install $(GO_IMPORT_PATH)/fdb/directory
 
-$(GO_PACKAGE_OUTDIR)/fdb.a: $(GO_SRC) bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb.a: fdb_go_path $(GO_SRC) $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb"
-	@go install fdb
+	@go install $(GO_IMPORT_PATH)/fdb
 
-bindings/go/src/fdb/generated.go: lib/libfdb_c.$(DLEXT) bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options
+$(GO_DEST)/fdb/generated.go: fdb_go_path lib/libfdb_c.$(DLEXT) bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options
 	@echo "Building       $@"
 	@go run bindings/go/src/_util/translate_fdb_options.go < fdbclient/vexillographer/fdb.options > $@
 
-godoc: $(GO_SRC)
+godoc: fdb_go_path $(GO_SRC)
 	@echo "Generating Go Documentation"
 	@rm -rf $(GODOC_DIR)/godoc
 	@mkdir -p $(GODOC_DIR)/godoc
 	@mkdir -p $(GODOC_DIR)/godoc/lib/godoc
-	@godoc -url "http://localhost:6060/pkg/fdb" > $(GODOC_DIR)/godoc/fdb.html
-	@godoc -url "http://localhost:6060/pkg/fdb/tuple" > $(GODOC_DIR)/godoc/fdb.tuple.html
-	@godoc -url "http://localhost:6060/pkg/fdb/subspace" > $(GODOC_DIR)/godoc/fdb.subspace.html
-	@godoc -url "http://localhost:6060/pkg/fdb/directory" > $(GODOC_DIR)/godoc/fdb.directory.html
-	@cp $(GOPATH)/godoc-resources/* $(GODOC_DIR)/godoc/lib/godoc
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb" > $(GODOC_DIR)/godoc/fdb.html
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb/tuple" > $(GODOC_DIR)/godoc/fdb.tuple.html
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb/subspace" > $(GODOC_DIR)/godoc/fdb.subspace.html
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb/directory" > $(GODOC_DIR)/godoc/fdb.directory.html
+	@cp $(CURDIR)/bindings/go/godoc-resources/* $(GODOC_DIR)/godoc/lib/godoc
 	@echo "Mangling paths in Go Documentation"
 	@(find $(GODOC_DIR)/godoc/ -name *.html -exec sed -i '' -e 's_/lib_lib_' {} \;)
 	@(sed -i -e 's_a href="tuple/"_a href="fdb.tuple.html"_' $(GODOC_DIR)/godoc/fdb.html)
diff --git a/bindings/go/src/_stacktester/directory.go b/bindings/go/src/_stacktester/directory.go
index ac4ff1217b..fa66205fde 100644
--- a/bindings/go/src/_stacktester/directory.go
+++ b/bindings/go/src/_stacktester/directory.go
@@ -21,10 +21,10 @@
 package main
 
 import (
-	"fdb"
-	"fdb/tuple"
-	"fdb/subspace"
-	"fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
 	"strings"
 	"bytes"
 )
@@ -94,14 +94,14 @@ func (sm *StackMachine) maybePath() []string {
 }
 
 var createOps = map[string]bool {
-    "CREATE_SUBSPACE": true,
-    "CREATE_LAYER": true,
-    "CREATE_OR_OPEN": true,
-    "CREATE": true,
-    "OPEN": true,
-    "MOVE": true,
-    "MOVE_TO": true,
-    "OPEN_SUBSPACE": true,
+	"CREATE_SUBSPACE": true,
+	"CREATE_LAYER": true,
+	"CREATE_OR_OPEN": true,
+	"CREATE": true,
+	"OPEN": true,
+	"MOVE": true,
+	"MOVE_TO": true,
+	"OPEN_SUBSPACE": true,
 }
 
 func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, idx int, t fdb.Transactor, rt fdb.ReadTransactor) {
diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go
index 81a1c9e18b..615e851f86 100644
--- a/bindings/go/src/_stacktester/stacktester.go
+++ b/bindings/go/src/_stacktester/stacktester.go
@@ -24,8 +24,8 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/hex"
-	"fdb"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"log"
 	"fmt"
 	"os"
diff --git a/bindings/go/src/fdb/directory/allocator.go b/bindings/go/src/fdb/directory/allocator.go
index 69400fff8d..7303a45fee 100644
--- a/bindings/go/src/fdb/directory/allocator.go
+++ b/bindings/go/src/fdb/directory/allocator.go
@@ -23,8 +23,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	"encoding/binary"
 	"bytes"
 	"math/rand"
diff --git a/bindings/go/src/fdb/directory/directory.go b/bindings/go/src/fdb/directory/directory.go
index 72e82230a1..7125dc45a5 100644
--- a/bindings/go/src/fdb/directory/directory.go
+++ b/bindings/go/src/fdb/directory/directory.go
@@ -40,8 +40,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	"errors"
 )
 
@@ -140,15 +140,15 @@ type Directory interface {
 }
 
 func stringsEqual(a, b []string) bool {
-    if len(a) != len(b) {
-        return false
-    }
-    for i, v := range a {
-        if v != b[i] {
-            return false
-        }
-    }
-    return true
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
 }
 
 func moveTo(t fdb.Transactor, dl directoryLayer, path, newAbsolutePath []string) (DirectorySubspace, error) {
diff --git a/bindings/go/src/fdb/directory/directoryLayer.go b/bindings/go/src/fdb/directory/directoryLayer.go
index a5bfd4f102..f27caac418 100644
--- a/bindings/go/src/fdb/directory/directoryLayer.go
+++ b/bindings/go/src/fdb/directory/directoryLayer.go
@@ -23,9 +23,9 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"encoding/binary"
 	"bytes"
 	"fmt"
diff --git a/bindings/go/src/fdb/directory/directoryPartition.go b/bindings/go/src/fdb/directory/directoryPartition.go
index 253d9482db..cdacc1746e 100644
--- a/bindings/go/src/fdb/directory/directoryPartition.go
+++ b/bindings/go/src/fdb/directory/directoryPartition.go
@@ -23,9 +23,9 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 type directoryPartition struct {
diff --git a/bindings/go/src/fdb/directory/directorySubspace.go b/bindings/go/src/fdb/directory/directorySubspace.go
index 70641f4422..36437ea994 100644
--- a/bindings/go/src/fdb/directory/directorySubspace.go
+++ b/bindings/go/src/fdb/directory/directorySubspace.go
@@ -23,8 +23,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 )
 
 // DirectorySubspace represents a Directory that may also be used as a Subspace
diff --git a/bindings/go/src/fdb/directory/node.go b/bindings/go/src/fdb/directory/node.go
index a608abafcc..881a965880 100644
--- a/bindings/go/src/fdb/directory/node.go
+++ b/bindings/go/src/fdb/directory/node.go
@@ -23,8 +23,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	"bytes"
 )
 
diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go
index aee3e85cfe..fc4cdc563b 100644
--- a/bindings/go/src/fdb/doc.go
+++ b/bindings/go/src/fdb/doc.go
@@ -39,7 +39,7 @@ A basic interaction with the FoundationDB API is demonstrated below:
     package main
 
     import (
-        "github.com/apple/foundationdb/bindings/go/fdb"
+        "github.com/apple/foundationdb/bindings/go/src/fdb"
         "log"
         "fmt"
     )
diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go
index 3e31256597..b8df4793d7 100644
--- a/bindings/go/src/fdb/errors.go
+++ b/bindings/go/src/fdb/errors.go
@@ -23,7 +23,7 @@
 package fdb
 
 /*
- #define FDB_API_VERSION 200
+ #define FDB_API_VERSION 500
  #include <foundationdb/fdb_c.h>
 */
 import "C"
diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go
index 4f07a5c6f9..709310a499 100644
--- a/bindings/go/src/fdb/fdb_test.go
+++ b/bindings/go/src/fdb/fdb_test.go
@@ -23,7 +23,7 @@
 package fdb_test
 
 import (
-	"fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
 	"fmt"
 	"testing"
 )
diff --git a/bindings/go/src/fdb/subspace/subspace.go b/bindings/go/src/fdb/subspace/subspace.go
index 3e48842a1f..a123ea1d31 100644
--- a/bindings/go/src/fdb/subspace/subspace.go
+++ b/bindings/go/src/fdb/subspace/subspace.go
@@ -33,8 +33,8 @@
 package subspace
 
 import (
-	"fdb"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"bytes"
 	"errors"
 )
diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go
index 1f1c5959da..444f2a8cff 100644
--- a/bindings/go/src/fdb/tuple/tuple.go
+++ b/bindings/go/src/fdb/tuple/tuple.go
@@ -38,7 +38,7 @@ import (
 	"fmt"
 	"encoding/binary"
 	"bytes"
-	"fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
 )
 
 // A TupleElement is one of the types that may be encoded in FoundationDB
diff --git a/bindings/java/local.mk b/bindings/java/local.mk
index 83636de36d..30cac01290 100644
--- a/bindings/java/local.mk
+++ b/bindings/java/local.mk
@@ -144,6 +144,11 @@ define add_java_binding_targets
 	  @rm -r packages/jar$(1)_regular
 	  @cd bindings && jar uf $$(TOPDIR)/$$@ ../LICENSE
 
+    packages/fdb-java$(1)-$$(JARVER)-tests.jar: fdb_java$(1) versions.target
+	  @echo "Building       $$@"
+	  @rm -f $$@
+	  @cp $$(TOPDIR)/bindings/java/foundationdb-tests$(1).jar packages/fdb-java$(1)-$$(JARVER)-tests.jar
+
     packages/fdb-java$(1)-$$(JARVER)-sources.jar: $$(JAVA$(1)_GENERATED_SOURCES) versions.target
 	  @echo "Building       $$@"
 	  @rm -f $$@
@@ -165,7 +170,7 @@ define add_java_binding_targets
 	  @cd packages/bundle$(1)_regular && jar cf $(TOPDIR)/$$@ *
 	  @rm -rf packages/bundle$(1)_regular
 
-    fdb_java$(1)_release: packages/fdb-java$(1)-$$(JARVER)-bundle.jar
+    fdb_java$(1)_release: packages/fdb-java$(1)-$$(JARVER)-bundle.jar packages/fdb-java$(1)-$$(JARVER)-tests.jar
 
     fdb_java$(1)_release_clean:
 	  @echo "Cleaning       Java release"
diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/Database.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/Database.java
index 1026d04148..6e8a12be4b 100644
--- a/bindings/java/src-completable/main/com/apple/cie/foundationdb/Database.java
+++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/Database.java
@@ -44,8 +44,8 @@ public interface Database extends Disposable, TransactionContext {
 	 * Creates a {@link Transaction} that operates on this {@code Database}.<br>
 	 * <br>
 	 * Note: Java transactions automatically set the {@link TransactionOptions#setUsedDuringCommitProtectionDisable}
-	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after either
-	 *  {@link Transaction#reset} or {@link Transaction#onError} is called.
+	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after
+	 *  {@link Transaction#onError} is called.
 	 *
 	 * @return a newly created {@code Transaction} that reads from and writes to this {@code Database}.
 	 */
diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/AsyncIterator.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/AsyncIterator.java
index 5c8faf0062..c1067805c4 100644
--- a/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/AsyncIterator.java
+++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/AsyncIterator.java
@@ -61,7 +61,7 @@ public interface AsyncIterator<T> extends Iterator<T>, Disposable {
 	/**
 	 * Returns the next element in the sequence. This will not block if, since the
 	 *  last call to {@code next()}, {@link #onHasNext()} was called and the resulting
-	 *  <h1>FIXME!!!!</h1> has completed or the blocking call {@link #hasNext()} was called
+	 *  {@link CompletableFuture} has completed or the blocking call {@link #hasNext()} was called
 	 *  and has returned. It is legal, therefore, to make a call to {@code next()} without a
 	 *  preceding call to
 	 *  {@link #hasNext()} or {@link #onHasNext()}, but that invocation of {@code next()}
diff --git a/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/package-info.java b/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/package-info.java
index 38619798b4..8c116b32c1 100644
--- a/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/package-info.java
+++ b/bindings/java/src-completable/main/com/apple/cie/foundationdb/async/package-info.java
@@ -19,7 +19,7 @@
  */
 
 /**
- * Provides additional constructs for asynchronous programming against Java's CompletableFutures.
+ * Provides additional constructs for asynchronous programming against Java's {@link java.util.concurrent.CompletableFuture CompletableFuture}s.
  *
  */
 package com.apple.cie.foundationdb.async;
diff --git a/bindings/java/src-completable/main/overview.html.in b/bindings/java/src-completable/main/overview.html.in
index 9ec77aef8a..131035a151 100644
--- a/bindings/java/src-completable/main/overview.html.in
+++ b/bindings/java/src-completable/main/overview.html.in
@@ -24,10 +24,11 @@ and add it to your classpath.<br>
 <br>
 <h3>Getting started</h3>
 To start using FoundationDB from Java, create an instance of the 
-{@link FDB FoundationDB API interface} with the version of the
+{@link com.apple.cie.foundationdb.FDB FoundationDB API interface} with the version of the
 API that you want to use (this release of the FoundationDB Java API supports only version {@code 500}).
-With this API object you can then open {@link Cluster}s and
-{@link Database}s and start using {@link Transaction}s.
+With this API object you can then open {@link com.apple.cie.foundationdb.Cluster Cluster}s and
+{@link com.apple.cie.foundationdb.Database Database}s and start using
+{@link com.apple.cie.foundationdb.Transaction Transactions}s.
 Here we give an example. The example relies on a cluster file at the
 <a href="/documentation/api-general.html#default-cluster-file">default location</a> 
 for your platform and a running server.<br>
@@ -77,7 +78,7 @@ for information about how Tuples sort and can be used to efficiently model data.
 The {@link com.apple.cie.foundationdb.directory Directory API} is provided with the core
 Java API for FoundationDB. This layer is provided in some form in all official
 language bindings. The FoundationDB API provides directories as a tool for
-managing related {@link Subspace}s. Directories are a
+managing related {@link com.apple.cie.foundationdb.subspace.Subspace Subspace}s. Directories are a
 recommended approach for administering applications. Each application should 
 create or open at least one directory to manage its subspaces. Directories are 
 identified by hierarchical paths analogous to the paths in a Unix-like file system. 
diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java
new file mode 100644
index 0000000000..2ef9abb340
--- /dev/null
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java
@@ -0,0 +1,117 @@
+/*
+ * AbstractTester.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.Database;
+import com.apple.cie.foundationdb.FDB;
+
+import java.nio.charset.Charset;
+import java.util.Random;
+
+public abstract class AbstractTester {
+    public static final int API_VERSION = 500;
+    protected static final int NUM_RUNS = 25;
+    protected static final Charset ASCII = Charset.forName("ASCII");
+
+    protected TesterArgs args;
+    protected Random random;
+    protected TestResult result;
+    protected FDB fdb;
+
+    public AbstractTester() {
+        args = null;
+        random = new Random();
+        result = new TestResult(random);
+    }
+
+    public void runTest() {
+        Database db;
+
+        try {
+            db = fdb.open();
+        } catch (Exception e) {
+            result.addError(wrapAndPrintError(e, "fdb.open failed"));
+            return;
+        }
+
+        try {
+            testPerformance(db);
+        } catch (Exception e) {
+            result.addError(wrapAndPrintError(e, "Failed to complete all tests"));
+        }
+    }
+
+    public abstract void testPerformance(Database db);
+
+    public String multiVersionDescription() {
+        if (args == null) return "";
+
+        if (!args.useMultiversionApi()) {
+            return "multi-version API disabled";
+        } else if (args.useExternalClient()) {
+            if (args.putCallbacksOnExternalThread()) {
+                return "external client on external thread";
+            } else {
+                return "external client on main thread";
+            }
+        } else {
+            return "local client";
+        }
+    }
+
+    public void run(String[] argStrings) {
+        args = TesterArgs.parseArgs(argStrings);
+        if (args == null) return;
+
+        fdb = FDB.selectAPIVersion(API_VERSION);
+
+        // Validate argument combinations and set options.
+        if (!args.useMultiversionApi()) {
+            if (args.putCallbacksOnExternalThread() || args.useExternalClient()) {
+                throw new IllegalArgumentException("Invalid multi-version API argument combination");
+            }
+            fdb.options().setDisableMultiVersionClientApi();
+        }
+        if (args.putCallbacksOnExternalThread()) {
+            if (!args.useExternalClient()) {
+                throw new IllegalArgumentException("Cannot enable callbacks on external thread without using external client");
+            }
+            fdb.options().setCallbacksOnExternalThreads();
+        }
+        if (args.useExternalClient()) {
+            fdb.options().setDisableLocalClient();
+        }
+
+        try {
+            runTest();
+        } catch (Exception e) {
+            result.addError(e);
+        }
+
+        result.save(args.getOutputDirectory());
+    }
+
+    public RuntimeException wrapAndPrintError(Throwable t, String message) {
+        String errorMessage = message + ": " + t.getClass() + ": " + t.getMessage() + "\n";
+        t.printStackTrace();
+        return new RuntimeException(errorMessage, t);
+    }
+}
diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncPerformanceTester.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncPerformanceTester.java
new file mode 100644
index 0000000000..d47e516429
--- /dev/null
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncPerformanceTester.java
@@ -0,0 +1,28 @@
+/*
+ * AsyncPerformanceTester.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+public class AsyncPerformanceTester {
+
+    public static void main(String[] args) {
+        System.out.println("Running Java async performance test on Java version " + System.getProperty("java.version"));
+    }
+}
diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/PerformanceTester.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/PerformanceTester.java
new file mode 100644
index 0000000000..562f6831c9
--- /dev/null
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/PerformanceTester.java
@@ -0,0 +1,408 @@
+/*
+ * PerformanceTester.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.Database;
+import com.apple.cie.foundationdb.KeySelector;
+import com.apple.cie.foundationdb.Transaction;
+import com.apple.cie.foundationdb.TransactionContext;
+import com.apple.cie.foundationdb.async.AsyncUtil;
+import com.apple.cie.foundationdb.tuple.ByteArrayUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+public class PerformanceTester extends AbstractTester {
+    private final int keyCount;
+    private final int keySize;
+    private final int valueSize;
+
+    private final String keyFormat;
+    private final byte[] valueBytes;
+
+    public static final int DEFAULT_KEY_COUNT = 10_000;
+    public static final int DEFAULT_KEY_SIZE = 16;
+    public static final int DEFAULT_VALUE_SIZE = 100;
+
+    private enum Tests {
+        FUTURE_LATENCY("Java Completable API future throughput"),
+        SET("Java Completable API set throughput"),
+        CLEAR("Java Completable API clear throughput"),
+        CLEAR_RANGE("Java Completable API clear_range throughput"),
+        PARALLEL_GET("Java Completable API parallel get throughput"),
+        SERIAL_GET("Java Completable API serial get throughput"),
+        GET_RANGE("Java Completable API get_range throughput"),
+        GET_KEY("Java Completable API get_key throughput"),
+        GET_SINGLE_KEY_RANGE("Java Completable API get_single_key_range throughput"),
+        ALTERNATING_GET_SET("Java Completable API alternating get and set throughput"),
+        WRITE_TRANSACTION("Java Completable API single-key transaction throughput");
+
+        private String kpi;
+        private Function<? super Database, ? extends Double> function;
+
+        Tests(String kpi) {
+            this.kpi = kpi;
+        }
+
+        public void setFunction(Function<?super Database, ? extends Double> function) {
+            this.function = function;
+        }
+
+        public Function<? super Database, ? extends Double> getFunction() {
+            return function;
+        }
+
+        public String getKpi() {
+            return kpi;
+        }
+    }
+
+    public PerformanceTester() {
+        this(DEFAULT_KEY_COUNT, DEFAULT_KEY_SIZE, DEFAULT_VALUE_SIZE);
+    }
+
+    public PerformanceTester(int keyCount, int keySize, int valueSize) {
+        super();
+        this.keyCount = keyCount;
+        this.keySize = keySize;
+        this.valueSize = valueSize;
+
+        keyFormat = "%0" + keySize + "d";
+
+        valueBytes = new byte[valueSize];
+        Arrays.fill(valueBytes, (byte)'x');
+
+        // Initialize tests.
+        Tests.FUTURE_LATENCY.setFunction(db -> futureLatency(db, 100_000));
+        Tests.SET.setFunction(db -> set(db, 100_000));
+        Tests.CLEAR.setFunction(db -> clear(db, 100_000));
+        Tests.CLEAR_RANGE.setFunction(db -> clearRange(db, 100_000));
+        Tests.PARALLEL_GET.setFunction(db -> parallelGet(db, 10_000));
+        Tests.SERIAL_GET.setFunction(db -> serialGet(db, 2_000));
+        Tests.GET_RANGE.setFunction(db -> getRange(db, 1_000));
+        Tests.GET_KEY.setFunction(db -> getKey(db, 2_000));
+        Tests.GET_SINGLE_KEY_RANGE.setFunction(db -> getSingleKeyRange(db, 2_000));
+        Tests.ALTERNATING_GET_SET.setFunction(db -> alternatingGetSet(db, 2_000));
+        Tests.WRITE_TRANSACTION.setFunction(db -> writeTransaction(db, 1_000));
+    }
+
+    @Override
+    public void testPerformance(Database db) {
+        insertData(db);
+
+        List<String> testsToRun;
+        if (args.getTestsToRun().isEmpty()) {
+            testsToRun = Arrays.stream(Tests.values()).map(Tests::name).map(String::toLowerCase).sorted().collect(Collectors.toList());
+        } else {
+            testsToRun = args.getTestsToRun();
+        }
+
+        for (String test : testsToRun) {
+            Tests testObj;
+            try {
+                testObj = Tests.valueOf(test.toUpperCase());
+            } catch (IllegalArgumentException e) {
+                result.addError(new IllegalArgumentException("Test " + test + " not implemented"));
+                continue;
+            }
+
+            Function<? super Database, ? extends Double> function = testObj.getFunction();
+
+            try {
+                Thread.sleep(5_000);
+            } catch (InterruptedException e) {
+                result.addError(wrapAndPrintError(e, "Interrupted while sleeping"));
+            }
+
+            System.out.println("Running test " + test);
+
+            List<Double> results = new ArrayList<>(NUM_RUNS);
+
+            for (int i = 0; i < NUM_RUNS; i++) {
+                try {
+                    results.add(function.apply(db));
+                } catch (Exception e) {
+                    result.addError(wrapAndPrintError(e, "Performance test failed: " + test));
+                    break;
+                }
+            }
+
+            if (results.size() == NUM_RUNS) {
+                Collections.sort(results);
+                result.addKpi(String.format("%s (%s)", testObj.getKpi(), multiVersionDescription()), results.get(results.size()/2).intValue(), "keys/s");
+            }
+        }
+    }
+
+    public void insertData(Database db) {
+        System.out.println("Loading database");
+
+        db.run(tr -> {
+            byte[] subspacePrefix = args.getSubspace().pack();
+            if (subspacePrefix.length == 0) {
+                // Clear user space.
+                tr.clear(new byte[0], new byte[]{(byte)0xff});
+            } else {
+                tr.clear(args.getSubspace().range());
+            }
+            return null;
+        });
+
+        int keysPerActor = 100_000 / (keySize + valueSize);
+        int numActors = (int)Math.ceil(keyCount*1.0/keysPerActor);
+
+        List<CompletableFuture<Void>> futures = IntStream.range(0, numActors).mapToObj(i -> {
+            int startKey = keysPerActor * i;
+            int endKey = (i + 1 == numActors) ? (keyCount) : (keysPerActor * (i+1));
+            return db.runAsync(tr -> {
+                IntStream.range(startKey, endKey).forEach(keyIndex -> tr.set(key(keyIndex), value(keyIndex)));
+                return CompletableFuture.completedFuture((Void)null);
+            });
+        }).collect(Collectors.toList());
+
+        try {
+            AsyncUtil.whenAll(futures).get();
+        } catch (InterruptedException | ExecutionException e) {
+            result.addError(wrapAndPrintError(e, "Data insertion failed"));
+        }
+
+        // Give the database time to re-balance
+        try {
+            Thread.sleep(15_000);
+        } catch (InterruptedException e) {
+            result.addError(wrapAndPrintError(e, "Interrupted while waiting for quiescence"));
+        }
+    }
+
+    public Double futureLatency(Database db, int count) {
+        return db.run(tr -> {
+            tr.options().setRetryLimit(5);
+            tr.getReadVersion().join();
+
+            long start = System.nanoTime();
+
+            for (int i = 0; i < count; i++) {
+                tr.getReadVersion().join();
+            }
+
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double clear(Database db, int count) {
+        Transaction tr = db.createTransaction();
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.clear(randomKey());
+        }
+        long end = System.nanoTime();
+        tr.cancel();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double clearRange(Database db, int count) {
+        Transaction tr = db.createTransaction();
+
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            int keyIndex = randomKeyIndex();
+            tr.clear(key(keyIndex), key(keyIndex+1));
+        }
+        long end = System.nanoTime();
+        tr.cancel();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double set(Database db, int count) {
+        Transaction tr = db.createTransaction();
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            int keyIndex = randomKeyIndex();
+            tr.set(key(keyIndex), value(keyIndex));
+        }
+        long end = System.nanoTime();
+        tr.cancel();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double parallelGet(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            long start = System.nanoTime();
+
+            List<CompletableFuture<byte[]>> futures = IntStream.range(0, count)
+                    .mapToObj(ignore -> tr.get(randomKey()))
+                    .collect(Collectors.toList());
+            AsyncUtil.whenAll(futures).join();
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double alternatingGetSet(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            long start = System.nanoTime();
+
+            List<CompletableFuture<byte[]>> futures = IntStream.range(0, count)
+                    .mapToObj(ignore -> {
+                        int keyIndex = randomKeyIndex();
+                        byte[] keyBytes = key(keyIndex);
+                        byte[] valBytes = value(keyIndex);
+
+                        tr.set(keyBytes, valBytes);
+                        return tr.get(keyBytes);
+                    }).collect(Collectors.toList());
+            AsyncUtil.whenAll(futures).join();
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double serialGet(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+
+            List<byte[]> keys;
+            if (count > keyCount/2) {
+                keys = Stream.generate(this::randomKey).limit(count).collect(Collectors.toList());
+            } else {
+                Set<Integer> keySet = new HashSet<>();
+                while (keySet.size() < count) {
+                    keySet.add(randomKeyIndex());
+                }
+                keys = keySet.stream().map(this::key).collect(Collectors.toList());
+            }
+
+            long start = System.nanoTime();
+            for (byte[] key : keys) {
+                tr.get(key).join();
+            }
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double getRange(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            int startIndex = random.nextInt(keyCount - count);
+
+            long start = System.nanoTime();
+            tr.getRange(key(startIndex), key(startIndex+count)).asList().join();
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double getKey(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+
+            long start = System.nanoTime();
+            for (int i = 0; i < count; i++) {
+                tr.getKey(new KeySelector(randomKey(), true, random.nextInt(20) - 10)).join();
+            }
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double getSingleKeyRange(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            long start = System.nanoTime();
+            for (int i = 0; i < count; i++) {
+                int keyIndex = randomKeyIndex();
+                tr.getRange(key(keyIndex), key(keyIndex + 1)).asList().join();
+            }
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double writeTransaction(TransactionContext tcx, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tcx.run(tr -> {
+                int keyIndex = randomKeyIndex();
+                tr.set(key(keyIndex), value(keyIndex));
+                return null;
+            });
+        }
+        long end = System.nanoTime();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public byte[] key(int i) {
+        return ByteArrayUtil.join(args.getSubspace().pack(), String.format(keyFormat, i).getBytes(ASCII));
+    }
+
+    public int randomKeyIndex() {
+        return random.nextInt(keyCount);
+    }
+
+    public byte[] randomKey() {
+        return key(randomKeyIndex());
+    }
+
+    public byte[] value(int key) {
+        return valueBytes;
+    }
+
+    public static void main(String[] args) {
+        System.out.println("Running Java performance test on Java version " + System.getProperty("java.version"));
+        try {
+            new PerformanceTester().run(args);
+        } catch (IllegalArgumentException e) {
+            System.out.println("Could not run test due to malformed arguments.");
+            System.out.println(e.getMessage());
+            System.exit(1);
+        } catch (Exception e) {
+            System.out.println("Fatal error encountered during run: " + e);
+            e.printStackTrace();
+            System.exit(2);
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/RYWBenchmark.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/RYWBenchmark.java
new file mode 100644
index 0000000000..5b090b6328
--- /dev/null
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/RYWBenchmark.java
@@ -0,0 +1,233 @@
+/*
+ * RYWBenchmark.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.Database;
+import com.apple.cie.foundationdb.Transaction;
+import com.apple.cie.foundationdb.tuple.ByteArrayUtil;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class RYWBenchmark extends AbstractTester {
+    private int keyCount;
+
+    public static final int DEFAULT_KEY_COUNT = 10_000;
+    public static final int DEFAULT_KEY_SIZE = 16;
+
+    private final String keyFormat;
+
+    private enum Tests {
+        GET_SINGLE("RYW Java Completable: get single cached value throughput"),
+        GET_MANY_SEQUENTIAL("RYW Java Completable: get sequential cached value throughput"),
+        GET_RANGE_BASIC("RYW Java Completable: get range cached values throughput"),
+        SINGLE_CLEAR_GET_RANGE("RYW Java Completable: get range cached values with clears throughput"),
+        CLEAR_RANGE_GET_RANGE("RYW Java Completable: get range cached values with clear ranges throughput"),
+        INTERLEAVED_SETS_GETS("RYW Java Completable: interleaved sets and gets on a single key throughput");
+
+        private String kpi;
+        private Function<? super Transaction, ? extends Double> function;
+
+        Tests(String kpi) {
+            this.kpi = kpi;
+        }
+
+        public void setFunction(Function<?super Transaction, ? extends Double> function) {
+            this.function = function;
+        }
+
+        public Function<? super Transaction, ? extends Double> getFunction() {
+            return function;
+        }
+
+        public String getKpi() {
+            return kpi;
+        }
+    }
+
+    public RYWBenchmark() {
+        this(DEFAULT_KEY_COUNT, DEFAULT_KEY_SIZE);
+    }
+
+    public RYWBenchmark(int keyCount, int keySize) {
+        super();
+        this.keyCount = keyCount;
+
+        keyFormat = "%0" + keySize + "d";
+
+        Tests.GET_SINGLE.setFunction(tr -> getSingle(tr, 10_000));
+        Tests.GET_MANY_SEQUENTIAL.setFunction(tr -> getManySequential(tr, 10_000));
+        Tests.GET_RANGE_BASIC.setFunction(tr -> getRangeBasic(tr, 1_000));
+        Tests.SINGLE_CLEAR_GET_RANGE.setFunction(tr -> singleClearGetRange(tr, 1_000));
+        Tests.CLEAR_RANGE_GET_RANGE.setFunction(tr -> clearRangeGetRange(tr, 1_000));
+        Tests.INTERLEAVED_SETS_GETS.setFunction(tr -> interleavedSetsGets(tr, 10_000));
+    }
+
+    @Override
+    public void testPerformance(Database db) {
+        Transaction tr = db.createTransaction();
+        insertData(tr);
+
+        List<String> testsToRun;
+        if (args.getTestsToRun().isEmpty()) {
+            testsToRun = Arrays.stream(Tests.values()).map(Tests::name).map(String::toLowerCase).sorted().collect(Collectors.toList());
+        } else {
+            testsToRun = args.getTestsToRun();
+        }
+
+        for (String test : testsToRun) {
+            Tests testObj;
+            try {
+                testObj = Tests.valueOf(test.toUpperCase());
+            } catch (IllegalArgumentException e) {
+                result.addError(new IllegalArgumentException("Test " + test + " not implemented"));
+                continue;
+            }
+
+            Function<? super Transaction, ? extends Double> function = testObj.getFunction();
+
+            try {
+                Thread.sleep(5_000);
+            } catch (InterruptedException e) {
+                result.addError(wrapAndPrintError(e, "Interrupted while sleeping"));
+            }
+
+            System.out.println("Running test " + test);
+
+            List<Double> results = new ArrayList<>(NUM_RUNS);
+
+            for (int i = 0; i < NUM_RUNS; i++) {
+                try {
+                    results.add(function.apply(tr));
+                } catch (Exception e) {
+                    result.addError(wrapAndPrintError(e, "Performance test failed: " + test));
+                    break;
+                }
+            }
+
+            if (results.size() == NUM_RUNS) {
+                Collections.sort(results);
+                result.addKpi(String.format("%s", testObj.getKpi()), results.get(results.size() / 2).intValue(), "keys/s");
+            }
+        }
+
+        tr.cancel();
+    }
+
+    public Double getSingle(Transaction tr, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.get(key(5001)).join();
+        }
+        long end = System.nanoTime();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double getManySequential(Transaction tr, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.get(key(i)).join();
+        }
+        long end = System.nanoTime();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double getRangeBasic(Transaction tr, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+           tr.getRange(key(0), key(keyCount)).asList().join();
+        }
+        long end = System.nanoTime();
+
+        return count * 1_000_000_000.0 * keyCount/(end - start);
+    }
+
+    public Double singleClearGetRange(Transaction tr, int count) {
+        for (int i = 0; i < keyCount; i += 2) {
+            tr.clear(("" + i).getBytes(ASCII));
+        }
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.getRange(key(0), key(keyCount)).asList().join();
+        }
+        long end = System.nanoTime();
+
+        Double kpi = count * 1_000_000_000.0 * keyCount / 2 / (end - start);
+        insertData(tr);
+        return kpi;
+    }
+
+    public Double clearRangeGetRange(Transaction tr, int count) {
+        for (int i = 0; i < keyCount; i += 4) {
+            tr.clear(key(i), key(i+1));
+        }
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.getRange(key(0), key(keyCount)).asList().join();
+        }
+        long end = System.nanoTime();
+
+        Double kpi = count * 1_000_000_000.0 * keyCount * 3 / 4 / (end - start);
+        insertData(tr);
+        return kpi;
+    }
+
+    public Double interleavedSetsGets(Transaction tr, int count) {
+        long start = System.nanoTime();
+        byte[] keyBytes = "foo".getBytes(ASCII);
+        tr.set(keyBytes, "1".getBytes(ASCII));
+        for (int i = 0; i < count; i++) {
+            int old = Integer.parseInt(new String(tr.get(keyBytes).join(), ASCII));
+            tr.set(keyBytes, ("" + (old + 1)).getBytes(ASCII));
+        }
+        long end = System.nanoTime();
+
+        return count * 1_000_000_000.0/(end - start);
+    }
+
+    public void insertData(Transaction tr) {
+        tr.clear(new byte[0], new byte[]{(byte)0xff}); // Clear user space.
+        for (int i = 0; i < keyCount; i++) {
+            tr.set(key(i), "foo".getBytes(ASCII));
+        }
+    }
+
+    public byte[] key(int i) {
+        return ByteArrayUtil.join(args.getSubspace().pack(), String.format(keyFormat, i).getBytes(ASCII));
+    }
+
+    public static void main(String[] args) {
+        System.out.println("Running Java RYW benchmark on Java version " + System.getProperty("java.version"));
+        try {
+            new RYWBenchmark().run(args);
+        } catch (IllegalArgumentException e) {
+            System.out.println("Could not run test due to malformed arguments.");
+            System.exit(1);
+        } catch (Exception e) {
+            System.out.println("Fatal error encountered during run: " + e);
+            e.printStackTrace();
+            System.exit(2);
+        }
+    }
+}
diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TestResult.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TestResult.java
new file mode 100644
index 0000000000..2ea112d50f
--- /dev/null
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TestResult.java
@@ -0,0 +1,151 @@
+/*
+ * TestResult.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+
+public class TestResult {
+    private long id;
+    private Map<String,Map<String,Object>> kpis;
+    private List<Throwable> errors;
+
+    public TestResult(Random r) {
+        id = Math.abs(r.nextLong());
+        kpis = new TreeMap<String,Map<String,Object>>(); // Tree map because we will have to print this out.
+        errors = new ArrayList<Throwable>();
+    }
+
+    public void addKpi(String name, Number value, String units) {
+        TreeMap<String,Object> kpi = new TreeMap<String,Object>();
+        kpi.put("value", value);
+        kpi.put("units", units);
+        kpis.put(name, kpi);
+    }
+
+    public void addError(Throwable t) {
+        errors.add(t);
+    }
+
+    public void save(String directory) {
+        String file = "javacompletableresult-" + id + ".json";
+        if(directory.length() > 0) {
+            file = directory + "/" + file;
+        }
+
+        // TODO: Should we use a really JSON library?
+
+        StringBuilder outputBuilder = new StringBuilder();
+        outputBuilder.append('{');
+
+        // Add KPIs:
+        outputBuilder.append("\"kpis\": {");
+        boolean firstKpi = true;
+        for (Map.Entry<String,Map<String,Object>> kpi : kpis.entrySet()) {
+            if (firstKpi) {
+                firstKpi = false;
+            } else {
+                outputBuilder.append(", ");
+            }
+
+            outputBuilder.append("\"");
+            outputBuilder.append(kpi.getKey());
+            outputBuilder.append("\": {");
+
+            boolean firstEntry = true;
+
+            for (Map.Entry<String,Object> entry : kpi.getValue().entrySet()) {
+                if (firstEntry) {
+                    firstEntry = false;
+                } else {
+                    outputBuilder.append(", ");
+                }
+
+                outputBuilder.append("\"");
+                outputBuilder.append(entry.getKey());
+                outputBuilder.append("\": ");
+
+                Object value = entry.getValue();
+                if (value instanceof String) {
+                    outputBuilder.append("\"");
+                    outputBuilder.append((String)value);
+                    outputBuilder.append("\"");
+                } else {
+                    outputBuilder.append(value.toString());
+                }
+            }
+
+            outputBuilder.append("}");
+        }
+        outputBuilder.append("}, ");
+
+        // Add errors:
+        outputBuilder.append("\"errors\":[");
+        boolean firstError = true;
+        for (Throwable t : errors) {
+            if (firstError) {
+                firstError = false;
+            } else {
+                outputBuilder.append(", ");
+            }
+
+            StringBuilder msgBuilder = new StringBuilder();
+            msgBuilder.append(t.getClass().toString());
+            msgBuilder.append(": ");
+            msgBuilder.append(t.getMessage()); // Escaping quotes. Yeah, this won't work in the general case....
+            StackTraceElement[] stackTraceElements = t.getStackTrace();
+            for (StackTraceElement element : stackTraceElements) {
+                msgBuilder.append("\n    ");
+                msgBuilder.append(element.toString());
+            }
+            outputBuilder.append('"');
+            outputBuilder.append(msgBuilder.toString()
+                    .replace("\\", "\\\\")
+                    .replace("\"", "\\\"")
+                    .replace("\t", "\\t")
+                    .replace("\r", "\\r")
+                    .replace("\n", "\\n")
+                    .replace("\f", "\\f")
+                    .replace("\b", "\\b")
+            );
+            outputBuilder.append('"');
+        }
+        outputBuilder.append("]");
+
+        outputBuilder.append('}');
+
+        BufferedWriter writer = null;
+        try  {
+            writer = new BufferedWriter(new FileWriter(file));
+            writer.write(outputBuilder.toString());
+            writer.close();
+        } catch (IOException e) {
+            System.out.println("Could not write results to file " + file);
+            throw new RuntimeException("Could not save results: " + e.getMessage(), e);
+        }
+    }
+}
diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TesterArgs.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TesterArgs.java
new file mode 100644
index 0000000000..a2b27457b2
--- /dev/null
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TesterArgs.java
@@ -0,0 +1,153 @@
+/*
+ * TesterArgs.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.subspace.Subspace;
+import com.apple.cie.foundationdb.tuple.Tuple;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class TesterArgs {
+    private String outputDirectory;
+    private boolean multiversionApi;
+    private boolean callbacksOnExternalThread;
+    private boolean externalClient;
+    private Subspace subspace;
+    private List<String> testsToRun;
+
+    private TesterArgs(String outputDirectory, boolean multiversionApi, boolean callbacksOnExternalThread, boolean externalClient, Subspace subspace, List<String> testsToRun) {
+        this.outputDirectory = outputDirectory;
+        this.multiversionApi = multiversionApi;
+        this.callbacksOnExternalThread = callbacksOnExternalThread;
+        this.externalClient = externalClient;
+        this.subspace = subspace;
+        this.testsToRun = testsToRun;
+    }
+
+    public static void printUsage() {
+        String usage = "Arguments:  [-o/--output-directory DIR] [--disable-multiversion-api] [--enable-callbacks-on-external-threads] [--use-external-client] [--tests-to-run TEST [TEST ...]] [-h/--help]\n"
+                     + "\n"
+                     + "Arguments:\n"
+                     + "   -o/--output-directory DIR                Directory to store JSON output. If not set, the current directory is used.\n"
+                     + "   --disable-multiversion-api               Disables the multi-version client API\n"
+                     + "   --enable-callbacks-on-external-threads   Allows callbacks to be called on threads created by the client library.\n"
+                     + "   --use-external-client                    Connect to the server using an external client.\n"
+                     + "   --tests-to-run TEST [TEST ...]           List of test names to run.\n"
+                     + "   -h/--help                                Print this help message and then quit.\n";
+
+        System.out.print(usage);
+    }
+
+    /**
+     * Parses the argument strings into a <code>TesterArgs</code> instance.
+     * This will return <code>null</code> if the args include an argument telling
+     * it to print the help message and it will throw an {@link IllegalArgumentException}
+     * if it can't parse the arguments.
+     *
+     * @param args command-line args
+     * @return built instance or <code>null</code>
+     * @throws IllegalArgumentException if the arguments can't be parsed
+     */
+    public static TesterArgs parseArgs(String[] args) {
+        String outputDirectory = "";
+        boolean multiversionApi = true;
+        boolean callbacksOnExternalThread = false;
+        boolean externalClient = false;
+        Subspace subspace = new Subspace();
+        List<String> testsToRun = new ArrayList<String>();
+
+        for (int i = 0; i < args.length; i++) {
+            String arg = args[i];
+            if (arg.equals("-o") || arg.equals("--output-directory")) {
+                if (i + 1 < args.length) {
+                    outputDirectory = args[++i];
+                } else {
+                    System.out.println("No output directory specified for argument " + arg + "\n");
+                    printUsage();
+                    throw new IllegalArgumentException("No output directory specified for argument " + arg);
+                }
+            } else if (arg.equals("--subspace")) {
+                if (i + 1 < args.length) {
+                    subspace = new Subspace(Tuple.from(args[++i]));
+                } else {
+                    System.out.println("No subspace specified for argument " + args + "\n");
+                    printUsage();
+                    throw new IllegalArgumentException("Not subspace specified for argument " + arg);
+                }
+            } else if (arg.equals("--disable-multiversion-api")) {
+                multiversionApi = false;
+            } else if (arg.equals("--enable-callbacks-on-external-threads")) {
+                callbacksOnExternalThread = true;
+            } else if (arg.equals("--use-external-client")) {
+                externalClient = true;
+            } else if (arg.equals("--tests-to-run")) {
+                if (i + 1 < args.length && args[i + 1].charAt(0) != '-') {
+                    int j;
+                    for (j = i + 1; j < args.length && args[j].charAt(0) != '-'; j++) {
+                        testsToRun.add(args[j]);
+                    }
+                    i = j;
+                } else {
+                    System.out.println("No tests specified with argument " + arg + "\n");
+                    printUsage();
+                    throw new IllegalArgumentException("No tests specified with argument " + arg);
+                }
+            } else if (arg.equals("-h") || arg.equals("--help")) {
+                printUsage();
+                return null;
+            } else {
+                System.out.println("Unknown argument " + arg + "\n");
+                printUsage();
+                throw new IllegalArgumentException("Unknown argument " + arg);
+            }
+        }
+
+        return new TesterArgs(outputDirectory, multiversionApi, callbacksOnExternalThread, externalClient, subspace, testsToRun);
+    }
+
+    // Accessors.
+
+    public String getOutputDirectory() {
+        return outputDirectory;
+    }
+
+    public boolean useMultiversionApi() {
+        return multiversionApi;
+    }
+
+    public boolean putCallbacksOnExternalThread() {
+        return callbacksOnExternalThread;
+    }
+
+    public boolean useExternalClient() {
+        return externalClient;
+    }
+
+    public Subspace getSubspace() {
+        return subspace;
+    }
+
+    public List<String> getTestsToRun() {
+        return testsToRun;
+    }
+}
\ No newline at end of file
diff --git a/bindings/java/src/main/com/apple/cie/foundationdb/Database.java b/bindings/java/src/main/com/apple/cie/foundationdb/Database.java
index c02e66c1a0..c222f4e033 100644
--- a/bindings/java/src/main/com/apple/cie/foundationdb/Database.java
+++ b/bindings/java/src/main/com/apple/cie/foundationdb/Database.java
@@ -46,8 +46,8 @@ public interface Database extends Disposable, TransactionContext {
 	 * Creates a {@link Transaction} that operates on this {@code Database}.<br>
 	 * <br>
 	 * Note: Java transactions automatically set the {@link TransactionOptions#setUsedDuringCommitProtectionDisable}
-	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after either
-	 *  {@link Transaction#reset} or {@link Transaction#onError} is called.
+	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after
+	 *  {@link Transaction#onError} is called.
 	 *
 	 * @return a newly created {@code Transaction} that reads from and writes to this {@code Database}.
 	 */
diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in
index 1244f5860d..892a5aa8c1 100644
--- a/bindings/java/src/main/overview.html.in
+++ b/bindings/java/src/main/overview.html.in
@@ -24,11 +24,12 @@ and add it to your classpath.<br>
 <br>
 <h3>Getting started</h3>
 To start using FoundationDB from Java, create an instance of the 
-{@link FDB FoundationDB API interface} with the version of the
+{@link com.apple.cie.foundationdb.FDB FoundationDB API interface} with the version of the
 API that you want to use (this release of the FoundationDB Java API supports only version {@code 500}).
-With this API object you can then open {@link Cluster}s and
-{@link Database}s and start using {@link Transaction}s.
-Here we give an example. The example relies on a cluster file at the
+With this API object you can then open {@link com.apple.cie.foundationdb.Cluster}s and
+{@link com.apple.cie.foundationdb.Database}s and start using
+{@link com.apple.cie.foundationdb.Transaction}s. Here we give an example. The example relies on a
+cluster file at the
 <a href="/documentation/api-general.html#default-cluster-file">default location</a> 
 for your platform and a running server.<br>
 <br>
@@ -77,7 +78,7 @@ for information about how Tuples sort and can be used to efficiently model data.
 The {@link com.apple.cie.foundationdb.directory Directory API} is provided with the core
 Java API for FoundationDB. This layer is provided in some form in all official
 language bindings. The FoundationDB API provides directories as a tool for
-managing related {@link Subspace}s. Directories are a
+managing related {@link com.apple.cie.foundationdb.subspace.Subspace Subspace}s. Directories are a
 recommended approach for administering applications. Each application should 
 create or open at least one directory to manage its subspaces. Directories are 
 identified by hierarchical paths analogous to the paths in a Unix-like file system. 
@@ -87,12 +88,12 @@ for the corresponding subspace. In effect, directories provide a level of indire
 for access to subspaces.
 <br>
 <h3>{@link com.apple.cie.foundationdb.async.Future Future}s and asynchronous operation</h3>
-Asynchronous FoundationDB operations return {@link Future}s.
-A {@link Future} can be used in a blocking way using the
-{@link Future#get() get()} method or in a
+Asynchronous FoundationDB operations return {@link com.apple.cie.foundationdb.async.Future Future}s.
+A {@link com.apple.cie.foundationdb.async.Future Future} can be used in a blocking way using the
+{@link com.apple.cie.foundationdb.async.Future#get() get()} method or in a
 fully-asynchronous way using the 
-{@link Future#map(Function) map()} and
-{@link Future#flatMap(Function) flatMap()}
+{@link com.apple.cie.foundationdb.async.Future#map(Function) map()} and
+{@link com.apple.cie.foundationdb.async.Future#flatMap(Function) flatMap()}
 methods. Generally, the blocking style is more straightforward and the asynchronous style
 is more efficient. Mixing the two styles correctly can be tricky, so consider choosing
 one or the other. See the {@linkplain com.apple.cie.foundationdb.async async Package documentation}
diff --git a/bindings/nodejs/binding.gyp b/bindings/nodejs/binding.gyp
index d24034df76..6771942577 100644
--- a/bindings/nodejs/binding.gyp
+++ b/bindings/nodejs/binding.gyp
@@ -2,7 +2,7 @@
   'targets': [
     {
       'target_name': 'fdblib',
-      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp' ],
+      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp', 'src/FdbUtil.cpp' ],
       'include_dirs': ['../c'],
       'conditions': [
         ['OS=="linux"', {
diff --git a/bindings/nodejs/binding.gyp.npmsrc b/bindings/nodejs/binding.gyp.npmsrc
index f6b6e5e691..4e39c9a049 100644
--- a/bindings/nodejs/binding.gyp.npmsrc
+++ b/bindings/nodejs/binding.gyp.npmsrc
@@ -2,7 +2,7 @@
   'targets': [
     {
       'target_name': 'fdblib',
-      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp' ],
+      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp', 'src/FdbUtil.cpp' ],
       'conditions': [
         ['OS=="linux"', {
           'link_settings': { 'libraries': ['-lfdb_c'] },
diff --git a/bindings/nodejs/fdb_node.target b/bindings/nodejs/fdb_node.target
index 9712d768e6..14c7d6684d 100644
--- a/bindings/nodejs/fdb_node.target
+++ b/bindings/nodejs/fdb_node.target
@@ -136,12 +136,14 @@ copy "$(TargetPath)" "modules\$(NodeVersionDir)\fdblib.node"
     <ClCompile Include="src\Cluster.cpp" />
     <ClCompile Include="src\FdbV8Wrapper.cpp" />
     <ClCompile Include="src\FdbOptions.cpp" />
+    <ClCompile Include="src\FdbUtil.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="src\Cluster.h" />
     <ClInclude Include="src\Database.h" />
     <ClInclude Include="src\FdbError.h" />
     <ClInclude Include="src\FdbOptions.h" />
+    <ClInclude Include="src\FdbUtil.h" />
     <ClInclude Include="src\NodeCallback.h" />
     <ClInclude Include="src\Transaction.h" />
     <ClInclude Include="src\Version.h" />
diff --git a/bindings/nodejs/include.mk b/bindings/nodejs/include.mk
index c6e7135f4c..2f4243eb20 100644
--- a/bindings/nodejs/include.mk
+++ b/bindings/nodejs/include.mk
@@ -46,7 +46,7 @@ bindings/nodejs/fdb_node.stamp: bindings/nodejs/src/FdbOptions.g.cpp bindings/no
 	for ver in $(NODE_VERSIONS); do \
 		MMVER=`echo $$ver | sed -e 's,\., ,g' | awk '{print $$1 "." $$2}'` && \
 		mkdir modules/$$MMVER && \
-		node-gyp configure --target=$$ver && \
+		node-gyp configure --dist-url=https://nodejs.org/dist --target=$$ver && \
 		node-gyp -v build && \
 		cp build/Release/fdblib.node modules/$${MMVER} ; \
 	done
@@ -67,6 +67,7 @@ bindings/nodejs/package.json: bindings/nodejs/package.json.in $(ALL_MAKEFILES) v
 	@m4 -DVERSION=$(NPMVER) $< > $@
 	@echo "Updating       Node dependencies"
 	@cd bindings/nodejs && \
+	npm config set registry "https://registry.npmjs.org/" && \
 	npm update
 
 fdb_node_npm: fdb_node versions.target bindings/nodejs/README.md bindings/nodejs/lib/*.js bindings/nodejs/src/* bindings/nodejs/binding.gyp LICENSE
diff --git a/bindings/nodejs/lib/tuple.js b/bindings/nodejs/lib/tuple.js
index 1fe1d1dcc5..5758791407 100644
--- a/bindings/nodejs/lib/tuple.js
+++ b/bindings/nodejs/lib/tuple.js
@@ -24,6 +24,7 @@
 var assert = require('assert');
 var buffer = require('./bufferConversion');
 var fdbUtil = require('./fdbUtil');
+var fdb = require('./fdbModule');
 var FDBError = require('./error');
 
 var sizeLimits = new Array(8);
@@ -83,7 +84,7 @@ function Float(value) {
 			return this.rawData;
 		} else {
 			var buf = new Buffer(4);
-			buf.writeFloatBE(this.value, 0);
+			buf.writeFloatBE(fdb.toFloat(this.value), 0);
 			return buf;
 		}
 	};
@@ -230,7 +231,7 @@ function encode(item, buf, pos) {
 		if (isNaN(item.value) && item.rawData !== undefined) {
 			item.rawData.copy(outBuf, 1, 0, 4);
 		} else {
-			outBuf.writeFloatBE(item.value, 1);
+			outBuf.writeFloatBE(fdb.toFloat(item.value), 1);
 		}
 		adjustFloat(outBuf, 1, true);
 		return outBuf;
diff --git a/bindings/nodejs/src/FdbUtil.cpp b/bindings/nodejs/src/FdbUtil.cpp
new file mode 100644
index 0000000000..6928982868
--- /dev/null
+++ b/bindings/nodejs/src/FdbUtil.cpp
@@ -0,0 +1,42 @@
+/*
+ * FdbUtil.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <node.h>
+#include "FdbUtil.h"
+
+using namespace v8;
+
+Handle<Value> ToFloat(const Arguments &args) {
+    HandleScope scope;
+
+    if (args.Length() != 1) {
+        return ThrowException(Exception::TypeError(String::NewSymbol("Wrong number of arguments (must be exactly 1)")));
+    }
+
+    if (!args[0]->IsNumber()) {
+        return ThrowException(Exception::TypeError(String::NewSymbol("Argument is not a Number")));
+    }
+
+    float value = (float)args[0]->NumberValue();
+    Handle<Value> jsValue = Number::New(value);
+
+    return scope.Close(jsValue);
+}
diff --git a/bindings/nodejs/src/FdbUtil.h b/bindings/nodejs/src/FdbUtil.h
new file mode 100644
index 0000000000..a8ad05a01a
--- /dev/null
+++ b/bindings/nodejs/src/FdbUtil.h
@@ -0,0 +1,29 @@
+/*
+ * FdbUtil.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef FDB_NODE_FDB_UTIL_H
+#define FDB_NODE_FDB_UTIL_H
+
+#include <node.h>
+
+v8::Handle<v8::Value> ToFloat(const v8::Arguments &args);
+
+#endif
diff --git a/bindings/nodejs/src/FdbV8Wrapper.cpp b/bindings/nodejs/src/FdbV8Wrapper.cpp
index 4b7eea8371..659b26a0a9 100644
--- a/bindings/nodejs/src/FdbV8Wrapper.cpp
+++ b/bindings/nodejs/src/FdbV8Wrapper.cpp
@@ -33,6 +33,7 @@
 #include "Version.h"
 #include "FdbError.h"
 #include "FdbOptions.h"
+#include "FdbUtil.h"
 
 uv_thread_t fdbThread;
 
@@ -140,6 +141,7 @@ void init(Handle<Object> target){
 	target->Set(String::NewSymbol("options"), FdbOptions::CreateOptions(FdbOptions::NetworkOption));
 	target->Set(String::NewSymbol("streamingMode"), FdbOptions::CreateEnum(FdbOptions::StreamingMode));
 	target->Set(String::NewSymbol("atomic"), FdbOptions::CreateOptions(FdbOptions::MutationType));
+	target->Set(String::NewSymbol("toFloat"), FunctionTemplate::New(ToFloat)->GetFunction());
 }
 
 #if NODE_VERSION_AT_LEAST(0, 8, 0)
diff --git a/bindings/nodejs/tests/tuple_test.js b/bindings/nodejs/tests/tuple_test.js
index e430c964c4..56efde8d78 100755
--- a/bindings/nodejs/tests/tuple_test.js
+++ b/bindings/nodejs/tests/tuple_test.js
@@ -18,7 +18,8 @@
  * limitations under the License.
  */
 
-var fdb = require('../lib/fdb.js').apiVersion(200);
+var fdb = require('../lib/fdb.js').apiVersion(500);
+var fdbModule = require('../lib/fdbModule.js');
 
 console.log(fdb.tuple.pack([-Math.pow(2,53)]));
 console.log(fdb.tuple.pack([-Math.pow(2,53)+1]));
@@ -76,3 +77,25 @@ tuples = [
 ];
 tuples.sort(fdb.tuple.compare);
 console.log(tuples);
+
+// Float overruns.
+const floats = [ 2.037036e90, -2.037036e90, 4.9090935e-91, -4.9090935e-91, 2.345624805922133125e14, -2.345624805922133125e14 ];
+for (var i = 0; i < floats.length; i++) {
+    var f = floats[i];
+    console.log(f + " -> " + fdb.tuple.Float.fromBytes((new fdb.tuple.Float(f)).toBytes()).value);
+}
+
+// Float type errors.
+try {
+    console.log((new fdb.tuple.Float("asdf")).toBytes());
+} catch (e) {
+    console.log("Caught!");
+    console.log(e);
+}
+
+try {
+    console.log(fdbModule.toFloat(3.14, 2.718));
+} catch (e) {
+    console.log("Caught!");
+    console.log(e);
+}
diff --git a/design/tuple.md b/design/tuple.md
new file mode 100644
index 0000000000..aed8858f32
--- /dev/null
+++ b/design/tuple.md
@@ -0,0 +1,196 @@
+# FDB Tuple layer typecodes
+
+This document is intended to be the system of record for the allocation of typecodes in the Tuple layer. The source code isn’t good enough because a typecode might be added to one language (or by a customer) before another.
+
+Status: Standard means that all of our language bindings implement this typecode
+Status: Reserved means that this typecode is not yet used in our standard language bindings, but may be in use by third party bindings or specific applications
+Status: Deprecated means that a previous layer used this type, but issues with that type code have led us to mark this type code as not to be used.
+
+
+### **Null Value**
+
+Typecode: 0x00  
+Length: 0 bytes  
+Status: Standard
+
+### **Byte String**
+
+Typecode: 0x01  
+Length: Variable (terminated by` [\x00]![\xff]`)  
+Encoding: `b'\x01' + value.replace(b'\x00', b'\x00\xFF') + b'\x00'`  
+Test case: `pack(“foo\x00bar”) == b'\x01foo\x00\xffbar\x00'`  
+Status: Standard
+
+In other words, byte strings are null terminated with null values occurring in the string escaped in an order-preserving way.
+
+### **Unicode String**
+
+Typecode: 0x02  
+Length: Variable (terminated by [\x00]![\xff])  
+Encoding: `b'\x02' + value.encode('utf-8').replace(b'\x00', b'\x00\xFF') + b'\x00'`  
+Test case: `pack( u"F\u00d4O\u0000bar" ) == b'\x02F\xc3\x94O\x00\xffbar\x00'`  
+Status: Standard
+
+This is the same way that byte strings are encoded, but first, the unicode string is encoded in UTF-8.
+
+### **(DEBRECATED) Nested Tuple**
+
+Typecodes: 0x03-0x04  
+Length: Variable (terminated by 0x04 type code)  
+Status: Deprecated  
+
+This encoding was used by a few layers. However, it had ordering problems when one tuple was a prefix of another and the type of the first element in the longer tuple was either null or a byte string. For an example, consider the empty tuple and the tuple containing only null. In the old scheme, the empty tuple would be encoded as `\x03\x04` while the tuple containing only null would be encoded as `\x03\x00\x04`, so the second tuple would sort first based on their bytes, which is incorrect semantically.
+
+### **Nested Tuple**
+
+Typecodes: 0x05  
+Length: Variable (terminated by `[\x00]![\xff]` at beginning of nested element)  
+Encoding: `b'\x05' + ''.join(map(lambda x: b'\x00\xff' if x is None else pack(x), value)) + b'\x00'`  
+Test case: `pack( (“foo\x00bar”, None, ()) ) == b'\x05\x01foo\x00\xffbar\x00\x00\xff\x05\x00\x00'`  
+Status: Standard
+
+The list is ended with a 0x00 byte. Nulls within the tuple are encoded as `\x00\xff`. There is no other null escaping. In particular, 0x00 bytes that are within the nested types can be left as-is as they are passed over when decoding the interior types. To show how this fixes the bug in the previous version of nested tuples, the empty tuple is now encoded as `\x05\x00` while the tuple containing only null is encoded as `\x05\x00\xff\x00`, so the first tuple will sort first.
+
+### **Negative arbitrary-precision Integer**
+
+Typecodes: 0x0a, 0x0b  
+Encoding: Not defined yet  
+Status: Reserved; 0x0b used in Python and Java
+
+These typecodes are reserved for encoding integers larger than 8 bytes. Presumably the type code would be followed by some encoding of the length, followed by the big endian one’s complement number. Reserving two typecodes for each of positive and negative numbers is probably overkill, but until there’s a design in place we might as well not use them. In the Python and Java implementations, 0x0b stores negative numbers which are expressed with between 9 and 255 bytes. The first byte following the type code (0x0b) is a single byte expressing the number of bytes in the integer (with its bits flipped to preserve order), followed by that number of bytes representing the number in big endian order in one's complement.
+
+### **Integer**
+
+Typecodes: 0x0c - 0x1c  
+&nbsp;0x0c is an 8 byte negative number  
+&nbsp;0x13 is a 1 byte negative number  
+&nbsp;0x14 is a zero  
+&nbsp;0x15 is a 1 byte positive number  
+&nbsp;0x1c is an 8 byte positive number  
+Length: Depends on typecode (0-8 bytes)  
+Encoding: positive numbers are big endian  
+ negative numbers are big endian one’s complement (so -1 is 0x13 0xfe)  
+Test case: `pack( -5551212 ) == b'\x11\xabK\x93'`  
+Status: Standard
+
+There is some variation in the ability of language bindings to encode and decode values at the outside of the possible range, because of different native representations of integers. 
+
+### **Positive arbitrary-precision Integer**
+
+Typecodes: 0x1d, 0x1e  
+Encoding: Not defined yet  
+Status: Reserved; 0x1d used in Python and Java
+
+These typecodes are reserved for encoding integers larger than 8 bytes. Presumably the type code would be followed by some encoding of the length, followed by the big endian one’s complement number. Reserving two typecodes for each of positive and negative numbers is probably overkill, but until there’s a design in place we might as well not use them. In the Python and Java implementations, 0x1d stores positive numbers which are expressed with between 9 and 255 bytes. The first byte following the type code (0x1d) is a single byte expressing the number of bytes in the integer, followed by that number of bytes representing the number in big endian order.
+
+### **IEEE Binary Floating Point**
+
+Typecodes:   
+&nbsp;0x20 - float (32 bits)  
+&nbsp;0x21 - double (64 bits)  
+&nbsp;0x22 - long double (80 bits)  
+Length: 4 - 10 bytes  
+Test case: `pack( -42f ) == b'=\xd7\xff\xff'`  
+Encoding: Big-endian IEEE binary representation, followed by the following transformation:  
+```python
+ if ord(rep[0])&0x80: # Check sign bit
+    # Flip all bits, this is easier in most other languages!
+    return "".join( chr(0xff^ord(r)) for r in rep )
+ else:
+    # Flip just the sign bit
+    return chr(0x80^ord(rep[0])) + rep[1:]
+```
+Status: Standard (float and double) ; Reserved (long double)
+
+The binary representation should not be assumed to be canonicalized (as to multiple representations of NaN, for example) by a reader. This order sorts all numbers in the following way:
+
+* All negative NaN values with order determined by mantissa bits (which are semantically meaningless)
+* Negative inifinity
+* All real numbers in the standard order (except that -0.0 < 0.0)
+* Positive infinity
+* All positive NaN values with order determined by mantissa bits
+
+This should be equivalent to the standard IEEE total ordering.
+
+### **Arbitrary-precision Decimal**
+
+Typecodes: 0x23, 0x24  
+Length: Arbitrary  
+Encoding: Scale followed by arbitrary precision integer  
+Status: Reserved
+
+This encoding format has been used by layers. Note that this encoding makes almost no guarantees about ordering properties of tuple-encoded values and should thus generally be avoided.
+
+### **(DEPRECATED) True Value**
+
+Typecode: 0x25  
+Length: 0 bytes  
+Status: Deprecated
+
+### **False Value**
+
+Typecode: 0x26  
+Length: 0 bytes  
+Status: Standard
+
+### **True Value**
+
+Typecode: 0x27  
+Length: 0 bytes  
+Status: Standard
+
+Note that false will sort before true with the given encoding.
+
+### **RFC 4122 UUID**
+
+Typecode: 0x30  
+Length: 16 bytes  
+Encoding: Network byte order as defined in the rfc: [_http://www.ietf.org/rfc/rfc4122.txt_](http://www.ietf.org/rfc/rfc4122.txt)  
+Status: Standard
+
+This is equivalent to the unsigned byte ordering of the UUID bytes in big-endian order.
+
+### **64 bit identifier**
+
+Typecode: 0x31  
+Length: 8 bytes  
+Encoding: Big endian unsigned 8-byte integer (typically random or perhaps semi-sequential)  
+Status: Reserved
+
+There’s definitely some question of whether this deserves to be separated from a plain old 64 bit integer, but a separate type was desired in one of the third-party bindings. This type has not been ported over to the first-party bindings.
+
+### **80 Bit versionstamp**
+
+Typecode: 0x32  
+Length: 10 bytes  
+Encoding: Big endian 10-byte integer. First/high 8 bytes are a database version, next two are batch version.  
+Status: Reserved
+
+### **96 Bit Versionstamp**
+
+Typecode: 0x33  
+Length: 12 bytes  
+Encoding: Big endian 12-byte integer. First/high 8 bytes are a database version, next two are batch version, next two are ordering within transaction.  
+Status: Reserved
+
+The two versionstamp typecodes are reserved for future work adding compatibility between the tuple layer and versionstamp operations. Note that the first 80 bits of the 96 bit versionstamp are the same as the contents of the 80 bit versionstamp, and they correspond to what the `SET_VERSIONSTAMP_KEY` mutation will write into a database key , i.e., the first 8 bytes are a big-endian, unsigned version corresponding to the commit version of a transaction, and the next to bytes are a big-endian, unsigned batch number ordering transactions are committed at the same version. The final two bytes of the 96 bit versionstamp are written by the client and should order writes within a single transaction, thereby providing a global order for all versions.
+
+### **User type codes**
+
+Typecode: 0x40 - 0x4f  
+Length: Variable (user defined)  
+Encoding: User defined  
+Status: Reserved
+
+These type codes may be used by third party extenders without coordinating with us. If used in shipping software, the software should use the directory layer and specify a specific layer name when opening its directories to eliminate the possibility of conflicts.
+
+The only way in which future official, otherwise backward-compatible versions of the tuple layer would be expected to use these type codes is to implement some kind of actual extensibility point for this purpose - they will not be used for standard types.
+
+### **Escape Character**
+
+Typecode: 0xff  
+Length: N/A  
+Encoding: N/A  
+Status: Reserved
+
+This type code is not used for anything. However, several of the other tuple types depend on this type code not being used as a type code for other types in order to correctly escape bytes in an order-preserving way. Therefore, it would be a Very Bad Idea™ for future development to start using this code for anything else.
diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 22c78f5227..cfbedc5e5f 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -23,6 +23,7 @@
 #include "flow/serialize.h"
 #include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
+#include "flow/SignalSafeUnwind.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/BackupAgent.h"
@@ -1820,6 +1821,7 @@ extern uint8_t *g_extra_memory;
 
 int main(int argc, char* argv[]) {
 	platformInit();
+	initSignalSafeUnwind();
 
 	int	status = FDB_EXIT_SUCCESS;
 
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index 3db220f7bd..a0f35f5106 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -30,6 +30,7 @@
 #include "fdbclient/FDBOptions.g.h"
 
 #include "flow/DeterministicRandom.h"
+#include "flow/SignalSafeUnwind.h"
 #include "fdbrpc/TLSConnection.h"
 #include "fdbrpc/Platform.h"
 
@@ -436,9 +437,9 @@ void initHelp() {
 		"clear a range of keys from the database",
 		"All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK);
 	helpMap["configure"] = CommandHelp(
-		"configure [new] <single|double|triple|three_data_hall|three_datacenter|fast_recovery_double|fast_recovery_triple|ssd|memory|proxies=<PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
+		"configure [new] <single|double|triple|three_data_hall|three_datacenter|ssd|memory|proxies=<PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
 		"change database configuration",
-		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n  fast_recovery_double - two copies of data on the storage servers, three copies of the data on the logs, non-copying recovery if one log is missing.\n  fast_recovery_triple - three copies of data on the storage servers, four copies of the data on the logs, non-copying recovery if one log is missing.\n\nStorage engine:\n  ssd - B-Tree storage engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
+		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
 	helpMap["coordinators"] = CommandHelp(
 		"coordinators auto|<ADDRESS>+ [description=new_cluster_description]",
 		"change cluster coordinators or description",
@@ -504,6 +505,7 @@ void initHelp() {
 		"If no addresses are specified, populates the list of processes which can be killed. Processes cannot be killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is unresponsive.\n\nFor each IP:port pair in <ADDRESS>*, attempt to kill the specified process.");
 
 	hiddenCommands.insert("expensive_data_check");
+	hiddenCommands.insert("datadistribution");
 }
 
 void printVersion() {
@@ -1672,7 +1674,18 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 			state double worstFreeSpaceRatio = 1.0;
 			try {
 				for (auto proc : processesMap.obj()){
+					bool storageServer = false;
 					StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
+					for (StatusObjectReader role : rolesArray) {
+						if (role["role"].get_str() == "storage") {
+							storageServer = true;
+							break;
+						}
+					}
+					// Skip non-storage servers in free space calculation
+					if (!storageServer)
+						continue;
+
 					StatusObjectReader process(proc.second);
 					std::string addrStr;
 					if (!process.get("address", addrStr)) {
@@ -1681,6 +1694,9 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 					}
 					NetworkAddress addr = NetworkAddress::parse(addrStr);
 					bool excluded = (process.has("excluded") && process.last().get_bool()) || addressExcluded(exclusions, addr);
+					ssTotalCount++;
+					if (excluded)
+						ssExcludedCount++;
 
 					if(!excluded) {
 						StatusObjectReader disk;
@@ -1703,15 +1719,6 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 
 						worstFreeSpaceRatio = std::min(worstFreeSpaceRatio, double(free_bytes)/total_bytes);
 					}
-
-					for (StatusObjectReader role : rolesArray) {
-						if (role["role"].get_str() == "storage") {
-							if (excluded)
-								ssExcludedCount++;
-							ssTotalCount++;
-							break;
-						}
-					}
 				}
 			}
 			catch (...)  // std::exception
@@ -1895,7 +1902,7 @@ void onoff_generator(const char* text, const char *line, std::vector<std::string
 }
 
 void configure_generator(const char* text, const char *line, std::vector<std::string>& lc) {
-	const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "fast_recovery_double", "fast_recovery_triple", "ssd", "ssd-1", "ssd-2", "memory", "proxies=", "logs=", "resolvers=", NULL};
+	const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "proxies=", "logs=", "resolvers=", NULL};
 	array_generator(text, line, opts, lc);
 }
 
@@ -2210,35 +2217,45 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 			state UID randomID = g_random->randomUniqueID();
 			TraceEvent(SevInfo, "CLICommandLog", randomID).detail("command", printable(StringRef(line)));
 
-			bool err, partial;
-			state std::vector<std::vector<StringRef>> parsed = parseLine(line, err, partial);
-			if (err) {
-				LogCommand(line, randomID, "ERROR: malformed escape sequence");
-				is_error = true;
-				continue;
-			}
-			if (partial) {
-				LogCommand(line, randomID, "ERROR: unterminated quote");
-				is_error = true;
-				continue;
+			bool malformed, partial;
+			state std::vector<std::vector<StringRef>> parsed = parseLine(line, malformed, partial);
+			if (malformed) LogCommand(line, randomID, "ERROR: malformed escape sequence");
+			if (partial) LogCommand(line, randomID, "ERROR: unterminated quote");
+			if (malformed || partial) {
+				if (parsed.size() > 0) {
+					// Denote via a special token that the command was a parse failure.
+					auto& last_command = parsed.back();
+					last_command.insert(last_command.begin(), StringRef((const uint8_t*)"parse_error", strlen("parse_error")));
+				}
 			}
 
 			state bool multi = parsed.size() > 1;
+			is_error = false;
 
 			state std::vector<std::vector<StringRef>>::iterator iter;
 			for (iter = parsed.begin(); iter != parsed.end(); ++iter) {
 				state std::vector<StringRef> tokens = *iter;
 
-				if (opt.exec.present() && is_error) {
+				if (is_error) {
 					printf("WARNING: the previous command failed, the remaining commands will not be executed.\n");
-					return 1;
+					break;
 				}
 
-				is_error = false;
-
 				if (!tokens.size())
 					continue;
 
+				if (tokencmp(tokens[0], "parse_error")) {
+					printf("ERROR: Command failed to completely parse.\n");
+					if (tokens.size() > 1) {
+						printf("ERROR: Not running partial or malformed command:");
+						for (auto t = tokens.begin() + 1; t != tokens.end(); ++t)
+							printf(" %s", formatStringRef(*t, true).c_str());
+						printf("\n");
+					}
+					is_error = true;
+					continue;
+				}
+
 				if (multi) {
 					printf(">>>");
 					for (auto t = tokens.begin(); t != tokens.end(); ++t)
@@ -2717,6 +2734,25 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "datadistribution")) {
+					if (tokens.size() != 2) {
+						printf("Usage: datadistribution <on|off>\n");
+						is_error = true;
+					} else {
+						if(tokencmp(tokens[1], "on")) {
+							int _ = wait(setDDMode(db, 1));
+							printf("Data distribution is enabled\n");
+						} else if(tokencmp(tokens[1], "off")) {
+							int _ = wait(setDDMode(db, 0));
+							printf("Data distribution is disabled\n");
+						} else {
+							printf("Usage: datadistribution <on|off>\n");
+							is_error = true;
+						}
+					}
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "option")) {
 					if (tokens.size() == 2 || tokens.size() > 4) {
 						printUsage(tokens[0]);
@@ -2841,6 +2877,7 @@ ACTOR Future<Void> timeExit(double duration) {
 
 int main(int argc, char **argv) {
 	platformInit();
+	initSignalSafeUnwind();
 	Error::init();
 
 	registerCrashHandler();
diff --git a/fdbclient/BackupAgent.h b/fdbclient/BackupAgent.h
index b5689d5fd3..7cf8c33bb0 100644
--- a/fdbclient/BackupAgent.h
+++ b/fdbclient/BackupAgent.h
@@ -25,7 +25,7 @@
 #include "flow/flow.h"
 #include "NativeAPI.h"
 #include "TaskBucket.h"
-#include "flow/Notified.h"
+#include "Notified.h"
 #include <fdbrpc/IAsyncFile.h>
 #include "KeyBackedTypes.h"
 #include <ctime>
diff --git a/fdbclient/FailureMonitorClient.actor.cpp b/fdbclient/FailureMonitorClient.actor.cpp
index b263fb883a..46b0dd740f 100644
--- a/fdbclient/FailureMonitorClient.actor.cpp
+++ b/fdbclient/FailureMonitorClient.actor.cpp
@@ -22,12 +22,20 @@
 #include "fdbrpc/FailureMonitor.h"
 #include "ClusterInterface.h"
 
+struct FailureMonitorClientState : ReferenceCounted<FailureMonitorClientState> {
+	std::set<NetworkAddress> knownAddrs;
+	double serverFailedTimeout;
+
+	FailureMonitorClientState() {
+		serverFailedTimeout = CLIENT_KNOBS->FAILURE_TIMEOUT_DELAY;
+	}
+};
+
 ACTOR Future<Void> failureMonitorClientLoop( 
 	SimpleFailureMonitor* monitor, 
 	ClusterInterface controller,
-	double* pServerFailedTimeout,
-	bool trackMyStatus,
-	std::set<NetworkAddress>* knownAddrs)
+	Reference<FailureMonitorClientState> fmState,
+	bool trackMyStatus)
 {
 	state Version version = 0;
 	state Future<FailureMonitoringReply> request = Never();
@@ -37,7 +45,7 @@ ACTOR Future<Void> failureMonitorClientLoop(
 	state double waitfor = 0;
 
 	monitor->setStatus(controller.failureMonitoring.getEndpoint().address, FailureStatus(false));
-	knownAddrs->insert( controller.failureMonitoring.getEndpoint().address );
+	fmState->knownAddrs.insert( controller.failureMonitoring.getEndpoint().address );
 
 	//The cluster controller's address (controller.failureMonitoring.getEndpoint().address) is treated specially because we can declare that it is down independently
 	//of the response from the cluster controller. It still needs to be in knownAddrs in case the cluster controller changes, so the next cluster controller resets its state
@@ -51,14 +59,14 @@ ACTOR Future<Void> failureMonitorClientLoop(
 					requestTimeout = Never();
 					if (reply.allOthersFailed) {
 						// Reset all systems *not* mentioned in the reply to the default (failed) state
-						knownAddrs->erase( controller.failureMonitoring.getEndpoint().address );
+						fmState->knownAddrs.erase( controller.failureMonitoring.getEndpoint().address );
 						std::set<NetworkAddress> changedAddresses;
 						for(int c=0; c<reply.changes.size(); c++)
 							changedAddresses.insert( reply.changes[c].address );
-						for(auto it : *knownAddrs)
+						for(auto it : fmState->knownAddrs)
 							if (!changedAddresses.count( it ))
 								monitor->setStatus( it, FailureStatus() );
-						knownAddrs->clear();
+						fmState->knownAddrs.clear();
 					} else {
 						ASSERT( version != 0 );
 					}
@@ -66,20 +74,20 @@ ACTOR Future<Void> failureMonitorClientLoop(
 					if( monitor->getState( controller.failureMonitoring.getEndpoint() ).isFailed() )
 						TraceEvent("FailureMonitoringServerUp").detail("OldServer",controller.id());
 					monitor->setStatus( controller.failureMonitoring.getEndpoint().address, FailureStatus(false) );
-					knownAddrs->insert( controller.failureMonitoring.getEndpoint().address );
+					fmState->knownAddrs.insert( controller.failureMonitoring.getEndpoint().address );
 
 					//if (version != reply.failureInformationVersion)
 					//	printf("Client '%s': update from %lld to %lld (%d changes, aof=%d)\n", g_network->getLocalAddress().toString().c_str(), version, reply.failureInformationVersion, reply.changes.size(), reply.allOthersFailed);
 
 					version = reply.failureInformationVersion;
-					*pServerFailedTimeout = reply.considerServerFailedTimeoutMS * .001;
+					fmState->serverFailedTimeout = reply.considerServerFailedTimeoutMS * .001;
 					for(int c=0; c<reply.changes.size(); c++) {
 						//printf("Client '%s': status of '%s' is now '%s'\n", g_network->getLocalAddress().toString().c_str(), reply.changes[c].address.toString().c_str(), reply.changes[c].status.failed ? "Failed" : "OK");
 						monitor->setStatus( reply.changes[c].address, reply.changes[c].status );
 						if (reply.changes[c].status != FailureStatus())
-							knownAddrs->insert( reply.changes[c].address );
+							fmState->knownAddrs.insert( reply.changes[c].address );
 						else
-							knownAddrs->erase( reply.changes[c].address );
+							fmState->knownAddrs.erase( reply.changes[c].address );
 						ASSERT( reply.changes[c].address != controller.failureMonitoring.getEndpoint().address || !reply.changes[c].status.failed );
 					}
 					before = now();
@@ -91,7 +99,7 @@ ACTOR Future<Void> failureMonitorClientLoop(
 					requestTimeout = Never();
 					TraceEvent(SevWarn, "FailureMonitoringServerDown").detail("OldServerID",controller.id());
 					monitor->setStatus( controller.failureMonitoring.getEndpoint().address, FailureStatus(true) );
-					knownAddrs->erase( controller.failureMonitoring.getEndpoint().address );
+					fmState->knownAddrs.erase( controller.failureMonitoring.getEndpoint().address );
 				}
 				when( Void _ = wait( nextRequest ) ) {
 					g_network->setCurrentTask(TaskDefaultDelay);
@@ -111,7 +119,7 @@ ACTOR Future<Void> failureMonitorClientLoop(
 						req.senderStatus = FailureStatus(false);
 					request = controller.failureMonitoring.getReply( req, TaskFailureMonitor );
 					if(!controller.failureMonitoring.getEndpoint().isLocal())
-						requestTimeout = delay( *pServerFailedTimeout, TaskFailureMonitor );
+						requestTimeout = delay( fmState->serverFailedTimeout, TaskFailureMonitor );
 				}
 			}
 		}
@@ -125,11 +133,10 @@ ACTOR Future<Void> failureMonitorClientLoop(
 
 ACTOR Future<Void> failureMonitorClient( Reference<AsyncVar<Optional<struct ClusterInterface>>> ci, bool trackMyStatus ) {
 	state SimpleFailureMonitor* monitor = static_cast<SimpleFailureMonitor*>( &IFailureMonitor::failureMonitor() );
-	state std::set<NetworkAddress> knownAddrs;
-	state double serverFailedTimeout = CLIENT_KNOBS->FAILURE_TIMEOUT_DELAY;
+	state Reference<FailureMonitorClientState> fmState = Reference<FailureMonitorClientState>(new FailureMonitorClientState()); 
 
 	loop {
-		state Future<Void> client = ci->get().present() ? failureMonitorClientLoop(monitor, ci->get().get(), &serverFailedTimeout, trackMyStatus, &knownAddrs) : Void();
+		state Future<Void> client = ci->get().present() ? failureMonitorClientLoop(monitor, ci->get().get(), fmState, trackMyStatus) : Void();
 		Void _ = wait( ci->onChange() );
 	}
 }
\ No newline at end of file
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index e69ab9859e..7aeabaa445 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -87,7 +87,7 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 		return out;
 	}
 
-	std::string redundancy, log_replicas, log_recovery_anti_quorum, dc="1", minDC="1";
+	std::string redundancy, log_replicas;
 	IRepPolicyRef storagePolicy;
 	IRepPolicyRef tLogPolicy;
 
@@ -95,36 +95,23 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 	if (mode == "single") {
 		redundancy="1";
 		log_replicas="1";
-		log_recovery_anti_quorum="0";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyOne());
 
-	} else if(mode == "double") {
+	} else if(mode == "double" || mode == "fast_recovery_double") {
 		redundancy="2";
 		log_replicas="2";
-		log_recovery_anti_quorum="0";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
-	} else if(mode == "triple") {
+	} else if(mode == "triple" || mode == "fast_recovery_triple") {
 		redundancy="3";
 		log_replicas="3";
-		log_recovery_anti_quorum="0";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
-	} else if(mode == "fast_recovery_double") {
-		redundancy="2";
-		log_replicas="3";
-		log_recovery_anti_quorum="1";
-		storagePolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
-		tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
-	} else if(mode == "fast_recovery_triple") {
-		redundancy="3";
-		log_replicas="4";
-		log_recovery_anti_quorum="1";
-		storagePolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
-		tLogPolicy = IRepPolicyRef(new PolicyAcross(4, "zoneid", IRepPolicyRef(new PolicyOne())));
 	} else if(mode == "two_datacenter") {
-		redundancy="3"; log_replicas="3"; log_recovery_anti_quorum="0"; dc="2"; minDC="1";
+		redundancy="3";
+		log_replicas="3";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
 	} else if(mode == "three_datacenter") {
-		redundancy="3"; log_replicas="3"; log_recovery_anti_quorum="0"; dc="3"; minDC="2";
+		redundancy="3";
+		log_replicas="3";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAnd({
 			IRepPolicyRef(new PolicyAcross(3, "dcid", IRepPolicyRef(new PolicyOne()))),
 			IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())))
@@ -132,7 +119,6 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 	} else if(mode == "three_data_hall") {
 		redundancy="3";
 		log_replicas="4";
-		log_recovery_anti_quorum="0";
 		storagePolicy = IRepPolicyRef(new PolicyAcross(3, "data_hall", IRepPolicyRef(new PolicyOne())));
 		tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "data_hall",
 			IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))
@@ -144,9 +130,6 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 			out[p+"storage_quorum"] = redundancy;
 		out[p+"log_replicas"] = log_replicas;
 		out[p+"log_anti_quorum"] = "0";
-		out[p+"log_recovery_anti_quorum"] = log_recovery_anti_quorum;
-		out[p+"replica_datacenters"] = dc;
-		out[p+"min_replica_datacenters"] = minDC;
 
 		BinaryWriter policyWriter(IncludeVersion());
 		serializeReplicationPolicy(policyWriter, storagePolicy);
@@ -214,9 +197,7 @@ ConfigurationResult::Type buildConfiguration( std::string const& configMode, std
 bool isCompleteConfiguration( std::map<std::string, std::string> const& options ) {
 	std::string p = configKeysPrefix.toString();
 
-	return options.count( p+"min_replica_datacenters" ) == 1 &&
-			options.count( p+"replica_datacenters" ) == 1 &&
-			options.count( p+"log_replicas" ) == 1 &&
+	return 	options.count( p+"log_replicas" ) == 1 &&
 			options.count( p+"log_anti_quorum" ) == 1 &&
 			options.count( p+"storage_quorum" ) == 1 &&
 			options.count( p+"storage_replicas" ) == 1 &&
@@ -307,10 +288,10 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 		result.auto_replication = "double";
 		storage_replication = 2;
 		log_replication = 2;
-	} else if( result.old_replication == "double" ) {
+	} else if( result.old_replication == "double" || result.old_replication == "fast_recovery_double" ) {
 		storage_replication = 2;
 		log_replication = 2;
-	} else if( result.old_replication == "triple" ) {
+	} else if( result.old_replication == "triple" || result.old_replication == "fast_recovery_triple" ) {
 		storage_replication = 3;
 		log_replication = 3;
 	} else if( result.old_replication == "two_datacenter" ) {
@@ -319,12 +300,6 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 	} else if( result.old_replication == "three_datacenter" ) {
 		storage_replication = 3;
 		log_replication = 3;
-	} else if( result.old_replication == "fast_recovery_double" ) {
-		storage_replication = 2;
-		log_replication = 3;
-	} else if( result.old_replication == "fast_recovery_triple" ) {
-		storage_replication = 3;
-		log_replication = 4;
 	} else
 		return ConfigureAutoResult();
 
@@ -705,7 +680,9 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
 
 			if(g_network->isSimulated()) {
 				for(int i = 0; i < (desiredCoordinators.size()/2)+1; i++) {
-					g_simulator.protectedAddresses.insert( desiredCoordinators[i] );
+					auto address = NetworkAddress(desiredCoordinators[i].ip,desiredCoordinators[i].port,true,false);
+					g_simulator.protectedAddresses.insert(address);
+					TraceEvent("ProtectCoordinator").detail("Address", address).backtrace();
 				}
 			}
 
@@ -1067,6 +1044,39 @@ ACTOR Future<vector<AddressExclusion>> getExcludedServers( Database cx ) {
 	}
 }
 
+ACTOR Future<int> setDDMode( Database cx, int mode ) {
+	state Transaction tr(cx);
+	state int oldMode = -1;
+	state BinaryWriter wr(Unversioned());
+	wr << mode;
+
+	loop {
+		try {
+			Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
+			if (oldMode < 0) {
+				oldMode = 1;
+				if (old.present()) {
+					BinaryReader rd(old.get(), Unversioned());
+					rd >> oldMode;
+				}
+			}
+			if (!mode) {
+				BinaryWriter wrMyOwner(Unversioned());
+				wrMyOwner << dataDistributionModeLock;
+				tr.set( moveKeysLockOwnerKey, wrMyOwner.toStringRef() );
+			}
+
+			tr.set( dataDistributionModeKey, wr.toStringRef() );
+
+			Void _ = wait( tr.commit() );
+			return oldMode;
+		} catch (Error& e) {
+			TraceEvent("setDDModeRetrying").error(e);
+			Void _ = wait (tr.onError(e));
+		}
+	}
+}
+
 ACTOR Future<Void> waitForExcludedServers( Database cx, vector<AddressExclusion> excl ) {
 	state std::set<AddressExclusion> exclusions( excl.begin(), excl.end() );
 
diff --git a/fdbclient/ManagementAPI.h b/fdbclient/ManagementAPI.h
index e3cf772b49..590ddd8c1a 100644
--- a/fdbclient/ManagementAPI.h
+++ b/fdbclient/ManagementAPI.h
@@ -153,6 +153,8 @@ Future<Void> unlockDatabase( Database const& cx, UID const& id );
 Future<Void> checkDatabaseLock( Transaction* const& tr, UID const& id );
 Future<Void> checkDatabaseLock( Reference<ReadYourWritesTransaction> const& tr, UID const& id );
 
+Future<int> setDDMode( Database const& cx, int const& mode );
+
 // Gets the cluster connection string
 Future<std::vector<NetworkAddress>> getCoordinators( Database const& cx );
 #endif
\ No newline at end of file
diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h
index d501e89fbb..b848087d95 100644
--- a/fdbclient/MasterProxyInterface.h
+++ b/fdbclient/MasterProxyInterface.h
@@ -33,7 +33,7 @@ struct MasterProxyInterface {
 	RequestStream< struct CommitTransactionRequest > commit;
 	RequestStream< struct GetReadVersionRequest > getConsistentReadVersion;  // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent
 															     //   (at some point between when this request is sent and when its response is received, the latest version reported committed)
-	RequestStream< ReplyPromise<vector<StorageServerInterface>> > getKeyServersLocations;
+	RequestStream< ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> > getKeyServersLocations;
 	RequestStream< struct GetStorageServerRejoinInfoRequest > getStorageServerRejoinInfo;
 
 	RequestStream<ReplyPromise<Void>> waitFailure;
diff --git a/fdbserver/MetricLogger.actor.cpp b/fdbclient/MetricLogger.actor.cpp
similarity index 100%
rename from fdbserver/MetricLogger.actor.cpp
rename to fdbclient/MetricLogger.actor.cpp
diff --git a/fdbclient/MetricLogger.h b/fdbclient/MetricLogger.h
new file mode 100755
index 0000000000..96720c363c
--- /dev/null
+++ b/fdbclient/MetricLogger.h
@@ -0,0 +1,25 @@
+/*
+ * MetricLogger.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "NativeAPI.h"
+
+Future<Void> runMetrics( Future<Database> const& fcx, Key const& metricsPrefix );
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 230de8486f..2f2fefee6d 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -602,7 +602,7 @@ Reference<LocationInfo> DatabaseContext::setCachedLocation( const KeyRangeRef& k
 		attempts++;
 		auto r = locationCache.randomRange();
 		Key begin = r.begin(), end = r.end();  // insert invalidates r, so can't be passed a mere reference into it
-		if( begin >= keyServersPrefix )
+		if( begin >= keyServersPrefix && attempts > maxEvictionAttempts / 2)
 			continue;
 		locationCache.insert( KeyRangeRef(begin, end), Reference<LocationInfo>() );
 	}
@@ -875,7 +875,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) {
 	if (!networkOptions.logClientInfo.present())
 		networkOptions.logClientInfo = true;
 
-	g_network = newNet2(NetworkAddress(), false, useMetrics);
+	g_network = newNet2(NetworkAddress(), false, useMetrics || networkOptions.traceDirectory.present());
 	FlowTransport::createInstance(transportId);
 	Net2FileSystem::newFileSystem();
 
@@ -1102,22 +1102,33 @@ ACTOR Future< pair<KeyRange,Reference<LocationInfo>> > getKeyLocation( Database
 
 	state vector<StorageServerInterface> serverInterfaces;
 	state KeyRangeRef range;
-
+	
 	// We assume that not only /FF/keyServers but /FF/serverList is present on the keyServersLocations since we now need both of them to terminate our search. Currently this is guaranteed because nothing after /FF/keyServers is split.
 	if ( ( key.startsWith( serverListPrefix) && (!isBackward || key.size() > serverListPrefix.size()) ) ||
 		( key.startsWith( keyServersPrefix ) && (!isBackward || key.size() > keyServersPrefix.size()) )) {
 		if( info.debugID.present() )
-			g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.Before");
+			g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.Before");	
 		loop {
 			choose {
 				when ( Void _ = wait( cx->onMasterProxiesChanged() ) ) {}
-				when ( vector<StorageServerInterface> s = wait( loadBalance( cx->getMasterProxies(), &MasterProxyInterface::getKeyServersLocations, ReplyPromise<vector<StorageServerInterface>>(), info.taskID ) ) ) {
+				when ( vector<pair<KeyRangeRef, vector<StorageServerInterface>>> keyServersShards = wait( loadBalance( cx->getMasterProxies(), &MasterProxyInterface::getKeyServersLocations, ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>>(), info.taskID ) ) ) {
 					if( info.debugID.present() )
 						g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.After");
-					ASSERT( s.size() );  // There should always be storage servers, except on version 0 which should not get to this function
-					range = KeyRangeRef( keyServersPrefix, allKeys.end );
-					serverInterfaces = s;
-					break;
+					ASSERT( keyServersShards.size() );  // There should always be storage servers, except on version 0 which should not get to this function
+
+					Reference<LocationInfo> cachedLocation;
+					for (pair<KeyRangeRef, vector<StorageServerInterface>> keyServersShard : keyServersShards) {
+						auto locationInfo = cx->setCachedLocation(keyServersShard.first, keyServersShard.second);
+
+						if (isBackward ? (keyServersShard.first.begin < key && keyServersShard.first.end >= key) : keyServersShard.first.contains(key)) {
+							range = keyServersShard.first;
+							cachedLocation = locationInfo;
+						}
+					}
+
+					ASSERT(isBackward ? (range.begin < key && range.end >= key) : range.contains(key));
+
+					return make_pair(range, cachedLocation);
 				}
 			}
 		}
@@ -1654,6 +1665,15 @@ Future<Key> resolveKey( Database const& cx, KeySelector const& key, Version cons
 ACTOR Future<Standalone<RangeResultRef>> getRangeFallback( Database cx, Version version,
 	KeySelector begin, KeySelector end, GetRangeLimits limits, bool reverse, TransactionInfo info )
 {
+	if(version == latestVersion) {
+		state Transaction transaction(cx);
+		transaction.setOption(FDBTransactionOptions::CAUSAL_READ_RISKY);
+		transaction.setOption(FDBTransactionOptions::LOCK_AWARE);
+		transaction.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		Version ver = wait( transaction.getReadVersion() );
+		version = ver;
+	}
+
 	Future<Key> fb = resolveKey(cx, begin, version, info);
 	state Future<Key> fe = resolveKey(cx, end, version, info);
 
@@ -1849,15 +1869,8 @@ ACTOR Future<Standalone<RangeResultRef>> getRange( Database cx, Future<Version>
 				cx->invalidateCache( beginServer.second );
 
 				if (e.code() == error_code_wrong_shard_server) {
-					if (version == latestVersion) {
-						// latestVersion queries are only for keyServersPrefix/*, which shard is guaranteed not to split,
-						//   so we should always be able to use the fast path--try again
-						TEST(true); //Latest version retry fast path
-						TraceEvent("LatestVersionRetryFastPath").detail("KeyBegin", printable(begin.getKey())).detail("KeyEnd", printable(end.getKey()));
-					} else {
 						Standalone<RangeResultRef> result = wait( getRangeFallback(cx, version, originalBegin, originalEnd, originalLimits, reverse, info ) );
 						return result;
-					}
 				}
 
 				Void _ = wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID));
@@ -2424,6 +2437,7 @@ ACTOR static Future<Void> commitDummyTransaction( Database cx, KeyRange range, T
 			tr.info.taskID = info.taskID;
 			tr.setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
 			tr.setOption( FDBTransactionOptions::CAUSAL_WRITE_RISKY );
+			tr.setOption( FDBTransactionOptions::LOCK_AWARE );
 			tr.addReadConflictRange(range);
 			tr.addWriteConflictRange(range);
 			Void _ = wait( tr.commit() );
diff --git a/flow/Notified.h b/fdbclient/Notified.h
similarity index 96%
rename from flow/Notified.h
rename to fdbclient/Notified.h
index 4496b88e11..fd32476a38 100644
--- a/flow/Notified.h
+++ b/fdbclient/Notified.h
@@ -18,11 +18,11 @@
  * limitations under the License.
  */
 
-#ifndef FLOW_NOTIFIED_H
-#define FLOW_NOTIFIED_H
+#ifndef FDBCLIENT_NOTIFIED_H
+#define FDBCLIENT_NOTIFIED_H
 #pragma once
 
-#include "fdbclient/FDBTypes.h"
+#include "FDBTypes.h"
 #include "flow/TDMetric.actor.h"
 
 struct NotifiedVersion {
@@ -78,4 +78,4 @@ private:
 	VersionMetricHandle val;
 };
 
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp
index aea6279574..35d25c6364 100644
--- a/fdbclient/SystemData.cpp
+++ b/fdbclient/SystemData.cpp
@@ -33,7 +33,8 @@ const KeyRef afterAllKeys = LiteralStringRef("\xff\xff\x00");
 const KeyRangeRef keyServersKeys( LiteralStringRef("\xff/keyServers/"), LiteralStringRef("\xff/keyServers0") );
 const KeyRef keyServersPrefix = keyServersKeys.begin;
 const KeyRef keyServersEnd = keyServersKeys.end;
-const KeyRef keyServersKeyServersKey = LiteralStringRef("\xff/keyServers/\xff/keyServers/");
+const KeyRangeRef keyServersKeyServersKeys ( LiteralStringRef("\xff/keyServers/\xff/keyServers/"), LiteralStringRef("\xff/keyServers/\xff/keyServers0"));
+const KeyRef keyServersKeyServersKey = keyServersKeyServersKeys.begin;
 
 const Key keyServersKey( const KeyRef& k ) {
 	return k.withPrefix( keyServersPrefix );
diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h
index 395f3ac121..22f92fd55d 100644
--- a/fdbclient/SystemData.h
+++ b/fdbclient/SystemData.h
@@ -34,7 +34,7 @@ extern const KeyRangeRef allKeys; // '' to systemKeys.end
 extern const KeyRef afterAllKeys;
 
 //    "\xff/keyServers/[[begin]]" := "[[vector<serverID>, vector<serverID>]]"
-extern const KeyRangeRef keyServersKeys;
+extern const KeyRangeRef keyServersKeys, keyServersKeyServersKeys;
 extern const KeyRef keyServersPrefix, keyServersEnd, keyServersKeyServersKey;
 const Key keyServersKey( const KeyRef& k );
 const KeyRef keyServersKey( const KeyRef& k, Arena& arena );
diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj
old mode 100644
new mode 100755
index f4b0723505..4a43526153
--- a/fdbclient/fdbclient.vcxproj
+++ b/fdbclient/fdbclient.vcxproj
@@ -38,6 +38,8 @@
       <EnableCompile Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">false</EnableCompile>
       <EnableCompile Condition="'$(Configuration)|$(Platform)'=='Release|X64'">false</EnableCompile>
     </ActorCompiler>
+    <ClInclude Include="MetricLogger.h" />
+    <ActorCompiler Include="MetricLogger.actor.cpp" />
     <ClInclude Include="FailureMonitorClient.h" />
     <ClInclude Include="FDBOptions.g.h" />
     <ClInclude Include="FDBOptions.h" />
@@ -57,6 +59,7 @@
     <ClInclude Include="MultiVersionTransaction.h" />
     <ClInclude Include="MutationList.h" />
     <ClInclude Include="NativeAPI.h" />
+    <ClInclude Include="Notified.h" />
     <ClInclude Include="ReadYourWrites.h" />
     <ActorCompiler Include="RunTransaction.actor.h" />
     <ClInclude Include="RYWIterator.h" />
@@ -201,4 +204,4 @@
   <Target Name="MyPreCompileSteps" AfterTargets="CLCompile">
     <Exec Command="&quot;$(SolutionDir)bin\$(Configuration)\coveragetool.exe&quot; &quot;$(OutDir)coverage.$(TargetName).xml&quot; @(ActorCompiler -> '%(RelativeDir)%(Filename)%(Extension)', ' ') @(CLInclude -> '%(RelativeDir)%(Filename)%(Extension)', ' ') @(CLCompile -> '%(RelativeDir)%(Filename)%(Extension)', ' ')" />
   </Target>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 6e70dd9eec..c194784a52 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -94,7 +94,7 @@ description is not currently required but encouraged.
     <Option name="disable_client_statistics_logging" code="70"
             description="Disables logging of client statistics, such as sampled transaction activity." />
     <Option name="enable_slow_task_profiling" code="71"
-            description="Enables slow task profiling. Requires trace logging to be enabled." />
+            description="Enables debugging feature to perform slow task profiling. Requires trace logging to be enabled. WARNING: this feature is not recommended for use in production." />
     <Option name="supported_client_versions" code="1000"
             paramType="String" paramDescription="[release version],[source version],[protocol version];..."
             description="This option is set automatically to communicate the list of supported clients to the active client."
diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp
index f9df4584d3..7432317321 100644
--- a/fdbmonitor/fdbmonitor.cpp
+++ b/fdbmonitor/fdbmonitor.cpp
@@ -253,7 +253,8 @@ public:
 	uint32_t restart_delay_reset_interval;
 	double last_start;
 	bool quiet;
-	bool delete_wd40_env;
+	//bool delete_wd40_env;
+	const char *delete_envvars;
 	bool deconfigured;
 	bool kill_on_configuration_change;
 
@@ -261,7 +262,7 @@ public:
 	int pipes[2][2];
 
 	Command() : argv(NULL) { }
-	Command(const CSimpleIni& ini, std::string _section, uint64_t id, fdb_fd_set fds, int* maxfd) : section(_section), argv(NULL), quiet(false), delete_wd40_env(false), fds(fds), deconfigured(false), kill_on_configuration_change(true) {
+	Command(const CSimpleIni& ini, std::string _section, uint64_t id, fdb_fd_set fds, int* maxfd) : section(_section), argv(NULL), quiet(false), delete_envvars(NULL), fds(fds), deconfigured(false), kill_on_configuration_change(true) {
 		char _ssection[strlen(section.c_str()) + 22];
 		snprintf(_ssection, strlen(section.c_str()) + 22, "%s.%llu", section.c_str(), id);
 		ssection = _ssection;
@@ -351,10 +352,8 @@ public:
 		if (q && !strcmp(q, "true"))
 			quiet = true;
 
-		const char* dwe = get_value_multi(ini, "delete_wd40_env", ssection.c_str(), section.c_str(), "general", NULL);
-		if(dwe && !strcmp(dwe, "true")) {
-			delete_wd40_env = true;
-		}
+		const char* del_env = get_value_multi(ini, "delete_envvars", ssection.c_str(), section.c_str(), "general", NULL);
+		delete_envvars = del_env;
 
 		const char* kocc = get_value_multi(ini, "kill_on_configuration_change", ssection.c_str(), section.c_str(), "general", NULL);
 		if(kocc && strcmp(kocc, "true")) {
@@ -373,7 +372,7 @@ public:
 
 		for (auto i : keys) {
 			if (!strcmp(i.pItem, "command") || !strcmp(i.pItem, "restart_delay") || !strcmp(i.pItem, "initial_restart_delay") || !strcmp(i.pItem, "restart_backoff") ||
-				!strcmp(i.pItem, "restart_delay_reset_interval") || !strcmp(i.pItem, "disable_lifecycle_logging") || !strcmp(i.pItem, "delete_wd40_env") ||
+				!strcmp(i.pItem, "restart_delay_reset_interval") || !strcmp(i.pItem, "disable_lifecycle_logging") || !strcmp(i.pItem, "delete_envvars") ||
 				!strcmp(i.pItem, "kill_on_configuration_change"))
 			{
 				continue;
@@ -408,7 +407,7 @@ public:
 	}
 	void update(const Command& other) {
 		quiet = other.quiet;
-		delete_wd40_env = other.delete_wd40_env;
+		delete_envvars = other.delete_envvars;
 		initial_restart_delay = other.initial_restart_delay;
 		max_restart_delay = other.max_restart_delay;
 		restart_backoff = other.restart_backoff;
@@ -474,12 +473,21 @@ void start_process(Command* cmd, uint64_t id, uid_t uid, gid_t gid, int delay, s
 		signal(SIGINT, SIG_DFL);
 		signal(SIGTERM, SIG_DFL);
 
-		if(cmd->delete_wd40_env) {
-			/* remove WD40 environment variables */
-			if(unsetenv("WD40_BV") || unsetenv("WD40_IS_MY_DADDY") || unsetenv("CONF_BUILD_VERSION")) {
-				log_err("unsetenv", errno, "Failed to remove parent environment variables");
-				exit(1);
-			}
+		if(cmd->delete_envvars != NULL && std::strlen(cmd->delete_envvars) > 0) {
+			std::string vars(cmd->delete_envvars);
+			size_t start = 0;
+			do {
+				size_t bound = vars.find(" ", start);
+				std::string var = vars.substr(start, bound - start);
+				log_msg(LOG_INFO, "Deleting parent environment variable: \"%s\"\n", var.c_str());
+				if(unsetenv(var.c_str())) {
+					log_err("unsetenv", errno, "Failed to remove parent environment variable: %s\n", var.c_str());
+					exit(1);
+				}
+				start = bound;
+				while(vars[start] == ' ')
+					start++;
+			} while(start <= vars.length());
 		}
 
 		dup2( cmd->pipes[0][1], fileno(stdout) );
diff --git a/fdbrpc/AsyncFileKAIO.actor.h b/fdbrpc/AsyncFileKAIO.actor.h
index 023af64697..81cc3870f1 100644
--- a/fdbrpc/AsyncFileKAIO.actor.h
+++ b/fdbrpc/AsyncFileKAIO.actor.h
@@ -268,6 +268,7 @@ public:
 		}
 
 		lastFileSize = nextFileSize = size;
+
 		return Void();
 	}
 
@@ -503,7 +504,7 @@ private:
 		}
 
 		void setIOTimeout(double timeout) {
-			ioTimeout = timeout;
+			ioTimeout = fabs(timeout);
 			timeoutWarnOnly = timeout < 0;
 		}
 
@@ -550,6 +551,7 @@ private:
 	static Context ctx;
 
 	explicit AsyncFileKAIO(int fd, int flags, std::string const& filename) : fd(fd), flags(flags), filename(filename), failed(false) {
+
 		if( !g_network->isSimulated() ) {
 			countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename);
 			countFileLogicalReads.init( LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename);
diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h
index 951147b162..912beb5e9c 100644
--- a/fdbrpc/AsyncFileNonDurable.actor.h
+++ b/fdbrpc/AsyncFileNonDurable.actor.h
@@ -213,8 +213,9 @@ public:
 			//If we are in the process of deleting a file, we can't let someone else modify it at the same time.  We therefore block the creation of new files until deletion is complete
 			state std::map<std::string, Future<Void>>::iterator deletedFile = filesBeingDeleted.find(filename);
 			if(deletedFile != filesBeingDeleted.end()) {
-				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete").detail("Filename", filename);
+				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete1").detail("Filename", filename);
 				Void _ = wait( deletedFile->second || shutdown );
+				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete2").detail("Filename", filename);
 				if(shutdown.isReady())
 					throw io_error().asInjectedFault();
 			}
@@ -425,7 +426,7 @@ private:
 			std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(offset, length, true, writeEnded);
 
 			if(BUGGIFY_WITH_PROB(0.001))
-				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY));
+				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture());
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, length) || self->killed.getFuture());
 
@@ -547,7 +548,7 @@ private:
 			std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(size, -1, true, truncateEnded);
 
 			if(BUGGIFY_WITH_PROB(0.001))
-				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY));
+				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture());
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, 0) || self->killed.getFuture());
 
@@ -711,35 +712,44 @@ private:
 
 	//Finishes all outstanding actors on an AsyncFileNonDurable and then deletes it
 	ACTOR Future<Void> deleteFile(AsyncFileNonDurable *self) {
-		//We must run on the main thread (instead of a SQLite coroutine).  We don't want to signal any promises from a coroutine, so we switch at the beginning
-		//of this ACTOR
-		Void _ = wait(self->returnToMainThread());
+		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state int currentTaskID = g_network->getCurrentTask();
+		state std::string filename = self->filename;
 
-		//Make sure all writes have gone through.
-		Promise<bool> startSyncPromise = self->startSyncPromise;
-		self->startSyncPromise = Promise<bool>();
-		startSyncPromise.send(true);
+		Void _ = wait( g_simulator.onMachine( currentProcess ) );
+		try {
+			//Make sure all writes have gone through.
+			Promise<bool> startSyncPromise = self->startSyncPromise;
+			self->startSyncPromise = Promise<bool>();
+			startSyncPromise.send(true);
 
-		std::vector<Future<Void>> outstandingModifications;
+			std::vector<Future<Void>> outstandingModifications;
 
-		for(auto itr = self->pendingModifications.ranges().begin(); itr != self->pendingModifications.ranges().end(); ++itr)
-			if(itr->value().isValid() && !itr->value().isReady())
-				outstandingModifications.push_back(itr->value());
+			for(auto itr = self->pendingModifications.ranges().begin(); itr != self->pendingModifications.ranges().end(); ++itr)
+				if(itr->value().isValid() && !itr->value().isReady())
+					outstandingModifications.push_back(itr->value());
 
-		//Ignore errors here so that all modifications can finish
-		Void _ = wait(waitForAllReady(outstandingModifications));
+			//Ignore errors here so that all modifications can finish
+			Void _ = wait(waitForAllReady(outstandingModifications));
 
-		//Make sure we aren't in the process of killing the file
-		if(self->killed.isSet())
-			Void _ = wait(self->killComplete.getFuture());
+			//Make sure we aren't in the process of killing the file
+			if(self->killed.isSet())
+				Void _ = wait(self->killComplete.getFuture());
 
-		//Remove this file from the filesBeingDeleted map so that new files can be created with this filename
-		g_simulator.getMachineByNetworkAddress( self->openedAddress )->closingFiles.erase(self->getFilename());
-		AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
-		//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
+			//Remove this file from the filesBeingDeleted map so that new files can be created with this filename
+			g_simulator.getMachineByNetworkAddress( self->openedAddress )->closingFiles.erase(self->getFilename());
+			g_simulator.getMachineByNetworkAddress( self->openedAddress )->deletingFiles.erase(self->getFilename());
+			AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
+			//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
 
-		delete self;
-		return Void();
+			delete self;
+			Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+			return Void();
+		} catch( Error &e ) {
+			state Error err = e;
+			Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+			throw err;
+		}
 	}
 };
 
diff --git a/fdbrpc/AsyncFileWriteChecker.cpp b/fdbrpc/AsyncFileWriteChecker.cpp
new file mode 100644
index 0000000000..d1963af549
--- /dev/null
+++ b/fdbrpc/AsyncFileWriteChecker.cpp
@@ -0,0 +1,24 @@
+/*
+ * AsyncFileWriteChecker.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "AsyncFileWriteChecker.h"
+
+int AsyncFileWriteChecker::checksumHistoryBudget;
+int AsyncFileWriteChecker::checksumHistoryPageSize = 4096;
diff --git a/fdbrpc/AsyncFileWriteChecker.h b/fdbrpc/AsyncFileWriteChecker.h
new file mode 100644
index 0000000000..ab8bdb6fe6
--- /dev/null
+++ b/fdbrpc/AsyncFileWriteChecker.h
@@ -0,0 +1,151 @@
+/*
+ * AsyncFileWriteChecker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "IAsyncFile.h"
+
+#if VALGRIND
+#include <memcheck.h>
+#endif
+
+class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFileWriteChecker> {
+public:
+	void addref() { ReferenceCounted<AsyncFileWriteChecker>::addref(); }
+	void delref() { ReferenceCounted<AsyncFileWriteChecker>::delref(); }
+
+	// For read() and write(), the data buffer must remain valid until the future is ready
+	Future<int> read( void* data, int length, int64_t offset ) {
+		return map(m_f->read(data, length, offset), [=](int r) { updateChecksumHistory(false, offset, length, (uint8_t *)data); return r; });
+	}
+	Future<Void> readZeroCopy( void** data, int* length, int64_t offset ) {
+		return map(m_f->readZeroCopy(data, length, offset), [=](Void r) { updateChecksumHistory(false, offset, *length, (uint8_t *)data); return r; });
+	}
+
+	Future<Void> write( void const* data, int length, int64_t offset ) {
+		updateChecksumHistory(true, offset, length, (uint8_t *)data);
+		return m_f->write(data, length, offset);
+	}
+
+	Future<Void> truncate( int64_t size )  {
+		return map(m_f->truncate(size), [=](Void r) {
+			// Truncate the page checksum history if it is in use
+			if( (size / checksumHistoryPageSize) < checksumHistory.size() ) {
+				int oldCapacity = checksumHistory.capacity();
+				checksumHistory.resize(size / checksumHistoryPageSize);
+				checksumHistoryBudget -= (checksumHistory.capacity() - oldCapacity);
+			}
+			return r;
+		});
+	}
+
+	Future<Void> sync() { return m_f->sync(); }
+	Future<Void> flush() { return m_f->flush(); }
+	Future<int64_t> size() { return m_f->size(); }
+	std::string getFilename() { return m_f->getFilename(); }
+	void releaseZeroCopy( void* data, int length, int64_t offset )  { return m_f->releaseZeroCopy(data, length, offset); }
+	int64_t debugFD() { return m_f->debugFD(); }
+
+	AsyncFileWriteChecker(Reference<IAsyncFile> f) : m_f(f) {
+		// Initialize the static history budget the first time (and only the first time) a file is opened.
+		static int _ = checksumHistoryBudget = FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY;
+
+		// Adjust the budget by the initial capacity of history, which should be 0 but maybe not for some implementations.
+		checksumHistoryBudget -= checksumHistory.capacity();
+	}
+
+
+	virtual ~AsyncFileWriteChecker() {
+		checksumHistoryBudget += checksumHistory.capacity();
+	}
+
+private:
+	Reference<IAsyncFile> m_f;
+
+	struct WriteInfo {
+		WriteInfo() : checksum(0), timestamp(0) {}
+		uint32_t checksum;
+		uint32_t timestamp;
+	};
+
+	std::vector<WriteInfo> checksumHistory;
+	// This is the most page checksum history blocks we will use across all files.
+	static int checksumHistoryBudget;
+	static int checksumHistoryPageSize;
+
+	// Update or check checksum(s) in history for any full pages covered by this operation
+	void updateChecksumHistory(bool write, int64_t offset, int len, uint8_t *buf) {
+		// Check or set each full block in the the range
+		int page = offset / checksumHistoryPageSize;            // First page number
+		int slack = offset % checksumHistoryPageSize;           // Bytes after most recent page boundary
+		uint8_t *start = buf;                                   // Position in buffer to start checking from
+		// If offset is not page-aligned, move to next page and adjust start
+		if(slack != 0) {
+			++page;
+			start += (checksumHistoryPageSize - slack);
+		}
+		int pageEnd = (offset + len) / checksumHistoryPageSize; // Last page plus 1
+
+		// Make sure history is large enough or limit pageEnd
+		if(checksumHistory.size() < pageEnd) {
+			if(checksumHistoryBudget > 0) {
+				// Resize history and update budget based on capacity change
+				auto initialCapacity = checksumHistory.capacity();
+				checksumHistory.resize(checksumHistory.size() + std::min<int>(checksumHistoryBudget, pageEnd - checksumHistory.size()));
+				checksumHistoryBudget -= (checksumHistory.capacity() - initialCapacity);
+			}
+
+			// Limit pageEnd to end of history, which works whether or not all of the desired
+			// history slots were allocated.
+			pageEnd = checksumHistory.size();
+		}
+
+		while(page < pageEnd) {
+			uint32_t checksum = hashlittle(start, checksumHistoryPageSize, 0xab12fd93);
+			WriteInfo &history = checksumHistory[page];
+			//printf("%d %d %u %u\n", write, page, checksum, history.checksum);
+
+#if VALGRIND
+			// It's possible we'll read or write a page where not all of the data is defined, but the checksum of the page is still valid
+			VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE(&checksum, sizeof(uint32_t));
+#endif
+
+			// For writes, just update the stored sum
+			if(write) {
+				history.timestamp = (uint32_t)now();
+				history.checksum = checksum;
+			}
+			else {
+				if(history.checksum != 0 && history.checksum != checksum) {
+					// For reads, verify the stored sum if it is not 0.  If it fails, clear it.
+					TraceEvent (SevError, "AsyncFileLostWriteDetected")
+						.detail("Filename", m_f->getFilename())
+						.detail("PageNumber", page)
+						.detail("ChecksumOfPage", checksum)
+						.detail("ChecksumHistory", history.checksum)
+						.detail("LastWriteTime", history.timestamp)
+						.error(checksum_failed());
+					history.checksum = 0;
+				}
+			}
+
+			start += checksumHistoryPageSize;
+			++page;
+		}
+	}
+};
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index 7175ceceb2..e3b772e6a2 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -137,7 +137,7 @@ class TransportData {
 public:
 	TransportData(uint64_t transportId) 
 	  : endpointNotFoundReceiver(endpoints),
-	    pingReceiver(endpoints),
+		pingReceiver(endpoints),
 		warnAlwaysForLargePacket(true),
 		lastIncompatibleMessage(0),
 		transportId(transportId)
@@ -204,7 +204,9 @@ struct ConnectPacket {
 static_assert( sizeof(ConnectPacket) == CONNECT_PACKET_V2_SIZE, "ConnectPacket packed incorrectly" );
 #pragma pack( pop )
 
-static Future<Void> connectionReader( TransportData* const& transport, Reference<IConnection> const& conn, bool const& isOutgoing, Promise<NetworkAddress> const& onPeerAddress );
+static Future<Void> connectionReader( TransportData* const& transport, Reference<IConnection> const& conn, Peer* const& peer, Promise<Peer*> const& onConnected );
+
+static PacketID sendPacket( TransportData* self, ISerializeSource const& what, const Endpoint& destination, bool reliable );
 
 struct Peer : NonCopyable {
 	// FIXME: Peers don't die!
@@ -215,12 +217,14 @@ struct Peer : NonCopyable {
 	ReliablePacketList reliable;
 	AsyncTrigger dataToSend;  // Triggered when unsent.empty() becomes false
 	Future<Void> connect;
+	AsyncVar<bool> incompatibleDataRead;
+	bool compatible;
 	bool outgoingConnectionIdle;  // We don't actually have a connection open and aren't trying to open one because we don't have anything to send
 	double lastConnectTime;
 	double reconnectionDelay;
 
 	explicit Peer( TransportData* transport, NetworkAddress const& destination, bool doConnect = true ) 
-		: transport(transport), destination(destination), outgoingConnectionIdle(!doConnect), lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME)
+		: transport(transport), destination(destination), outgoingConnectionIdle(!doConnect), lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), compatible(true)
 	{
 		if(doConnect) {
 			connect = connectionKeeper(this);
@@ -293,8 +297,7 @@ struct Peer : NonCopyable {
 		}
 	}
 
-	ACTOR static Future<Void> connectionMonitor( Peer* peer ) {
-
+	ACTOR static Future<Void> connectionMonitor( Peer *peer ) {
 		state RequestStream< ReplyPromise<Void> > remotePing( Endpoint( peer->destination, WLTOKEN_PING_PACKET ) );
 
 		loop {
@@ -305,9 +308,11 @@ struct Peer : NonCopyable {
 			state ReplyPromise<Void> reply;
 			FlowTransport::transport().sendUnreliable( SerializeSource<ReplyPromise<Void>>(reply), remotePing.getEndpoint() );
 
+			peer->incompatibleDataRead.set(false);
 			choose {
 				when (Void _ = wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) { TraceEvent("ConnectionTimeout").detail("WithAddr", peer->destination); throw connection_failed(); }
 				when (Void _ = wait( reply.getFuture() )) {}
+				when (Void _ = wait( peer->incompatibleDataRead.onChange())) {}
 			}
 		}
 	}
@@ -364,18 +369,17 @@ struct Peer : NonCopyable {
 					Reference<IConnection> _conn = wait( timeout( INetworkConnections::net()->connect(self->destination), FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT, Reference<IConnection>() ) );
 					if (_conn) {
 						conn = _conn;
-						TraceEvent("ConnEstablishedTo", conn->getDebugID()).detail("PeerAddr", self->destination);
+						TraceEvent("ConnectionExchangingConnectPacket", conn->getDebugID()).detail("PeerAddr", self->destination);
 						self->prependConnectPacket();
 					} else {
-						TraceEvent("ConnTimedOut", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination);
+						TraceEvent("ConnectionTimedOut", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination);
 						throw connection_failed();
 					}
 
-					reader = connectionReader( self->transport, conn, true, Promise<NetworkAddress>() );
+					reader = connectionReader( self->transport, conn, self, Promise<Peer*>());
 				} else {
 					self->outgoingConnectionIdle = false;
 				}
-				self->transport->countConnEstablished++;
 
 				Void _ = wait( connectionWriter( self, conn ) || reader || connectionMonitor(self) );
 
@@ -389,12 +393,17 @@ struct Peer : NonCopyable {
 				self->discardUnreliablePackets();
 				reader = Future<Void>();
 				bool ok = e.code() == error_code_connection_failed || e.code() == error_code_actor_cancelled || ( g_network->isSimulated() && e.code() == error_code_checksum_failed );
-				TraceEvent(ok ? SevInfo : SevError, "ConnectionClosed", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination).error(e, true);
 
-				if (ok)
-					self->transport->countConnClosedWithoutError++;
-				else
-					self->transport->countConnClosedWithError++;
+				if(self->compatible) {
+					TraceEvent(ok ? SevInfo : SevError, "ConnectionClosed", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination).error(e, true);
+					if (ok)
+						self->transport->countConnClosedWithoutError++;
+					else
+						self->transport->countConnClosedWithError++;
+				}
+				else {
+					TraceEvent(ok ? SevInfo : SevError, "IncompatibleConnectionClosed", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination).error(e, true);
+				}
 
 				if (conn) {
 					conn->close();
@@ -408,8 +417,6 @@ struct Peer : NonCopyable {
 	}
 };
 
-static PacketID sendPacket( TransportData* self, ISerializeSource const& what, const Endpoint& destination, bool reliable );
-
 ACTOR static void deliver( TransportData* self, Endpoint destination, ArenaReader reader, bool inReadSocket ) {
 	int priority = self->endpoints.getPriority(destination.token);
 	if (priority < TaskReadSocket || !inReadSocket) {
@@ -449,7 +456,7 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
 	uint8_t* p = unprocessed_begin;
 
 	bool checksumEnabled = true;
-	if (!g_network->isSimulated() && transport->localAddress.isTLS() && peerAddress.isTLS()) {
+	if (transport->localAddress.isTLS() || peerAddress.isTLS()) {
 		checksumEnabled = false;
 	}
 
@@ -533,7 +540,9 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
 ACTOR static Future<Void> connectionReader(
 		TransportData* transport,
 		Reference<IConnection> conn, 
-		bool isOutgoing, Promise<NetworkAddress> onPeerAddress ) {
+		Peer *peer,
+		Promise<Peer*> onConnected) 
+{
 	// This actor exists whenever there is an open or opening connection, whether incoming or outgoing
 	// For incoming connections conn is set and peer is initially NULL; for outgoing connections it is the reverse
 
@@ -542,12 +551,14 @@ ACTOR static Future<Void> connectionReader(
 	state uint8_t* unprocessed_end = NULL;
 	state uint8_t* buffer_end = NULL;
 	state bool expectConnectPacket = true;
+	state bool compatible = false;
 	state NetworkAddress peerAddress;
 	state uint64_t peerProtocolVersion = 0;
 
 	peerAddress = conn->getPeerAddress();
-	if (!isOutgoing)
+	if (peer == nullptr) { 
 		ASSERT( !peerAddress.isPublic() );
+	}
 
 	loop {
 		loop {
@@ -592,7 +603,8 @@ ACTOR static Future<Void> connectionReader(
 									.detail("LocalVersion", currentProtocolVersion)
 									.detail("RejectedVersion", p->protocolVersion)
 									.detail("VersionMask", compatibleProtocolVersionMask)
-									.detail("Peer", p->canonicalRemotePort ? NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort ) : conn->getPeerAddress());
+									.detail("Peer", p->canonicalRemotePort ? NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort ) : conn->getPeerAddress())
+									.detail("ConnectionId", connectionId);
 								transport->lastIncompatibleMessage = now();
 							}
 							if(!transport->incompatiblePeers.count(addr)) {
@@ -601,7 +613,20 @@ ACTOR static Future<Void> connectionReader(
 						} else if(connectionId > 1) {
 							transport->multiVersionConnections[connectionId] = now() + FLOW_KNOBS->CONNECTION_ID_TIMEOUT;
 						}
-						throw incompatible_protocol_version();
+
+						compatible = false;
+						if(p->protocolVersion < 0x0FDB00A551000000LL) {
+							// Older versions expected us to hang up. It may work even if we don't hang up here, but it's safer to keep the old behavior.
+							throw incompatible_protocol_version();
+						}
+					}
+					else {
+						compatible = true;
+						TraceEvent("ConnectionEstablished", conn->getDebugID())
+							.detail("Peer", conn->getPeerAddress())
+							.detail("ConnectionId", connectionId);
+
+						transport->countConnEstablished++;
 					}
 
 					if(connectionId > 1) {
@@ -611,21 +636,29 @@ ACTOR static Future<Void> connectionReader(
 					expectConnectPacket = false;
 
 					peerProtocolVersion = p->protocolVersion;
-					if (isOutgoing) {
+					if (peer != nullptr) {
 						// Outgoing connection; port information should be what we expect
 						TraceEvent("ConnectedOutgoing").detail("PeerAddr", NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort ) );
+						peer->compatible = compatible;
 						ASSERT( p->canonicalRemotePort == peerAddress.port );
 					} else {
 						if (p->canonicalRemotePort) {
 							peerAddress = NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort, true, peerAddress.isTLS() );
 						}
-						onPeerAddress.send( peerAddress );
+						peer = transport->getPeer(peerAddress);
+						peer->compatible = compatible;
+						onConnected.send( peer );
 						Void _ = wait( delay(0) );  // Check for cancellation
 					}
 				}
 			}
-			if (!expectConnectPacket)
+			if (compatible) {
 				scanPackets( transport, unprocessed_begin, unprocessed_end, arena, peerAddress, peerProtocolVersion );
+			}
+			else if(!expectConnectPacket) {
+				unprocessed_begin = unprocessed_end;
+				peer->incompatibleDataRead.set(true);
+			}
 
 			if (readWillBlock)
 				break;
@@ -640,12 +673,11 @@ ACTOR static Future<Void> connectionReader(
 
 ACTOR static Future<Void> connectionIncoming( TransportData* self, Reference<IConnection> conn ) {
 	try {
-		state Promise<NetworkAddress> onPeerAddress;
-		state Future<Void> reader = connectionReader( self, conn, false, onPeerAddress );
+		state Promise<Peer*> onConnected;
+		state Future<Void> reader = connectionReader( self, conn, nullptr, onConnected );
 		choose {
 			when( Void _ = wait( reader ) ) { ASSERT(false); return Void(); }
-			when( NetworkAddress pa = wait( onPeerAddress.getFuture() ) ) {
-				Peer* p = self->getPeer( pa, false );
+			when( Peer *p = wait( onConnected.getFuture() ) ) {
 				p->onIncomingConnection( conn, reader );
 			}
 			when( Void _ = wait( delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT) ) ) {
@@ -777,7 +809,7 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 		return (PacketID)NULL;
 	} else {
 		bool checksumEnabled = true;
-		if (!g_network->isSimulated() && self->localAddress.isTLS() && destination.address.isTLS()) {
+		if (self->localAddress.isTLS() || destination.address.isTLS()) {
 			checksumEnabled = false;
 		}
 
@@ -785,9 +817,9 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 
 		Peer* peer = self->getPeer(destination.address);
 
-		// If there isn't an open connection or public address, we can't send
-		if (peer->outgoingConnectionIdle && !destination.address.isPublic()) {
-			TEST(true);  // Can't send to private address without an open connection
+		// If there isn't an open connection, a public address, or the peer isn't compatible, we can't send
+		if ((peer->outgoingConnectionIdle && !destination.address.isPublic()) || (!peer->compatible && destination.token != WLTOKEN_PING_PACKET)) {
+			TEST(true);  // Can't send to private address without a compatible open connection
 			return (PacketID)NULL;
 		}
 
diff --git a/fdbrpc/IAsyncFile.actor.cpp b/fdbrpc/IAsyncFile.actor.cpp
index a746e20bfd..04dc2b7733 100644
--- a/fdbrpc/IAsyncFile.actor.cpp
+++ b/fdbrpc/IAsyncFile.actor.cpp
@@ -27,25 +27,36 @@
 
 IAsyncFile::IAsyncFile(){};
 
-ACTOR static Future<Void> incrementalDeleteHelper( std::string filename, int64_t truncateAmt, double interval ){
+ACTOR static Future<Void> incrementalDeleteHelper( std::string filename, bool mustBeDurable, int64_t truncateAmt, double interval ) {
+	state Reference<IAsyncFile> file;
+	state int64_t remainingFileSize;
+	state bool exists = fileExists(filename);
 
-	state Reference<IAsyncFile> f = wait(
-		IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READWRITE, 0));
-	state int64_t filesize = wait(f->size());
-	state int64_t i = filesize;
+	if(exists) {
+		Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED, 0));
+		file = f;
 
-	Void _ = wait(IAsyncFileSystem::filesystem()->deleteFile(filename, true));
-	for( ;i > 0; i -= truncateAmt ){
-		Void _ = wait(f->truncate(i));
-		Void _ = wait(f->sync());
-		Void _ = wait(delay(interval));
+		int64_t fileSize = wait(file->size());
+		remainingFileSize = fileSize;
 	}
+
+	Void _ = wait(IAsyncFileSystem::filesystem()->deleteFile(filename, mustBeDurable));
+
+	if(exists) {
+		for( ; remainingFileSize > 0; remainingFileSize -= truncateAmt ){
+			Void _ = wait(file->truncate(remainingFileSize));
+			Void _ = wait(file->sync());
+			Void _ = wait(delay(interval));
+		}
+	}
+
 	return Void();
 }
 
-Future<Void> IAsyncFile::incrementalDelete( std::string filename){
+Future<Void> IAsyncFile::incrementalDelete( std::string filename, bool mustBeDurable ) {
 	return uncancellable(incrementalDeleteHelper(
 		filename,
+		mustBeDurable,
 		FLOW_KNOBS->INCREMENTAL_DELETE_TRUNCATE_AMOUNT,
 		FLOW_KNOBS->INCREMENTAL_DELETE_INTERVAL));
 }
@@ -63,6 +74,6 @@ TEST_CASE( "fileio/incrementalDelete" ) {
 	Void _ = wait(f->truncate(fileSize));
 	//close the file by deleting the reference
 	f.clear();
-	Void _ = wait(IAsyncFile::incrementalDelete(filename));
+	Void _ = wait(IAsyncFile::incrementalDelete(filename, true));
 	return Void();
 }
diff --git a/fdbrpc/IAsyncFile.h b/fdbrpc/IAsyncFile.h
index 8f4ec32c71..4da4fb7a8b 100644
--- a/fdbrpc/IAsyncFile.h
+++ b/fdbrpc/IAsyncFile.h
@@ -57,8 +57,10 @@ public:
 	virtual Future<Void> flush() { return Void();  }      // Sends previous writes to the OS if they have been buffered in memory, but does not make them power safe
 	virtual Future<int64_t> size() = 0;
 	virtual std::string getFilename() = 0;
-	//start an actor to truncate the file repeatedly so that the operating system doesn't delete it all at once
-	static Future<Void> incrementalDelete( std::string filename);
+
+	// Unlinks a file and then deletes it slowly by truncating the file repeatedly.
+	// If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
+	static Future<Void> incrementalDelete( std::string filename, bool mustBeDurable );
 
 	// Attempt to read the *length bytes at offset without copying.  If successful, a pointer to the
 	//   requested bytes is written to *data, and the number of bytes successfully read is
diff --git a/fdbrpc/ITLSPlugin.h b/fdbrpc/ITLSPlugin.h
index cc05c4053d..f6d1b57791 100644
--- a/fdbrpc/ITLSPlugin.h
+++ b/fdbrpc/ITLSPlugin.h
@@ -120,7 +120,7 @@ struct ITLSPolicy {
 // remaining arguments must be pairs of (const char*); the first of
 // each pair must be a valid XML attribute name, and the second a
 // valid XML attribute value. The final parameter must be NULL.
-typedef void (*ITLSLogFunc)(const char* event, void* uid, bool is_error, ...);
+typedef void (*ITLSLogFunc)(const char* event, void* uid, int is_error, ...);
 
 struct ITLSPlugin {
 	virtual void addref() = 0;
diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp
index 5c661b79ee..7e53cabeac 100644
--- a/fdbrpc/Net2FileSystem.cpp
+++ b/fdbrpc/Net2FileSystem.cpp
@@ -37,6 +37,7 @@
 #include "AsyncFileKAIO.actor.h"
 #include "flow/AsioReactor.h"
 #include "flow/Platform.h"
+#include "AsyncFileWriteChecker.h"
 
 // Opens a file for asynchronous I/O
 Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename, int64_t flags, int64_t mode )
@@ -54,12 +55,17 @@ Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename
 	if ( (flags & IAsyncFile::OPEN_EXCLUSIVE) ) ASSERT( flags & IAsyncFile::OPEN_CREATE );
 	if (!(flags & IAsyncFile::OPEN_UNCACHED))
 		return AsyncFileCached::open(filename, flags, mode);
+
+	Future<Reference<IAsyncFile>> f;
 #ifdef __linux__
 	if ( (flags & IAsyncFile::OPEN_UNBUFFERED) && !(flags & IAsyncFile::OPEN_NO_AIO) )
-		return AsyncFileKAIO::open(filename, flags, mode, NULL);
+		f = AsyncFileKAIO::open(filename, flags, mode, NULL);
+	else
 #endif
-
-	return Net2AsyncFile::open(filename, flags, mode, static_cast<boost::asio::io_service*> ((void*) g_network->global(INetwork::enASIOService)));
+	f = Net2AsyncFile::open(filename, flags, mode, static_cast<boost::asio::io_service*> ((void*) g_network->global(INetwork::enASIOService)));
+	if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
+		f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+	return f;
 }
 
 // Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
diff --git a/fdbrpc/ReplicationPolicy.cpp b/fdbrpc/ReplicationPolicy.cpp
index 621ba793a8..f51b5efb5b 100644
--- a/fdbrpc/ReplicationPolicy.cpp
+++ b/fdbrpc/ReplicationPolicy.cpp
@@ -60,7 +60,7 @@ bool IReplicationPolicy::validateFull(
 		}
 		else if (validate(fromServers->getGroupEntries(), fromServers)) {
 			if (g_replicationdebug > 2) {
-				printf("Error: Validated unsolved policy with all%5lu servers\n", fromServers->size());
+				printf("Error: Validated unsolved policy with all%5d servers\n", fromServers->size());
 			}
 			valid = false;
 		}
@@ -194,7 +194,7 @@ bool PolicyAcross::validate(
 				count ++;
 			}
 			else if (g_replicationdebug > 4) {
-				printf("Across invalid solution:%5lu key: %-7s value: (%3lu) %-10s policy: %-10s => %s\n", itValid.second.size(), _attribKey.c_str(), itValid.first._id, fromServers->valueText(itValid.first).c_str(), _policy->name().c_str(), _policy->info().c_str());
+				printf("Across invalid solution:%5lu key: %-7s value: (%3d) %-10s policy: %-10s => %s\n", itValid.second.size(), _attribKey.c_str(), itValid.first._id, fromServers->valueText(itValid.first).c_str(), _policy->name().c_str(), _policy->info().c_str());
 				if (g_replicationdebug > 5) {
 					for (auto& entry : itValid.second) {
 						printf("   entry: %s\n", fromServers->getEntryInfo(entry).c_str());
diff --git a/fdbrpc/TLSConnection.actor.cpp b/fdbrpc/TLSConnection.actor.cpp
index 275f9fc6fc..a2ca17d7fa 100644
--- a/fdbrpc/TLSConnection.actor.cpp
+++ b/fdbrpc/TLSConnection.actor.cpp
@@ -273,7 +273,7 @@ Reference<ITLSPolicy> TLSOptions::get_policy() {
 	return policy;
 }
 
-static void TLSConnectionLogFunc( const char* event, void* uid_ptr, bool is_error, ... ) {
+static void TLSConnectionLogFunc( const char* event, void* uid_ptr, int is_error, ... ) {
 	UID uid;
 
 	if ( uid_ptr )
diff --git a/fdbrpc/fdbrpc.vcxproj b/fdbrpc/fdbrpc.vcxproj
index 03aed07ef7..608437900c 100644
--- a/fdbrpc/fdbrpc.vcxproj
+++ b/fdbrpc/fdbrpc.vcxproj
@@ -30,6 +30,7 @@
     <ClCompile Include="libb64\cdecode.c" />
     <ClCompile Include="md5\md5.c" />
     <ClCompile Include="Platform.cpp" />
+    <ClCompile Include="AsyncFileWriteChecker.cpp" />
     <ClCompile Include="libcoroutine\Common.c" />
     <ClCompile Include="libcoroutine\Coro.c" />
     <ClCompile Include="Locality.cpp" />
diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp
index 89b5df40db..9e30626193 100644
--- a/fdbrpc/sim2.actor.cpp
+++ b/fdbrpc/sim2.actor.cpp
@@ -31,6 +31,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/Replication.h"
 #include "fdbrpc/ReplicationUtils.h"
+#include "AsyncFileWriteChecker.h"
 
 
 using std::min;
@@ -49,14 +50,13 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l
 			h2 = p->fault_injection_r;
 
 		if (h1 < p->fault_injection_p1*std::numeric_limits<uint32_t>::max()) {
-			TEST(true);
-			TEST(error_code == error_code_io_timeout);
-			TEST(error_code == error_code_io_error);
-			TEST(error_code == error_code_platform_error);
+			TEST(true);                                     // A fault was injected
+			TEST(error_code == error_code_io_timeout);      // An io timeout was injected
+			TEST(error_code == error_code_io_error);        // An io error was injected
+			TEST(error_code == error_code_platform_error);  // A platform error was injected.
 			TraceEvent(SevWarn, "FaultInjected").detail("Context", context).detail("File", file).detail("Line", line).detail("ErrorCode", error_code);
 			if(error_code == error_code_io_timeout) {
 				g_network->setGlobal(INetwork::enASIOTimedOut, (flowGlobalType)true);
-				g_pSimulator->getCurrentProcess()->io_timeout_injected = true;
 			}
 			return true;
 		}
@@ -65,6 +65,29 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l
 	return false;
 }
 
+void ISimulator::displayWorkers() const
+{
+	std::map<std::string, std::vector<ISimulator::ProcessInfo*>>	zoneMap;
+
+	// Create a map of zone Id
+	for (auto processInfo : getAllProcesses()) {
+		std::string dataHall = processInfo->locality.dataHallId().present() ? processInfo->locality.dataHallId().get().printable() : "[unset]";
+		std::string zoneId = processInfo->locality.zoneId().present() ? processInfo->locality.zoneId().get().printable() : "[unset]";
+		zoneMap[format("%-8s  %s", dataHall.c_str(), zoneId.c_str())].push_back(processInfo);
+	}
+
+	printf("DataHall  ZoneId\n");
+	printf("                  Address   Name      Class        Excluded Failed Rebooting Role                                              DataFolder\n");
+	for (auto& zoneRecord : zoneMap) {
+		printf("\n%s\n", zoneRecord.first.c_str());
+		for (auto& processInfo : zoneRecord.second) {
+			printf("                  %9s %-10s%-13s%-8s %-6s %-9s %-48s %-40s\n",
+			processInfo->address.toString().c_str(), processInfo->name, processInfo->startingClass.toString().c_str(), (processInfo->excluded ? "True" : "False"), (processInfo->failed ? "True" : "False"), (processInfo->rebooting ? "True" : "False"), getRoles(processInfo->address).c_str(), processInfo->dataFolder);
+		}
+	}
+
+	return;
+}
 
 namespace std {
 template<>
@@ -863,16 +886,29 @@ public:
 		// This is a _rudimentary_ simulation of the untrustworthiness of non-durable deletes and the possibility of
 		// rebooting during a durable one.  It isn't perfect: for example, on real filesystems testing
 		// for the existence of a non-durably deleted file BEFORE a reboot will show that it apparently doesn't exist.
-		g_simulator.getCurrentProcess()->machine->openFiles.erase(filename);
+		if(g_simulator.getCurrentProcess()->machine->openFiles.count(filename)) {
+			g_simulator.getCurrentProcess()->machine->openFiles.erase(filename);
+			g_simulator.getCurrentProcess()->machine->deletingFiles.insert(filename);
+		}
 		if ( mustBeDurable || g_random->random01() < 0.5 ) {
-			Void _ = wait( ::delay(0.05 * g_random->random01()) );
-			if (!self->getCurrentProcess()->rebooting) {
-				auto f = IAsyncFileSystem::filesystem(self->net2)->deleteFile(filename, false);
-				ASSERT( f.isReady() );
+			state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+			state int currentTaskID = g_network->getCurrentTask();
+			Void _ = wait( g_simulator.onMachine( currentProcess ) );
+			try {
 				Void _ = wait( ::delay(0.05 * g_random->random01()) );
-				TEST( true );  // Simulated durable delete
+				if (!currentProcess->rebooting) {
+					auto f = IAsyncFileSystem::filesystem(self->net2)->deleteFile(filename, false);
+					ASSERT( f.isReady() );
+					Void _ = wait( ::delay(0.05 * g_random->random01()) );
+					TEST( true );  // Simulated durable delete
+				}
+				Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+				return Void();
+			} catch( Error &e ) {
+				state Error err = e;
+				Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+				throw err;
 			}
-			return Void();
 		} else {
 			TEST( true );  // Simulated non-durable delete
 			return Void();
@@ -946,11 +982,13 @@ public:
 		machine.processes.push_back(m);
 		currentlyRebootingProcesses.erase(address);
 		addressMap[ m->address ] = m;
+		m->excluded = g_simulator.isExcluded(address);
 
 		m->setGlobal(enTDMetrics, (flowGlobalType) &m->tdmetrics);
 		m->setGlobal(enNetworkConnections, (flowGlobalType) m->network);
+		m->setGlobal(enASIOTimedOut, (flowGlobalType) false);
 
-		TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId());
+		TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId()).detail("Excluded", m->excluded);
 
 		// FIXME: Sometimes, connections to/from this process will explicitly close
 
@@ -962,14 +1000,17 @@ public:
 
 		for (auto processInfo : getAllProcesses()) {
 			// Add non-test processes (ie. datahall is not be set for test processes)
-			if (processInfo->startingClass != ProcessClass::TesterClass) {
-				// Do not kill protected processes
-				if (protectedAddresses.count(processInfo->address))
-					processesLeft.push_back(processInfo);
-				else if (processInfo->isAvailable())
+			if (processInfo->isAvailableClass()) {
+				// Ignore excluded machines
+				if (processInfo->excluded)
+					;
+				// Mark all of the unavailable as dead
+				else if (!processInfo->isAvailable())
+					processesDead.push_back(processInfo);
+				else if (protectedAddresses.count(processInfo->address))
 					processesLeft.push_back(processInfo);
 				else
-					processesDead.push_back(processInfo);
+					processesLeft.push_back(processInfo);
 			}
 		}
 		return canKillProcesses(processesLeft, processesDead, KillInstantly, NULL);
@@ -979,16 +1020,20 @@ public:
 	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const
 	{
 		bool	canSurvive = true;
+		int		nQuorum = ((desiredCoordinators+1)/2)*2-1;
+
 		KillType	newKt = kt;
 		if ((kt == KillInstantly) || (kt == InjectFaults) || (kt == RebootAndDelete) || (kt == RebootProcessAndDelete))
 		{
 			LocalityGroup	processesLeft, processesDead;
 			std::vector<LocalityData>	localitiesDead, localitiesLeft, badCombo;
+			std::set<Optional<Standalone<StringRef>>>	uniqueMachines;
 			ASSERT(storagePolicy);
 			ASSERT(tLogPolicy);
 			for (auto processInfo : availableProcesses) {
 				processesLeft.add(processInfo->locality);
 				localitiesLeft.push_back(processInfo->locality);
+				uniqueMachines.insert(processInfo->locality.machineId());
 			}
 			for (auto processInfo : deadProcesses) {
 				processesDead.add(processInfo->locality);
@@ -1014,17 +1059,22 @@ public:
 			}
 			// Reboot and Delete if remaining machines do NOT fulfill policies
 			else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (!processesLeft.validate(tLogPolicy))) {
-				auto newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
+				newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
 				canSurvive = false;
 				TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("tLogPolicy", tLogPolicy->info()).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("RemainingZones", ::describeZones(localitiesLeft)).detail("RemainingDataHalls", ::describeDataHalls(localitiesLeft)).detail("Reason", "tLogPolicy does not validates against remaining processes.");
 			}
 			else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (!processesLeft.validate(storagePolicy))) {
-				auto newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
+				newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
 				canSurvive = false;
 				TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("storagePolicy", storagePolicy->info()).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("RemainingZones", ::describeZones(localitiesLeft)).detail("RemainingDataHalls", ::describeDataHalls(localitiesLeft)).detail("Reason", "storagePolicy does not validates against remaining processes.");
 			}
+			else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (nQuorum > uniqueMachines.size())) {
+				newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
+				canSurvive = false;
+				TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("storagePolicy", storagePolicy->info()).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("RemainingZones", ::describeZones(localitiesLeft)).detail("RemainingDataHalls", ::describeDataHalls(localitiesLeft)).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
+			}
 			else {
-				TraceEvent("CanSurviveKills").detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("DeadZones", ::describeZones(localitiesDead)).detail("DeadDataHalls", ::describeDataHalls(localitiesDead)).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
+				TraceEvent("CanSurviveKills").detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("DeadZones", ::describeZones(localitiesDead)).detail("DeadDataHalls", ::describeDataHalls(localitiesDead)).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("ZonesLeft", ::describeZones(localitiesLeft)).detail("DataHallsLeft", ::describeDataHalls(localitiesLeft)).detail("ValidateRemaining", processesLeft.validate(tLogPolicy));
 			}
 		}
 		if (newKillType) *newKillType = newKt;
@@ -1032,7 +1082,7 @@ public:
 	}
 
 	virtual void destroyProcess( ISimulator::ProcessInfo *p ) {
-		TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("zoneId", p->locality.zoneId()).backtrace();
+		TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("zoneId", p->locality.zoneId());
 		currentlyRebootingProcesses.insert(std::pair<NetworkAddress, ProcessInfo*>(p->address, p));
 		std::vector<ProcessInfo*>& processes = machines[ p->locality.zoneId().get() ].processes;
 		if( p != processes.back() ) {
@@ -1048,12 +1098,12 @@ public:
 		TEST( kt == InjectFaults ); // Simulated machine was killed with faults
 
 		if (kt == KillInstantly) {
-			TraceEvent(SevWarn, "FailMachine").detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).backtrace();
+			TraceEvent(SevWarn, "FailMachine", machine->locality.zoneId()).detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).detail("Protected", protectedAddresses.count(machine->address)).backtrace();
 			// This will remove all the "tracked" messages that came from the machine being killed
 			latestEventCache.clear();
 			machine->failed = true;
 		} else if (kt == InjectFaults) {
-			TraceEvent(SevWarn, "FaultMachine").detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).backtrace();
+			TraceEvent(SevWarn, "FaultMachine", machine->locality.zoneId()).detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).detail("Protected", protectedAddresses.count(machine->address)).backtrace();
 			should_inject_fault = simulator_should_inject_fault;
 			machine->fault_injection_r = g_random->randomUniqueID().first();
 			machine->fault_injection_p1 = 0.1;
@@ -1064,8 +1114,10 @@ public:
 		ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting);
 	}
 	virtual void rebootProcess( ProcessInfo* process, KillType kt ) {
-		if( kt == RebootProcessAndDelete && protectedAddresses.count(process->address) )
+		if( kt == RebootProcessAndDelete && protectedAddresses.count(process->address) ) {
+			TraceEvent("RebootChanged").detail("ZoneId", process->locality.describeZone()).detail("KillType", RebootProcess).detail("OrigKillType", kt).detail("Reason", "Protected process");
 			kt = RebootProcess;
+		}
 		doReboot( process, kt );
 	}
 	virtual void rebootProcess(Optional<Standalone<StringRef>> zoneId, bool allProcesses ) {
@@ -1105,11 +1157,12 @@ public:
 		auto ktOrig = kt;
 		if (killIsSafe) ASSERT( kt == ISimulator::RebootAndDelete );  // Only types of "safe" kill supported so far
 
-		TEST(true); // Trying to killing a machine 
+		TEST(true); // Trying to killing a machine
 		TEST(kt == KillInstantly); // Trying to kill instantly
 		TEST(kt == InjectFaults);  // Trying to kill by injecting faults
 
 		if(speedUpSimulation && !forceKill) {
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detailext("ZoneId", zoneId).detail("Reason", "Unforced kill within speedy simulation.").backtrace();
 			return false;
 		}
 
@@ -1124,20 +1177,37 @@ public:
 				processesOnMachine++;
 		}
 
+		// Do nothing, if no processes to kill
+		if (processesOnMachine == 0) {
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detailext("ZoneId", zoneId).detail("Reason", "The target had no processes running.").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
+			return false;
+		}
+
 		// Check if machine can be removed, if requested
 		if ((kt == KillInstantly) || (kt == InjectFaults) || (kt == RebootAndDelete) || (kt == RebootProcessAndDelete))
 		{
 			std::vector<ProcessInfo*>	processesLeft, processesDead;
+			int	protectedWorker = 0, unavailable = 0, excluded = 0;
 
 			for (auto machineRec : machines) {
 				for (auto processInfo : machineRec.second.processes) {
 					// Add non-test processes (ie. datahall is not be set for test processes)
-					if (processInfo->startingClass != ProcessClass::TesterClass) {
-						if (protectedAddresses.count(processInfo->address))
-							processesLeft.push_back(processInfo);
-						else if (processInfo->isAvailable() && (machineRec.second.zoneId != zoneId)) {
-							processesLeft.push_back(processInfo);
+					if (processInfo->isAvailableClass()) {
+						// Do not include any excluded machines
+						if (processInfo->excluded) {
+							processesDead.push_back(processInfo);
+							excluded ++;
 						}
+						else if (!processInfo->isAvailable()) {
+							processesDead.push_back(processInfo);
+							unavailable ++;
+						}
+						else if (protectedAddresses.count(processInfo->address)) {
+							processesLeft.push_back(processInfo);
+							protectedWorker ++;
+						}
+						else if (machineRec.second.zoneId != zoneId)
+							processesLeft.push_back(processInfo);
 						// Add processes from dead machines and datacenter machines to dead group
 						else
 							processesDead.push_back(processInfo);
@@ -1148,16 +1218,25 @@ public:
 				if ((kt != Reboot) && (!killIsSafe)) {
 					kt = Reboot;
 				}
-				TraceEvent("ChangedKillMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
+				TraceEvent("ChangedKillMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("Protected", protectedWorker).detail("Unavailable", unavailable).detail("Excluded", excluded).detail("ProtectedTotal", protectedAddresses.size()).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
 			}
 			else if ((kt == KillInstantly) || (kt == InjectFaults)) {
 				TraceEvent("DeadMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
 				for (auto process : processesLeft) {
 					TraceEvent("DeadMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
 				}
+				for (auto process : processesDead) {
+					TraceEvent("DeadMachineVictims", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", describe(*process));
+				}
 			}
 			else {
 				TraceEvent("ClearMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
+				for (auto process : processesLeft) {
+					TraceEvent("ClearMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
+				}
+				for (auto process : processesDead) {
+					TraceEvent("ClearMachineVictims", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", describe(*process));
+				}
 			}
 		}
 
@@ -1166,21 +1245,30 @@ public:
 		// Check if any processes on machine are rebooting
 		if( processesOnMachine != processesPerMachine && kt >= RebootAndDelete ) {
 			TEST(true); //Attempted reboot, but the target did not have all of its processes running
-			TraceEvent(SevWarn, "AbortedReboot", zoneId).detailext("ZoneId", zoneId).detail("Reason", "The target did not have all of its processes running.").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
+			return false;
+		}
+
+		// Check if any processes on machine are rebooting
+		if ( processesOnMachine != processesPerMachine) {
+			TEST(true); //Attempted reboot, but the target did not have all of its processes running
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
 			return false;
 		}
 
 		TraceEvent("KillMachine", zoneId).detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KilledMachines", killedMachines).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig).detail("killIsSafe", killIsSafe);
 		if (kt < RebootAndDelete ) {
+			if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr)
+				killProcess_internal( machines[zoneId].machineProcess, kt );
 			for (auto& process : machines[zoneId].processes) {
-				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString());
+				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()).detail("failed", process->failed).detail("excluded", process->excluded).detail("rebooting", process->rebooting);
 				if (process->startingClass != ProcessClass::TesterClass)
 					killProcess_internal( process, kt );
 			}
 		}
 		else if ( kt == Reboot || killIsSafe) {
 			for (auto& process : machines[zoneId].processes) {
-				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString());
+				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()).detail("failed", process->failed).detail("excluded", process->excluded).detail("rebooting", process->rebooting);
 				if (process->startingClass != ProcessClass::TesterClass)
 					doReboot(process, kt );
 			}
@@ -1196,13 +1284,16 @@ public:
 		int	dcProcesses = 0;
 
 		// Switch to a reboot, if anything protected on machine
-		for (auto& process : processes) {
-			auto processDcId = process->locality.dcId();
-			auto processZoneId = process->locality.zoneId();
+		for (auto& procRecord : processes) {
+			auto processDcId = procRecord->locality.dcId();
+			auto processZoneId = procRecord->locality.zoneId();
 			ASSERT(processZoneId.present());
 			if (processDcId.present() && (processDcId == dcId)) {
-				if (protectedAddresses.count(process->address))
+				if ((kt != Reboot) && (protectedAddresses.count(procRecord->address))) {
 					kt = Reboot;
+					TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig)
+						.detail("Reason", "Datacenter has protected process").detail("ProcessAddress", procRecord->address).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("excluded", procRecord->excluded).detail("Process", describe(*procRecord));
+				}
 				datacenterZones[processZoneId.get()] ++;
 				dcProcesses ++;
 			}
@@ -1215,13 +1306,17 @@ public:
 			for (auto machineRec : machines) {
 				for (auto processInfo : machineRec.second.processes) {
 					// Add non-test processes (ie. datahall is not be set for test processes)
-					if (processInfo->startingClass != ProcessClass::TesterClass) {
-						// Do not kill protected processes
-						if (protectedAddresses.count(processInfo->address))
+					if (processInfo->isAvailableClass()) {
+						// Mark all of the unavailable as dead
+						if (processInfo->excluded)
+							processesDead.push_back(processInfo);
+						else if (!processInfo->isAvailable())
+							processesDead.push_back(processInfo);
+						else if (protectedAddresses.count(processInfo->address))
 							processesLeft.push_back(processInfo);
-						else if (processInfo->isAvailable() && (datacenterZones.find(machineRec.second.zoneId) == datacenterZones.end())) {
+						// Keep all not in the datacenter zones
+						else if (datacenterZones.find(machineRec.second.zoneId) == datacenterZones.end())
 							processesLeft.push_back(processInfo);
-						}
 						else
 							processesDead.push_back(processInfo);
 					}
@@ -1229,7 +1324,7 @@ public:
 			}
 
 			if (!canKillProcesses(processesLeft, processesDead, kt, &kt)) {
-				TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", ktOrig).detail("NewKillType", kt);
+				TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig);
 			}
 			else {
 				TraceEvent("DeadDataCenter").detailext("DataCenter", dcId).detail("KillType", kt).detail("DcZones", datacenterZones.size()).detail("DcProcesses", dcProcesses).detail("ProcessesDead", processesDead.size()).detail("ProcessesLeft", processesLeft.size()).detail("tLogPolicy", storagePolicy->info()).detail("storagePolicy", storagePolicy->info());
@@ -1244,10 +1339,13 @@ public:
 			.detail("DcZones", datacenterZones.size())
 			.detail("DcProcesses", dcProcesses)
 			.detailext("DCID", dcId)
-			.detail("KillType", kt);
+			.detail("KillType", kt)
+			.detail("OrigKillType", ktOrig);
 
 		for (auto& datacenterZone : datacenterZones)
-			killMachine( datacenterZone.first, kt, (kt == RebootAndDelete), true);
+		killMachine( datacenterZone.first, kt, (kt == RebootAndDelete), true);
+// ahm  If above doesn't work, go conservative
+//	killMachine( datacenterZone.first, kt, false, true);
 	}
 	virtual void clogInterface( uint32_t ip, double seconds, ClogMode mode = ClogDefault ) {
 		if (mode == ClogDefault) {
@@ -1391,6 +1489,7 @@ public:
 	std::map<Optional<Standalone<StringRef>>, MachineInfo > machines;
 	std::map<NetworkAddress, ProcessInfo*> addressMap;
 	std::map<ProcessInfo*, Promise<Void>> filesDeadMap;
+	std::set<AddressExclusion>	exclusionSet;
 
 	//tasks is guarded by ISimulator::mutex
 	std::priority_queue<Task, std::vector<Task>> tasks;
@@ -1424,6 +1523,9 @@ static double networkLatency() {
 }
 
 ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) {
+	TraceEvent("RebootingProcessAttempt").detailext("ZoneId", p->locality.zoneId()).detail("KillType", kt).detail("Process", p->toString()).detail("startingClass", p->startingClass.toString()).detail("failed", p->failed).detail("excluded", p->excluded).detail("rebooting", p->rebooting).detail("TaskDefaultDelay", TaskDefaultDelay);
+//	ASSERT(p->failed); //ahm
+
 	Void _ = wait( g_sim2.delay( 0, TaskDefaultDelay, p ) ); // Switch to the machine in question
 
 	try {
@@ -1436,7 +1538,7 @@ ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) {
 
 		if( p->rebooting )
 			return;
-		TraceEvent("RebootingMachine").detail("KillType", kt).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId()).detailext("DataHall", p->locality.dataHallId()).detail("Locality", p->locality.toString());
+		TraceEvent("RebootingProcess").detail("KillType", kt).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId()).detailext("DataHall", p->locality.dataHallId()).detail("Locality", p->locality.toString()).detail("failed", p->failed).detail("excluded", p->excluded).backtrace();
 		p->rebooting = true;
 		p->shutdownSignal.send( kt );
 	} catch (Error& e) {
@@ -1503,14 +1605,20 @@ Future< Reference<class IAsyncFile> > Sim2FileSystem::open( std::string filename
 				actualFilename = filename + ".part";
 				auto partFile = machineCache.find(actualFilename);
 				if(partFile != machineCache.end()) {
-					return AsyncFileDetachable::open(partFile->second);
+					Future<Reference<IAsyncFile>> f = AsyncFileDetachable::open(partFile->second);
+					if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
+						f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+					return f;
 				}
 			}
 			//Simulated disk parameters are shared by the AsyncFileNonDurable and the underlying SimpleFile.  This way, they can both keep up with the time to start the next operation
 			Reference<DiskParameters> diskParameters(new DiskParameters(FLOW_KNOBS->SIM_DISK_IOPS, FLOW_KNOBS->SIM_DISK_BANDWIDTH));
 			machineCache[actualFilename] = AsyncFileNonDurable::open(filename, actualFilename, SimpleFile::open(filename, flags, mode, diskParameters, false), diskParameters);
 		}
-		return AsyncFileDetachable::open( machineCache[actualFilename] );
+		Future<Reference<IAsyncFile>> f = AsyncFileDetachable::open( machineCache[actualFilename] );
+		if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
+			f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+		return f;
 	}
 	else
 		return AsyncFileCached::open(filename, flags, mode);
diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index 92e4594fbf..7643018b25 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -64,27 +64,41 @@ public:
 
 		uint64_t fault_injection_r;
 		double fault_injection_p1, fault_injection_p2;
-		bool io_timeout_injected;
 
 		ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddress address,
 					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder )
 			: name(name), locality(locality), startingClass(startingClass), address(address), dataFolder(dataFolder),
 				network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
 				rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
-				fault_injection_r(0), machine(0), io_timeout_injected(false)
-		{}
+				fault_injection_r(0), machine(0) {}
 
 		Future<KillType> onShutdown() { return shutdownSignal.getFuture(); }
 
 		bool isReliable() const { return !failed && fault_injection_p1 == 0 && fault_injection_p2 == 0; }
 		bool isAvailable() const { return !excluded && isReliable(); }
 
+		// Returns true if the class represents an acceptable worker
+		bool isAvailableClass() const {
+			switch (startingClass._class) {
+				case ProcessClass::UnsetClass: return true;
+				case ProcessClass::StorageClass: return true;
+				case ProcessClass::TransactionClass: return true;
+				case ProcessClass::ResolutionClass: return false;
+				case ProcessClass::ProxyClass: return false;
+				case ProcessClass::MasterClass: return false;
+				case ProcessClass::TesterClass: return false;
+				case ProcessClass::StatelessClass: return false;
+				case ProcessClass::LogClass: return true;
+				default: return false;
+			}
+		}
+
 		inline flowGlobalType global(int id) { return (globals.size() > id) ? globals[id] : NULL; };
 		inline void setGlobal(size_t id, flowGlobalType v) { globals.resize(std::max(globals.size(),id+1)); globals[id] = v; };
 
 		std::string toString() const {
-			return format("name: %s  address: %d.%d.%d.%d:%d  zone: %s  datahall: %s  class: %s  coord: %s data: %s",
-			name, (address.ip>>24)&0xff, (address.ip>>16)&0xff, (address.ip>>8)&0xff, address.ip&0xff, address.port, (locality.zoneId().present() ? locality.zoneId().get().printable().c_str() : "[unset]"), (locality.dataHallId().present() ? locality.dataHallId().get().printable().c_str() : "[unset]"), startingClass.toString().c_str(), coordinationFolder, dataFolder); }
+			return format("name: %s  address: %d.%d.%d.%d:%d  zone: %s  datahall: %s  class: %s  coord: %s data: %s  excluded: %d",
+			name, (address.ip>>24)&0xff, (address.ip>>16)&0xff, (address.ip>>8)&0xff, address.ip&0xff, address.port, (locality.zoneId().present() ? locality.zoneId().get().printable().c_str() : "[unset]"), (locality.dataHallId().present() ? locality.dataHallId().get().printable().c_str() : "[unset]"), startingClass.toString().c_str(), coordinationFolder, dataFolder, excluded); }
 
 		// Members not for external use
 		Promise<KillType> shutdownSignal;
@@ -94,6 +108,7 @@ public:
 		ProcessInfo* machineProcess;
 		std::vector<ProcessInfo*> processes;
 		std::map<std::string, Future<Reference<IAsyncFile>>> openFiles;
+		std::set<std::string> deletingFiles;
 		std::set<std::string> closingFiles;
 		Optional<Standalone<StringRef>>	zoneId;
 
@@ -134,6 +149,84 @@ public:
 	//virtual KillType getMachineKillState( UID zoneID ) = 0;
 	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const = 0;
 	virtual bool isAvailable() const = 0;
+	virtual void displayWorkers() const;
+
+	virtual void addRole(NetworkAddress const& address, std::string const& role) {
+		roleAddresses[address][role] ++;
+		TraceEvent("RoleAdd").detail("Address", address).detail("Role", role).detail("Roles", roleAddresses[address].size()).detail("Value", roleAddresses[address][role]);
+	}
+
+	virtual void removeRole(NetworkAddress const& address, std::string const& role) {
+		auto addressIt = roleAddresses.find(address);
+		if (addressIt != roleAddresses.end()) {
+			auto rolesIt = addressIt->second.find(role);
+			if (rolesIt != addressIt->second.end()) {
+				if (rolesIt->second > 1) {
+					rolesIt->second --;
+					TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", addressIt->second.size()).detail("Value", rolesIt->second).detail("Result", "Decremented Role");
+				}
+				else {
+					addressIt->second.erase(rolesIt);
+					if (addressIt->second.size()) {
+						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", addressIt->second.size()).detail("Value", 0).detail("Result", "Removed Role");
+					}
+					else {
+						roleAddresses.erase(addressIt);
+						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", 0).detail("Value", 0).detail("Result", "Removed Address");
+					}
+				}
+			}
+			else {
+				TraceEvent(SevWarn,"RoleRemove").detail("Address", address).detail("Role", role).detail("Result", "Role Missing");
+			}
+		}
+		else {
+			TraceEvent(SevWarn,"RoleRemove").detail("Address", address).detail("Role", role).detail("Result", "Address Missing");
+		}
+	}
+
+	virtual std::string getRoles(NetworkAddress const& address, bool skipWorkers = true) const {
+		auto addressIt = roleAddresses.find(address);
+		std::string roleText;
+		if (addressIt != roleAddresses.end()) {
+			for (auto& roleIt : addressIt->second) {
+				if ((!skipWorkers) || (roleIt.first != "Worker"))
+					roleText += roleIt.first + ((roleIt.second > 1) ? format("-%d ", roleIt.second) : " ");
+			}
+		}
+		if (roleText.empty())
+				roleText = "[unset]";
+		return roleText;
+	}
+
+	virtual void excludeAddress(NetworkAddress const& address) {
+		excludedAddresses[address]++;
+		TraceEvent("ExcludeAddress").detail("Address", address).detail("Value", excludedAddresses[address]);
+	}
+
+	virtual void includeAddress(NetworkAddress const& address) {
+		auto addressIt = excludedAddresses.find(address);
+		if (addressIt != excludedAddresses.end()) {
+			if (addressIt->second > 1) {
+				addressIt->second --;
+				TraceEvent("IncludeAddress").detail("Address", address).detail("Value", addressIt->second).detail("Result", "Decremented");
+			}
+			else {
+				excludedAddresses.erase(addressIt);
+				TraceEvent("IncludeAddress").detail("Address", address).detail("Value", 0).detail("Result", "Removed");
+			}
+		}
+		else {
+			TraceEvent(SevWarn,"IncludeAddress").detail("Address", address).detail("Result", "Missing");
+		}
+	}
+	virtual void includeAllAddresses() {
+		TraceEvent("IncludeAddressAll").detail("AddressTotal", excludedAddresses.size());
+		excludedAddresses.clear();
+	}
+	virtual bool isExcluded(NetworkAddress const& address) const {
+		return excludedAddresses.find(address) != excludedAddresses.end();
+	}
 
 	virtual void disableSwapToMachine(Optional<Standalone<StringRef>> zoneId ) {
 		swapsDisabled.insert(zoneId);
@@ -201,6 +294,8 @@ protected:
 
 private:
 	std::set<Optional<Standalone<StringRef>>> swapsDisabled;
+	std::map<NetworkAddress, int> excludedAddresses;
+	std::map<NetworkAddress, std::map<std::string, int>> roleAddresses;
 	bool allSwapsDisabled;
 };
 
diff --git a/fdbserver/ApplyMetadataMutation.h b/fdbserver/ApplyMetadataMutation.h
index 4506d52270..2910a810ca 100644
--- a/fdbserver/ApplyMetadataMutation.h
+++ b/fdbserver/ApplyMetadataMutation.h
@@ -25,10 +25,10 @@
 #include "fdbclient/MutationList.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/BackupAgent.h"
+#include "fdbclient/Notified.h"
 #include "IKeyValueStore.h"
 #include "LogSystem.h"
 #include "LogProtocolMessage.h"
-#include "flow/Notified.h"
 
 static bool isMetadataMutation(MutationRef const& m) {
 	// FIXME: This is conservative - not everything in system keyspace is necessarily processed by applyMetadataMutations
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index c7315c2813..fb2c5b0116 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -218,6 +218,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 		std::vector<LocalityData>							unavailableLocals;
 		LocalitySetRef																					logServerSet;
 		LocalityMap<std::pair<WorkerInterface, ProcessClass>>*	logServerMap;
+		UID 		functionId = g_nondeterministic_random->randomUniqueID();
 		bool		bCompleted = false;
 
 		logServerSet = Reference<LocalitySet>(new LocalityMap<std::pair<WorkerInterface, ProcessClass>>());
@@ -230,7 +231,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			}
 			else {
 				if (it.second.interf.locality.dataHallId().present())
-					TraceEvent(SevWarn,"GWFTADNotAvailable", id)
+					TraceEvent(SevWarn,"GWFTADNotAvailable", functionId)
 						.detail("Fitness", fitness)
 						.detailext("Zone", it.second.interf.locality.zoneId())
 						.detailext("DataHall", it.second.interf.locality.dataHallId())
@@ -243,7 +244,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 						.detail("Locality", it.second.interf.locality.toString())
 						.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 				unavailableLocals.push_back(it.second.interf.locality);
 			}
 		}
@@ -258,12 +260,13 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 				logServerMap->add(worker.first.locality, &worker);
 			}
 			if (logServerSet->size() < conf.tLogReplicationFactor) {
-				TraceEvent(SevWarn,"GWFTADTooFew", id)
+				TraceEvent(SevWarn,"GWFTADTooFew", functionId)
 					.detail("Fitness", fitness)
 					.detail("Processes", logServerSet->size())
 					.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 					.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-					.detail("DesiredLogs", conf.getDesiredLogs());
+					.detail("DesiredLogs", conf.getDesiredLogs())
+					.detail("InterfaceId", id);
 			}
 			else if (logServerSet->size() <= conf.getDesiredLogs()) {
 				ASSERT(conf.tLogPolicy);
@@ -275,12 +278,13 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 					break;
 				}
 				else {
-					TraceEvent(SevWarn,"GWFTADNotAcceptable", id)
+					TraceEvent(SevWarn,"GWFTADNotAcceptable", functionId)
 						.detail("Fitness", fitness)
 						.detail("Processes", logServerSet->size())
 						.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 				}
 			}
 			// Try to select the desired size, if larger
@@ -300,7 +304,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 						results.push_back(*object);
 						tLocalities.push_back(object->first.locality);
 					}
-					TraceEvent("GWFTADBestResults", id)
+					TraceEvent("GWFTADBestResults", functionId)
 						.detail("Fitness", fitness)
 						.detail("Processes", logServerSet->size())
 						.detail("BestCount", bestSet.size())
@@ -308,17 +312,19 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 						.detail("BestDataHalls", ::describeDataHalls(tLocalities))
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
 						.detail("TotalResults", results.size())
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 					bCompleted = true;
 					break;
 				}
 				else {
-					TraceEvent(SevWarn,"GWFTADNoBest", id)
+					TraceEvent(SevWarn,"GWFTADNoBest", functionId)
 						.detail("Fitness", fitness)
 						.detail("Processes", logServerSet->size())
 						.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 				}
 			}
 		}
@@ -331,7 +337,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 					tLocalities.push_back(object->first.locality);
 				}
 
-				TraceEvent(SevWarn, "GetTLogTeamFailed")
+				TraceEvent(SevWarn, "GetTLogTeamFailed", functionId)
 					.detail("Policy", conf.tLogPolicy->info())
 					.detail("Processes", logServerSet->size())
 					.detail("Workers", id_worker.size())
@@ -344,7 +350,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 					.detail("DesiredLogs", conf.getDesiredLogs())
 					.detail("RatingTests",SERVER_KNOBS->POLICY_RATING_TESTS)
 					.detail("checkStable", checkStable)
-					.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS).backtrace();
+					.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS)
+					.detail("InterfaceId", id).backtrace();
 
 			// Free the set
 			logServerSet->clear();
@@ -356,14 +363,25 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			id_used[result.first.locality.processId()]++;
 		}
 
-		TraceEvent("GetTLogTeamDone")
+		TraceEvent("GetTLogTeamDone", functionId)
 			.detail("Completed", bCompleted).detail("Policy", conf.tLogPolicy->info())
 			.detail("Results", results.size()).detail("Processes", logServerSet->size())
 			.detail("Workers", id_worker.size())
 			.detail("Replication", conf.tLogReplicationFactor)
 			.detail("Desired", conf.getDesiredLogs())
 			.detail("RatingTests",SERVER_KNOBS->POLICY_RATING_TESTS)
-			.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS);
+			.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS)
+			.detail("InterfaceId", id);
+
+		for (auto& result : results) {
+			TraceEvent("GetTLogTeamWorker", functionId)
+				.detail("Class", result.second.toString())
+				.detail("Address", result.first.address())
+				.detailext("Zone", result.first.locality.zoneId())
+				.detailext("DataHall", result.first.locality.dataHallId())
+				.detail("isExcludedServer", conf.isExcludedServer(result.first.address()))
+				.detail("isAvailable", IFailureMonitor::failureMonitor().getState(result.first.storage.getEndpoint()).isAvailable());
+		}
 
 		// Free the set
 		logServerSet->clear();
@@ -452,28 +470,23 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 		ProcessClass::Fitness resolverFit;
 		int proxyCount;
 		int resolverCount;
-		int datacenters;
 
-		InDatacenterFitness( ProcessClass::Fitness proxyFit, ProcessClass::Fitness resolverFit, int proxyCount, int resolverCount, int datacenters)
-			: proxyFit(proxyFit), resolverFit(resolverFit), proxyCount(proxyCount), resolverCount(resolverCount), datacenters(datacenters) {}
+		InDatacenterFitness( ProcessClass::Fitness proxyFit, ProcessClass::Fitness resolverFit, int proxyCount, int resolverCount)
+			: proxyFit(proxyFit), resolverFit(resolverFit), proxyCount(proxyCount), resolverCount(resolverCount) {}
 
-		InDatacenterFitness() : proxyFit( ProcessClass::NeverAssign ), resolverFit( ProcessClass::NeverAssign ), datacenters(10000000) {}
+		InDatacenterFitness() : proxyFit( ProcessClass::NeverAssign ), resolverFit( ProcessClass::NeverAssign ) {}
 
 		InDatacenterFitness( vector<std::pair<WorkerInterface, ProcessClass>> proxies, vector<std::pair<WorkerInterface, ProcessClass>> resolvers ) {
-			std::set<Optional<Standalone<StringRef>>> dcs;
 			proxyFit = ProcessClass::BestFit;
 			resolverFit = ProcessClass::BestFit;
 			for(auto it: proxies) {
-				dcs.insert(it.first.locality.dcId());
 				proxyFit = std::max(proxyFit, it.second.machineClassFitness( ProcessClass::Proxy ));
 			}
 			for(auto it: resolvers) {
-				dcs.insert(it.first.locality.dcId());
 				resolverFit = std::max(resolverFit, it.second.machineClassFitness( ProcessClass::Resolver ));
 			}
 			proxyCount = proxies.size();
 			resolverCount = resolvers.size();
-			datacenters = dcs.size();
 		}
 
 		InDatacenterFitness( vector<MasterProxyInterface> proxies, vector<ResolverInterface> resolvers, vector<ProcessClass> proxyClasses, vector<ProcessClass> resolverClasses ) {
@@ -491,12 +504,9 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 
 			proxyCount = proxies.size();
 			resolverCount = resolvers.size();
-			datacenters = dcs.size();
 		}
 
 		bool operator < (InDatacenterFitness const& r) const {
-			if(datacenters != r.datacenters) return datacenters < r.datacenters;
-
 			int lmax = std::max(resolverFit,proxyFit);
 			int lmin = std::min(resolverFit,proxyFit);
 			int rmax = std::max(r.resolverFit,r.proxyFit);
@@ -508,17 +518,16 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			return resolverCount > r.resolverCount;
 		}
 
-		bool operator == (InDatacenterFitness const& r) const { return proxyFit == r.proxyFit && resolverFit == r.resolverFit && datacenters == r.datacenters && proxyCount == r.proxyCount && resolverCount == r.resolverCount; }
+		bool operator == (InDatacenterFitness const& r) const { return proxyFit == r.proxyFit && resolverFit == r.resolverFit && proxyCount == r.proxyCount && resolverCount == r.resolverCount; }
 	};
 
 	struct AcrossDatacenterFitness {
 		ProcessClass::Fitness tlogFit;
 		int tlogCount;
-		int datacenters;
 
-		AcrossDatacenterFitness( ProcessClass::Fitness tlogFit, int tlogCount, int datacenters ) : tlogFit(tlogFit), tlogCount(tlogCount), datacenters(datacenters) {}
+		AcrossDatacenterFitness( ProcessClass::Fitness tlogFit, int tlogCount) : tlogFit(tlogFit), tlogCount(tlogCount) {}
 
-		AcrossDatacenterFitness() : tlogFit( ProcessClass::NeverAssign ), datacenters(0), tlogCount(0) {}
+		AcrossDatacenterFitness() : tlogFit( ProcessClass::NeverAssign ), tlogCount(0) {}
 
 		AcrossDatacenterFitness( vector<std::pair<WorkerInterface, ProcessClass>> tlogs ) {
 			std::set<Optional<Standalone<StringRef>>> dcs;
@@ -527,7 +536,6 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 				dcs.insert(it.first.locality.dcId());
 				tlogFit = std::max(tlogFit, it.second.machineClassFitness( ProcessClass::TLog ));
 			}
-			datacenters = dcs.size();
 			tlogCount = tlogs.size();
 		}
 
@@ -539,17 +547,15 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 				dcs.insert(tlogs[i].interf().locality.dcId());
 				tlogFit = std::max(tlogFit, processClasses[i].machineClassFitness( ProcessClass::TLog ));
 			}
-			datacenters = dcs.size();
 			tlogCount = tlogs.size();
 		}
 
 		bool operator < (AcrossDatacenterFitness const& r) const {
 			if(tlogFit != r.tlogFit) return tlogFit < r.tlogFit;
-			if(tlogCount != r.tlogCount) return tlogCount > r.tlogCount;
-			return datacenters > r.datacenters;
+			return tlogCount > r.tlogCount;
 		}
 
-		bool operator == (AcrossDatacenterFitness const& r) const { return datacenters == r.datacenters && tlogFit == r.tlogFit && tlogCount == r.tlogCount; }
+		bool operator == (AcrossDatacenterFitness const& r) const { return tlogFit == r.tlogFit && tlogCount == r.tlogCount; }
 	};
 
 	std::set<Optional<Standalone<StringRef>>> getDatacenters( DatabaseConfiguration const& conf, bool checkStable = false ) {
@@ -613,8 +619,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			.detail("desiredResolvers", req.configuration.getDesiredResolvers()).detail("actualResolvers", result.resolvers.size());
 
 		if( now() - startTime < SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY &&
-			( AcrossDatacenterFitness(tlogs) > AcrossDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), req.configuration.minDataCenters) ||
-			bestFitness > InDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_PROXY_FITNESS, (ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, req.configuration.getDesiredProxies(), req.configuration.getDesiredResolvers(), 1) ) ) {
+			( AcrossDatacenterFitness(tlogs) > AcrossDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs()) ||
+			bestFitness > InDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_PROXY_FITNESS, (ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, req.configuration.getDesiredProxies(), req.configuration.getDesiredResolvers()) ) ) {
 			throw operation_failed();
 		}
 
@@ -697,10 +703,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 		if(oldInFit < newInFit) return false;
 		if(oldMasterFit > newMasterFit || oldAcrossFit > newAcrossFit || oldInFit > newInFit) {
 			TraceEvent("BetterMasterExists", id).detail("oldMasterFit", oldMasterFit).detail("newMasterFit", newMasterFit)
-				.detail("oldAcrossFitD", oldAcrossFit.datacenters).detail("newAcrossFitD", newAcrossFit.datacenters)
 				.detail("oldAcrossFitC", oldAcrossFit.tlogCount).detail("newAcrossFitC", newAcrossFit.tlogCount)
 				.detail("oldAcrossFitT", oldAcrossFit.tlogFit).detail("newAcrossFitT", newAcrossFit.tlogFit)
-				.detail("oldInFitD", oldInFit.datacenters).detail("newInFitD", newInFit.datacenters)
 				.detail("oldInFitP", oldInFit.proxyFit).detail("newInFitP", newInFit.proxyFit)
 				.detail("oldInFitR", oldInFit.resolverFit).detail("newInFitR", newInFit.resolverFit)
 				.detail("oldInFitPC", oldInFit.proxyCount).detail("newInFitPC", newInFit.proxyCount)
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index 44ce52b1d3..0baefde375 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -438,7 +438,6 @@ struct DDTeamCollection {
 	Database cx;
 	UID masterId;
 	int teamSize;
-	int minDataCenters, desiredDataCenters;
 	IRepPolicyRef replicationPolicy;
 	KeyValueStoreType storeType;
 
@@ -481,15 +480,14 @@ struct DDTeamCollection {
 		PromiseStream<RelocateShard> const& output,
 		Reference<ShardsAffectedByTeamFailure> const& shardsAffectedByTeamFailure,
 		int teamSize,
-		int minDataCenters,
-		int desiredDataCenters,
 		IRepPolicyRef replicationPolicy,
 		KeyValueStoreType storeType,
-		PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > const& serverChanges )
+		PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > const& serverChanges,
+		Future<Void> readyToStart )
 		:cx(cx), masterId(masterId), lock(lock), output(output), shardsAffectedByTeamFailure(shardsAffectedByTeamFailure), doBuildTeams( true ), teamBuilder( Void() ),
-		 teamSize( teamSize ), minDataCenters( minDataCenters ), desiredDataCenters( desiredDataCenters ), replicationPolicy(replicationPolicy), storeType( storeType ), serverChanges(serverChanges),
+		 teamSize( teamSize ), replicationPolicy(replicationPolicy), storeType( storeType ), serverChanges(serverChanges),
 		 initialFailureReactionDelay( delay( BUGGIFY ? 0 : SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskDataDistribution  ) ), healthyTeamCount( 0 ),
-		 initializationDoneActor(logOnCompletion(initialFailureReactionDelay, this)), optimalTeamCount( 0 ), recruitingStream(0), restartRecruiting( SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY ),
+		 initializationDoneActor(logOnCompletion(readyToStart && initialFailureReactionDelay, this)), optimalTeamCount( 0 ), recruitingStream(0), restartRecruiting( SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY ),
 		 unhealthyServers(0)
 	{
 		TraceEvent("DDTrackerStarting", masterId)
@@ -958,7 +956,6 @@ struct DDTeamCollection {
 		int serverCount = 0;
 		int uniqueDataCenters = 0;
 		int uniqueMachines = 0;
-		std::set<Optional<Standalone<StringRef>>> dataCenters;
 		std::set<Optional<Standalone<StringRef>>> machines;
 
 		for(auto i = self->server_info.begin(); i != self->server_info.end(); ++i) {
@@ -966,15 +963,12 @@ struct DDTeamCollection {
 				++serverCount;
 				LocalityData& serverLocation = i->second->lastKnownInterface.locality;
 				machines.insert( serverLocation.zoneId() );
-				// Only add the datacenter if it's set or we don't care (less than two datacenters targeted)
-				if( serverLocation.dcId().present() || self->desiredDataCenters < 2 )
-					dataCenters.insert( serverLocation.dcId() );
 			}
 		}
 		uniqueMachines = machines.size();
 
 		// If there are too few machines to even build teams or there are too few represented datacenters, build no new teams
-		if( uniqueMachines >= self->teamSize && dataCenters.size() >= self->minDataCenters ) {
+		if( uniqueMachines >= self->teamSize ) {
 			desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER*serverCount;
 
 			// Count only properly sized teams against the desired number of teams. This is to prevent "emergency" merged teams (see MoveKeys)
@@ -989,7 +983,7 @@ struct DDTeamCollection {
 			}
 
 			TraceEvent("BuildTeamsBegin", self->masterId).detail("DesiredTeams", desiredTeams).detail("UniqueMachines", uniqueMachines)
-				.detail("TeamSize", self->teamSize).detail("Servers", serverCount).detail("DataCenters", dataCenters.size())
+				.detail("TeamSize", self->teamSize).detail("Servers", serverCount)
 				.detail("CurrentTrackedTeams", self->teams.size()).detail("TeamCount", teamCount);
 
 			if( desiredTeams > teamCount ) {
@@ -1767,14 +1761,13 @@ ACTOR Future<Void> dataDistributionTeamCollection(
 	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
 	MoveKeysLock lock,
 	PromiseStream<RelocateShard> output,
-	UID masterId, int teamSize, int minDataCenters, int desiredDataCenters,
+	UID masterId, int teamSize,
 	IRepPolicyRef replicationPolicy,
 	KeyValueStoreType storeType,
 	PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges,
 	Future<Void> readyToStart )
 {
-	state DDTeamCollection self( cx, masterId, lock, output, shardsAffectedByTeamFailure, teamSize, minDataCenters,
-		desiredDataCenters, replicationPolicy, storeType, serverChanges );
+	state DDTeamCollection self( cx, masterId, lock, output, shardsAffectedByTeamFailure, teamSize, replicationPolicy, storeType, serverChanges, readyToStart );
 
 	state Future<Void> loggingTrigger = Void();
 	state PromiseStream<Void> serverRemoved;
@@ -1885,54 +1878,6 @@ ACTOR Future<bool> isDataDistributionEnabled( Database cx ) {
 	}
 }
 
-ACTOR Future<int> disableDataDistribution( Database cx ) {
-	state Transaction tr(cx);
-	state int oldMode = -1;
-	state BinaryWriter wr(Unversioned());
-	wr << 0;
-
-	loop {
-		try {
-			Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
-			if (oldMode < 0) {
-				oldMode = 1;
-				if (old.present()) {
-					BinaryReader rd(old.get(), Unversioned());
-					rd >> oldMode;
-				}
-			}
-			// SOMEDAY: Write a wrapper in MoveKeys.h
-			BinaryWriter wrMyOwner(Unversioned()); wrMyOwner << dataDistributionModeLock;
-			tr.set( moveKeysLockOwnerKey, wrMyOwner.toStringRef() );
-			tr.set( dataDistributionModeKey, wr.toStringRef() );
-
-			Void _ = wait( tr.commit() );
-			return oldMode;
-		} catch (Error& e) {
-			TraceEvent("disableDDModeRetrying").error(e);
-			Void _ = wait ( tr.onError(e) );
-		}
-	}
-}
-
-ACTOR Future<Void> enableDataDistribution( Database cx, int mode ) {
-	state Transaction tr(cx);
-	state BinaryWriter wr(Unversioned());
-	wr << mode;
-
-	loop {
-		try {
-			Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
-			tr.set( dataDistributionModeKey, wr.toStringRef() );
-			Void _ = wait( tr.commit() );
-			return Void();
-		} catch (Error& e) {
-			TraceEvent("enableDDModeRetrying").error(e);
-			Void _ = wait( tr.onError(e) );
-		}
-	}
-}
-
 //Ensures that the serverKeys key space is properly coalesced
 //This method is only used for testing and is not implemented in a manner that is safe for large databases
 ACTOR Future<Void> debugCheckCoalescing(Database cx) {
@@ -2120,8 +2065,7 @@ ACTOR Future<Void> dataDistribution(
 			actors.push_back( popOldTags( cx, logSystem, recoveryCommitVersion) );
 			actors.push_back( reportErrorsExcept( dataDistributionTracker( initData, cx, shardsAffectedByTeamFailure, output, getShardMetrics, getAverageShardBytes.getFuture(), readyToStart, mi.id() ), "DDTracker", mi.id(), &normalDDQueueErrors() ) );
 			actors.push_back( reportErrorsExcept( dataDistributionQueue( cx, output, getShardMetrics, tci, shardsAffectedByTeamFailure, lock, getAverageShardBytes, mi, configuration.storageTeamSize, configuration.durableStorageQuorum, lastLimited ), "DDQueue", mi.id(), &normalDDQueueErrors() ) );
-			actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( initData, tci, cx, db, shardsAffectedByTeamFailure, lock, output, mi.id(), configuration.storageTeamSize, configuration.minDataCenters,
-				configuration.desiredDataCenters, configuration.storagePolicy, configuration.storageServerStoreType, serverChanges, readyToStart.getFuture() ), "DDTeamCollection", mi.id(), &normalDDQueueErrors() ) );
+			actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( initData, tci, cx, db, shardsAffectedByTeamFailure, lock, output, mi.id(), configuration.storageTeamSize, configuration.storagePolicy, configuration.storageServerStoreType, serverChanges, readyToStart.getFuture() ), "DDTeamCollection", mi.id(), &normalDDQueueErrors() ) );
 
 			Void _ = wait( waitForAll( actors ) );
 			return Void();
@@ -2153,11 +2097,10 @@ DDTeamCollection* testTeamCollection(int teamSize, IRepPolicyRef policy, int pro
 		PromiseStream<RelocateShard>(),
 		Reference<ShardsAffectedByTeamFailure>(new ShardsAffectedByTeamFailure()),
 		teamSize,
-		-1,
-		-1,
 		policy,
 		KeyValueStoreType(),
-		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>()
+		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>(),
+		Future<Void>(Void())
 	);
 
 	for(int id = 1; id <= processCount; id++) {
diff --git a/fdbserver/DataDistribution.h b/fdbserver/DataDistribution.h
index 12396d7bd8..4c1823c62f 100644
--- a/fdbserver/DataDistribution.h
+++ b/fdbserver/DataDistribution.h
@@ -210,7 +210,4 @@ struct ShardSizeBounds {
 ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize);
 
 //Determines the maximum shard size based on the size of the database
-int64_t getMaxShardSize( double dbSizeEstimate );
-
-Future<Void> enableDataDistribution( Database const& cx, int const& mode );
-Future<int> disableDataDistribution( Database const& cx );
\ No newline at end of file
+int64_t getMaxShardSize( double dbSizeEstimate );
\ No newline at end of file
diff --git a/fdbserver/DataDistributionTracker.actor.cpp b/fdbserver/DataDistributionTracker.actor.cpp
index ddad94203d..2b4a958373 100644
--- a/fdbserver/DataDistributionTracker.actor.cpp
+++ b/fdbserver/DataDistributionTracker.actor.cpp
@@ -376,15 +376,6 @@ Future<Void> shardMerger(
 	TEST(true);  // shard to be merged
 	ASSERT( keys.begin > allKeys.begin );
 
-	// We must not merge the keyServers shard
-	if (keys.begin == keyServersPrefix) {
-		TraceEvent(SevError, "LastShardMerge", self->masterId)
-			.detail("ShardKeyBegin", printable(keys.begin))
-			.detail("ShardKeyEnd", printable(keys.end))
-			.detail("TrackerID", trackerId);
-		ASSERT(false);
-	}
-
 	// This will merge shards both before and after "this" shard in keyspace.
 	int shardsMerged = 1;
 	bool forwardComplete = false;
@@ -394,7 +385,7 @@ Future<Void> shardMerger(
 	loop {
 		Optional<StorageMetrics> newMetrics;
 		if( !forwardComplete ) {
-			if( nextIter->range().end == keyServersPrefix ) {
+			if( nextIter->range().end == allKeys.end ) {
 				forwardComplete = true;
 				continue;
 			}
@@ -610,14 +601,6 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker *self,
 	state int lastBegin = -1;
 	state vector<UID> last;
 
-	//The ending shard does not have a shardTracker, so instead just track the size of the shard
-	Reference<AsyncVar<Optional<StorageMetrics>>> endShardSize( new AsyncVar<Optional<StorageMetrics>>() );
-	KeyRangeRef endShardRange( keyServersPrefix, allKeys.end );
-	ShardTrackedData endShardData;
-	endShardData.stats = endShardSize;
-	endShardData.trackBytes = trackShardBytes( self, endShardRange, endShardSize, g_random->randomUniqueID(), false );
-	self->shards.insert( endShardRange, endShardData );
-
 	state int s;
 	for(s=0; s<initData->shards.size(); s++) {
 		state InitialDataDistribution::Team src = initData->shards[s].value.first;
@@ -637,8 +620,7 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker *self,
 
 			if (lastBegin >= 0) {
 				state KeyRangeRef keys( initData->shards[lastBegin].begin, initData->shards[s].begin );
-				if (keys.begin < keyServersPrefix) // disallow spliting of keyServers shard
-					restartShardTrackers( self, keys );
+				restartShardTrackers( self, keys );
 				shardsAffectedByTeamFailure->defineShard( keys );
 				shardsAffectedByTeamFailure->moveShard( keys, last );
 			}
@@ -648,7 +630,7 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker *self,
 		Void _ = wait( yield( TaskDataDistribution ) );
 	}
 
-	Future<Void> initialSize = changeSizes( self, KeyRangeRef(allKeys.begin, keyServersPrefix), 0 );
+	Future<Void> initialSize = changeSizes( self, KeyRangeRef(allKeys.begin, allKeys.end), 0 );
 	self->readyToStart.send(Void());
 	Void _ = wait( initialSize );
 	self->maxShardSizeUpdater = updateMaxShardSize( self->cx->dbName, self->dbSizeEstimate, self->maxShardSize );
diff --git a/fdbserver/DatabaseConfiguration.cpp b/fdbserver/DatabaseConfiguration.cpp
index 9fb7ce998b..ebca2d7828 100644
--- a/fdbserver/DatabaseConfiguration.cpp
+++ b/fdbserver/DatabaseConfiguration.cpp
@@ -29,7 +29,6 @@ DatabaseConfiguration::DatabaseConfiguration()
 void DatabaseConfiguration::resetInternal() {
 	// does NOT reset rawConfiguration
 	initialized = false;
-	minDataCenters = desiredDataCenters = -1;
 	masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = durableStorageQuorum = storageTeamSize = -1;
 	tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
 	autoMasterProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES;
@@ -56,9 +55,6 @@ void DatabaseConfiguration::setDefaultReplicationPolicy() {
 
 bool DatabaseConfiguration::isValid() const {
 	return initialized &&
-		minDataCenters >= 1 &&
-		desiredDataCenters >= 1 &&
-		minDataCenters <= desiredDataCenters &&
 		tLogWriteAntiQuorum >= 0 &&
 		tLogReplicationFactor >= 1 &&
 		durableStorageQuorum >= 1 &&
@@ -81,19 +77,18 @@ std::map<std::string, std::string> DatabaseConfiguration::toMap() const {
 	std::map<std::string, std::string> result;
 
 	if( initialized ) {
-		if( tLogReplicationFactor == durableStorageQuorum &&
-			durableStorageQuorum == storageTeamSize &&
+		if( durableStorageQuorum == storageTeamSize &&
 			tLogWriteAntiQuorum == 0 ) {
-			if( durableStorageQuorum == 1 && desiredDataCenters == 1 && minDataCenters == 1 )
+			if( tLogReplicationFactor == 1 && durableStorageQuorum == 1 )
 				result["redundancy_mode"] = "single";
-			else if( durableStorageQuorum == 2 && desiredDataCenters == 1 && minDataCenters == 1 )
+			else if( tLogReplicationFactor == 2 && durableStorageQuorum == 2 )
 				result["redundancy_mode"] = "double";
-			else if( durableStorageQuorum == 3 && desiredDataCenters == 1 && minDataCenters == 1 )
+			else if( tLogReplicationFactor == 3 && durableStorageQuorum == 3 )
 				result["redundancy_mode"] = "triple";
-			else if( durableStorageQuorum == 3 && desiredDataCenters == 2 && minDataCenters == 1 )
-				result["redundancy_mode"] = "two_datacenter";
-			else if( durableStorageQuorum == 3 && desiredDataCenters == 3 && minDataCenters == 2 )
-				result["redundancy_mode"] = "three_datacenter";
+			else if( tLogReplicationFactor == 3 && durableStorageQuorum == 2 )
+				result["redundancy_mode"] = "fast_recovery_double";
+			else if( tLogReplicationFactor == 4 && durableStorageQuorum == 3 )
+				result["redundancy_mode"] = "fast_recovery_triple";
 			else
 				result["redundancy_mode"] = "custom";
 		} else
@@ -129,8 +124,6 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	int type;
 
 	if (ck == LiteralStringRef("initialized")) initialized = true;
-	else if (ck == LiteralStringRef("min_replica_datacenters")) parse(&minDataCenters, value);
-	else if (ck == LiteralStringRef("replica_datacenters")) parse(&desiredDataCenters, value);
 	else if (ck == LiteralStringRef("proxies")) parse(&masterProxyCount, value);
 	else if (ck == LiteralStringRef("resolvers")) parse(&resolverCount, value);
 	else if (ck == LiteralStringRef("logs")) parse(&desiredTLogCount, value);
diff --git a/fdbserver/DatabaseConfiguration.h b/fdbserver/DatabaseConfiguration.h
index 0c74907815..adf39663d0 100644
--- a/fdbserver/DatabaseConfiguration.h
+++ b/fdbserver/DatabaseConfiguration.h
@@ -43,14 +43,11 @@ struct DatabaseConfiguration {
 	std::string toString() const;
 	std::map<std::string, std::string> toMap() const;
 
-	// SOMEDAY: think about changing desiredDataCenters to minDataCenters
 	// SOMEDAY: think about changing storageTeamSize to durableStorageQuorum
-	int32_t minMachinesRequired() const { return std::max(tLogReplicationFactor, std::max(storageTeamSize, desiredDataCenters)); }
+	int32_t minMachinesRequired() const { return std::max(tLogReplicationFactor, storageTeamSize); }
 	int32_t maxMachineFailuresTolerated() const { return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, durableStorageQuorum - 1); }
 
 	// Redundancy Levels
-	int32_t minDataCenters;
-	int32_t desiredDataCenters;
 	IRepPolicyRef storagePolicy;
 
 	// MasterProxy Servers
diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp
index cb5ae12325..275fc92a34 100644
--- a/fdbserver/DiskQueue.actor.cpp
+++ b/fdbserver/DiskQueue.actor.cpp
@@ -298,11 +298,11 @@ public:
 				.detail("File0Size", self->files[0].size).detail("File1Size", self->files[1].size)
 				.detail("File0Name", self->files[0].dbgFilename).detail("SyncedFiles", syncFiles.size());*/
 
-			committed.send(Void());
 			if(g_random->random01() < 0.01) {
 				//occasionally delete all the ready future in the AndFuture
 				self->lastCommit.cleanup();
 			}
+			committed.send(Void());
 		} catch (Error& e) {
 			delete pageMem;
 			TEST(true);  // push error
@@ -405,8 +405,8 @@ public:
 				TraceEvent("DiskQueueShutdownDeleting", self->dbgid)
 					.detail("File0", self->filename(0))
 					.detail("File1", self->filename(1));
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename(0), false ) );
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename(1), true ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename(0), false ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename(1), true ) );
 			}
 			TraceEvent("DiskQueueShutdownComplete", self->dbgid)
 				.detail("DeleteFiles", deleteFiles)
@@ -419,8 +419,8 @@ public:
 		}
 
 		if( error.code() != error_code_actor_cancelled ) {
-			if (!self->stopped.isSet()) self->stopped.send(Void());
-			if (!self->error.isSet()) self->error.send(Never());
+			if (self->stopped.canBeSet()) self->stopped.send(Void());
+			if (self->error.canBeSet()) self->error.send(Never());
 			delete self;
 		}
 	}
diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp
index 54ebc3ee5a..d081ee0fe4 100644
--- a/fdbserver/KeyValueStoreMemory.actor.cpp
+++ b/fdbserver/KeyValueStoreMemory.actor.cpp
@@ -22,8 +22,8 @@
 #include "IKeyValueStore.h"
 #include "IDiskQueue.h"
 #include "flow/IndexedSet.h"
-#include "flow/Notified.h"
 #include "flow/ActorCollection.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
 
 #define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1)
@@ -705,4 +705,4 @@ IKeyValueStore* keyValueStoreMemory( std::string const& basename, UID logID, int
 
 IKeyValueStore* keyValueStoreLogSystem( class IDiskQueue* queue, UID logID, int64_t memoryLimit, bool disableSnapshot ) {
 	return new KeyValueStoreMemory( queue, logID, memoryLimit, disableSnapshot );
-}
\ No newline at end of file
+}
diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp
index 8b905bbc2f..1693aa390b 100644
--- a/fdbserver/KeyValueStoreSQLite.actor.cpp
+++ b/fdbserver/KeyValueStoreSQLite.actor.cpp
@@ -65,9 +65,7 @@ struct SpringCleaningStats {
 };
 
 struct PageChecksumCodec {
-	PageChecksumCodec(std::string const &filename, int verifyChecksumHistorySize) : pageSize(0), reserveSize(0), filename(filename), silent(false) {
-		checksumHistory.resize(verifyChecksumHistorySize);
-	}
+	PageChecksumCodec(std::string const &filename) : pageSize(0), reserveSize(0), filename(filename), silent(false) {}
 
 	int pageSize;
 	int reserveSize;
@@ -80,7 +78,6 @@ struct PageChecksumCodec {
 		uint32_t part2;
 		std::string toString() { return format("0x%08x%08x", part1, part2); }
 	};
-	std::vector<std::pair<Pgno, SumType>> checksumHistory;
 
 	// Calculates and then either stores or verifies a checksum.
 	// The checksum is read/stored at the end of the page buffer.
@@ -118,33 +115,6 @@ struct PageChecksumCodec {
 			return false;
 		}
 
-		// Update or check sum in history if the history buffer isn't empty and if we're not in a simulated injected fault situation
-		if(!checksumHistory.empty() &&
-			(!g_network->isSimulated() || (!g_simulator.getCurrentProcess()->fault_injection_p1 && !g_simulator.getCurrentProcess()->rebooting))
-		) {
-			auto &bucket = checksumHistory[pageNumber % checksumHistory.size()];
-			if(write) {
-				// For writes, put this pagenumber and sum into the bucket
-				bucket.first = pageNumber;
-				bucket.second = *sumOut;
-			}
-			else {
-				// For reads, see if the bucket has the right page number, if so then verify sum
-				if(bucket.first == pageNumber && bucket.second != *pSumInPage) {
-					TraceEvent (SevError, "SQLitePageChecksumDetectedLostWrite")
-						.detail("CodecPageSize", pageSize)
-						.detail("CodecReserveSize", reserveSize)
-						.detail("Filename", filename)
-						.detail("PageNumber", pageNumber)
-						.detail("PageSize", pageLen)
-						.detail("ChecksumInPage", pSumInPage->toString())
-						.detail("ChecksumHistory", bucket.second.toString())
-						.error(checksum_failed());
-					return false;
-				}
-			}
-		}
-
 		return true;
 	}
 
@@ -246,7 +216,7 @@ struct SQLiteDB : NonCopyable {
 				ASSERT(false);
 			}
 			// Always start with a new pager codec with default options.
-			pPagerCodec = new PageChecksumCodec(filename, SERVER_KNOBS->SQLITE_PAGER_CHECKSUM_HISTORY);
+			pPagerCodec = new PageChecksumCodec(filename);
 			sqlite3BtreePagerSetCodec(btree, PageChecksumCodec::codec, PageChecksumCodec::sizeChange, PageChecksumCodec::free, pPagerCodec);
 		}
 	}
@@ -257,7 +227,7 @@ struct SQLiteDB : NonCopyable {
 			// Our exceptions don't propagate through sqlite, so we don't know for sure if the error that caused this was
 			// an injected fault.  Assume that if fault injection is happening, this is an injected fault.
 			Error err = io_error();
-			if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))
+			if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->machine->machineProcess->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))
 				err = err.asInjectedFault();
 
 			if (db)
@@ -1853,8 +1823,8 @@ private:
 			self->logging.cancel();
 			Void _ = wait( self->readThreads->stop() && self->writeThread->stop() );
 			if (deleteOnClose) {
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename, true ) );
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename + "-wal", false ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename, true ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename + "-wal", false ) );
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "KVDoCloseError", self->logID)
diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index 42c6741980..16d1e72a8c 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -48,7 +48,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE,                     1e5 );
 	init( MAX_MESSAGE_SIZE,            std::max<int>(LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE, 1e5 + 2e4 + 1) + 8 ); // VALUE_SIZE_LIMIT + SYSTEM_KEY_SIZE_LIMIT + 9 bytes (4 bytes for length, 4 bytes for sequence number, and 1 byte for mutation type)
 	init( TLOG_MESSAGE_BLOCK_BYTES,                             10e6 );
-	init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR,      double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) );
+	init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR,      double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) ); //1.0121466709838096006362758832473
 	init( PEEK_TRACKER_EXPIRATION_TIME,                          600 ); if( randomize && BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = 0.1;
 	init( PARALLEL_GET_MORE_REQUESTS,                             32 ); if( randomize && BUGGIFY ) PARALLEL_GET_MORE_REQUESTS = 2;
 	init( MAX_QUEUE_COMMIT_BYTES,                               15e6 ); if( randomize && BUGGIFY ) MAX_QUEUE_COMMIT_BYTES = 5000;
@@ -57,7 +57,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( MAX_VERSIONS_IN_FLIGHT,                          100000000 );
 	init( VERSIONS_PER_SECOND,                               1000000 );
 	init( MAX_READ_TRANSACTION_LIFE_VERSIONS,      5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_READ_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 0.1 * VERSIONS_PER_SECOND); else if( randomize && BUGGIFY ) MAX_READ_TRANSACTION_LIFE_VERSIONS = 10 * VERSIONS_PER_SECOND;
-	init( MAX_WRITE_TRANSACTION_LIFE_VERSIONS,     5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_WRITE_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 0.5 * VERSIONS_PER_SECOND);
+	init( MAX_WRITE_TRANSACTION_LIFE_VERSIONS,     5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_WRITE_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 1 * VERSIONS_PER_SECOND);
 	init( MAX_COMMIT_BATCH_INTERVAL,                             0.5 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 2.0; // Each master proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly
 
 	// Data distribution queue
@@ -163,7 +163,6 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( DISK_METRIC_LOGGING_INTERVAL,                          5.0 );
 	init( SOFT_HEAP_LIMIT,                                     300e6 );
 
-	init( SQLITE_PAGER_CHECKSUM_HISTORY,                           0 );
 	init( SQLITE_PAGE_SCAN_ERROR_LIMIT,                        10000 );
 	init( SQLITE_BTREE_PAGE_USABLE,                          4096 - 8);  // pageSize - reserveSize for page checksum
 
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index 9f097da8e4..b8a38ad3a8 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -137,7 +137,6 @@ public:
 	int SQLITE_FRAGMENT_PRIMARY_PAGE_USABLE;
 	int SQLITE_FRAGMENT_OVERFLOW_PAGE_USABLE;
 	double SQLITE_FRAGMENT_MIN_SAVINGS;
-	int SQLITE_PAGER_CHECKSUM_HISTORY;
 
 	// KeyValueStoreSqlite spring cleaning
 	double CLEANING_INTERVAL;
diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h
index 8876765d20..14ac8e2f8b 100644
--- a/fdbserver/LogSystem.h
+++ b/fdbserver/LogSystem.h
@@ -115,6 +115,7 @@ struct ILogSystem {
 		bool parallelGetMore;
 		int sequence;
 		Deque<Future<TLogPeekReply>> futureResults;
+		Future<Void> interfaceChanged;
 
 		ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf, Tag tag, Version begin, Version end, bool returnIfBlocked, bool parallelGetMore );
 
diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp
index 3d32e6b808..a2dbb32772 100644
--- a/fdbserver/LogSystemPeekCursor.actor.cpp
+++ b/fdbserver/LogSystemPeekCursor.actor.cpp
@@ -119,6 +119,10 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
 		throw internal_error();
 	}
 
+	if(!self->interfaceChanged.isValid()) {
+		self->interfaceChanged = self->interf->onChange();
+	}
+
 	loop {
 		try {
 			while(self->futureResults.size() < SERVER_KNOBS->PARALLEL_GET_MORE_REQUESTS && self->interf->get().present()) {
@@ -139,7 +143,9 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
 					//TraceEvent("SPC_getMoreB", self->randomID).detail("has", self->hasMessage()).detail("end", res.end).detail("popped", res.popped.present() ? res.popped.get() : 0);
 					return Void();
 				}
-				when( Void _ = wait( self->interf->onChange() ) ) {
+				when( Void _ = wait( self->interfaceChanged ) ) {
+					self->interfaceChanged = self->interf->onChange();
+					self->randomID = g_random->randomUniqueID();
 					self->sequence = 0;
 					self->futureResults.clear();
 				}
@@ -150,6 +156,7 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
 				return Void();
 			} else if(e.code() == error_code_timed_out) {
 				TraceEvent("PeekCursorTimedOut", self->randomID);
+				self->interfaceChanged = self->interf->onChange();
 				self->randomID = g_random->randomUniqueID();
 				self->sequence = 0;
 				self->futureResults.clear();
diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp
index de7fd4d89d..55c5e8eafe 100644
--- a/fdbserver/MasterProxyServer.actor.cpp
+++ b/fdbserver/MasterProxyServer.actor.cpp
@@ -31,9 +31,9 @@
 #include "LogSystemDiskQueueAdapter.h"
 #include "IKeyValueStore.h"
 #include "fdbclient/SystemData.h"
-#include "flow/Notified.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbrpc/batcher.actor.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "ConflictSet.h"
 #include "flow/Stats.h"
@@ -1001,17 +1001,49 @@ ACTOR static Future<Void> readRequestServer(
 	TraceEvent("ProxyReadyForReads", proxy.id());
 
 	loop choose{
-		when(ReplyPromise<vector<StorageServerInterface>> req = waitNext(proxy.getKeyServersLocations.getFuture())) {
-			// SOMEDAY: keep ssis around?
-			vector<UID> src, dest;
-			decodeKeyServersValue(commitData->txnStateStore->readValue(keyServersKeyServersKey).get().get(), src, dest);
-			vector<StorageServerInterface> ssis;
-			ssis.reserve(src.size());
-			for (auto const& id : src) {
-				ssis.push_back(decodeServerListValue(commitData->txnStateStore->readValue(serverListKeyFor(id)).get().get()));
+		when(ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> req = waitNext(proxy.getKeyServersLocations.getFuture())) {
+			Standalone<VectorRef<KeyValueRef>> keyServersBegin = commitData->txnStateStore->readRange(KeyRangeRef(allKeys.begin, keyServersKeyServersKeys.begin), -1).get();
+			Standalone<VectorRef<KeyValueRef>> keyServersEnd = commitData->txnStateStore->readRange(KeyRangeRef(keyServersKeyServersKeys.end, allKeys.end), 2).get();
+			Standalone<VectorRef<KeyValueRef>> keyServersShardBoundaries = commitData->txnStateStore->readRange(KeyRangeRef(keyServersBegin[0].key, keyServersEnd[1].key)).get();
+
+			Standalone<VectorRef<KeyValueRef>> serverListBegin = commitData->txnStateStore->readRange(KeyRangeRef(allKeys.begin, keyServersKey(serverListKeys.begin)), -1).get();
+			Standalone<VectorRef<KeyValueRef>> serverListEnd = commitData->txnStateStore->readRange(KeyRangeRef(keyServersKey(serverListKeys.end), allKeys.end), 2).get();
+			Standalone<VectorRef<KeyValueRef>> serverListShardBoundaries = commitData->txnStateStore->readRange(KeyRangeRef(serverListBegin[0].key, serverListEnd[1].key)).get();
+
+			bool ignoreFirstServerListShard = false;
+			if (keyServersShardBoundaries.back().key > serverListShardBoundaries.front().key)
+				ignoreFirstServerListShard = true;
+
+			// shards include all keyServers and serverLists information
+			vector<pair<KeyRangeRef, vector<StorageServerInterface>>> shards;
+			int reserveSize = keyServersShardBoundaries.size() + serverListShardBoundaries.size() - 2 - (ignoreFirstServerListShard ? 1 : 0);
+			shards.reserve(reserveSize);
+
+			for (int i = 0; i < keyServersShardBoundaries.size() - 1; i++) {
+				vector<UID> src, dest;
+				decodeKeyServersValue(keyServersShardBoundaries[i].value, src, dest);
+				vector<StorageServerInterface> ssis;
+				ssis.reserve(src.size());
+				for (auto const& id : src) {
+					ssis.push_back(decodeServerListValue(commitData->txnStateStore->readValue(serverListKeyFor(id)).get().get()));
+				}
+
+				shards.push_back(std::make_pair(KeyRangeRef(keyServersShardBoundaries[i].key.removePrefix(keyServersPrefix), keyServersShardBoundaries[i + 1].key.removePrefix(keyServersPrefix)), ssis));
 			}
 
-			req.send(ssis);
+			for (int i = ignoreFirstServerListShard ? 1 : 0 ; i < serverListShardBoundaries.size() - 1; i++) {
+				vector<UID> src, dest;
+				decodeKeyServersValue(serverListShardBoundaries[i].value, src, dest);
+				vector<StorageServerInterface> ssis;
+				ssis.reserve(src.size());
+				for (auto const& id : src) {
+					ssis.push_back(decodeServerListValue(commitData->txnStateStore->readValue(serverListKeyFor(id)).get().get()));
+				}
+
+				shards.push_back(std::make_pair(KeyRangeRef(serverListShardBoundaries[i].key.removePrefix(keyServersPrefix), serverListShardBoundaries[i + 1].key.removePrefix(keyServersPrefix)), ssis));
+			}
+
+			req.send(shards);
 		}
 		when(GetStorageServerRejoinInfoRequest req = waitNext(proxy.getStorageServerRejoinInfo.getFuture())) {
 			if (commitData->txnStateStore->readValue(serverListKeyFor(req.id)).get().present()) {
diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp
index c3f1809423..0fb00a5021 100644
--- a/fdbserver/MoveKeys.actor.cpp
+++ b/fdbserver/MoveKeys.actor.cpp
@@ -828,8 +828,7 @@ void seedShardServers(
 
 	// We have to set this range in two blocks, because the master tracking of "keyServersLocations" depends on a change to a specific
 	//   key (keyServersKeyServersKey)
-	krmSetPreviouslyEmptyRange( tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), keyServersPrefix), keyServersValue( serverIds ), Value() );
-	krmSetPreviouslyEmptyRange( tr, arena, keyServersPrefix, KeyRangeRef(keyServersPrefix, allKeys.end), keyServersValue( serverIds ), Value() );
+	krmSetPreviouslyEmptyRange( tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), keyServersValue( serverIds ), Value() );
 
 	for(int s=0; s<servers.size(); s++)
 		krmSetPreviouslyEmptyRange( tr, arena, serverKeysPrefixFor( servers[s].id() ), allKeys, serverKeysTrue, serverKeysFalse );
diff --git a/fdbserver/OldTLogServer.actor.cpp b/fdbserver/OldTLogServer.actor.cpp
index c8903a4b36..50634d838c 100644
--- a/fdbserver/OldTLogServer.actor.cpp
+++ b/fdbserver/OldTLogServer.actor.cpp
@@ -23,11 +23,11 @@
 #include "flow/Stats.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/NativeAPI.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/SystemData.h"
 #include "WorkerInterface.h"
 #include "TLogInterface.h"
-#include "flow/Notified.h"
 #include "Knobs.h"
 #include "IKeyValueStore.h"
 #include "flow/ActorCollection.h"
@@ -1448,4 +1448,4 @@ namespace oldTLog {
 		Void _ = wait( tLogStart( &self, LogSystemConfig(), Version(0), Version(0), std::vector<Tag>(), true, tli, ReplyPromise<TLogInterface>(), recoveryCount ) || removed );
 		throw internal_error();  // tLogStart doesn't return without an error
 	}
-}
\ No newline at end of file
+}
diff --git a/fdbserver/Orderer.actor.h b/fdbserver/Orderer.actor.h
index bcd5ee5cf2..0d1d1048b5 100644
--- a/fdbserver/Orderer.actor.h
+++ b/fdbserver/Orderer.actor.h
@@ -27,8 +27,8 @@
 #elif !defined(FDBSERVER_ORDERER_ACTOR_H)
 	#define FDBSERVER_ORDERER_ACTOR_H
 
+#include "fdbclient/Notified.h"
 #include "flow/actorcompiler.h"
-#include "flow/Notified.h"
 
 template <class Seq>
 class Orderer {
@@ -71,4 +71,4 @@ private:
 	Promise<Void> shutdown;  // Never set, only broken on destruction
 };
 
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp
index 38f295f9e0..5fea70df12 100644
--- a/fdbserver/QuietDatabase.actor.cpp
+++ b/fdbserver/QuietDatabase.actor.cpp
@@ -139,6 +139,7 @@ ACTOR Future<vector<StorageServerInterface>> getStorageServers( Database cx, boo
 	state Transaction tr( cx );
 	if (use_system_priority)
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 	loop {
 		try {
 			Standalone<RangeResultRef> serverList = wait( tr.getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) );
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 7a0849dd8d..073fa76fb1 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -198,15 +198,16 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 		bool runBackupAgents)
 {
 	state ISimulator::ProcessInfo *simProcess = g_simulator.getCurrentProcess();
-	state int cycles =0;
+	state UID randomId = g_nondeterministic_random->randomUniqueID();
+	state int cycles = 0;
 
 	loop {
 		auto waitTime = SERVER_KNOBS->MIN_REBOOT_TIME + (SERVER_KNOBS->MAX_REBOOT_TIME - SERVER_KNOBS->MIN_REBOOT_TIME) * g_random->random01();
 		cycles ++;
-		TraceEvent("SimulatedFDBDWait").detail("Cycles", cycles)
-			.detail("ProcessAddress", NetworkAddress(ip, port, true, false))
+		TraceEvent("SimulatedFDBDPreWait").detail("Cycles", cycles).detail("RandomId", randomId)
+			.detail("Address", NetworkAddress(ip, port, true, false))
 			.detailext("ZoneId", localities.zoneId())
-			.detail("waitTime", waitTime);
+			.detail("waitTime", waitTime).detail("Port", port);
 
 		Void _ = wait( delay( waitTime ) );
 
@@ -215,12 +216,13 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 		state Future<ISimulator::KillType> onShutdown = process->onShutdown();
 
 		try {
-			TraceEvent("SimulatedRebooterStarting", localities.zoneId()).detail("Cycles", cycles)
+			TraceEvent("SimulatedRebooterStarting", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
 				.detailext("ZoneId", localities.zoneId())
 				.detailext("DataHall", localities.dataHallId())
-				.detail("ProcessAddress", process->address.toString())
+				.detail("Address", process->address.toString())
+				.detail("Excluded", process->excluded)
 				.detail("UsingSSL", useSSL);
-			TraceEvent("ProgramStart").detail("Cycles", cycles)
+			TraceEvent("ProgramStart").detail("Cycles", cycles).detail("RandomId", randomId)
 				.detail("SourceVersion", getHGVersion())
 				.detail("Version", FDB_VT_VERSION)
 				.detail("PackageName", FDB_VT_PACKAGE_NAME)
@@ -246,7 +248,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 			} catch (Error& e) {
 				// If in simulation, if we make it here with an error other than io_timeout but enASIOTimedOut is set then somewhere an io_timeout was converted to a different error.
 				if(g_network->isSimulated() && e.code() != error_code_io_timeout && (bool)g_network->global(INetwork::enASIOTimedOut))
-					TraceEvent(SevError, "IOTimeoutErrorSuppressed").detail("ErrorCode", e.code()).backtrace();
+					TraceEvent(SevError, "IOTimeoutErrorSuppressed").detail("ErrorCode", e.code()).detail("RandomId", randomId).backtrace();
 
 				if (onShutdown.isReady() && onShutdown.isError()) throw onShutdown.getError();
 				if(e.code() != error_code_actor_cancelled)
@@ -255,15 +257,16 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 				TraceEvent(e.code() == error_code_actor_cancelled || e.code() == error_code_file_not_found || destructed ? SevInfo : SevError, "SimulatedFDBDTerminated", localities.zoneId()).error(e, true);
 			}
 
-			TraceEvent("SimulatedFDBDDone", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDDone", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
+				.detail("Excluded", process->excluded)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", onShutdown.isReady() ? onShutdown.get() : ISimulator::None);
 
 			if (!onShutdown.isReady())
 				onShutdown = ISimulator::InjectFaults;
 		} catch (Error& e) {
-			TraceEvent(destructed ? SevInfo : SevError, "SimulatedFDBDRebooterError", localities.zoneId()).error(e, true);
+			TraceEvent(destructed ? SevInfo : SevError, "SimulatedFDBDRebooterError", localities.zoneId()).detail("RandomId", randomId).error(e, true);
 			onShutdown = e;
 		}
 
@@ -273,6 +276,11 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 			process->rebooting = true;
 			process->shutdownSignal.send(ISimulator::None);
 		}
+		TraceEvent("SimulatedFDBDWait", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+			.detail("Address", process->address)
+			.detail("Excluded", process->excluded)
+			.detail("Rebooting", process->rebooting)
+			.detailext("ZoneId", localities.zoneId());
 		Void _ = wait( g_simulator.onProcess( simProcess ) );
 
 		Void _ = wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY));  // One last chance for the process to clean up?
@@ -280,22 +288,24 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 		g_simulator.destroyProcess( process );  // Leak memory here; the process may be used in other parts of the simulation
 
 		auto shutdownResult = onShutdown.get();
-		TraceEvent("SimulatedFDBDShutdown", localities.zoneId()).detail("Cycles", cycles)
-			.detail("ProcessAddress", process->address)
+		TraceEvent("SimulatedFDBDShutdown", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+			.detail("Address", process->address)
+			.detail("Excluded", process->excluded)
 			.detailext("ZoneId", localities.zoneId())
 			.detail("KillType", shutdownResult);
 
 		if( shutdownResult < ISimulator::RebootProcessAndDelete ) {
-			TraceEvent("SimulatedFDBDLowerReboot", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDLowerReboot", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
+				.detail("Excluded", process->excluded)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", shutdownResult);
 			return onShutdown.get();
 		}
 
 		if( onShutdown.get() == ISimulator::RebootProcessAndDelete ) {
-			TraceEvent("SimulatedFDBDRebootAndDelete", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDRebootAndDelete", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", shutdownResult);
 			*coordFolder = joinPath(baseFolder, g_random->randomUniqueID().toString());
@@ -311,8 +321,8 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 			}
 		}
 		else {
-			TraceEvent("SimulatedFDBDJustRepeat", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDJustRepeat", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", shutdownResult);
 		}
@@ -346,6 +356,7 @@ ACTOR Future<Void> simulatedMachine(
 	state int bootCount = 0;
 	state std::vector<std::string> myFolders;
 	state std::vector<std::string> coordFolders;
+	state UID randomId = g_nondeterministic_random->randomUniqueID();
 
 	try {
 		CSimpleIni ini;
@@ -382,6 +393,7 @@ ACTOR Future<Void> simulatedMachine(
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
 				Reference<ClusterConnectionFile> clusterFile(useSeedFile ? new ClusterConnectionFile(path, connStr.toString()) : new ClusterConnectionFile(path));
 				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, i + 1, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents));
+				TraceEvent("SimulatedMachineProcess", randomId).detail("Address", NetworkAddress(ips[i], i+1, true, false)).detailext("ZoneId", localities.zoneId()).detailext("DataHall", localities.dataHallId()).detail("Folder", myFolders[i]);
 			}
 
 			TEST( bootCount >= 1 ); // Simulated machine rebooted
@@ -389,7 +401,7 @@ ACTOR Future<Void> simulatedMachine(
 			TEST( bootCount >= 3 ); // Simulated machine rebooted three times
 			++bootCount;
 
-			TraceEvent("SimulatedMachineStart")
+			TraceEvent("SimulatedMachineStart", randomId)
 				.detail("Folder0", myFolders[0])
 				.detail("CFolder0", coordFolders[0])
 				.detail("MachineIPs", toIPVectorString(ips))
@@ -405,7 +417,7 @@ ACTOR Future<Void> simulatedMachine(
 
 			Void _ = wait( waitForAll( processes ) );
 
-			TraceEvent("SimulatedMachineRebootStart")
+			TraceEvent("SimulatedMachineRebootStart", randomId)
 				.detail("Folder0", myFolders[0])
 				.detail("CFolder0", coordFolders[0])
 				.detail("MachineIPs", toIPVectorString(ips))
@@ -437,7 +449,12 @@ ACTOR Future<Void> simulatedMachine(
 				ASSERT( it.second.isReady() && !it.second.isError() );
 			}
 
-			TraceEvent("SimulatedMachineRebootAfterKills")
+			for( auto it : g_simulator.getMachineById(localities.zoneId())->deletingFiles ) {
+				filenames.insert( it );
+				closingStr += it + ", ";
+			}
+
+			TraceEvent("SimulatedMachineRebootAfterKills", randomId)
 				.detail("Folder0", myFolders[0])
 				.detail("CFolder0", coordFolders[0])
 				.detail("MachineIPs", toIPVectorString(ips))
@@ -466,12 +483,12 @@ ACTOR Future<Void> simulatedMachine(
 						openFiles += *it + ", ";
 						i++;
 					}
-					TraceEvent("MachineFilesOpen").detail("PAddr", toIPVectorString(ips)).detail("OpenFiles", openFiles);
+					TraceEvent("MachineFilesOpen", randomId).detail("PAddr", toIPVectorString(ips)).detail("OpenFiles", openFiles);
 				} else
 					break;
 
 				if( shutdownDelayCount++ >= 50 ) {  // Worker doesn't shut down instantly on reboot
-					TraceEvent(SevError, "SimulatedFDBDFilesCheck")
+					TraceEvent(SevError, "SimulatedFDBDFilesCheck", randomId)
 						.detail("PAddrs", toIPVectorString(ips))
 						.detailext("ZoneId", localities.zoneId())
 						.detailext("DataHall", localities.dataHallId());
@@ -482,8 +499,8 @@ ACTOR Future<Void> simulatedMachine(
 				backoff = std::min( backoff + 1.0, 6.0 );
 			}
 
-			TraceEvent("SimulatedFDBDFilesClosed")
-				.detail("ProcessAddress", toIPVectorString(ips))
+			TraceEvent("SimulatedFDBDFilesClosed", randomId)
+				.detail("Address", toIPVectorString(ips))
 				.detailext("ZoneId", localities.zoneId())
 				.detailext("DataHall", localities.dataHallId());
 
@@ -505,7 +522,7 @@ ACTOR Future<Void> simulatedMachine(
 
 			auto rebootTime = g_random->random01() * MACHINE_REBOOT_TIME;
 
-			TraceEvent("SimulatedMachineShutdown")
+			TraceEvent("SimulatedMachineShutdown", randomId)
 				.detail("Swap", swap)
 				.detail("KillType", killType)
 				.detail("RebootTime", rebootTime)
@@ -525,7 +542,7 @@ ACTOR Future<Void> simulatedMachine(
 
 				if( myFolders != toRebootFrom ) {
 					TEST( true ); // Simulated machine swapped data folders
-					TraceEvent("SimulatedMachineFolderSwap")
+					TraceEvent("SimulatedMachineFolderSwap", randomId)
 						.detail("OldFolder0", myFolders[0]).detail("NewFolder0", toRebootFrom[0])
 						.detail("MachineIPs", toIPVectorString(ips));
 				}
@@ -643,93 +660,157 @@ ACTOR Future<Void> restartSimulatedSystem(vector<Future<Void>> *systemActors, st
 	return Void();
 }
 
-std::string randomConfiguration( int physicalDatacenters ) {
-	int r = std::min(g_random->randomInt(0, 6), 3);
-//	r = 1; //ahm
+struct SimulationConfig {
+	explicit SimulationConfig(int extraDB);
+	int extraDB;
 
-	// See also random configuration choices in ConfigureDatabase workload
+	DatabaseConfiguration db;
 
-	std::string startingConfig = "new";
-	if (r == 0) {
+	void set_config(std::string config);
+
+	// Simulation layout
+	int datacenters;
+	int machine_count;  // Total, not per DC.
+	int processes_per_machine;
+	int coordinators;
+
+	std::string toString();
+
+private:
+	void generateNormalConfig();
+};
+
+SimulationConfig::SimulationConfig(int extraDB) : extraDB(extraDB) {
+	generateNormalConfig();
+}
+
+void SimulationConfig::set_config(std::string config) {
+	// The only mechanism we have for turning "single" into what single means
+	// is buildConfiguration()... :/
+	std::map<std::string, std::string> hack_map;
+	ASSERT( buildConfiguration(config, hack_map) );
+	for(auto kv : hack_map) db.set( kv.first, kv.second );
+}
+
+StringRef StringRefOf(const char* s) {
+  return StringRef((uint8_t*)s, strlen(s));
+}
+
+void SimulationConfig::generateNormalConfig() {
+	set_config("new");
+	datacenters = g_random->randomInt( 1, 4 );
+	if (g_random->random01() < 0.25) db.desiredTLogCount = g_random->randomInt(1,7);
+	if (g_random->random01() < 0.25) db.masterProxyCount = g_random->randomInt(1,7);
+	if (g_random->random01() < 0.25) db.resolverCount = g_random->randomInt(1,7);
+	if (g_random->random01() < 0.5) {
+		set_config("ssd");
+	} else {
+		set_config("memory");
+	}
+
+	int replication_type = std::min(g_random->randomInt( 1, 6 ), 3);
+	//replication_type = 1;  //ahm
+	switch (replication_type) {
+	case 0: {
 		TEST( true );  // Simulated cluster using custom redundancy mode
-		int storage_replicas = g_random->randomInt(1,5);
-		startingConfig += " storage_replicas:=" + format("%d", storage_replicas);
-		startingConfig += " storage_quorum:=" + format("%d", storage_replicas);
-		int log_replicas = g_random->randomInt(1,5);
-		startingConfig += " log_replicas:=" + format("%d", log_replicas);
-		int log_anti_quorum = g_random->randomInt(0, log_replicas);
-		startingConfig += " log_anti_quorum:=" + format("%d", log_anti_quorum);
-		startingConfig += " replica_datacenters:=1";
-		startingConfig += " min_replica_datacenters:=1";
+		int storage_servers = g_random->randomInt(1,5);
+		int replication_factor = g_random->randomInt(1,5);
+		int anti_quorum = g_random->randomInt(0, db.tLogReplicationFactor);
+		// Go through buildConfiguration, as it sets tLogPolicy/storagePolicy.
+		set_config(format("storage_replicas:=%d storage_quorum:=%d "
+		                  "log_replicas:=%d log_anti_quorum:=%1 "
+		                  "replica_datacenters:=1 min_replica_datacenters:=1",
+		                  storage_servers, storage_servers,
+		                  replication_factor, anti_quorum));
+		break;
 	}
-	else if (r == 1) {
+	case 1: {
 		TEST( true );  // Simulated cluster running in single redundancy mode
-		startingConfig += " single";
+		set_config("single");
+		break;
 	}
-	else if( r == 2 ) {
+	case 2: {
 		TEST( true );  // Simulated cluster running in double redundancy mode
-		startingConfig += " double";
+		set_config("double");
+		break;
 	}
-	else if( r == 3 ) {
-		if( physicalDatacenters == 1 ) {
+	case 3: {
+		if( datacenters == 1 ) {
 			TEST( true );  // Simulated cluster running in triple redundancy mode
-			startingConfig += " triple";
+			set_config("triple");
 		}
-		else if( physicalDatacenters == 2 ) {
+		else if( datacenters == 2 ) {
 			TEST( true );  // Simulated cluster running in 2 datacenter mode
-			startingConfig += " two_datacenter";
+			set_config("two_datacenter");
 		}
-		else if( physicalDatacenters == 3 ) {
+		else if( datacenters == 3 ) {
 			TEST( true );  // Simulated cluster running in 3 data-hall mode
-			startingConfig += " three_data_hall";
+			set_config("three_data_hall");
 		}
 		else {
 			ASSERT( false );
 		}
+		break;
+	}
+	default:
+		ASSERT(false);  // Programmer forgot to adjust cases.
 	}
 
-	if (g_random->random01() < 0.25) startingConfig += " logs=" + format("%d", g_random->randomInt(1,7));
-	if (g_random->random01() < 0.25) startingConfig += " proxies=" + format("%d", g_random->randomInt(1,7));
-	if (g_random->random01() < 0.25) startingConfig += " resolvers=" + format("%d", g_random->randomInt(1,7));
+	machine_count = g_random->randomInt( std::max( 2+datacenters, db.minMachinesRequired() ), extraDB ? 6 : 10 );
+	processes_per_machine = g_random->randomInt(1, (extraDB ? 14 : 28)/machine_count + 2 );
+	coordinators = BUGGIFY ? g_random->randomInt(1, machine_count+1) : std::min( machine_count, db.maxMachineFailuresTolerated()*2 + 1 );
+}
 
-	startingConfig += g_random->random01() < 0.5 ? " ssd" : " memory";
-	return startingConfig;
+std::string SimulationConfig::toString() {
+	std::stringstream config;
+	std::map<std::string, std::string>&& dbconfig = db.toMap();
+	config << "new";
+
+	if (dbconfig["redundancy_mode"] != "custom") {
+		config << " " << dbconfig["redundancy_mode"];
+	} else {
+		config << " " << "log_replicas:=" << db.tLogReplicationFactor;
+		config << " " << "log_anti_quorum:=" << db.tLogWriteAntiQuorum;
+		config << " " << "storage_replicas:=" << db.storageTeamSize;
+		config << " " << "storage_quorum:=" << db.durableStorageQuorum;
+	}
+
+	config << " logs=" << db.getDesiredLogs();
+	config << " proxies=" << db.getDesiredProxies();
+	config << " resolvers=" << db.getDesiredResolvers();
+
+	config << " " << dbconfig["storage_engine"];
+	return config.str();
 }
 
 void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseFolder,
 							int* pTesterCount, Optional<ClusterConnectionString> *pConnString,
 							Standalone<StringRef> *pStartingConfiguration, int extraDB)
 {
-	int dataCenters = g_random->randomInt( 1, 4 );
-
 	// SOMEDAY: this does not test multi-interface configurations
-	std::string startingConfigString = randomConfiguration(dataCenters);
-	std::map<std::string,std::string> startingConfigMap;
-	ASSERT( buildConfiguration( startingConfigString, startingConfigMap ) == ConfigurationResult::SUCCESS );
+	SimulationConfig simconfig(extraDB);
+	std::string startingConfigString = simconfig.toString();
 
-	DatabaseConfiguration startingConfig;
-	for(auto kv : startingConfigMap) startingConfig.set( kv.first, kv.second );
-	g_simulator.storagePolicy = startingConfig.storagePolicy;
-	g_simulator.tLogPolicy = startingConfig.tLogPolicy;
-	g_simulator.tLogWriteAntiQuorum = startingConfig.tLogWriteAntiQuorum;
+	g_simulator.storagePolicy = simconfig.db.storagePolicy;
+	g_simulator.tLogPolicy = simconfig.db.tLogPolicy;
+	g_simulator.tLogWriteAntiQuorum = simconfig.db.tLogWriteAntiQuorum;
 	ASSERT(g_simulator.storagePolicy);
 	ASSERT(g_simulator.tLogPolicy);
 	TraceEvent("simulatorConfig").detail("tLogPolicy", g_simulator.tLogPolicy->info()).detail("storagePolicy", g_simulator.storagePolicy->info()).detail("tLogWriteAntiQuorum", g_simulator.tLogWriteAntiQuorum).detail("ConfigString", startingConfigString);
 
-	int machineCount = g_random->randomInt( std::max( 2+dataCenters, startingConfig.minMachinesRequired() ), extraDB ? 6 : 10 );
+	const int dataCenters = simconfig.datacenters;
+	const int machineCount = simconfig.machine_count;
+	const int coordinatorCount = simconfig.coordinators;
+	const int processesPerMachine = simconfig.processes_per_machine;
 
 	// half the time, when we have more than 4 machines that are not the first in their dataCenter, assign classes
 	bool assignClasses = machineCount - dataCenters > 4 && g_random->random01() < 0.5;
-	int processesPerMachine = g_random->randomInt(1, (extraDB ? 14 : 28)/machineCount + 2 );
 
 	// Use SSL half the time
 	bool sslEnabled = g_random->random01() < 0.05;
 	TEST( sslEnabled ); // SSL enabled
 	TEST( !sslEnabled ); // SSL disabled
 
-	// Pick coordination processes.
-	int coordinatorCount = BUGGIFY ? g_random->randomInt(1, machineCount+1) : std::min( machineCount, startingConfig.maxMachineFailuresTolerated()*2 + 1 );
-
 	vector<NetworkAddress> coordinatorAddresses;
 	for( int dc = 0; dc < dataCenters; dc++ ) {
 		int machines = machineCount / dataCenters + (dc < machineCount % dataCenters); // add remainder of machines to first datacenter
@@ -744,8 +825,8 @@ void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseF
 
 	g_random->randomShuffle(coordinatorAddresses);
 	for(int i = 0; i < (coordinatorAddresses.size()/2)+1; i++) {
-		TraceEvent("ProtectMachine").detail("Address", coordinatorAddresses[i]).detail("Coordinators", coordinatorAddresses.size()).backtrace();
-		g_simulator.protectedAddresses.insert(coordinatorAddresses[i]);
+		TraceEvent("ProtectCoordinator").detail("Address", coordinatorAddresses[i]).detail("Coordinators", describe(coordinatorAddresses)).backtrace();
+		g_simulator.protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip,coordinatorAddresses[i].port,true,false));
 	}
 	g_random->randomShuffle(coordinatorAddresses);
 
@@ -812,12 +893,12 @@ void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseF
 	}
 
 	g_simulator.desiredCoordinators = coordinatorCount;
-	g_simulator.killableMachines = startingConfig.maxMachineFailuresTolerated();
-	g_simulator.neededDatacenters = startingConfig.minDataCenters;
-	g_simulator.killableDatacenters = startingConfig.minDataCenters - 1;
+	g_simulator.killableMachines = simconfig.db.maxMachineFailuresTolerated();
+	g_simulator.neededDatacenters = 1;
+	g_simulator.killableDatacenters = 0;
 	g_simulator.physicalDatacenters = dataCenters;
 	g_simulator.maxCoordinatorsInDatacenter = ((coordinatorCount-1)/dataCenters) + 1;
-	g_simulator.machinesNeededForProgress = startingConfig.minMachinesRequired() + nonVersatileMachines;
+	g_simulator.machinesNeededForProgress = simconfig.db.minMachinesRequired() + nonVersatileMachines;
 	g_simulator.processesPerMachine = processesPerMachine;
 
 	TraceEvent("SetupSimulatorSettings")
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 4345f5ec77..f3a11cfd1a 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -947,6 +947,7 @@ ACTOR static Future<double> doGrvProbe(Transaction *tr, Optional<FDBTransactionO
 
 	loop {
 		try {
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			if(priority.present()) {
 				tr->setOption(priority.get());
 			}
@@ -969,6 +970,7 @@ ACTOR static Future<double> doReadProbe(Future<double> grvProbe, Transaction *tr
 	state double start = timer_monotonic();
 
 	loop {
+		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		try {
 			Optional<Standalone<StringRef> > _ = wait(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793")));
 			return timer_monotonic() - start;
@@ -993,6 +995,7 @@ ACTOR static Future<double> doCommitProbe(Future<double> grvProbe, Transaction *
 
 	loop {
 		try {
+			tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 			tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 			tr->makeSelfConflicting();
 			Void _ = wait(tr->commit());
@@ -1022,9 +1025,7 @@ ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, cons
 	return Void();
 }
 
-ACTOR static Future<StatusObject> latencyProbeFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, StatusArray *messages, std::set<std::string> *incomplete_reasons) {
-	Database cx = openDBOnServer(db, TaskDefaultEndpoint, true, true); // Open a new database connection that is lock-aware
-
+ACTOR static Future<StatusObject> latencyProbeFetcher(Database cx, StatusArray *messages, std::set<std::string> *incomplete_reasons) {
 	state Transaction trImmediate(cx);
 	state Transaction trDefault(cx);
 	state Transaction trBatch(cx);
@@ -1777,9 +1778,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		if (configuration.present()){
 			// Do the latency probe by itself to avoid interference from other status activities
-			state Future<StatusObject> latencyProbe = latencyProbeFetcher(db, &messages, &status_incomplete_reasons);
-
-			StatusObject latencyProbeResults = wait(latencyProbe);
+			StatusObject latencyProbeResults = wait(latencyProbeFetcher(cx, &messages, &status_incomplete_reasons));
 
 			statusObj["database_available"] = latencyProbeResults.count("immediate_priority_transaction_start_seconds") && latencyProbeResults.count("read_seconds") && latencyProbeResults.count("commit_seconds");
 			if (!latencyProbeResults.empty()) {
diff --git a/fdbserver/StorageMetrics.actor.h b/fdbserver/StorageMetrics.actor.h
index d4407af686..669be605b9 100644
--- a/fdbserver/StorageMetrics.actor.h
+++ b/fdbserver/StorageMetrics.actor.h
@@ -54,7 +54,7 @@ struct StorageMetricSample {
 				bck_split.decrementNonEnd();
 
 				KeyRef split = keyBetween(KeyRangeRef(bck_split != sample.begin() ? std::max<KeyRef>(*bck_split,range.begin) : range.begin, *it));
-				if( split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT )
+				if(!front || (getEstimate(KeyRangeRef(range.begin, split)) > 0 && split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT))
 					return split;
 			}
 
@@ -63,7 +63,7 @@ struct StorageMetricSample {
 				++it;
 
 				KeyRef split = keyBetween(KeyRangeRef(*fwd_split, it != sample.end() ? std::min<KeyRef>(*it, range.end) : range.end));
-				if( split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT )
+				if(front || (getEstimate(KeyRangeRef(split, range.end)) > 0 && split.size() <= CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT))
 					return split;
 
 				fwd_split = it;
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 9b0dc08652..8ba666b9c2 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -23,11 +23,11 @@
 #include "flow/Stats.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/NativeAPI.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/SystemData.h"
 #include "WorkerInterface.h"
 #include "TLogInterface.h"
-#include "flow/Notified.h"
 #include "Knobs.h"
 #include "IKeyValueStore.h"
 #include "flow/ActorCollection.h"
@@ -43,9 +43,6 @@ using std::make_pair;
 using std::min;
 using std::max;
 
-//FIXME: also defined in worker, move to someplace available to both
-#define DUMPTOKEN( name ) TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token)
-
 struct TLogQueueEntryRef {
 	UID id;
 	Version version;
@@ -203,198 +200,6 @@ struct CompareFirst {
 	}
 };
 
-struct LogData : NonCopyable, public ReferenceCounted<LogData> {
-	struct TagData {
-		std::deque<std::pair<Version, LengthPrefixedStringRef>> version_messages;
-		bool nothing_persistent;				// true means tag is *known* to have no messages in persistentData.  false means nothing.
-		bool popped_recently;					// `popped` has changed since last updatePersistentData
-		Version popped;				// see popped version tracking contract below
-		bool update_version_sizes;
-
-		TagData( Version popped, bool nothing_persistent, bool popped_recently, Tag tag ) : nothing_persistent(nothing_persistent), popped(popped), popped_recently(popped_recently), update_version_sizes(tag != txsTag) {}
-
-		TagData(TagData&& r) noexcept(true) : version_messages(std::move(r.version_messages)), nothing_persistent(r.nothing_persistent), popped_recently(r.popped_recently), popped(r.popped), update_version_sizes(r.update_version_sizes) {}
-		void operator= (TagData&& r) noexcept(true) {
-			version_messages = std::move(r.version_messages);
-			nothing_persistent = r.nothing_persistent;
-			popped_recently = r.popped_recently;
-			popped = r.popped;
-			update_version_sizes = r.update_version_sizes;
-		}
-
-		// Erase messages not needed to update *from* versions >= before (thus, messages with toversion <= before)
-		ACTOR Future<Void> eraseMessagesBefore( TagData *self, Version before, int64_t* gBytesErased, Reference<LogData> tlogData, int taskID ) {
-			while(!self->version_messages.empty() && self->version_messages.front().first < before) {
-				Version version = self->version_messages.front().first;
-				std::pair<int, int> &sizes = tlogData->version_sizes[version];
-				int64_t messagesErased = 0;
-
-				while(!self->version_messages.empty() && self->version_messages.front().first == version) {
-					auto const& m = self->version_messages.front();
-					++messagesErased;
-
-					if(self->update_version_sizes) {
-						sizes.first -= m.second.expectedSize();
-					}
-
-					self->version_messages.pop_front();
-				}
-
-				tlogData->bytesDurable += (messagesErased * sizeof(std::pair<Version, LengthPrefixedStringRef>) * SERVER_KNOBS->VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS) >> 10;
-				*gBytesErased += (messagesErased * sizeof(std::pair<Version, LengthPrefixedStringRef>) * SERVER_KNOBS->VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS) >> 10;
-				Void _ = wait(yield(taskID));
-			}
-
-			return Void();
-		}
-
-		Future<Void> eraseMessagesBefore(Version before, int64_t* gBytesErased, Reference<LogData> tlogData, int taskID) {
-			return eraseMessagesBefore(this, before, gBytesErased, tlogData, taskID);
-		}
-	};
-
-	/*
-	Popped version tracking contract needed by log system to implement ILogCursor::popped():
-
-		- Log server tracks for each (possible) tag a popped_version
-		Impl: TagData::popped (in memory) and persistTagPoppedKeys (in persistentData)
-		- popped_version(tag) is <= the maximum version for which log server (or a predecessor) is ever asked to pop the tag
-		Impl: Only increased by tLogPop() in response to either a pop request or recovery from a predecessor
-		- popped_version(tag) is > the maximum version for which log server is unable to peek messages due to previous pops (on this server or a predecessor)
-		Impl: Increased by tLogPop() atomically with erasing messages from memory; persisted by updatePersistentData() atomically with erasing messages from store; messages are not erased from queue where popped_version is not persisted
-		- LockTLogReply returns all tags which either have messages, or which have nonzero popped_versions
-		Impl: tag_data is present for all such tags
-		- peek(tag, v) returns the popped_version for tag if that is greater than v
-		Impl: Check tag_data->popped (after all waits)
-	*/
-
-	bool stopped, initialized;
-	DBRecoveryCount recoveryCount;
-
-	VersionMetricHandle persistentDataVersion, persistentDataDurableVersion;  // The last version number in the portion of the log (written|durable) to persistentData
-	NotifiedVersion version, queueCommittedVersion;
-	Version queueCommittingVersion;
-	Version knownCommittedVersion;
-
-	Deque<std::pair<Version, Standalone<VectorRef<uint8_t>>>> messageBlocks;
-	Map< Tag, TagData > tag_data;
-
-	Map<Version, std::pair<int,int>> version_sizes;
-
-	CounterCollection cc;
-	Counter bytesInput;
-	Counter bytesDurable;
-
-	UID logId;
-	Version newPersistentDataVersion;
-	Future<Void> removed;
-	TLogInterface tli;
-	PromiseStream<Future<Void>> addActor;
-
-	explicit LogData(TLogInterface interf) : knownCommittedVersion(0), tli(interf), logId(interf.id()),
-			cc("TLog", interf.id().toString()),
-			bytesInput("bytesInput", cc),
-			bytesDurable("bytesDurable", cc),
-			// These are initialized differently on init() or recovery
-			recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion)
-	{
-		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
-		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
-		version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
-		queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
-
-		specialCounter(cc, "version", [this](){ return this->version.get(); });
-	}
-
-	LogEpoch epoch() const { return recoveryCount; }
-};
-
-struct TLogData : NonCopyable {
-	AsyncTrigger newLogData;
-	Deque<UID> queueOrder;
-	std::map<UID, Reference<LogData>> id_data;
-
-	UID dbgid;
-
-	IKeyValueStore* persistentData;
-	IDiskQueue* rawPersistentQueue;
-	TLogQueue *persistentQueue;
-
-	int64_t diskQueueCommitBytes;
-	AsyncVar<bool> largeDiskQueueCommitBytes; //becomes true when diskQueueCommitBytes is greater than MAX_QUEUE_COMMIT_BYTES
-
-	Reference<AsyncVar<ServerDBInfo>> dbInfo;
-
-	NotifiedVersion queueCommitEnd;
-	Version queueCommitBegin;
-	AsyncTrigger newVersion;
-
-	int64_t instanceID;
-	int64_t bytesInput;
-	int64_t bytesDurable;
-
-	Version prevVersion;
-
-	struct peekTrackerData {
-		std::map<int, Promise<Version>> sequence_version;
-		double lastUpdate;
-	};
-
-	std::map<UID, peekTrackerData> peekTracker;
-	WorkerCache<TLogInterface> tlogCache;
-
-	Future<Void> updatePersist; //SOMEDAY: integrate the recovery and update storage so that only one of them is committing to persistant data.
-	Future<Void> oldLogServer;
-
-	PromiseStream<Future<Void>> sharedActors;
-
-	TLogData(UID dbgid, IKeyValueStore* persistentData, IDiskQueue * persistentQueue, Reference<AsyncVar<ServerDBInfo>> const& dbInfo)
-			: dbgid(dbgid), instanceID(g_random->randomUniqueID().first()),
-			  persistentData(persistentData), rawPersistentQueue(persistentQueue), persistentQueue(new TLogQueue(persistentQueue, dbgid)),
-			  dbInfo(dbInfo), queueCommitBegin(0), queueCommitEnd(0), prevVersion(0),
-			  diskQueueCommitBytes(0), largeDiskQueueCommitBytes(false),
-			  bytesInput(0), bytesDurable(0), updatePersist(Void())
-		{
-			//specialCounter(cc, "kvstoreBytesUsed", [this](){ return this->persistentData->getStorageBytes().used; });
-			//specialCounter(cc, "kvstoreBytesFree", [this](){ return this->persistentData->getStorageBytes().free; });
-			//specialCounter(cc, "kvstoreBytesAvailable", [this](){ return this->persistentData->getStorageBytes().available; });
-			//specialCounter(cc, "kvstoreBytesTotal", [this](){ return this->persistentData->getStorageBytes().total; });
-			//specialCounter(cc, "queueDiskBytesUsed", [this](){ return this->rawPersistentQueue->getStorageBytes().used; });
-			//specialCounter(cc, "queueDiskBytesFree", [this](){ return this->rawPersistentQueue->getStorageBytes().free; });
-			//specialCounter(cc, "queueDiskBytesAvailable", [this](){ return this->rawPersistentQueue->getStorageBytes().available; });
-			//specialCounter(cc, "queueDiskBytesTotal", [this](){ return this->rawPersistentQueue->getStorageBytes().total; });
-		}
-};
-
-ACTOR Future<Void> tLogLock( TLogData* self, ReplyPromise< TLogLockResult > reply, Reference<LogData> logData ) {
-	state Version stopVersion = logData->version.get();
-
-	TEST(true); // TLog stopped by recovering master
-	TEST( logData->stopped );
-	TEST( !logData->stopped );
-
-	TraceEvent("TLogStop", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get());
-
-	logData->stopped = true;
-
-	// Lock once the current version has been committed
-	Void _ = wait( logData->queueCommittedVersion.whenAtLeast( stopVersion ) );
-
-	ASSERT(stopVersion == logData->version.get());
-
-	TLogLockResult result;
-	result.end = stopVersion;
-	result.knownCommittedVersion = logData->knownCommittedVersion;
-	for( auto & tag : logData->tag_data )
-		result.tags.push_back( tag.key );
-
-	TraceEvent("TLogStop2", self->dbgid).detail("logId", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get()).detail("tags", describe(result.tags));
-
-
-	reply.send( result );
-	return Void();
-}
-
 KeyRange prefixRange( KeyRef prefix ) {
 	Key end = strinc(prefix);
 	return KeyRangeRef( prefix, end );
@@ -452,6 +257,225 @@ static Version decodeTagMessagesKey( StringRef key ) {
 	return bigEndian64( BinaryReader::fromStringRef<Version>( stripTagMessagesKey(key), Unversioned() ) );
 }
 
+struct TLogData : NonCopyable {
+	AsyncTrigger newLogData;
+	Deque<UID> queueOrder;
+	std::map<UID, Reference<struct LogData>> id_data;
+
+	UID dbgid;
+
+	IKeyValueStore* persistentData;
+	IDiskQueue* rawPersistentQueue;
+	TLogQueue *persistentQueue;
+
+	int64_t diskQueueCommitBytes;
+	AsyncVar<bool> largeDiskQueueCommitBytes; //becomes true when diskQueueCommitBytes is greater than MAX_QUEUE_COMMIT_BYTES
+
+	Reference<AsyncVar<ServerDBInfo>> dbInfo;
+
+	NotifiedVersion queueCommitEnd;
+	Version queueCommitBegin;
+	AsyncTrigger newVersion;
+
+	int64_t instanceID;
+	int64_t bytesInput;
+	int64_t bytesDurable;
+
+	Version prevVersion;
+
+	struct peekTrackerData {
+		std::map<int, Promise<Version>> sequence_version;
+		double lastUpdate;
+	};
+
+	std::map<UID, peekTrackerData> peekTracker;
+	WorkerCache<TLogInterface> tlogCache;
+
+	Future<Void> updatePersist; //SOMEDAY: integrate the recovery and update storage so that only one of them is committing to persistant data.
+	Future<Void> oldLogServer;
+
+	PromiseStream<Future<Void>> sharedActors;
+	bool terminated;
+
+	TLogData(UID dbgid, IKeyValueStore* persistentData, IDiskQueue * persistentQueue, Reference<AsyncVar<ServerDBInfo>> const& dbInfo)
+			: dbgid(dbgid), instanceID(g_random->randomUniqueID().first()),
+			  persistentData(persistentData), rawPersistentQueue(persistentQueue), persistentQueue(new TLogQueue(persistentQueue, dbgid)),
+			  dbInfo(dbInfo), queueCommitBegin(0), queueCommitEnd(0), prevVersion(0),
+			  diskQueueCommitBytes(0), largeDiskQueueCommitBytes(false),
+			  bytesInput(0), bytesDurable(0), updatePersist(Void()), terminated(false)
+		{
+		}
+};
+
+struct LogData : NonCopyable, public ReferenceCounted<LogData> {
+	struct TagData {
+		std::deque<std::pair<Version, LengthPrefixedStringRef>> version_messages;
+		bool nothing_persistent;				// true means tag is *known* to have no messages in persistentData.  false means nothing.
+		bool popped_recently;					// `popped` has changed since last updatePersistentData
+		Version popped;				// see popped version tracking contract below
+		bool update_version_sizes;
+
+		TagData( Version popped, bool nothing_persistent, bool popped_recently, Tag tag ) : nothing_persistent(nothing_persistent), popped(popped), popped_recently(popped_recently), update_version_sizes(tag != txsTag) {}
+
+		TagData(TagData&& r) noexcept(true) : version_messages(std::move(r.version_messages)), nothing_persistent(r.nothing_persistent), popped_recently(r.popped_recently), popped(r.popped), update_version_sizes(r.update_version_sizes) {}
+		void operator= (TagData&& r) noexcept(true) {
+			version_messages = std::move(r.version_messages);
+			nothing_persistent = r.nothing_persistent;
+			popped_recently = r.popped_recently;
+			popped = r.popped;
+			update_version_sizes = r.update_version_sizes;
+		}
+
+		// Erase messages not needed to update *from* versions >= before (thus, messages with toversion <= before)
+		ACTOR Future<Void> eraseMessagesBefore( TagData *self, Version before, int64_t* gBytesErased, Reference<LogData> tlogData, int taskID ) {
+			while(!self->version_messages.empty() && self->version_messages.front().first < before) {
+				Version version = self->version_messages.front().first;
+				std::pair<int, int> &sizes = tlogData->version_sizes[version];
+				int64_t messagesErased = 0;
+
+				while(!self->version_messages.empty() && self->version_messages.front().first == version) {
+					auto const& m = self->version_messages.front();
+					++messagesErased;
+
+					if(self->update_version_sizes) {
+						sizes.first -= m.second.expectedSize();
+					}
+
+					self->version_messages.pop_front();
+				}
+
+				int64_t bytesErased = (messagesErased * sizeof(std::pair<Version, LengthPrefixedStringRef>) * SERVER_KNOBS->VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS) >> 10;
+				tlogData->bytesDurable += bytesErased;
+				*gBytesErased += bytesErased;
+				Void _ = wait(yield(taskID));
+			}
+
+			return Void();
+		}
+
+		Future<Void> eraseMessagesBefore(Version before, int64_t* gBytesErased, Reference<LogData> tlogData, int taskID) {
+			return eraseMessagesBefore(this, before, gBytesErased, tlogData, taskID);
+		}
+	};
+
+	/*
+	Popped version tracking contract needed by log system to implement ILogCursor::popped():
+
+		- Log server tracks for each (possible) tag a popped_version
+		Impl: TagData::popped (in memory) and persistTagPoppedKeys (in persistentData)
+		- popped_version(tag) is <= the maximum version for which log server (or a predecessor) is ever asked to pop the tag
+		Impl: Only increased by tLogPop() in response to either a pop request or recovery from a predecessor
+		- popped_version(tag) is > the maximum version for which log server is unable to peek messages due to previous pops (on this server or a predecessor)
+		Impl: Increased by tLogPop() atomically with erasing messages from memory; persisted by updatePersistentData() atomically with erasing messages from store; messages are not erased from queue where popped_version is not persisted
+		- LockTLogReply returns all tags which either have messages, or which have nonzero popped_versions
+		Impl: tag_data is present for all such tags
+		- peek(tag, v) returns the popped_version for tag if that is greater than v
+		Impl: Check tag_data->popped (after all waits)
+	*/
+
+	bool stopped, initialized;
+	DBRecoveryCount recoveryCount;
+
+	VersionMetricHandle persistentDataVersion, persistentDataDurableVersion;  // The last version number in the portion of the log (written|durable) to persistentData
+	NotifiedVersion version, queueCommittedVersion;
+	Version queueCommittingVersion;
+	Version knownCommittedVersion;
+
+	Deque<std::pair<Version, Standalone<VectorRef<uint8_t>>>> messageBlocks;
+	Map< Tag, TagData > tag_data;
+
+	Map<Version, std::pair<int,int>> version_sizes;
+
+	CounterCollection cc;
+	Counter bytesInput;
+	Counter bytesDurable;
+
+	UID logId;
+	Version newPersistentDataVersion;
+	Future<Void> removed;
+	TLogInterface tli;
+	PromiseStream<Future<Void>> addActor;
+	TLogData* tLogData;
+	Future<Void> recovery;
+
+	explicit LogData(TLogData* tLogData, TLogInterface interf) : tLogData(tLogData), knownCommittedVersion(0), tli(interf), logId(interf.id()),
+			cc("TLog", interf.id().toString()),
+			bytesInput("bytesInput", cc),
+			bytesDurable("bytesDurable", cc),
+			// These are initialized differently on init() or recovery
+			recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), recovery(Void())
+	{
+		startRole(interf.id(), UID(), "TLog");
+
+		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
+		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
+		version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
+		queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
+
+		specialCounter(cc, "version", [this](){ return this->version.get(); });
+		specialCounter(cc, "sharedBytesInput", [tLogData](){ return tLogData->bytesInput; });
+		specialCounter(cc, "sharedBytesDurable", [tLogData](){ return tLogData->bytesDurable; });
+		specialCounter(cc, "kvstoreBytesUsed", [tLogData](){ return tLogData->persistentData->getStorageBytes().used; });
+		specialCounter(cc, "kvstoreBytesFree", [tLogData](){ return tLogData->persistentData->getStorageBytes().free; });
+		specialCounter(cc, "kvstoreBytesAvailable", [tLogData](){ return tLogData->persistentData->getStorageBytes().available; });
+		specialCounter(cc, "kvstoreBytesTotal", [tLogData](){ return tLogData->persistentData->getStorageBytes().total; });
+		specialCounter(cc, "queueDiskBytesUsed", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().used; });
+		specialCounter(cc, "queueDiskBytesFree", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().free; });
+		specialCounter(cc, "queueDiskBytesAvailable", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().available; });
+		specialCounter(cc, "queueDiskBytesTotal", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().total; });
+	}
+
+	~LogData() {
+		tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue();
+		TraceEvent("TLogBytesWhenRemoved", tli.id()).detail("sharedBytesInput", tLogData->bytesInput).detail("sharedBytesDurable", tLogData->bytesDurable).detail("localBytesInput", bytesInput.getValue()).detail("localBytesDurable", bytesDurable.getValue());
+
+		ASSERT(tLogData->bytesDurable <= tLogData->bytesInput);
+		endRole(tli.id(), "TLog", "Error", true);
+
+		if(!tLogData->terminated) {
+			Key logIdKey = BinaryWriter::toValue(logId,Unversioned());
+			tLogData->persistentData->clear( singleKeyRange(logIdKey.withPrefix(persistCurrentVersionKeys.begin)) );
+			tLogData->persistentData->clear( singleKeyRange(logIdKey.withPrefix(persistRecoveryCountKeys.begin)) );
+			Key msgKey = logIdKey.withPrefix(persistTagMessagesKeys.begin);
+			tLogData->persistentData->clear( KeyRangeRef( msgKey, strinc(msgKey) ) );
+			Key poppedKey = logIdKey.withPrefix(persistTagPoppedKeys.begin);
+			tLogData->persistentData->clear( KeyRangeRef( poppedKey, strinc(poppedKey) ) );
+		}
+	}
+
+	LogEpoch epoch() const { return recoveryCount; }
+};
+
+ACTOR Future<Void> tLogLock( TLogData* self, ReplyPromise< TLogLockResult > reply, Reference<LogData> logData ) {
+	state Version stopVersion = logData->version.get();
+
+	TEST(true); // TLog stopped by recovering master
+	TEST( logData->stopped );
+	TEST( !logData->stopped );
+
+	TraceEvent("TLogStop", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get());
+
+	logData->stopped = true;
+	logData->recovery = Void();
+
+	// Lock once the current version has been committed
+	Void _ = wait( logData->queueCommittedVersion.whenAtLeast( stopVersion ) );
+
+	ASSERT(stopVersion == logData->version.get());
+
+	TLogLockResult result;
+	result.end = stopVersion;
+	result.knownCommittedVersion = logData->knownCommittedVersion;
+	for( auto & tag : logData->tag_data )
+		result.tags.push_back( tag.key );
+
+	TraceEvent("TLogStop2", self->dbgid).detail("logId", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get()).detail("tags", describe(result.tags));
+
+
+	reply.send( result );
+	return Void();
+}
+
 void updatePersistentPopped( TLogData* self, Reference<LogData> logData, Tag tag, LogData::TagData& data ) {
 	if (!data.popped_recently) return;
 	self->persistentData->set(KeyValueRef( persistTagPoppedKey(logData->logId, tag), persistTagPoppedValue(data.popped) ));
@@ -526,13 +550,19 @@ ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logD
 	Void _ = wait(yield(TaskUpdateStorage));
 
 	while(!logData->messageBlocks.empty() && logData->messageBlocks.front().first <= newPersistentDataVersion) {
-		logData->bytesDurable += logData->messageBlocks.front().second.size() * SERVER_KNOBS->TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
-		self->bytesDurable += logData->messageBlocks.front().second.size() * SERVER_KNOBS->TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
+		int64_t bytesErased = int64_t(logData->messageBlocks.front().second.size()) * SERVER_KNOBS->TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
+		logData->bytesDurable += bytesErased;
+		self->bytesDurable += bytesErased;
 		logData->messageBlocks.pop_front();
 		Void _ = wait(yield(TaskUpdateStorage));
 	}
 
+	if(logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) {
+		TraceEvent(SevError, "BytesDurableTooLarge", logData->logId).detail("sharedBytesInput", self->bytesInput).detail("sharedBytesDurable", self->bytesDurable).detail("localBytesInput", logData->bytesInput.getValue()).detail("localBytesDurable", logData->bytesDurable.getValue());
+	}
+
 	ASSERT(logData->bytesDurable.getValue() <= logData->bytesInput.getValue());
+	ASSERT(self->bytesDurable <= self->bytesInput);
 
 	if( self->queueCommitEnd.get() > 0 )
 		self->persistentQueue->pop( newPersistentDataVersion+1 ); // SOMEDAY: this can cause a slow task (~0.5ms), presumably from erasing too many versions. Should we limit the number of versions cleared at a time?
@@ -559,8 +589,6 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
 	state int totalSize = 0;
 
 	if(logData->stopped) {
-		//FIXME: do we really want to make everything durable for this generation once we pass the threshold?
-		//FIXME: do we need yields?
 		if (self->bytesInput - self->bytesDurable >= SERVER_KNOBS->TLOG_SPILL_THRESHOLD) {
 			while(logData->persistentDataDurableVersion != logData->version.get()) {
 				std::vector<std::pair<std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator, std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator>> iters;
@@ -595,9 +623,15 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
 				} else {
 					Void _ = wait( delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL, TaskUpdateStorage) );
 				}
+
+				if( logData->removed.isReady() ) {
+					break;
+				}
 			}
 
-			self->queueOrder.pop_front();
+			if(logData->persistentDataDurableVersion == logData->version.get()) {
+				self->queueOrder.pop_front();
+			}
 			Void _ = wait( delay(0.0, TaskUpdateStorage) );
 		} else {
 			Void _ = wait( delay(BUGGIFY ? SERVER_KNOBS->BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL : SERVER_KNOBS->TLOG_STORAGE_MIN_UPDATE_INTERVAL, TaskUpdateStorage) );
@@ -1112,11 +1146,11 @@ ACTOR Future<Void> respondToRecovered( TLogInterface tli, Future<Void> recovery
 
 ACTOR Future<Void> cleanupPeekTrackers( TLogData* self ) {
 	loop {
-		double minExpireTime = SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME;
+		double minTimeUntilExpiration = SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME;
 		auto it = self->peekTracker.begin();
 		while(it != self->peekTracker.end()) {
-			double expireTime = SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME - now()-it->second.lastUpdate;
-			if(expireTime < 1.0e-6) {
+			double timeUntilExpiration = it->second.lastUpdate + SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME - now();
+			if(timeUntilExpiration < 1.0e-6) {
 				for(auto seq : it->second.sequence_version) {
 					if(!seq.second.isSet()) {
 						seq.second.sendError(timed_out());
@@ -1124,12 +1158,12 @@ ACTOR Future<Void> cleanupPeekTrackers( TLogData* self ) {
 				}
 				it = self->peekTracker.erase(it);
 			} else {
-				minExpireTime = std::min(minExpireTime, expireTime);
+				minTimeUntilExpiration = std::min(minTimeUntilExpiration, timeUntilExpiration);
 				++it;
 			}
 		}
 
-		Void _ = wait( delay(minExpireTime) );
+		Void _ = wait( delay(minTimeUntilExpiration) );
 	}
 }
 
@@ -1179,7 +1213,22 @@ ACTOR Future<Void> serveTLogInterface( TLogData* self, TLogInterface tli, Refere
 	}
 }
 
-ACTOR Future<Void> tLogCore( TLogData* self, Reference<LogData> logData, Future<Void> recovery ) {
+void removeLog( TLogData* self, Reference<LogData> logData ) {
+	TraceEvent("TLogRemoved", logData->logId).detail("input", logData->bytesInput.getValue()).detail("durable", logData->bytesDurable.getValue());
+	logData->stopped = true;
+	logData->recovery = Void();
+
+	logData->addActor = PromiseStream<Future<Void>>(); //there could be items still in the promise stream if one of the actors threw an error immediately
+	self->id_data.erase(logData->logId);
+
+	if(self->id_data.size() || (self->oldLogServer.isValid() && !self->oldLogServer.isReady())) {
+		return;
+	} else {
+		throw worker_removed();
+	}
+}
+
+ACTOR Future<Void> tLogCore( TLogData* self, Reference<LogData> logData ) {
 	if(logData->removed.isReady()) {
 		Void _ = wait(delay(0)); //to avoid iterator invalidation in restorePersistentState when removed is already ready
 		ASSERT(logData->removed.isError());
@@ -1188,27 +1237,8 @@ ACTOR Future<Void> tLogCore( TLogData* self, Reference<LogData> logData, Future<
 			throw logData->removed.getError();
 		}
 
-		double inputSum = 0;
-		double durableSum = 0;
-		for(auto it : self->id_data) {
-			inputSum += it.second->bytesInput.getValue();
-			durableSum += it.second->bytesDurable.getValue();
-			TraceEvent("QueueMetrics1", self->dbgid).detail("logId", it.first).detail("input", it.second->bytesInput.getValue()).detail("durable", it.second->bytesDurable.getValue());
-		}
-
-		TraceEvent("RemoveMetrics1", self->dbgid).detail("logId", logData->logId).detail("inputSum", inputSum).detail("durableSum", durableSum).detail("nonDurableSum", inputSum - durableSum)
-			.detail("input", self->bytesInput).detail("durable", self->bytesDurable).detail("nonDurable", self->bytesInput - self->bytesDurable).detail("durableIncrease", logData->bytesInput.getValue() - logData->bytesDurable.getValue());
-
-		self->bytesDurable += logData->bytesInput.getValue() - logData->bytesDurable.getValue();
-		logData->stopped = true;
-		self->id_data.erase(logData->logId);
-		TraceEvent("TLogRemoved", logData->logId);
-
-		if(self->id_data.size() || (self->oldLogServer.isValid() && !self->oldLogServer.isReady())) {
-			return Void();
-		} else {
-			throw logData->removed.getError();
-		}
+		removeLog(self, logData);
+		return Void();
 	}
 
 	TraceEvent("newLogData", self->dbgid).detail("logId", logData->logId);
@@ -1219,15 +1249,15 @@ ACTOR Future<Void> tLogCore( TLogData* self, Reference<LogData> logData, Future<
 	state Future<Void> warningCollector = timeoutWarningCollector( warningCollectorInput.getFuture(), 1.0, "TLogQueueCommitSlow", self->dbgid );
 	state Future<Void> error = actorCollection( logData->addActor.getFuture() );
 
-	if( recovery.isValid() && !recovery.isReady()) {
-		logData->addActor.send( recovery );
+	if( logData->recovery.isValid() && !logData->recovery.isReady()) {
+		logData->addActor.send( logData->recovery );
 	}
 
 	logData->addActor.send( waitFailureServer(logData->tli.waitFailure.getFuture()) );
-	logData->addActor.send( respondToRecovered(logData->tli, recovery) );
+	logData->addActor.send( respondToRecovered(logData->tli, logData->recovery) );
 	logData->addActor.send( logData->removed );
 	//FIXME: update tlogMetrics to include new information, or possibly only have one copy for the shared instance
-	logData->addActor.send( traceCounters("TLogMetrics", logData->logId, SERVER_KNOBS->STORAGE_LOGGING_DELAY, &logData->cc, self->dbgid.toString() + "/TLogMetrics"));
+	logData->addActor.send( traceCounters("TLogMetrics", logData->logId, SERVER_KNOBS->STORAGE_LOGGING_DELAY, &logData->cc, logData->logId.toString() + "/TLogMetrics"));
 	logData->addActor.send( serveTLogInterface(self, logData->tli, logData, warningCollectorInput) );
 
 	try {
@@ -1237,29 +1267,8 @@ ACTOR Future<Void> tLogCore( TLogData* self, Reference<LogData> logData, Future<
 		if( e.code() != error_code_worker_removed )
 			throw;
 
-		double inputSum = 0;
-		double durableSum = 0;
-		for(auto it : self->id_data) {
-			inputSum += it.second->bytesInput.getValue();
-			durableSum += it.second->bytesDurable.getValue();
-			TraceEvent("QueueMetrics2", self->dbgid).detail("logId", it.first).detail("input", it.second->bytesInput.getValue()).detail("durable", it.second->bytesDurable.getValue());
-		}
-
-		TraceEvent("RemoveMetrics2", self->dbgid).detail("logId", logData->logId).detail("inputSum", inputSum).detail("durableSum", durableSum).detail("nonDurableSum", inputSum - durableSum)
-			.detail("input", self->bytesInput).detail("durable", self->bytesDurable).detail("nonDurable", self->bytesInput - self->bytesDurable).detail("durableIncrease", logData->bytesInput.getValue() - logData->bytesDurable.getValue());
-
-		self->bytesDurable += logData->bytesInput.getValue() - logData->bytesDurable.getValue();
-		logData->stopped = true;
-
-		logData->addActor = PromiseStream<Future<Void>>(); //there could be items still in the promise stream if one of the actors threw an error immediately
-		self->id_data.erase(logData->logId);
-		TraceEvent("TLogRemoved", logData->logId);
-
-		if(self->id_data.size() || (self->oldLogServer.isValid() && !self->oldLogServer.isReady())) {
-			return Void();
-		} else {
-			throw;
-		}
+		removeLog(self, logData);
+		return Void();
 	}
 }
 
@@ -1357,7 +1366,7 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
 		DUMPTOKEN( recruited.getQueuingMetrics );
 		DUMPTOKEN( recruited.confirmRunning );
 
-		logData = Reference<LogData>( new LogData(recruited) );
+		logData = Reference<LogData>( new LogData(self, recruited) );
 		logData->stopped = true;
 		self->id_data[id1] = logData;
 
@@ -1405,8 +1414,13 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
 				when( TLogQueueEntry qe = wait( self->persistentQueue->readNext() ) ) {
 					if(!self->queueOrder.size() || self->queueOrder.back() != qe.id) self->queueOrder.push_back(qe.id);
 					if(qe.id != lastId) {
-						logData = self->id_data[qe.id];
 						lastId = qe.id;
+						auto it = self->id_data.find(qe.id);
+						if(it != self->id_data.end()) {
+							logData = it->second;
+						} else {
+							logData = Reference<LogData>();
+						}
 					} else {
 						ASSERT( qe.version >= lastVer );
 						lastVer = qe.version;
@@ -1415,19 +1429,21 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
 					//TraceEvent("TLogRecoveredQE", self->dbgid).detail("logId", qe.id).detail("ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size())
 					//	.detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("version", logData->version.get());
 
-					logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
-					if( qe.version > logData->version.get() ) {
-						commitMessages(logData, qe.version, qe.arena(), qe.messages, qe.tags, self->bytesInput);
-						logData->version.set( qe.version );
-						logData->queueCommittedVersion.set( qe.version );
+					if(logData) {
+						logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
+						if( qe.version > logData->version.get() ) {
+							commitMessages(logData, qe.version, qe.arena(), qe.messages, qe.tags, self->bytesInput);
+							logData->version.set( qe.version );
+							logData->queueCommittedVersion.set( qe.version );
 
-						while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
-							TEST(true);  // Flush excess data during TLog queue recovery
-							TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid).detail("BytesInput", self->bytesInput).detail("BytesDurable", self->bytesDurable).detail("Version", logData->version.get()).detail("PVer", logData->persistentDataVersion);
+							while (self->bytesInput - self->bytesDurable >= recoverMemoryLimit) {
+								TEST(true);  // Flush excess data during TLog queue recovery
+								TraceEvent("FlushLargeQueueDuringRecovery", self->dbgid).detail("BytesInput", self->bytesInput).detail("BytesDurable", self->bytesDurable).detail("Version", logData->version.get()).detail("PVer", logData->persistentDataVersion);
 
-							choose {
-								when( Void _ = wait( updateStorage(self) ) ) {}
-								when( Void _ = wait( allRemoved ) ) { throw worker_removed(); }
+								choose {
+									when( Void _ = wait( updateStorage(self) ) ) {}
+									when( Void _ = wait( allRemoved ) ) { throw worker_removed(); }
+								}
 							}
 						}
 					}
@@ -1447,7 +1463,7 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
 			TraceEvent("TLogZeroVersion", self->dbgid).detail("logId", it.first);
 			it.second->queueCommittedVersion.set(it.second->version.get());
 		}
-		self->sharedActors.send( tLogCore( self, it.second, Void() ) );
+		self->sharedActors.send( tLogCore( self, it.second ) );
 	}
 
 	return Void();
@@ -1455,6 +1471,7 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
 
 bool tlogTerminated( TLogData* self, IKeyValueStore* persistentData, TLogQueue* persistentQueue, Error const& e ) {
 	// Dispose the IKVS (destroying its data permanently) only if this shutdown is definitely permanent.  Otherwise just close it.
+	self->terminated = true;
 	if (e.code() == error_code_worker_removed || e.code() == error_code_recruitment_failed) {
 		persistentData->dispose();
 		persistentQueue->dispose();
@@ -1480,7 +1497,7 @@ ACTOR Future<Void> recoverTagFromLogSystem( TLogData* self, Reference<LogData> l
 	state Version tagPopped = 0;
 	state Version lastVer = 0;
 
-	TraceEvent("LogRecoveringTagBegin", self->dbgid).detail("Tag", tag).detail("recoverAt", endVersion);
+	TraceEvent("LogRecoveringTagBegin", logData->logId).detail("Tag", tag).detail("recoverAt", endVersion);
 
 	while (tagAt <= endVersion) {
 		loop {
@@ -1499,18 +1516,18 @@ ACTOR Future<Void> recoverTagFromLogSystem( TLogData* self, Reference<LogData> l
 			}
 		}
 
-		TraceEvent("LogRecoveringTagResults", logData->logId).detail("Tag", tag);
+		//TraceEvent("LogRecoveringTagResults", logData->logId).detail("Tag", tag);
 
 		Version ver = 0;
 		BinaryWriter wr( Unversioned() );
 		int writtenBytes = 0;
 		while (true) {
 			bool foundMessage = r->hasMessage();
-			//TraceEvent("LogRecoveringMsg").detail("Tag", tag).detail("foundMessage", foundMessage).detail("ver", r->version().toString());
+			//TraceEvent("LogRecoveringMsg", logData->logId).detail("Tag", tag).detail("foundMessage", foundMessage).detail("ver", r->version().toString());
 			if (!foundMessage || r->version().version != ver) {
 				ASSERT(r->version().version > lastVer);
 				if (ver) {
-					//TraceEvent("LogRecoveringTagVersion", self->dbgid).detail("Tag", tag).detail("Ver", ver).detail("Bytes", wr.getLength());
+					//TraceEvent("LogRecoveringTagVersion", logData->logId).detail("Tag", tag).detail("Ver", ver).detail("Bytes", wr.getLength());
 					writtenBytes += 100 + wr.getLength();
 					self->persistentData->set( KeyValueRef( persistTagMessagesKey( logData->logId, tag, ver ), wr.toStringRef() ) );
 				}
@@ -1547,6 +1564,8 @@ ACTOR Future<Void> recoverTagFromLogSystem( TLogData* self, Reference<LogData> l
 	Void _ = wait(tLogPop( self, TLogPopRequest(tagPopped, tag), logData ));
 
 	updatePersistentPopped( self, logData, tag, logData->tag_data.find(tag)->value );
+
+	TraceEvent("LogRecoveringTagComplete", logData->logId).detail("Tag", tag).detail("recoverAt", endVersion);
 	return Void();
 }
 
@@ -1579,56 +1598,62 @@ ACTOR Future<Void> recoverFromLogSystem( TLogData* self, Reference<LogData> logD
 	state Future<Void> recoveryDone = Never();
 	state Future<Void> commitTimeout = delay(SERVER_KNOBS->LONG_TLOG_COMMIT_TIME);
 
-	loop {
-		choose {
-			when(Void _ = wait(copyDone)) {
-				recoverFutures.clear();
-				for(auto tag : recoverTags )
-					recoverFutures.push_back(recoverTagFromLogSystem(self, logData, 0, knownCommittedVersion, tag, uncommittedBytes, logSystem));
-				copyDone = Never();
-				recoveryDone =  waitForAll(recoverFutures);
+	try {
+		loop {
+			choose {
+				when(Void _ = wait(copyDone)) {
+					recoverFutures.clear();
+					for(auto tag : recoverTags )
+						recoverFutures.push_back(recoverTagFromLogSystem(self, logData, 0, knownCommittedVersion, tag, uncommittedBytes, logSystem));
+					copyDone = Never();
+					recoveryDone =  waitForAll(recoverFutures);
 
-				Void __ = wait( committing );
-				Void __ = wait( self->updatePersist );
-				committing = self->persistentData->commit();
-				commitTimeout = delay(SERVER_KNOBS->LONG_TLOG_COMMIT_TIME);
-				uncommittedBytes->set(0);
-				Void __ = wait( committing );
-				TraceEvent("TLogCommitCopyData", self->dbgid);
+					Void __ = wait( committing );
+					Void __ = wait( self->updatePersist );
+					committing = self->persistentData->commit();
+					commitTimeout = delay(SERVER_KNOBS->LONG_TLOG_COMMIT_TIME);
+					uncommittedBytes->set(0);
+					Void __ = wait( committing );
+					TraceEvent("TLogCommitCopyData", logData->logId);
 
-				if(!copyComplete.isSet())
-					copyComplete.send(Void());
-			}
-			when(Void _ = wait(recoveryDone)) { break; }
-			when(Void _ = wait(commitTimeout)) {
-				TEST(true); // We need to commit occasionally if this process is long to avoid running out of memory.
-				// We let one, but not more, commits pipeline with the network transfer
-				Void __ = wait( committing );
-				Void __ = wait( self->updatePersist );
-				committing = self->persistentData->commit();
-				commitTimeout = delay(SERVER_KNOBS->LONG_TLOG_COMMIT_TIME);
-				uncommittedBytes->set(0);
-				TraceEvent("TLogCommitRecoveryData", self->dbgid).detail("MemoryUsage", DEBUG_DETERMINISM ? 0 : getMemoryUsage());
-			}
-			when(Void _ = wait(uncommittedBytes->onChange())) {
-				if(uncommittedBytes->get() >= SERVER_KNOBS->LARGE_TLOG_COMMIT_BYTES)
-					commitTimeout = Void();
+					if(!copyComplete.isSet())
+						copyComplete.send(Void());
+				}
+				when(Void _ = wait(recoveryDone)) { break; }
+				when(Void _ = wait(commitTimeout)) {
+					TEST(true); // We need to commit occasionally if this process is long to avoid running out of memory.
+					// We let one, but not more, commits pipeline with the network transfer
+					Void __ = wait( committing );
+					Void __ = wait( self->updatePersist );
+					committing = self->persistentData->commit();
+					commitTimeout = delay(SERVER_KNOBS->LONG_TLOG_COMMIT_TIME);
+					uncommittedBytes->set(0);
+					//TraceEvent("TLogCommitRecoveryData", self->dbgid).detail("MemoryUsage", DEBUG_DETERMINISM ? 0 : getMemoryUsage());
+				}
+				when(Void _ = wait(uncommittedBytes->onChange())) {
+					if(uncommittedBytes->get() >= SERVER_KNOBS->LARGE_TLOG_COMMIT_BYTES)
+						commitTimeout = Void();
+				}
 			}
 		}
+
+		Void _ = wait( committing );
+		Void _ = wait( self->updatePersist );
+		Void _ = wait( self->persistentData->commit() );
+
+		TraceEvent("TLogRecoveryComplete", logData->logId).detail("Locality", self->dbInfo->get().myLocality.toString());
+		TEST(true);  // tLog restore from old log system completed
+
+		return Void();
+	} catch( Error &e ) {
+		TraceEvent("TLogRecoveryError", logData->logId).error(e,true);
+		if(!copyComplete.isSet())
+			copyComplete.sendError(worker_removed());
+		throw;
 	}
-
-	Void _ = wait( committing );
-	Void _ = wait( self->updatePersist );
-	Void _ = wait( self->persistentData->commit() );
-
-	TraceEvent("TLogRecoveryComplete", self->dbgid).detail("Locality", self->dbInfo->get().myLocality.toString());
-	TEST(true);  // tLog restore from old log system completed
-
-	return Void();
 }
 
 ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, LocalityData locality ) {
-	state Future<Void> recovery = Void();
 	state TLogInterface recruited;
 	recruited.locality = locality;
 	recruited.initEndpoints();
@@ -1642,16 +1667,15 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
 
 	for(auto it : self->id_data) {
 		it.second->stopped = true;
+		it.second->recovery = Void();
 	}
 
-	state Reference<LogData> logData = Reference<LogData>( new LogData(recruited) );
+	state Reference<LogData> logData = Reference<LogData>( new LogData(self, recruited) );
 	self->id_data[recruited.id()] = logData;
 	logData->recoveryCount = req.epoch;
 	logData->removed = rejoinMasters(self, recruited, req.epoch);
 	self->queueOrder.push_back(recruited.id());
 
-	Void _ = wait( delay(0.0) ); // if multiple recruitment requests were already in the promise stream make sure they are all started before any are removed
-
 	TraceEvent("TLogStart", logData->logId);
 
 	try {
@@ -1671,7 +1695,7 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
 
 			state Promise<Void> copyComplete;
 			TraceEvent("TLogRecover", self->dbgid).detail("logId", logData->logId).detail("at", req.recoverAt).detail("known", req.knownCommittedVersion).detail("tags", describe(req.recoverTags));
-			recovery = recoverFromLogSystem( self, logData, req.recoverFrom, req.recoverAt, req.knownCommittedVersion, req.recoverTags, copyComplete );
+			logData->recovery = recoverFromLogSystem( self, logData, req.recoverFrom, req.recoverAt, req.knownCommittedVersion, req.recoverTags, copyComplete );
 			Void _ = wait(copyComplete.getFuture() || logData->removed );
 		} else {
 			// Brand new tlog, initialization has already been done by caller
@@ -1686,38 +1710,17 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
 			throw;
 		}
 
-		double inputSum = 0;
-		double durableSum = 0;
-		for(auto it : self->id_data) {
-			inputSum += it.second->bytesInput.getValue();
-			durableSum += it.second->bytesDurable.getValue();
-			TraceEvent("QueueMetrics3", self->dbgid).detail("logId", it.first).detail("input", it.second->bytesInput.getValue()).detail("durable", it.second->bytesDurable.getValue());
-		}
+		Void _ = wait( delay(0.0) ); // if multiple recruitment requests were already in the promise stream make sure they are all started before any are removed
 
-		TraceEvent("RemoveMetrics3", self->dbgid).detail("logId", logData->logId).detail("inputSum", inputSum).detail("durableSum", durableSum).detail("nonDurableSum", inputSum - durableSum)
-			.detail("input", self->bytesInput).detail("durable", self->bytesDurable).detail("nonDurable", self->bytesInput - self->bytesDurable).detail("durableIncrease", logData->bytesInput.getValue() - logData->bytesDurable.getValue());
-
-		//FIXME: check that bytes durable cannot be increased by a dead log, check that bytes durable never becomes larger that bytes input
-		//FIXME: factor error handling code
-		self->bytesDurable += logData->bytesInput.getValue() - logData->bytesDurable.getValue();
-		logData->stopped = true;
-
-		logData->addActor = PromiseStream<Future<Void>>(); //there could be items still in the promise stream if one of the actors threw an error immediately
-		self->id_data.erase(logData->logId);
-		TraceEvent("TLogRemoved", logData->logId);
-
-		if(self->id_data.size() || (self->oldLogServer.isValid() && !self->oldLogServer.isReady())) {
-			return Void();
-		} else {
-			throw;
-		}
+		removeLog(self, logData);
+		return Void();
 	}
 
 	TraceEvent("TLogReady", logData->logId);
 
 	req.reply.send( recruited );
 
-	Void _ = wait( tLogCore( self, logData, recovery ) );
+	Void _ = wait( tLogCore( self, logData ) );
 	return Void();
 }
 
@@ -1754,11 +1757,13 @@ ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQ
 			}
 		}
 	} catch (Error& e) {
-		TraceEvent("TLogError", tlogId).error(e);
-		if(e.code() != error_code_actor_cancelled) {
-			while(!tlogRequests.isEmpty()) {
-				tlogRequests.getFuture().pop().reply.sendError(e);
-			}
+		TraceEvent("TLogError", tlogId).error(e, true);
+		while(!tlogRequests.isEmpty()) {
+			tlogRequests.getFuture().pop().reply.sendError(recruitment_failed());
+		}
+
+		for( auto& it : self.id_data ) {
+			it.second->recovery = Void();
 		}
 
 		if (tlogTerminated( &self, persistentData, self.persistentQueue, e )) {
diff --git a/fdbserver/WorkerInterface.h b/fdbserver/WorkerInterface.h
index 079d58e044..cf41ffe92f 100644
--- a/fdbserver/WorkerInterface.h
+++ b/fdbserver/WorkerInterface.h
@@ -32,6 +32,8 @@
 #include "fdbrpc/MultiInterface.h"
 #include "fdbclient/ClientWorkerInterface.h"
 
+#define DUMPTOKEN( name ) TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token)
+
 struct WorkerInterface {
 	ClientWorkerInterface clientInterface;
 	LocalityData locality;
@@ -292,7 +294,6 @@ Future<Void> tLog( class IKeyValueStore* const& persistentData, class IDiskQueue
 Future<Void> debugQueryServer( DebugQueryRequest const& req );
 Future<Void> monitorServerDBInfo( Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> const& ccInterface, Reference<ClusterConnectionFile> const&, LocalityData const&, Reference<AsyncVar<ServerDBInfo>> const& dbInfo );
 Future<Void> resolver( ResolverInterface const& proxy, InitializeResolverRequest const&, Reference<AsyncVar<ServerDBInfo>> const& db );
-Future<Void> runMetrics( Future<Database> const& fcx, Key const& metricsPrefix );
 
 void registerThreadForProfiling();
 void updateCpuProfiler(ProfilerRequest req);
diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp
index d4b40d4e03..32bb6d4cad 100644
--- a/fdbserver/fdbserver.actor.cpp
+++ b/fdbserver/fdbserver.actor.cpp
@@ -107,6 +107,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_NOBOX,                "-q",                          SO_NONE },
 	{ OPT_NOBOX,                "--no_dialog",                 SO_NONE },
 #endif
+	{ OPT_KVFILE,               "--kvfile",                    SO_REQ_SEP },
 	{ OPT_TESTFILE,             "-f",                          SO_REQ_SEP },
 	{ OPT_TESTFILE,             "--testfile",                  SO_REQ_SEP },
 	{ OPT_RESTARTING,           "-R",                          SO_NONE },
@@ -817,7 +818,7 @@ int main(int argc, char* argv[]) {
 
 		//Enables profiling on this thread (but does not start it)
 		registerThreadForProfiling();
-
+		
 		std::string commandLine;
 		for (int a = 0; a<argc; a++) {
 			if (a) commandLine += ' ';
@@ -1439,7 +1440,7 @@ int main(int argc, char* argv[]) {
 		// Initialize the thread pool
 		CoroThreadPool::init();
 		// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now.
-		TraceEvent::setNetworkThread();
+		TraceEvent::setNetworkThread(); 
 
 		if (role == Simulation || role == CreateTemplateDatabase) {
 			//startOldSimulator();
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 505711161b..fffe61cc4e 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -24,9 +24,9 @@
 #include "flow/Trace.h"
 #include "fdbrpc/FailureMonitor.h"
 #include "fdbclient/NativeAPI.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
 #include "ConflictSet.h"
-#include "flow/Notified.h"
 #include "DataDistribution.h"
 #include "Knobs.h"
 #include <iterator>
@@ -263,10 +263,6 @@ ACTOR Future<Void> newSeedServers( Reference<MasterData> self, vector<StorageSer
 			req.criticalRecruitment = true;
 			for(auto s = servers->begin(); s != servers->end(); ++s)
 				req.excludeMachines.push_back(s->locality.zoneId());
-			if( dataCenters.size() < self->configuration.minDataCenters ) {
-				for(auto dc = dataCenters.begin(); dc != dataCenters.end(); ++dc)
-					req.excludeDCs.push_back(*dc);
-			}
 
 			TraceEvent("MasterRecruitingInitialStorageServer", self->dbgid)
 				.detail("ExcludingMachines", req.excludeMachines.size())
@@ -454,7 +450,7 @@ ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster( Reference<Mast
 				}
 			}
 		}
-		when ( ReplyPromise<vector<StorageServerInterface>> req = waitNext( parent->provisionalProxies[0].getKeyServersLocations.getFuture() ) ) {
+		when ( ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> req = waitNext( parent->provisionalProxies[0].getKeyServersLocations.getFuture() ) ) {
 			req.send(Never());
 		}
 		when ( Void _ = wait( waitFailure ) ) { throw worker_removed(); }
diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index b0407bf45b..0fa7a70391 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -29,12 +29,12 @@
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/NativeAPI.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/MasterProxyInterface.h"
 #include "fdbclient/DatabaseContext.h"
 #include "WorkerInterface.h"
 #include "TLogInterface.h"
 #include "MoveKeys.h"
-#include "flow/Notified.h"
 #include "Knobs.h"
 #include "WaitFailure.h"
 #include "IKeyValueStore.h"
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 3a253aa278..ba453a2f6d 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -24,6 +24,7 @@
 #include "flow/TDMetric.actor.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/NativeAPI.h"
+#include "fdbclient/MetricLogger.h"
 #include "WorkerInterface.h"
 #include "IKeyValueStore.h"
 #include "WaitFailure.h"
@@ -77,10 +78,16 @@ struct ErrorInfo {
 
 Error checkIOTimeout(Error const &e) {
 	// Convert all_errors to io_timeout if global timeout bool was set
-	if((bool)g_network->global(INetwork::enASIOTimedOut)) {
+	bool timeoutOccurred = (bool)g_network->global(INetwork::enASIOTimedOut);
+	// In simulation, have to check global timed out flag for both this process and the machine process on which IO is done
+	if(g_network->isSimulated() && !timeoutOccurred)
+		timeoutOccurred = g_pSimulator->getCurrentProcess()->machine->machineProcess->global(INetwork::enASIOTimedOut);
+
+	if(timeoutOccurred) {
+		TEST(true); // Timeout occurred
 		Error timeout = io_timeout();
-		// If this error was injected OR if the timeout was injected then make the resulting io_timeout injected
-		if(e.isInjectedFault() || (g_network->isSimulated() && g_pSimulator->getCurrentProcess()->io_timeout_injected) )
+		// Preserve injectedness of error
+		if(e.isInjectedFault())
 			timeout = timeout.asInjectedFault();
 		return timeout;
 	}
@@ -124,14 +131,15 @@ ACTOR Future<Void> workerHandleErrors(FutureStream<ErrorInfo> errors) {
 	loop choose {
 		when( ErrorInfo _err = waitNext(errors) ) {
 			ErrorInfo err = _err;
-			err.error = checkIOTimeout(err.error);  // Possibly convert error to io_timeout
-
 			bool ok =
 				err.error.code() == error_code_success ||
 				err.error.code() == error_code_please_reboot ||
 				err.error.code() == error_code_actor_cancelled ||
 				err.error.code() == error_code_coordinators_changed;  // The worker server was cancelled
 
+			if(!ok)
+				err.error = checkIOTimeout(err.error);  // Possibly convert error to io_timeout
+
 			endRole(err.id, err.context, "Error", ok, err.error);
 
 			if (err.error.code() == error_code_please_reboot || err.error.code() == error_code_io_timeout) throw err.error;
@@ -164,8 +172,6 @@ ACTOR Future<Void> loadedPonger( FutureStream<LoadedPingRequest> pings ) {
 	}
 }
 
-#define DUMPTOKEN( name ) TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token)
-
 StringRef fileStoragePrefix = LiteralStringRef("storage-");
 StringRef fileLogDataPrefix = LiteralStringRef("log-");
 StringRef fileLogQueuePrefix = LiteralStringRef("logqueue-");
@@ -179,7 +185,7 @@ std::string filenameFromSample( KeyValueStoreType storeType, std::string folder,
 	if( storeType == KeyValueStoreType::SSD_BTREE_V1 )
 		return joinPath( folder, sample_filename );
 	else if ( storeType == KeyValueStoreType::SSD_BTREE_V2 )
-		return joinPath(folder, sample_filename); 
+		return joinPath(folder, sample_filename);
 	else if( storeType == KeyValueStoreType::MEMORY )
 		return joinPath( folder, sample_filename.substr(0, sample_filename.size() - 5) );
 
@@ -190,7 +196,7 @@ std::string filenameFromId( KeyValueStoreType storeType, std::string folder, std
 	if( storeType == KeyValueStoreType::SSD_BTREE_V1)
 		return joinPath( folder, prefix + id.toString() + ".fdb" );
 	else if (storeType == KeyValueStoreType::SSD_BTREE_V2)
-		return joinPath(folder, prefix + id.toString() + ".sqlite"); 
+		return joinPath(folder, prefix + id.toString() + ".sqlite");
 	else if( storeType == KeyValueStoreType::MEMORY )
 		return joinPath( folder, prefix + id.toString() + "-" );
 
@@ -350,6 +356,7 @@ void startRole(UID roleId, UID workerId, std::string as, std::map<std::string, s
 	g_roles.insert({as, roleId.shortString()});
 	StringMetricHandle(LiteralStringRef("Roles")) = roleString(g_roles, false);
 	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
+	if (g_network->isSimulated()) g_simulator.addRole(g_network->getLocalAddress(), as);
 }
 
 void endRole(UID id, std::string as, std::string reason, bool ok, Error e) {
@@ -381,6 +388,7 @@ void endRole(UID id, std::string as, std::string reason, bool ok, Error e) {
 	g_roles.erase({as, id.shortString()});
 	StringMetricHandle(LiteralStringRef("Roles")) = roleString(g_roles, false);
 	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
+	if (g_network->isSimulated()) g_simulator.removeRole(g_network->getLocalAddress(), as);
 }
 
 ACTOR Future<Void> monitorServerDBInfo( Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> ccInterface, Reference<ClusterConnectionFile> connFile, LocalityData locality, Reference<AsyncVar<ServerDBInfo>> dbInfo ) {
@@ -589,7 +597,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 				std::map<std::string, std::string> details;
 				details["StorageEngine"] = s.storeType.toString();
-				startRole( s.storeID, interf.id(), "TLog", details, "Restored" );
+				startRole( s.storeID, interf.id(), "SharedTLog", details, "Restored" );
 
 				Promise<Void> oldLog;
 				Future<Void> tl = tLog( kv, queue, dbInfo, locality, tlog.isReady() ? tlogRequests : PromiseStream<InitializeTLogRequest>(), s.storeID, true, oldLog );
@@ -598,7 +606,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				if(tlog.isReady()) {
 					tlog = oldLog.getFuture() || tl;
 				}
-				errorForwarders.add( forwardError( errors, "TLog", s.storeID, tl ) );
+				errorForwarders.add( forwardError( errors, "SharedTLog", s.storeID, tl ) );
 			}
 		}
 
@@ -616,7 +624,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 					Reference<IAsyncFile> checkFile = wait( IAsyncFileSystem::filesystem()->open( joinPath(folder, validationFilename), IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_READWRITE, 0600 ) );
 					Void _ = wait( checkFile->sync() );
 				}
-			
+
 				if(g_network->isSimulated()) {
 					TraceEvent("SimulatedReboot").detail("Deletion", rebootReq.deleteData );
 					if( rebootReq.deleteData ) {
@@ -655,9 +663,9 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 					std::map<std::string, std::string> details;
 					details["ForMaster"] = req.recruitmentID.shortString();
 					details["StorageEngine"] = req.storeType.toString();
-					
+
 					//FIXME: start role for every tlog instance, rather that just for the shared actor, also use a different role type for the shared actor
-					startRole( logId, interf.id(), "TLog", details );
+					startRole( logId, interf.id(), "SharedTLog", details );
 
 					std::string filename = filenameFromId( req.storeType, folder, fileLogDataPrefix.toString(), logId );
 					IKeyValueStore* data = openKVStore( req.storeType, filename, logId, memoryLimit );
@@ -667,7 +675,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 					tlog = tLog( data, queue, dbInfo, locality, tlogRequests, logId, false, Promise<Void>() );
 					tlog = handleIOErrors( tlog, data, logId );
 					tlog = handleIOErrors( tlog, queue, logId );
-					errorForwarders.add( forwardError( errors, "TLog", logId, tlog ) );
+					errorForwarders.add( forwardError( errors, "SharedTLog", logId, tlog ) );
 				}
 			}
 			when( InitializeStorageRequest req = waitNext(interf.storage.getFuture()) ) {
@@ -815,6 +823,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 		endRole(interf.id(), "Worker", "WorkerError", ok, e);
 		errorForwarders.clear(false);
+		tlog = Void();
 
 		if (e.code() != error_code_actor_cancelled) { // We get cancelled e.g. when an entire simulation times out, but in that case we won't be restarted and don't need to wait for shutdown
 			stopping.send(Void());
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index df9da038c2..fae6f05c0c 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -171,6 +171,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 				state DatabaseConfiguration configuration;
 
 				state Transaction tr(cx);
+				tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 				loop {
 					try {
 						Standalone<RangeResultRef> res = wait( tr.getRange(configKeys, 1000) );
@@ -247,15 +248,15 @@ struct ConsistencyCheckWorkload : TestWorkload
 				}
 
 				//Get a list of key servers; verify that the TLogs and master all agree about who the key servers are
-				state Promise<vector<StorageServerInterface>> keyServerPromise;
+				state Promise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> keyServerPromise;
 				bool keyServerResult = wait(self->getKeyServers(cx, self, keyServerPromise));
 				if(keyServerResult)
 				{
-					state vector<StorageServerInterface> storageServers = keyServerPromise.getFuture().get();
+					state vector<pair<KeyRangeRef, vector<StorageServerInterface>>> keyServers = keyServerPromise.getFuture().get();
 
 					//Get the locations of all the shards in the database
 					state Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise;
-					bool keyLocationResult = wait(self->getKeyLocations(cx, storageServers, self, keyLocationPromise));
+					bool keyLocationResult = wait(self->getKeyLocations(cx, keyServers, self, keyLocationPromise));
 					if(keyLocationResult)
 					{
 						state Standalone<VectorRef<KeyValueRef>> keyLocations = keyLocationPromise.getFuture().get();
@@ -268,7 +269,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 			catch(Error &e)
 			{
 				if(e.code() == error_code_past_version || e.code() == error_code_future_version || e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed || e.code() == error_code_server_request_queue_full)
-					TraceEvent("ConsistencyCheck_Retry").error(e);
+					TraceEvent("ConsistencyCheck_Retry").error(e); // FIXME: consistency check does not retry in this case
 				else
 					self->testFailure(format("Error %d - %s", e.code(), e.what()));
 			}
@@ -285,6 +286,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 		loop
 		{
 			state Transaction tr(cx);
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 			try
 			{
 				Version version = wait(tr.getReadVersion());
@@ -300,18 +302,18 @@ struct ConsistencyCheckWorkload : TestWorkload
 	//Get a list of storage servers from the master and compares them with the TLogs.
 	//If this is a quiescent check, then each master proxy needs to respond, otherwise only one needs to respond.
 	//Returns false if there is a failure (in this case, keyServersPromise will never be set)
-	ACTOR Future<bool> getKeyServers(Database cx, ConsistencyCheckWorkload *self, Promise<vector<StorageServerInterface>> keyServersPromise)
+	ACTOR Future<bool> getKeyServers(Database cx, ConsistencyCheckWorkload *self, Promise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> keyServersPromise)
 	{
-		state vector<StorageServerInterface> keyServers;
+		state vector<pair<KeyRangeRef, vector<StorageServerInterface>>> keyServers;
 
 		loop
 		{
 			state Reference<ProxyInfo> proxyInfo = wait(cx->getMasterProxiesFuture());
 
 			//Try getting key server locations from the master proxies
-			state vector<Future<ErrorOr<vector<StorageServerInterface>>>> keyServerLocationFutures;
+			state vector<Future<ErrorOr<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>>>> keyServerLocationFutures;
 			for(int i = 0; i < proxyInfo->size(); i++)
-				keyServerLocationFutures.push_back(proxyInfo->get(i,&MasterProxyInterface::getKeyServersLocations).getReplyUnlessFailedFor(ReplyPromise<vector<StorageServerInterface>>(), 2, 0));
+				keyServerLocationFutures.push_back(proxyInfo->get(i,&MasterProxyInterface::getKeyServersLocations).getReplyUnlessFailedFor(ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>>(), 2, 0));
 
 			choose {
 				when( Void _ = wait(waitForAll(keyServerLocationFutures)) ) {
@@ -320,21 +322,21 @@ struct ConsistencyCheckWorkload : TestWorkload
 					state bool successful = true;
 					for(int i = 0; i < keyServerLocationFutures.size(); i++)
 					{
-						ErrorOr<vector<StorageServerInterface>> interfaces = keyServerLocationFutures[i].get();
+						ErrorOr<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> shards = keyServerLocationFutures[i].get();
 
 						//If performing quiescent check, then all master proxies should be reachable.  Otherwise, only one needs to be reachable
-						if(self->performQuiescentChecks && !interfaces.present())
+						if(self->performQuiescentChecks && !shards.present())
 						{
 							TraceEvent("ConsistencyCheck_MasterProxyUnavailable").detail("MasterProxyID", proxyInfo->getId(i));
 							self->testFailure("Master proxy unavailable");
 							return false;
 						}
 
-						//Get the list of interfaces if one was returned.  If not doing a quiescent check, we can break if it is.
-						//If we are doing a quiescent check, then we only need to do this for the first interface.
-						if(interfaces.present() && (i == 0 || !self->performQuiescentChecks))
+						//Get the list of shards if one was returned.  If not doing a quiescent check, we can break if it is.
+						//If we are doing a quiescent check, then we only need to do this for the first shard.
+						if(shards.present() && (i == 0 || !self->performQuiescentChecks))
 						{
-							keyServers = interfaces.get();
+							keyServers = shards.get();
 							if(!self->performQuiescentChecks)
 								break;
 						}
@@ -364,96 +366,108 @@ struct ConsistencyCheckWorkload : TestWorkload
 
 	//Retrieves the locations of all shards in the database
 	//Returns false if there is a failure (in this case, keyLocationPromise will never be set)
-	ACTOR Future<bool> getKeyLocations(Database cx, vector<StorageServerInterface> storageServers, ConsistencyCheckWorkload *self, Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise)
+	ACTOR Future<bool> getKeyLocations(Database cx, vector<pair<KeyRangeRef, vector<StorageServerInterface>>> shards, ConsistencyCheckWorkload *self, Promise<Standalone<VectorRef<KeyValueRef>>> keyLocationPromise)
 	{
 		state Standalone<VectorRef<KeyValueRef>> keyLocations;
 		state Key beginKey = allKeys.begin;
+		state int i = 0;
 
 		//If the responses are too big, we may use multiple requests to get the key locations.  Each request begins where the last left off
-		while(beginKey < allKeys.end)
+		for ( ; i < shards.size(); i++)
 		{
-			try
+			// skip serverList shards
+			if (!shards[i].first.begin.startsWith(keyServersPrefix)) {
+				break;
+			}
+
+			state Key endKey = shards[i].first.end.startsWith(keyServersPrefix) ? shards[i].first.end.removePrefix(keyServersPrefix) : allKeys.end;
+
+			while(beginKey < endKey)
 			{
-				Version version = wait(self->getVersion(cx, self));
-
-				GetKeyValuesRequest req;
-				Key prefixBegin = beginKey.withPrefix(keyServersPrefix);
-				req.begin = firstGreaterOrEqual(prefixBegin);
-				req.end = firstGreaterOrEqual(keyServersEnd);
-				req.limit = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT;
-				req.limitBytes = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES;
-				req.version = version;
-
-				//Try getting the shard locations from the key servers
-				state vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
-				for(int i = 0; i < storageServers.size(); i++)
+				try
 				{
-					resetReply(req);
-					keyValueFutures.push_back(storageServers[i].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
-				}
+					Version version = wait(self->getVersion(cx, self));
 
-				Void _ = wait(waitForAll(keyValueFutures));
+					GetKeyValuesRequest req;
+					Key prefixBegin = beginKey.withPrefix(keyServersPrefix);
+					req.begin = firstGreaterOrEqual(prefixBegin);
+					req.end = firstGreaterOrEqual(keyServersEnd);
+					req.limit = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT;
+					req.limitBytes = SERVER_KNOBS->MOVE_KEYS_KRM_LIMIT_BYTES;
+					req.version = version;
 
-				int firstValidStorageServer = -1;
-
-				//Read the shard location results
-				for(int i = 0; i < keyValueFutures.size(); i++)
-				{
-					ErrorOr<GetKeyValuesReply> reply = keyValueFutures[i].get();
-
-					if(!reply.present())
+					//Try getting the shard locations from the key servers
+					state vector<Future<ErrorOr<GetKeyValuesReply>>> keyValueFutures;
+					for(int j = 0; j < shards[i].second.size(); j++)
 					{
-						//If the storage server didn't reply in a quiescent database, then the check fails
-						if(self->performQuiescentChecks)
+						resetReply(req);
+						keyValueFutures.push_back(shards[i].second[j].getKeyValues.getReplyUnlessFailedFor(req, 2, 0));
+					}
+
+					Void _ = wait(waitForAll(keyValueFutures));
+
+					int firstValidStorageServer = -1;
+
+					//Read the shard location results
+					for(int j = 0; j < keyValueFutures.size(); j++)
+					{
+						ErrorOr<GetKeyValuesReply> reply = keyValueFutures[j].get();
+
+						if(!reply.present())
 						{
-							TraceEvent("ConsistencyCheck_KeyServerUnavailable").detail("StorageServer", storageServers[i].id().toString().c_str());
-							self->testFailure("Key server unavailable");
-							return false;
+							//If the storage server didn't reply in a quiescent database, then the check fails
+							if(self->performQuiescentChecks)
+							{
+								TraceEvent("ConsistencyCheck_KeyServerUnavailable").detail("StorageServer", shards[i].second[j].id().toString().c_str());
+								self->testFailure("Key server unavailable");
+								return false;
+							}
+
+							//If no storage servers replied, then throw all_alternatives_failed to force a retry
+							else if(firstValidStorageServer < 0 && j == keyValueFutures.size() - 1)
+								throw all_alternatives_failed();
 						}
 
-						//If no storage servers replied, then throw all_alternatives_failed to force a retry
-						else if(firstValidStorageServer < 0 && i == keyValueFutures.size() - 1)
-							throw all_alternatives_failed();
+						//If this is the first storage server, store the locations to send back to the caller
+						else if(firstValidStorageServer < 0)
+							firstValidStorageServer = j;
+
+						//Otherwise, compare the data to the results from the first storage server.  If they are different, then the check fails
+						else if(reply.get().data != keyValueFutures[firstValidStorageServer].get().get().data || reply.get().more != keyValueFutures[firstValidStorageServer].get().get().more)
+						{
+							TraceEvent("ConsistencyCheck_InconsistentKeyServers").detail("StorageServer1", shards[i].second[firstValidStorageServer].id())
+								.detail("StorageServer2", shards[i].second[j].id());
+							self->testFailure("Key servers inconsistent");
+							return false;
+						}
 					}
 
-					//If this is the first storage server, store the locations to send back to the caller
-					else if(firstValidStorageServer < 0)
-						firstValidStorageServer = i;
+					auto keyValueResponse = keyValueFutures[firstValidStorageServer].get().get();
+					Standalone<RangeResultRef> currentLocations = krmDecodeRanges( keyServersPrefix, KeyRangeRef(beginKey, endKey), RangeResultRef( keyValueResponse.data, keyValueResponse.more) );
 
-					//Otherwise, compare the data to the results from the first storage server.  If they are different, then the check fails
-					else if(reply.get().data != keyValueFutures[firstValidStorageServer].get().get().data || reply.get().more != keyValueFutures[firstValidStorageServer].get().get().more)
-					{
-						TraceEvent("ConsistencyCheck_InconsistentKeyServers").detail("StorageServer1", storageServers[firstValidStorageServer].id())
-							.detail("StorageServer2", storageServers[i].id());
-						self->testFailure("Key servers inconsistent");
-						return false;
-					}
+					//Push all but the last item, which will be pushed as the first item next iteration
+					keyLocations.append_deep(keyLocations.arena(), currentLocations.begin(), currentLocations.size() - 1);
+
+					//Next iteration should pick up where we left off
+					ASSERT(currentLocations.size() > 1);
+					beginKey = currentLocations.end()[-1].key;
+
+					//If this is the last iteration, then push the allKeys.end KV pair
+					if(beginKey == allKeys.end)
+						keyLocations.push_back_deep(keyLocations.arena(), currentLocations.end()[-1]);
+				}
+				catch(Error &e)
+				{
+					//If we failed because of a version problem, then retry
+					if(e.code() == error_code_past_version || e.code() == error_code_future_version || e.code() == error_code_past_version)
+						TraceEvent("ConsistencyCheck_RetryGetKeyLocations").error(e);
+					else
+						throw;
 				}
-
-				auto keyValueResponse = keyValueFutures[firstValidStorageServer].get().get();
-				Standalone<RangeResultRef> currentLocations = krmDecodeRanges( keyServersPrefix, KeyRangeRef(beginKey, allKeys.end), RangeResultRef( keyValueResponse.data, keyValueResponse.more) );
-
-				//Push all but the last item, which will be pushed as the first item next iteration
-				keyLocations.append_deep(keyLocations.arena(), currentLocations.begin(), currentLocations.size() - 1);
-
-				//Next iteration should pick up where we left off
-				ASSERT(currentLocations.size() > 1);
-				beginKey = currentLocations.end()[-1].key;
-
-				//If this is the last iteration, then push the allKeys.end KV pair
-				if(beginKey == allKeys.end)
-					keyLocations.push_back_deep(keyLocations.arena(), currentLocations.end()[-1]);
-			}
-			catch(Error &e)
-			{
-				//If we failed because of a version problem, then retry
-				if(e.code() == error_code_past_version || e.code() == error_code_future_version || e.code() == error_code_past_version)
-					TraceEvent("ConsistencyCheck_RetryGetKeyLocations").error(e);
-				else
-					throw;
 			}
 		}
 
+
 		keyLocationPromise.send(keyLocations);
 		return true;
 	}
@@ -532,6 +546,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 
 	ACTOR Future<int64_t> getDatabaseSize(Database cx) {
 		state Transaction tr( cx );
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 		loop {
 			try {
 				StorageMetrics metrics = wait( tr.getStorageMetrics( KeyRangeRef(allKeys.begin, keyServersPrefix), 100000 ) );
@@ -1083,7 +1098,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 			for(auto id : stores.get()) {
 				if(!statefulProcesses[itr->first.address()].count(id)) {
 					TraceEvent("ConsistencyCheck_ExtraDataStore").detail("Address", itr->first.address()).detail("DataStoreID", id);
-					if(g_network->isSimulated()) {
+					if(g_network->isSimulated() && !g_simulator.speedUpSimulation) {
 						g_simulator.rebootProcess(g_simulator.getProcessByAddress(itr->first.address()), ISimulator::RebootProcess);
 					}
 
diff --git a/fdbserver/workloads/DummyWorkload.actor.cpp b/fdbserver/workloads/DummyWorkload.actor.cpp
index 77ab87a501..ee0bd55cf8 100644
--- a/fdbserver/workloads/DummyWorkload.actor.cpp
+++ b/fdbserver/workloads/DummyWorkload.actor.cpp
@@ -23,9 +23,13 @@
 
 // The workload that do nothing. It can be used for waiting for quiescence
 struct DummyWorkload : TestWorkload {
+	bool displayWorkers;
+	double displayDelay;
 
 	DummyWorkload(WorkloadContext const& wcx)
 	: TestWorkload(wcx) {
+		displayWorkers = getOption(options, LiteralStringRef("displayWorkers"), true);
+		displayDelay = getOption(options, LiteralStringRef("displayDelay"), 0.0);
 	}
 
 	virtual std::string description() {
@@ -33,6 +37,16 @@ struct DummyWorkload : TestWorkload {
 	}
 
 	virtual Future<Void> start(Database const& cx) {
+		if ((clientId == 0) && (displayWorkers)) {
+			return _start(this, cx);
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Void> _start( DummyWorkload* self, Database cx) {
+		if (self->displayDelay > 0.0)
+			Void _ = wait(delay(self->displayDelay));
+		g_simulator.displayWorkers();
 		return Void();
 	}
 
diff --git a/fdbserver/workloads/MachineAttrition.actor.cpp b/fdbserver/workloads/MachineAttrition.actor.cpp
index 41eb49a49d..5a9f198f1b 100644
--- a/fdbserver/workloads/MachineAttrition.actor.cpp
+++ b/fdbserver/workloads/MachineAttrition.actor.cpp
@@ -155,6 +155,7 @@ struct MachineAttritionWorkload : TestWorkload {
 				LocalityData targetMachine = self->machines.back();
 
 				TraceEvent("Assassination").detail("TargetMachine", targetMachine.toString())
+					.detailext("zoneId", targetMachine.zoneId())
 					.detail("Reboot", self->reboot).detail("killedMachines", killedMachines)
 					.detail("machinesToKill", self->machinesToKill).detail("machinesToLeave", self->machinesToLeave)
 					.detail("machines", self->machines.size()).detail("Replace", self->replacement);
@@ -166,8 +167,9 @@ struct MachineAttritionWorkload : TestWorkload {
 						g_simulator.killMachine( targetMachine.zoneId(), ISimulator::Reboot );
 					}
 				} else {
-					TraceEvent("WorkerKill").detail("MachineCount", self->machines.size());
-					if( g_random->random01() < 0.33 ) {
+					auto randomDouble = g_random->random01();
+					TraceEvent("WorkerKill").detail("MachineCount", self->machines.size()).detail("RandomValue", randomDouble);
+					if (randomDouble < 0.33 ) {
 						TraceEvent("RebootAndDelete").detail("TargetMachine", targetMachine.toString());
 						g_simulator.killMachine( targetMachine.zoneId(), ISimulator::RebootAndDelete );
 					} else {
diff --git a/fdbserver/workloads/RandomMoveKeys.actor.cpp b/fdbserver/workloads/RandomMoveKeys.actor.cpp
index 62dd950f16..1c06e24687 100644
--- a/fdbserver/workloads/RandomMoveKeys.actor.cpp
+++ b/fdbserver/workloads/RandomMoveKeys.actor.cpp
@@ -21,6 +21,7 @@
 #include "flow/actorcompiler.h"
 #include "fdbrpc/simulator.h"
 #include "fdbclient/StorageServerInterface.h"
+#include "fdbclient/ManagementAPI.h"
 #include "fdbserver/MoveKeys.h"
 #include "fdbclient/NativeAPI.h"
 #include "workloads.h"
@@ -65,15 +66,13 @@ struct MoveKeysWorkload : TestWorkload {
 				}
 			}
 
-			state int oldMode = wait( self->setDDMode( cx, 0 ) );
+			state int oldMode = wait( setDDMode( cx, 0 ) );
 			TraceEvent("RMKStartModeSetting");
 			Void _ = wait( timeout( reportErrors( self->worker( cx, self ), "moveKeysWorkloadWorkerError" ), self->testDuration, Void() ) );
 			// Always set the DD mode back, even if we die with an error
 			TraceEvent("RMKDoneMoving");
-			int _ = wait( self->setDDMode( cx, oldMode ) );
+			int _ = wait( setDDMode( cx, oldMode ) );
 			TraceEvent("RMKDoneModeSetting");
-			Void _ = wait( self->forceMasterFailure(cx, self) );
-			TraceEvent("RMKDoneKillingMaster");
 		}
 		return Void();
 	}
@@ -116,33 +115,6 @@ struct MoveKeysWorkload : TestWorkload {
 		return vector<StorageServerInterface>(t.begin(), t.end());
 	}
 
-	ACTOR Future<int> setDDMode( Database cx, int mode ) {
-		state Transaction tr(cx);
-		state int oldMode = -1;
-		state BinaryWriter wr(Unversioned());
-		wr << mode;
-
-		loop {
-			try {
-				Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
-				if (oldMode < 0) {
-					oldMode = 1;
-					if (old.present()) {
-						BinaryReader rd(old.get(), Unversioned());
-						rd >> oldMode;
-					}
-				}
-				tr.set( dataDistributionModeKey, wr.toStringRef() );
-
-				Void _ = wait( tr.commit() );
-				return oldMode;
-			} catch (Error& e) {
-				TraceEvent("setDDModeRetrying").error(e);
-				Void _ = wait (tr.onError(e));
-			}
-		}
-	}
-
 	ACTOR Future<Void> doMoveKeys(Database cx, MoveKeysWorkload *self, KeyRange keys, vector<StorageServerInterface> destinationTeam, 
 			MoveKeysLock lock, std::string dbName ) {
 		state TraceInterval relocateShardInterval("RelocateShard");
diff --git a/fdbserver/workloads/RemoveServersSafely.actor.cpp b/fdbserver/workloads/RemoveServersSafely.actor.cpp
index 1ee6a3c527..6d9e2e131c 100644
--- a/fdbserver/workloads/RemoveServersSafely.actor.cpp
+++ b/fdbserver/workloads/RemoveServersSafely.actor.cpp
@@ -68,9 +68,11 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 			AddressExclusion machineIp(it->address.ip);
 			AddressExclusion pAddr(it->address.ip, it->address.port);
 
-			processAddrs.push_back(pAddr);
 			TraceEvent("RemoveAndKill").detail("Step", "listAddresses")
 				.detail("Address", pAddr.toString()).detail("Process",describe(*it));
+
+			if (g_simulator.protectedAddresses.count(it->address) == 0)
+				processAddrs.push_back(pAddr);
 			machineProcesses[machineIp].insert(pAddr);
 
 			// add only one entry for each machine
@@ -149,6 +151,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 	{
 		std::vector<ISimulator::ProcessInfo*>	processes;
 		std::set<AddressExclusion>	processAddrs;
+		UID functionId = g_nondeterministic_random->randomUniqueID();
 
 		// Get the list of process network addresses
 		for (auto& netAddr : netAddrs) {
@@ -165,59 +168,112 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		// Get the list of processes matching network address
 		for (auto processInfo : g_simulator.getAllProcesses()) {
 			auto processNet = AddressExclusion(processInfo->address.ip, processInfo->address.port);
-			if (processAddrs.find(processNet) != processAddrs.end())
+			if (processAddrs.find(processNet) != processAddrs.end()) {
 				processes.push_back(processInfo);
+				TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcessItem").detail("ProcessAddress", processInfo->address).detail("Process", describe(*processInfo)).detail("failed", processInfo->failed).detail("excluded", processInfo->excluded).detail("rebooting", processInfo->rebooting).detail("Protected", g_simulator.protectedAddresses.count(processInfo->address));
+			}
+			else {
+				TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcessNoItem").detail("ProcessAddress", processInfo->address).detail("Process", describe(*processInfo)).detail("failed", processInfo->failed).detail("excluded", processInfo->excluded).detail("rebooting", processInfo->rebooting).detail("Protected", g_simulator.protectedAddresses.count(processInfo->address));
+			}
 		}
-		TraceEvent("RemoveAndKill").detail("Step", "getProcesses")
+		TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcesses")
+			.detail("netAddrSize",netAddrs.size()).detail("processAddrSize",processAddrs.size())
 			.detail("netAddrs",describe(netAddrs)).detail("processAddrs",describe(processAddrs))
 			.detail("Proceses", processes.size()).detail("MachineProcesses", machineProcesses.size());
 
-		// Processes may have been destroyed causing
-//		ASSERT(processAddrs.size() == processes.size());
 		return processes;
 	}
 
+	virtual std::vector<ISimulator::ProcessInfo*> excludeAddresses(std::set<AddressExclusion> const& procAddrs)
+	{
+		// Get the updated list of processes which may have changed due to reboots, deletes, etc
+		std::vector<ISimulator::ProcessInfo*>	procArray = getProcesses(procAddrs);
+
+		// Include all of the excluded machines because the first command of the next section is includeall
+		TraceEvent("RemoveAndKill").detail("Step", "exclude addresses").detail("AddrTotal", procAddrs.size()).detail("ProcTotal", procArray.size()).detail("Addresses", describe(procAddrs)).detail("ClusterAvailable", g_simulator.isAvailable());
+		for (auto& procAddr : procAddrs) {
+			g_simulator.excludeAddress(NetworkAddress(procAddr.ip, procAddr.port, true, false));
+		}
+		for (auto& procRecord : procArray) {
+			procRecord->excluded = true;
+			TraceEvent("RemoveAndKill").detail("Step", "ExcludeAddress").detail("ProcessAddress", procRecord->address).detail("Process", describe(*procRecord)).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("ClusterAvailable", g_simulator.isAvailable());
+		}
+		return procArray;
+	}
+
+	virtual std::vector<ISimulator::ProcessInfo*> includeAddresses(std::set<AddressExclusion> const& procAddrs)
+	{
+		// Get the updated list of processes which may have changed due to reboots, deletes, etc
+		std::vector<ISimulator::ProcessInfo*>	procArray = getProcesses(procAddrs);
+
+		// Include all of the excluded machines because the first command of the next section is includeall
+		TraceEvent("RemoveAndKill").detail("Step", "include addresses").detail("AddrTotal", procAddrs.size()).detail("ProcTotal", procArray.size()).detail("Addresses", describe(procAddrs)).detail("ClusterAvailable", g_simulator.isAvailable());
+		for (auto& procAddr : procAddrs) {
+			g_simulator.includeAddress(NetworkAddress(procAddr.ip, procAddr.port, true, false));
+		}
+		for (auto& procRecord : procArray) {
+			// Only change the exclusion member, if not failed since it will require a reboot to revive it
+			if (!procRecord->failed)
+				procRecord->excluded = false;
+			TraceEvent("RemoveAndKill").detail("Step", "IncludeAddress").detail("ProcessAddress", procRecord->address).detail("Process", describe(*procRecord)).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("ClusterAvailable", g_simulator.isAvailable());
+		}
+		return procArray;
+	}
+
 	virtual std::vector<ISimulator::ProcessInfo*> protectServers(std::set<AddressExclusion> const& killAddrs)
 	{
 		std::vector<ISimulator::ProcessInfo*>	processes;
 		std::set<AddressExclusion>	processAddrs;
-		std::vector<ISimulator::ProcessInfo*>	killProcesses, killableProcesses, processesLeft, processesDead;
+		std::vector<AddressExclusion> killableAddrs;
+		std::vector<ISimulator::ProcessInfo*>	killProcArray, killableProcesses, processesLeft, processesDead;
 
 		// Get the list of processes matching network address
-		for (auto processInfo : getProcesses(killAddrs)) {
-			// Add non-test processes
-			if (processInfo->startingClass != ProcessClass::TesterClass) {
-				if (g_simulator.protectedAddresses.count(processInfo->address))
-					processesLeft.push_back(processInfo);
-				// Add reliable machine processes to available group
-				else if (processInfo->isAvailable())
-					processesLeft.push_back(processInfo);
-				else
-					processesDead.push_back(processInfo);
-			}
+		for (auto processInfo : getServers()) {
+			auto processNet = AddressExclusion(processInfo->address.ip, processInfo->address.port);
+			// Mark all of the unavailable as dead
+			if (!processInfo->isAvailable())
+				processesDead.push_back(processInfo);
+			// Save all processes not specified within set
+			else if (killAddrs.find(processNet) == killAddrs.end())
+				processesLeft.push_back(processInfo);
+			else
+				killProcArray.push_back(processInfo);
 		}
-		killProcesses = processesLeft;
 
 		// Identify the largest set of processes which can be killed
-		int	kills, randomIndex;
-		for (int kills = 0; kills < killProcesses.size(); kills ++)
+		int	randomIndex;
+		bool bCanKillProcess;
+		ISimulator::ProcessInfo*	randomProcess;
+		auto deadProcess = processesDead.back();
+		for (int killsLeft = killProcArray.size(); killsLeft > 0; killsLeft --)
 		{
 			// Select a random kill process
-			randomIndex = g_random->randomInt(0, killProcesses.size());
-			processesDead.push_back(killProcesses[randomIndex]);
-			killProcesses[randomIndex] = killProcesses.back();
-			killProcesses.pop_back();
-			processesLeft.insert(processesLeft.end(), killProcesses.begin(), killProcesses.end());
+			randomIndex = g_random->randomInt(0, killsLeft);
+			randomProcess = killProcArray[randomIndex];
+			processesDead.push_back(randomProcess);
+			killProcArray[randomIndex] = killProcArray.back();
+			killProcArray.pop_back();
+			// Add all of the remaining processes the leftover array
+			processesLeft.insert(processesLeft.end(), killProcArray.begin(), killProcArray.end());
 
-			if (g_simulator.canKillProcesses(processesLeft, processesLeft, ISimulator::KillInstantly, NULL)) {
-				killableProcesses.push_back(processesDead.back());
+			// Check if we can kill the added process
+			bCanKillProcess = g_simulator.canKillProcesses(processesLeft, processesDead, ISimulator::KillInstantly, NULL);
+
+			// Remove the added processes
+			processesLeft.resize(processesLeft.size() - killProcArray.size());
+
+			if (bCanKillProcess) {
+				killableProcesses.push_back(randomProcess);
+				killableAddrs.push_back(AddressExclusion(randomProcess->address.ip, randomProcess->address.port));
+				TraceEvent("RemoveAndKill").detail("Step", "identifyVictim")
+					.detail("VictimCount", killableAddrs.size()).detail("Victim",randomProcess->toString())
+					.detail("Victims", describe(killableAddrs));
 			}
+			// Move the process to the keep array
 			else {
-				processesLeft.push_back(processesDead.back());
+				processesLeft.push_back(randomProcess);
 				processesDead.pop_back();
 			}
-			// Remove the added processes
-			processesLeft.resize(processesLeft.size() - killProcesses.size());
 		}
 
 		return killableProcesses;
@@ -229,217 +285,155 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 
 		// Removing the first set of machines might legitimately bring the database down, so a timeout is not an error
 		state std::vector<NetworkAddress> firstCoordinators;
-		state std::vector<ISimulator::ProcessInfo*>	killProcesses;
+		state std::vector<ISimulator::ProcessInfo*>	killProcArray;
+		state bool bClearedFirst;
 
-		TraceEvent("RemoveAndKill").detail("Step", "remove first list").detail("toKill1", describe(toKill1)).detail("KillTotal", toKill1.size())
-			.detail("ClusterAvailable", g_simulator.isAvailable());
-		Optional<Void> result = wait( timeout( removeAndKill( self, cx, toKill1, &firstCoordinators ), self->kill1Timeout ) );
+		TraceEvent("RemoveAndKill").detail("Step", "exclude list first").detail("toKill", describe(toKill1)).detail("KillTotal", toKill1.size()).detail("ClusterAvailable", g_simulator.isAvailable());
+		self->excludeAddresses(toKill1);
 
-		TraceEvent("RemoveAndKill").detail("Step", "remove result").detail("result", result.present() ? "succeeded" : "failed");
-		killProcesses = self->getProcesses(toKill1);
-		TraceEvent("RemoveAndKill").detail("Step", "unexclude first processes").detail("toKill1", describe(toKill1))
-			.detail("KillTotal", toKill1.size()).detail("Processes", killProcesses.size());
-		for (auto& killProcess : killProcesses) {
-			killProcess->excluded = false;
+		Optional<Void> result = wait( timeout( removeAndKill( self, cx, toKill1, NULL), self->kill1Timeout ) );
+
+		bClearedFirst = result.present();
+
+		TraceEvent("RemoveAndKill").detail("Step", "excluded list first").detail("excluderesult", bClearedFirst ? "succeeded" : "failed").detail("KillTotal", toKill1.size()).detail("Processes", killProcArray.size()).detail("toKill1", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable());
+
+		bClearedFirst=false;
+		// Include the servers, if unable to exclude
+		if (!bClearedFirst) {
+			// Get the updated list of processes which may have changed due to reboots, deletes, etc
+			TraceEvent("RemoveAndKill").detail("Step", "include all first").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable());
+			Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) );
+			self->includeAddresses(toKill1);
+			TraceEvent("RemoveAndKill").detail("Step", "included all first").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable());
 		}
 
-		//	virtual bool killMachine(Optional<Standalone<StringRef>> zoneId, KillType, bool killIsSafe = false, bool forceKill = false ) = 0;
-		//	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) = 0;
-
-		// Identify the unique machines within kill list
-		//		Remove any machine with a protected process (ie coordinator)
-		// Check if we can remove the machines and still have a healthy cluster
-		// 		If not, remove one random machine at a time until we have healthy cluster
-		// Mark all of the processes protected, store the list of modified processes
-		// Exclude the machines
-		// Remove the machines Kill instantly
-		// Unprotect the changed processes
-		killProcesses = self->protectServers(toKill2);
+		// Get the list of protected servers
+		killProcArray = self->protectServers(toKill2);
 
 		// Update the kill networks to the killable processes
-		toKill2 = self->getNetworks(killProcesses);
+		toKill2 = self->getNetworks(killProcArray);
+
+		TraceEvent("RemoveAndKill").detail("Step", "exclude list second").detail("KillTotal", toKill2.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
+		self->excludeAddresses(toKill2);
 
 		// The second set of machines is selected so that we can always make progress without it, even after the permitted number of other permanent failures
 		// so we expect to succeed after a finite amount of time
 		state Future<Void> disabler = disableConnectionFailuresAfter( self->kill2Timeout/2, "RemoveServersSafely" );
-		TraceEvent("RemoveAndKill").detail("Step", "remove second list").detail("toKill2", describe(toKill2)).detail("KillTotal", toKill2.size())
-			.detail("Processes", killProcesses.size()).detail("ClusterAvailable", g_simulator.isAvailable());
-		Void _ = wait( reportErrors( timeoutError( removeAndKill( self, cx, toKill2, NULL, firstCoordinators), self->kill2Timeout ), "RemoveServersSafelyError", UID() ) );
+		TraceEvent("RemoveAndKill").detail("Step", "exclude second list").detail("toKill2", describe(toKill2)).detail("KillTotal", toKill2.size())
+			.detail("Processes", killProcArray.size()).detail("ClusterAvailable", g_simulator.isAvailable());
+		Void _ = wait( reportErrors( timeoutError( removeAndKill( self, cx, toKill2, bClearedFirst ? &toKill1 : NULL), self->kill2Timeout ), "RemoveServersSafelyError", UID() ) );
 
-		killProcesses = self->getProcesses(toKill2);
-		TraceEvent("RemoveAndKill").detail("Step", "unexclude second processes").detail("toKill2", describe(toKill2))
-			.detail("KillTotal", toKill2.size()).detail("Processes", killProcesses.size());
-		for (auto& killProcess : killProcesses) {
-			killProcess->excluded = false;
+		TraceEvent("RemoveAndKill").detail("Step", "excluded second list").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
+
+		// Reinclude all of the machine, if buggified
+		if (BUGGIFY) {
+			// Get the updated list of processes which may have changed due to reboots, deletes, etc
+			TraceEvent("RemoveAndKill").detail("Step", "include all second").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
+			Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) );
+			self->includeAddresses(toKill2);
+			TraceEvent("RemoveAndKill").detail("Step", "included all second").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
 		}
 
-		TraceEvent("RemoveAndKill").detail("Step", "completed final remove").detail("KillTotal", toKill2.size()).detail("Excluded", killProcesses.size())
-			.detail("ClusterAvailable", g_simulator.isAvailable());
-
 		return Void();
 	}
 
-	ACTOR static Future<Void> removeAndKill( RemoveServersSafelyWorkload* self, Database cx, std::set<AddressExclusion> toKill, std::vector<NetworkAddress>* outSafeCoordinators, std::vector<NetworkAddress> firstCoordinators = std::vector<NetworkAddress>())
+	virtual std::vector<ISimulator::ProcessInfo*> killAddresses(std::set<AddressExclusion> const& killAddrs)
 	{
-		// First clear the exclusion list and exclude the given list
-		TraceEvent("RemoveAndKill").detail("Step", "include all").detail("coordinators", describe(firstCoordinators)).detail("ClusterAvailable", g_simulator.isAvailable());
-		Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) );
+		UID functionId = g_nondeterministic_random->randomUniqueID();
+		bool removeViaClear = !BUGGIFY;
+		std::vector<ISimulator::ProcessInfo*>	killProcArray;
+		std::vector<AddressExclusion>	toKillArray;
 
-		TraceEvent("RemoveAndKill").detail("Step", "getting coordinators").detail("ClusterAvailable", g_simulator.isAvailable());
+		std::copy(killAddrs.begin(), killAddrs.end(), std::back_inserter(toKillArray));
+		killProcArray = getProcesses(killAddrs);
 
-		std::vector<NetworkAddress> coordinators = wait( getCoordinators(cx) );
-		state std::vector<NetworkAddress> safeCoordinators;
-
-		TraceEvent("RemoveAndKill").detail("Step", "got coordinators").detail("coordinators", describe(coordinators));
-
-		ASSERT(coordinators.size() > (g_simulator.desiredCoordinators-1)/2);
-
-		if(firstCoordinators.size()) {
-			ASSERT(firstCoordinators.size() > (g_simulator.desiredCoordinators-1)/2);
-
-			//check that at least one coordinator from the first set is not excluded.
-			int firstSafeCount = 0;
-			for( auto it : firstCoordinators ) {
-				RemoveServersSafelyWorkload *tSelf = self;
-				if(std::none_of(toKill.begin(), toKill.end(), [tSelf, it](AddressExclusion exclusion){ return tSelf->killContainsProcess(exclusion, it); })) {
-					++firstSafeCount;
-				}
-			}
-
-			int idx = g_random->randomInt(0, firstCoordinators.size());
-			int startIndex = idx;
-			while(firstSafeCount <= (g_simulator.desiredCoordinators-1)/2 ) {
-				//remove a random coordinator from the kill list
-				auto addr = firstCoordinators[idx];
-
-				int removedCount = 0;
-				for (auto it = toKill.begin(); it != toKill.end(); ) {
-					if(self->killContainsProcess(*it, firstCoordinators[idx])) {
-						toKill.erase(it++);
-						removedCount++;
-					}
-					else {
-						it ++;
-					}
-				}
-				if(removedCount >= 1) {
-					firstSafeCount++;
-				}
-
-				idx = (idx + 1) % firstCoordinators.size();
-				ASSERT(idx != startIndex || firstSafeCount > (g_simulator.desiredCoordinators-1)/2);
-			}
-		}
-
-		//check that at least one coordinator is not excluded.
-		int safeCount = 0;
-		for( auto it : coordinators ) {
-			RemoveServersSafelyWorkload *tSelf = self;
-			if(std::none_of(toKill.begin(), toKill.end(), [tSelf, it](AddressExclusion exclusion){ return tSelf->killContainsProcess(exclusion, it); })) {
-				safeCoordinators.push_back(it);
-				++safeCount;
-			}
-		}
-
-		int idx = g_random->randomInt(0, coordinators.size());
-		int startIndex = idx;
-		while(safeCount <= (g_simulator.desiredCoordinators-1)/2) {
-			//remove a random coordinator from the kill list
-			auto addr = coordinators[idx];
-
-			int removedCount = 0;
-			for (auto it = toKill.begin(); it != toKill.end(); ) {
-				if(self->killContainsProcess(*it, coordinators[idx])) {
-					toKill.erase(it++);
-					removedCount++;
+		// Reboot and delete or kill the servers
+		if( killProcesses ) {
+			TraceEvent("RemoveAndKill", functionId).detail("Step", removeViaClear ? "ClearProcesses" : "IgnoreProcesses").detail("Addresses", describe(killAddrs))
+				.detail("Processes", killProcArray.size()).detail("ClusterAvailable", g_simulator.isAvailable()).detail("RemoveViaClear", removeViaClear);
+			for (auto& killProcess : killProcArray) {
+				if (g_simulator.protectedAddresses.count(killProcess->address))
+					TraceEvent("RemoveAndKill", functionId).detail("Step", "NoKill Process").detail("Process", describe(*killProcess)).detail("failed", killProcess->failed).detail("rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
+				else if (removeViaClear) {
+					g_simulator.rebootProcess( killProcess, ISimulator::RebootProcessAndDelete);
+					TraceEvent("RemoveAndKill", functionId).detail("Step", "Clear Process").detail("Process", describe(*killProcess)).detail("failed", killProcess->failed).detail("rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
 				}
+/*
 				else {
-					it ++;
+					g_simulator.killProcess( killProcess, ISimulator::KillInstantly );
+					TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("failed", killProcess->failed).detail("rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
 				}
+*/
 			}
-			if (removedCount >= 1) {
-				TraceEvent("ProtectMachine").detail("Address", addr).detail("Coordinators", coordinators.size()).backtrace();
-				g_simulator.protectedAddresses.insert(addr);
-				safeCoordinators.push_back(addr);
-				safeCount++;
+		}
+		else {
+			std::set<Optional<Standalone<StringRef>>> zoneIds;
+			bool killedMachine;
+			for (auto& killProcess : killProcArray) {
+				zoneIds.insert(killProcess->locality.zoneId());
+			}
+			TraceEvent("RemoveAndKill", functionId).detail("Step", removeViaClear ? "ClearMachines" : "KillMachines").detail("Addresses", describe(killAddrs)).detail("Processes", killProcArray.size()).detail("Zones", zoneIds.size()).detail("ClusterAvailable", g_simulator.isAvailable());
+			for (auto& zoneId : zoneIds) {
+				killedMachine = g_simulator.killMachine( zoneId, removeViaClear ? ISimulator::RebootAndDelete : ISimulator::KillInstantly, removeViaClear);
+				TraceEvent(killedMachine ? SevInfo : SevWarn, "RemoveAndKill").detail("Step", removeViaClear ? "Clear Machine" : "Kill Machine").detailext("ZoneId", zoneId).detail(removeViaClear ? "Cleared" : "Killed", killedMachine).detail("ClusterAvailable", g_simulator.isAvailable());
 			}
-
-			idx = (idx + 1) % coordinators.size();
-			ASSERT(idx != startIndex || safeCount > (g_simulator.desiredCoordinators-1)/2);
 		}
 
-		state std::vector<ISimulator::ProcessInfo*>	killProcesses;
+		return killProcArray;
+	}
 
-		killProcesses = self->getProcesses(toKill);
-		TraceEvent("RemoveAndKill").detail("Step", "exclude").detail("Addresses", describe(toKill))
-			.detail("AddressTotal", toKill.size()).detail("Processes", killProcesses.size())
-			.detail("SafeCount", safeCount).detail("ClusterAvailable", g_simulator.isAvailable());
-	//	ASSERT(toKill.size() == killProcesses.size());
-		for (auto& killProcess : killProcesses) {
-			killProcess->excluded = true;
-			TraceEvent("RemoveAndKill").detail("Step", "exclude process").detail("killProcess", describe(*killProcess));
+	ACTOR static Future<Void> removeAndKill( RemoveServersSafelyWorkload* self, Database cx, std::set<AddressExclusion> toKill, std::set<AddressExclusion>* pIncAddrs)
+	{
+		state UID functionId = g_nondeterministic_random->randomUniqueID();
+
+		// First clear the exclusion list and exclude the given list
+		TraceEvent("RemoveAndKill", functionId).detail("Step", "include all").detail("ClusterAvailable", g_simulator.isAvailable());
+		Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) );
+		TraceEvent("RemoveAndKill", functionId).detail("Step", "included all").detail("ClusterAvailable", g_simulator.isAvailable());
+		// Reinclude the addresses that were excluded, if present
+		if (pIncAddrs) {
+			self->includeAddresses(*pIncAddrs);
 		}
 
+		state std::vector<ISimulator::ProcessInfo*>	killProcArray;
 		state std::vector<AddressExclusion>	toKillArray;
-		std::copy(toKill.begin(), toKill.end(), std::back_inserter(toKillArray));
 
+		std::copy(toKill.begin(), toKill.end(), std::back_inserter(toKillArray));
+		killProcArray = self->getProcesses(toKill);
+
+		TraceEvent("RemoveAndKill", functionId).detail("Step", "Activate Server Exclusion").detail("KillAddrs", toKill.size()).detail("KillProcs", killProcArray.size()).detail("MissingProcs", toKill.size()!=killProcArray.size()).detail("toKill", describe(toKill)).detail("Addresses", describe(toKillArray)).detail("ClusterAvailable", g_simulator.isAvailable());
 		Void _ = wait( excludeServers( cx, toKillArray ) );
 
 		// We need to skip at least the quorum change if there's nothing to kill, because there might not be enough servers left
 		// alive to do a coordinators auto (?)
 		if (toKill.size()) {
 			// Wait for removal to be safe
-			TraceEvent("RemoveAndKill").detail("Step", "wait").detail("Addresses", describe(toKill));
+			TraceEvent("RemoveAndKill", functionId).detail("Step", "Wait For Server Exclusion").detail("Addresses", describe(toKill)).detail("ClusterAvailable", g_simulator.isAvailable());
 			Void _ = wait( waitForExcludedServers( cx, toKillArray ) );
 
-			// Change coordinators if necessary
-			TraceEvent("RemoveAndKill").detail("Step", "coordinators auto");
-			if(outSafeCoordinators != NULL) {
-				for(auto it : safeCoordinators) {
-					outSafeCoordinators->push_back(it);
-				}
-			}
+			TraceEvent("RemoveAndKill", functionId).detail("Step", "coordinators auto").detail("desiredCoordinators", g_simulator.desiredCoordinators).detail("ClusterAvailable", g_simulator.isAvailable());
+
+			// Setup the coordinators BEFORE the exclusion
+			// Otherwise, we may end up with NotEnoughMachinesForCoordinators
+			state int cycle=0;
+			state int nQuorum;
 			while (true) {
-				CoordinatorsResult::Type result = wait( changeQuorum( cx, autoQuorumChange(((g_simulator.desiredCoordinators+1)/2)*2-1) ) );
-				TraceEvent(result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES ? SevInfo : SevWarn, "RemoveAndKillQuorumChangeResult").detail("Step", "coordinators auto").detail("Result", (int)result);
+				cycle ++;
+				nQuorum = ((g_simulator.desiredCoordinators+1)/2)*2-1;
+				CoordinatorsResult::Type result = wait( changeQuorum( cx, autoQuorumChange(nQuorum) ) );
+				TraceEvent(result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES ? SevInfo : SevWarn, "RemoveAndKillQuorumChangeResult").detail("Step", "coordinators auto").detail("Result", (int)result).detail("attempt", cycle).detail("Quorum", nQuorum).detail("desiredCoordinators", g_simulator.desiredCoordinators);
 				if (result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES)
 					break;
 			}
 
-			TraceEvent("RemoveAndKill").detail("Step", "unexclude before clear").detail("Processes", killProcesses.size());
-			for (auto& killProcess : killProcesses) {
-				killProcess->excluded = false;
-			}
-
-			// Reboot and delete the servers
-			if( self->killProcesses ) {
-				TraceEvent("RemoveAndKill").detail("Step", "ClearProcesses").detail("Addresses", describe(toKill))
-					.detail("Processes", killProcesses.size()).detail("ClusterAvailable", g_simulator.isAvailable());
-				for (auto& killProcess : killProcesses) {
-					TraceEvent("RemoveAndKill").detail("Step", "Clear Process").detail("Process", describe(*killProcess)).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address) == 0);
-					g_simulator.rebootProcess( killProcess, ISimulator::RebootProcessAndDelete );
-				}
-			}
-			else {
-				std::set<Optional<Standalone<StringRef>>> zoneIds;
-				bool killedMachine;
-				for (auto& killProcess : killProcesses) {
-					zoneIds.insert(killProcess->locality.zoneId());
-				}
-				TraceEvent("RemoveAndKill").detail("Step", "ClearMachines").detail("Addresses", describe(toKill)).detail("Processes", killProcesses.size()).detail("Zones", zoneIds.size()).detail("ClusterAvailable", g_simulator.isAvailable());
-				for (auto& zoneId : zoneIds) {
-					killedMachine = g_simulator.killMachine( zoneId, ISimulator::RebootAndDelete, true );
-					TraceEvent(killedMachine ? SevInfo : SevWarn, "RemoveAndKill").detail("Step", "Clear Machine").detailext("ZoneId", zoneId).detail("Cleared", killedMachine).detail("ClusterAvailable", g_simulator.isAvailable());
-				}
-			}
+			self->killAddresses(toKill);
 		}
 		else
 		{
-			TraceEvent("RemoveAndKill").detail("Step", "nothing to clear");
+			TraceEvent("RemoveAndKill", functionId).detail("Step", "nothing to clear").detail("ClusterAvailable", g_simulator.isAvailable());
 		}
 
-		TraceEvent("RemoveAndKill").detail("Step", "done")
-			.detail("ClusterAvailable", g_simulator.isAvailable());
+		TraceEvent("RemoveAndKill", functionId).detail("Step", "done").detail("ClusterAvailable", g_simulator.isAvailable());
 
 		return Void();
 	}
@@ -448,7 +442,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
 		vector<ISimulator::ProcessInfo*> machines;
 		vector<ISimulator::ProcessInfo*> all = g_simulator.getAllProcesses();
 		for(int i = 0; i < all.size(); i++)
-			if (all[i]->name == std::string("Server") && all[i]->startingClass != ProcessClass::TesterClass )
+			if (all[i]->name == std::string("Server") && all[i]->isAvailableClass())
 				machines.push_back( all[i] );
 		return machines;
 	}
diff --git a/fdbserver/workloads/Rollback.actor.cpp b/fdbserver/workloads/Rollback.actor.cpp
index 058adb0359..6836dd210a 100644
--- a/fdbserver/workloads/Rollback.actor.cpp
+++ b/fdbserver/workloads/Rollback.actor.cpp
@@ -58,29 +58,32 @@ struct RollbackWorkload : TestWorkload {
 	ACTOR Future<Void> simulateFailure( Database cx, RollbackWorkload* self ) {
 		auto system = self->dbInfo->get();
 		auto tlogs = system.logSystemConfig.allPresentLogs();
-
-		if( tlogs.empty() ) {
+		
+		if( tlogs.empty() || system.client.proxies.empty() ) {
 			TraceEvent(SevInfo, "UnableToTriggerRollback").detail("Reason", "No tlogs in System Map");
 			return Void();
 		}
+
+		state MasterProxyInterface proxy = g_random->randomChoice( system.client.proxies );
+
 		int utIndex = g_random->randomInt(0, tlogs.size());
 		state NetworkAddress uncloggedTLog = tlogs[utIndex].address();
 
 		for(int t=0; t<tlogs.size(); t++)
 			if (t != utIndex)
-				if( tlogs[ t ].address().ip == system.master.address().ip ) {
-					TraceEvent(SevInfo, "UnableToTriggerRollback").detail("Reason", "master-clogged tLog shared IPs");
+				if( tlogs[ t ].address().ip == proxy.address().ip ) {
+					TraceEvent(SevInfo, "UnableToTriggerRollback").detail("Reason", "proxy-clogged tLog shared IPs");
 					return Void();
 				}
 
 		TraceEvent("AttemptingToTriggerRollback")
-			.detail("Master", system.master.address())
+			.detail("Proxy", proxy.address())
 			.detail("UncloggedTLog", uncloggedTLog);
 
 		for(int t=0; t<tlogs.size(); t++)
 			if (t != utIndex)
 				g_simulator.clogPair( 
-					system.master.address().ip,
+					proxy.address().ip,
 					tlogs[t].address().ip,
 					self->clogDuration );
 				//g_simulator.clogInterface( g_simulator.getProcess( system.tlogs[t].commit.getEndpoint() ), self->clogDuration, ClogAll );
@@ -89,12 +92,12 @@ struct RollbackWorkload : TestWorkload {
 		Void _ = wait( delay( self->clogDuration/3 ) );
 		auto system = self->dbInfo->get();
 
-		// Kill the master and the unclogged tlog
+		// Kill the proxy and the unclogged tlog
 		if (self->enableFailures) {
-			g_simulator.killProcess( g_simulator.getProcessByAddress( system.master.address() ), ISimulator::KillInstantly );
+			g_simulator.killProcess( g_simulator.getProcessByAddress( proxy.address() ), ISimulator::KillInstantly );
 			g_simulator.clogInterface( uncloggedTLog.ip, self->clogDuration, ClogAll );
 		} else {
-			g_simulator.clogInterface( system.master.address().ip, self->clogDuration, ClogAll );
+			g_simulator.clogInterface( proxy.address().ip, self->clogDuration, ClogAll );
 			g_simulator.clogInterface( uncloggedTLog.ip, self->clogDuration, ClogAll );
 		}
 		return Void();
diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp
old mode 100644
new mode 100755
index 0c052bda7b..73683d0535
--- a/flow/Knobs.cpp
+++ b/flow/Knobs.cpp
@@ -72,6 +72,8 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
 	init( MAX_OUTSTANDING,                                      64 );
 	init( MIN_SUBMIT,                                           10 );
 
+	init( PAGE_WRITE_CHECKSUM_HISTORY,                           0 ); if( randomize && BUGGIFY ) PAGE_WRITE_CHECKSUM_HISTORY = 10000000;
+
 	//AsyncFileNonDurable
 	init( MAX_PRIOR_MODIFICATION_DELAY,                        1.0 ); if( randomize && BUGGIFY ) MAX_PRIOR_MODIFICATION_DELAY = 10.0;
 
diff --git a/flow/Knobs.h b/flow/Knobs.h
index 04b00200fb..e1d80e53aa 100644
--- a/flow/Knobs.h
+++ b/flow/Knobs.h
@@ -90,6 +90,8 @@ public:
 	int MAX_OUTSTANDING;
 	int MIN_SUBMIT;
 
+	int PAGE_WRITE_CHECKSUM_HISTORY;
+
 	//AsyncFileNonDurable
 	double MAX_PRIOR_MODIFICATION_DELAY;
 
diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp
index a19b49c59c..ddbfb7ae08 100644
--- a/flow/Net2.actor.cpp
+++ b/flow/Net2.actor.cpp
@@ -52,7 +52,7 @@ using namespace boost::asio::ip;
 // These impact both communications and the deserialization of certain zookeeper, database and IKeyValueStore keys
 //                                                 xyzdev
 //                                                 vvvv
-uint64_t currentProtocolVersion        = 0x0FDB00A470010001LL;
+uint64_t currentProtocolVersion        = 0x0FDB00A551000001LL;
 uint64_t compatibleProtocolVersionMask = 0xffffffffffff0000LL;
 uint64_t minValidProtocolVersion       = 0x0FDB00A200060001LL;
 
@@ -550,7 +550,7 @@ void Net2::run() {
 
 	timeOffsetLogger = logTimeOffset();
 	startProfiling(this);
-	
+
 	// Get the address to the launch function
 	typedef void (*runCycleFuncPtr)();
 	runCycleFuncPtr runFunc = reinterpret_cast<runCycleFuncPtr>(reinterpret_cast<flowGlobalType>(g_network->global(INetwork::enRunCycleFunc)));
@@ -606,7 +606,7 @@ void Net2::run() {
 		taskBegin = timer_monotonic();
 		numYields = 0;
 		int minTaskID = TaskMaxPriority;
-		
+
 		while (!ready.empty()) {
 			++countTasks;
 			currentTaskID = ready.top().taskID;
@@ -671,7 +671,7 @@ void Net2::run() {
 		}
 #endif
 
-		if ((nnow-now) > FLOW_KNOBS->SLOW_LOOP_CUTOFF && g_nondeterministic_random->random01() < (nnow-now)*FLOW_KNOBS->SLOW_LOOP_SAMPLING_RATE) 
+		if ((nnow-now) > FLOW_KNOBS->SLOW_LOOP_CUTOFF && g_nondeterministic_random->random01() < (nnow-now)*FLOW_KNOBS->SLOW_LOOP_SAMPLING_RATE)
 			TraceEvent("SomewhatSlowRunLoopBottom").detail("Elapsed", nnow - now); // This includes the time spent running tasks
 
 		trackMinPriority( minTaskID, nnow );
diff --git a/flow/Platform.cpp b/flow/Platform.cpp
index 9e96af1110..98c3efe1b0 100644
--- a/flow/Platform.cpp
+++ b/flow/Platform.cpp
@@ -484,11 +484,6 @@ const char* getInterfaceName(uint32_t _ip) {
 void getNetworkTraffic(uint32_t ip, uint64_t& bytesSent, uint64_t& bytesReceived,
 					   uint64_t& outSegs, uint64_t& retransSegs) {
 	INJECT_FAULT( platform_error, "getNetworkTraffic" ); // Even though this function doesn't throw errors, the equivalents for other platforms do, and since all of our simulation testing is on Linux...
-	bytesSent = 0;
-	bytesReceived = 0;
-	outSegs = 0;
-	retransSegs = 0;
-
 	const char* ifa_name = getInterfaceName(ip);
 	if (!ifa_name)
 		return;
@@ -500,8 +495,8 @@ void getNetworkTraffic(uint32_t ip, uint64_t& bytesSent, uint64_t& bytesReceived
 	std::string iface;
 	std::string ignore;
 
-	bytesSent = 0;
-	bytesReceived = 0;
+	uint64_t bytesSentSum = 0;
+	uint64_t bytesReceivedSum = 0;
 
 	while (dev_stream.good()) {
 		dev_stream >> iface;
@@ -513,13 +508,20 @@ void getNetworkTraffic(uint32_t ip, uint64_t& bytesSent, uint64_t& bytesReceived
 			for (int i = 0; i < 7; i++) dev_stream >> ignore;
 			dev_stream >> sent;
 
-			bytesSent += sent;
-			bytesReceived += received;
+			bytesSentSum += sent;
+			bytesReceivedSum += received;
 
 			dev_stream.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
 		}
 	}
 
+	if(bytesSentSum > 0) {
+		bytesSent = bytesSentSum;
+	}
+	if(bytesReceivedSum > 0) {
+		bytesReceived = bytesReceivedSum;
+	}
+
 	std::ifstream snmp_stream("/proc/net/snmp", std::ifstream::in);
 
 	std::string label;
@@ -558,6 +560,8 @@ void getMachineLoad(uint64_t& idleTime, uint64_t& totalTime) {
 
 void getDiskStatistics(std::string const& directory, uint64_t& currentIOs, uint64_t& busyTicks, uint64_t& reads, uint64_t& writes, uint64_t& writeSectors, uint64_t& readSectors) {
 	INJECT_FAULT( platform_error, "getDiskStatistics" );
+	currentIOs = 0;
+
 	struct stat buf;
 	if (stat(directory.c_str(), &buf)) {
 		TraceEvent(SevError, "GetDiskStatisticsStatError").detail("Directory", directory).GetLastError();
@@ -653,12 +657,6 @@ void getDiskStatistics(std::string const& directory, uint64_t& currentIOs, uint6
 	}
 
 	if(!g_network->isSimulated()) TraceEvent(SevWarnAlways, "DeviceNotFound").detail("Directory", directory);
-	currentIOs = 0;
-	busyTicks = 0;
-	reads = 0;
-	writes = 0;
-	writeSectors = 0;
-	readSectors = 0;
 }
 
 dev_t getDeviceId(std::string path) {
@@ -685,10 +683,6 @@ dev_t getDeviceId(std::string path) {
 void getNetworkTraffic(uint32_t ip, uint64_t& bytesSent, uint64_t& bytesReceived,
 					   uint64_t& outSegs, uint64_t& retransSegs) {
 	INJECT_FAULT( platform_error, "getNetworkTraffic" );
-	bytesSent = 0;
-	bytesReceived = 0;
-	outSegs = 0;
-	retransSegs = 0;
 
 	const char* ifa_name = getInterfaceName(ip);
 	if (!ifa_name)
@@ -732,6 +726,7 @@ void getNetworkTraffic(uint32_t ip, uint64_t& bytesSent, uint64_t& bytesReceived
 				bytesSent = if2m->ifm_data.ifi_obytes;
 				bytesReceived = if2m->ifm_data.ifi_ibytes;
 				outSegs = if2m->ifm_data.ifi_opackets;
+				retransSegs = 0;
 				break;
 			}
 		}
@@ -758,8 +753,6 @@ void getDiskStatistics(std::string const& directory, uint64_t& currentIOs, uint6
 	INJECT_FAULT( platform_error, "getDiskStatistics" );
 	currentIOs = 0;
 	busyTicks = 0;
-	reads = 0;
-	writes = 0;
 	writeSectors = 0;
 	readSectors = 0;
 
@@ -1115,10 +1108,10 @@ SystemStatistics getSystemStatistics(std::string dataFolder, uint32_t ip, System
 			returnStats.machineCPUSeconds = (100 - DisplayValue.doubleValue) * returnStats.elapsed / 100.0;
 	}
 #elif defined(__unixish__)
-	uint64_t machineNowSent, machineNowReceived;
-	uint64_t machineOutSegs, machineRetransSegs;
-	uint64_t currentIOs, nowBusyTicks, nowReads, nowWrites, nowWriteSectors, nowReadSectors;
-	uint64_t clockIdleTime, clockTotalTime;
+	uint64_t machineNowSent = (*statState)->machineLastSent;
+	uint64_t machineNowReceived = (*statState)->machineLastReceived;
+	uint64_t machineOutSegs = (*statState)->machineLastOutSegs;
+	uint64_t machineRetransSegs = (*statState)->machineLastRetransSegs;
 
 	getNetworkTraffic(ip, machineNowSent, machineNowReceived, machineOutSegs, machineRetransSegs);
 	if( returnStats.initialized ) {
@@ -1132,6 +1125,13 @@ SystemStatistics getSystemStatistics(std::string dataFolder, uint32_t ip, System
 	(*statState)->machineLastOutSegs = machineOutSegs;
 	(*statState)->machineLastRetransSegs = machineRetransSegs;
 
+	uint64_t currentIOs;
+	uint64_t nowBusyTicks = (*statState)->lastBusyTicks;
+	uint64_t nowReads = (*statState)->lastReads;
+	uint64_t nowWrites = (*statState)->lastWrites;
+	uint64_t nowWriteSectors = (*statState)->lastWriteSectors; 
+	uint64_t nowReadSectors = (*statState)->lastReadSectors;
+
 	if(dataFolder != "") {
 		getDiskStatistics(dataFolder, currentIOs, nowBusyTicks, nowReads, nowWrites, nowWriteSectors, nowReadSectors);
 		returnStats.processDiskQueueDepth = currentIOs;
@@ -1151,6 +1151,9 @@ SystemStatistics getSystemStatistics(std::string dataFolder, uint32_t ip, System
 		(*statState)->lastReadSectors = nowReadSectors;
 	}
 
+	uint64_t clockIdleTime = (*statState)->lastClockIdleTime;
+	uint64_t clockTotalTime = (*statState)->lastClockTotalTime;
+
 	getMachineLoad(clockIdleTime, clockTotalTime);
 	returnStats.machineCPUSeconds = clockTotalTime - (*statState)->lastClockTotalTime != 0 ? ( 1 - ((clockIdleTime - (*statState)->lastClockIdleTime) / ((double)(clockTotalTime - (*statState)->lastClockTotalTime)))) * returnStats.elapsed : 0;
 	(*statState)->lastClockIdleTime = clockIdleTime;
diff --git a/flow/Platform.h b/flow/Platform.h
index ee4940bcfd..ae0e6571bc 100644
--- a/flow/Platform.h
+++ b/flow/Platform.h
@@ -372,12 +372,14 @@ dev_t getDeviceId(std::string path);
 
 // Version of CLang bundled with XCode doesn't yet include ia32intrin.h.
 #ifdef __APPLE__
+#if !(__has_builtin(__rdtsc))
 inline static uint64_t __rdtsc() {
 	uint64_t lo, hi;
 	asm( "rdtsc" : "=a" (lo), "=d" (hi) );
 	return( lo | (hi << 32) );
 }
 #endif
+#endif
 
 #ifdef _WIN32
 #include <intrin.h>
diff --git a/flow/SignalSafeUnwind.cpp b/flow/SignalSafeUnwind.cpp
index bb5244583d..a2cade1021 100644
--- a/flow/SignalSafeUnwind.cpp
+++ b/flow/SignalSafeUnwind.cpp
@@ -76,7 +76,11 @@ extern "C" int dl_iterate_phdr(
     } else {
         // This path is NOT async signal safe, and serves until and unless initSignalSafeUnwind() is called
         initChain();
-        return chain_dl_iterate_phdr(callback, data);
+
+		setProfilingEnabled(0);
+        int result = chain_dl_iterate_phdr(callback, data);
+		setProfilingEnabled(1);
+		return result;
     }
 }
 
diff --git a/flow/TDMetric.actor.h b/flow/TDMetric.actor.h
index 093b8ebc3a..6f5344985d 100755
--- a/flow/TDMetric.actor.h
+++ b/flow/TDMetric.actor.h
@@ -680,7 +680,7 @@ struct TimeDescriptor {
 };
 
 struct BaseMetric {
-	BaseMetric(MetricNameRef const &name) : metricName(name), pCollection(nullptr), registered(false) {
+	BaseMetric(MetricNameRef const &name) : metricName(name), pCollection(nullptr), registered(false), enabled(false) {
 		setConfig(false);
 	}
 	virtual ~BaseMetric() {
diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj
index 63d63194ad..9581936338 100644
--- a/flow/flow.vcxproj
+++ b/flow/flow.vcxproj
@@ -71,7 +71,6 @@
     <ClInclude Include="IThreadPool.h" />
     <ClInclude Include="Knobs.h" />
     <ClInclude Include="Net2Packet.h" />
-    <ClInclude Include="Notified.h" />
     <ClInclude Include="serialize.h" />
     <ClInclude Include="SimpleOpt.h" />
     <ClInclude Include="Stats.h" />
diff --git a/flow/flow.vcxproj.filters b/flow/flow.vcxproj.filters
index 91f8139364..9b8b9fdd73 100644
--- a/flow/flow.vcxproj.filters
+++ b/flow/flow.vcxproj.filters
@@ -64,9 +64,8 @@
     <ClInclude Include="network.h" />
     <ClInclude Include="AsioReactor.h" />
     <ClInclude Include="Net2Packet.h" />
-    <ClInclude Include="Notified.h" />
   </ItemGroup>
   <ItemGroup>
     <None Include="no_intellisense.opt" />
   </ItemGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h
index ba805654ef..221246cbcc 100644
--- a/flow/genericactors.actor.h
+++ b/flow/genericactors.actor.h
@@ -239,8 +239,12 @@ Future<Void> triggerOnError( Func what, Future<Void> signal ) {
 ACTOR template<class T>
 void uncancellable(Future<T> what, Promise<T> result)
 {
-	T val = wait(what);
-	result.send(val);
+	try {
+		T val = wait(what);
+		result.send(val);
+	} catch( Error &e ) {
+		result.sendError(e);
+	}
 }
 
 //Waits for a future to complete and cannot be cancelled
diff --git a/flow/local.mk b/flow/local.mk
index defc7af0b0..8701ff9e45 100644
--- a/flow/local.mk
+++ b/flow/local.mk
@@ -37,3 +37,27 @@ flow/hgVersion.h: FORCE
 
 lib/libflow.a: bin/coverage.flow.xml
 
+ifeq ($(RELEASE),true)
+  FLOWVER = $(VERSION)
+else
+  FLOWVER = $(VERSION)-PRERELEASE
+endif
+
+packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz: flow
+	@echo "Packaging      flow"
+	@rm -rf packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+	@mkdir -p packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/bin packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/flow
+	@cp lib/libflow.a packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib
+	@cp bin/actorcompiler.exe packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/bin
+	@find flow -name '*.h' -exec cp {} packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/flow \;
+	@tar czf packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz -C packages flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+	@rm -rf packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+
+FLOW: packages/flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz
+
+FLOW_clean:
+	@echo "Cleaning       flow"
+	@rm -rf packages/flow-*.tar.gz
+
+packages: FLOW
+packages_clean: FLOW_clean
diff --git a/packaging/foundationdb.conf b/packaging/foundationdb.conf
index 2cbeb53d29..4c6e43c0ee 100644
--- a/packaging/foundationdb.conf
+++ b/packaging/foundationdb.conf
@@ -15,7 +15,7 @@ restart_delay = 60
 # restart_backoff = 60
 # restart_delay_reset_interval = 60
 cluster_file = /etc/foundationdb/fdb.cluster
-# delete_wd40_env = false
+# delete_envvars =
 # kill_on_configuration_change = true
 
 ## Default parameters for individual fdbserver processes
diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
old mode 100644
new mode 100755
index 2250f6a628..4b364a242e
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{6E41960B-7E96-44D7-B835-E18E5361CAF4}'
+           Id='{77E8C5DC-24D9-46BD-863A-BD5B1D2BE2FE}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'
diff --git a/recipes/go-recipes/blob.go b/recipes/go-recipes/blob.go
index 4d54430814..7aa7dde7b2 100644
--- a/recipes/go-recipes/blob.go
+++ b/recipes/go-recipes/blob.go
@@ -21,10 +21,10 @@
 package main
 
 import (
-	"github.com/FoundationDB/fdb-go/fdb"
-	"github.com/FoundationDB/fdb-go/fdb/directory"
-	"github.com/FoundationDB/fdb-go/fdb/subspace"
-	"github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"fmt"
 )
 
@@ -76,7 +76,7 @@ func read_blob(t fdb.ReadTransactor, blob_subspace subspace.Subspace) ([]byte, e
 }
 
 func main() {
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/doc.go b/recipes/go-recipes/doc.go
index cdd6d55e04..45d59a72b4 100644
--- a/recipes/go-recipes/doc.go
+++ b/recipes/go-recipes/doc.go
@@ -21,44 +21,44 @@
 package main
 
 import (
-        "fmt"
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"encoding/json"
-        "github.com/FoundationDB/fdb-go/fdb"
-        "github.com/FoundationDB/fdb-go/fdb/directory"
-        "github.com/FoundationDB/fdb-go/fdb/subspace"
-        "github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
-        _, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
-                tr.ClearRange(sub)
-                return nil, nil
-        })
-        return err
+	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.ClearRange(sub)
+		return nil, nil
+	})
+	return err
 }
 
 func print_subspace(trtr fdb.Transactor, sub subspace.Subspace) {
-        trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
-                k := tr.GetRange(sub, fdb.RangeOptions{0, -1, false}).Iterator()
+	trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		k := tr.GetRange(sub, fdb.RangeOptions{0, -1, false}).Iterator()
 
-                for k.Advance() {
-                        fmt.Println(_unpack(k.MustGet().Value))
-                }
-                return nil, nil
-        })
+		for k.Advance() {
+			fmt.Println(_unpack(k.MustGet().Value))
+		}
+		return nil, nil
+	})
 }
 
 
 func _pack(t interface{}) []byte {
-        return tuple.Tuple{t}.Pack()
+	return tuple.Tuple{t}.Pack()
 }
 
 func _unpack(t []byte) tuple.Tuple {
-        i, e := tuple.Unpack(t)
-        if e != nil {return nil}
-        return i
+	i, e := tuple.Unpack(t)
+	if e != nil {return nil}
+	return i
 }
 
 const EmptyObject int = -1
@@ -67,16 +67,16 @@ const EmptyList int = -2
 func ToTuples(item interface{}) []tuple.Tuple {
 	switch i := item.(type) {
 	case []interface{}:
-                if len(i) == 0 {return []tuple.Tuple{tuple.Tuple{EmptyList}}}
+		if len(i) == 0 {return []tuple.Tuple{tuple.Tuple{EmptyList}}}
 		tuples := make([]tuple.Tuple, 0)
 		for i, v := range i {
 			for _, t := range ToTuples(v) {
 				tuples = append(tuples, append(tuple.Tuple{i}, t...))
 			}
 		}
-                return tuples
+		return tuples
 	case map[string]interface{}:
-                if len(i) == 0 {return []tuple.Tuple{tuple.Tuple{EmptyObject}}}
+		if len(i) == 0 {return []tuple.Tuple{tuple.Tuple{EmptyObject}}}
 		tuples := make([]tuple.Tuple, 0)
 		for k, v := range i {
 			for _, t := range ToTuples(v) {
@@ -91,44 +91,44 @@ func ToTuples(item interface{}) []tuple.Tuple {
 }
 
 func FromTuples(tuples []tuple.Tuple) interface{} {
-        //fmt.Println(tuples)
-        if len(tuples) == 0 {return nil}
-        first := tuples[0]
-        if len(first) == 1         {return first[0]}
-        if first[0] == EmptyObject {return make(map[string]interface{}, 0)}
-        if first[0] == EmptyList   {return make([]interface{}, 0)}
+	//fmt.Println(tuples)
+	if len(tuples) == 0 {return nil}
+	first := tuples[0]
+	if len(first) == 1         {return first[0]}
+	if first[0] == EmptyObject {return make(map[string]interface{}, 0)}
+	if first[0] == EmptyList   {return make([]interface{}, 0)}
 
-        group := make(map[string][]tuple.Tuple)
+	group := make(map[string][]tuple.Tuple)
 
-        for _, t := range tuples {
-                k := string(_pack(t[0]))
-                _, ok := group[k]
-                if !ok {group[k] = make([]tuple.Tuple, 0)}
-                group[k] = append(group[k], t[0:len(t)])
-        }
+	for _, t := range tuples {
+		k := string(_pack(t[0]))
+		_, ok := group[k]
+		if !ok {group[k] = make([]tuple.Tuple, 0)}
+		group[k] = append(group[k], t[0:len(t)])
+	}
 
-        switch first[0].(type) {
-                case int64:
-                        res := make([]interface{}, 0)
-                        for _, g := range group {
-                                subtup := make([]tuple.Tuple, 0)
-                                for _, t := range g {
-                                        subtup = append(subtup, t[1:len(t)])
-                                }
-                                res = append(res, FromTuples(subtup))
-                        }
-                        return res
-                default:
-                        res := make(map[string]interface{})
-                        for _, g := range group {
-                                subtup := make([]tuple.Tuple, 0)
-                                for _, t := range g {
-                                        subtup = append(subtup, t[1:len(t)])
-                                }
-                                res[g[0][0].(string)] = FromTuples(subtup)
-                        }
-                        return res
-        }
+	switch first[0].(type) {
+		case int64:
+			res := make([]interface{}, 0)
+			for _, g := range group {
+				subtup := make([]tuple.Tuple, 0)
+				for _, t := range g {
+						subtup = append(subtup, t[1:len(t)])
+				}
+				res = append(res, FromTuples(subtup))
+			}
+			return res
+		default:
+			res := make(map[string]interface{})
+			for _, g := range group {
+				subtup := make([]tuple.Tuple, 0)
+				for _, t := range g {
+						subtup = append(subtup, t[1:len(t)])
+				}
+				res[g[0][0].(string)] = FromTuples(subtup)
+			}
+			return res
+	}
 }
 
 type Doc struct {
@@ -138,16 +138,16 @@ type Doc struct {
 func (doc Doc) InsertDoc(trtr fdb.Transactor, docdata []byte) int {
 	var data interface{}
 	json.Unmarshal(docdata, &data)
-        docid := 0
+	docid := 0
 	switch d := data.(type) {
 		case map[string]interface{}:
 			temp, ok := d["doc_id"]
 			if !ok {
-                                docid = doc._GetNewID(trtr)
-                                d["doc_id"] = docid
-                        } else {
-                                docid = temp.(int)
-                        }
+				docid = doc._GetNewID(trtr)
+				d["doc_id"] = docid
+			} else {
+				docid = temp.(int)
+			}
 			tuples := ToTuples(d)
 			trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
 				for _, t := range tuples {
@@ -156,7 +156,7 @@ func (doc Doc) InsertDoc(trtr fdb.Transactor, docdata []byte) int {
 				return nil, nil
 			})
 	}
-        return docid
+		return docid
 }
 
 func (doc Doc) _GetNewID(trtr fdb.Transactor) int {
@@ -178,49 +178,40 @@ func (doc Doc) _GetNewID(trtr fdb.Transactor) int {
 }
 
 func (doc Doc) GetDoc(trtr fdb.Transactor, doc_id int) interface{} {
-        tuples := make([]tuple.Tuple, 0)
-        trtr.ReadTransact(func(tr fdb.ReadTransaction) (interface{}, error) {
-                kr, err := fdb.PrefixRange(doc.DocSS.Pack(tuple.Tuple{doc_id}))
-                if err != nil {panic(err)}
+	tuples := make([]tuple.Tuple, 0)
+	trtr.ReadTransact(func(tr fdb.ReadTransaction) (interface{}, error) {
+		kr, err := fdb.PrefixRange(doc.DocSS.Pack(tuple.Tuple{doc_id}))
+		if err != nil {panic(err)}
 
-                items := tr.GetRange(kr, fdb.RangeOptions{}).Iterator()
+		items := tr.GetRange(kr, fdb.RangeOptions{}).Iterator()
 
-                for items.Advance() {
-                        v := items.MustGet()
-                        tup, err := doc.DocSS.Unpack(v.Key)
-                        if err != nil {panic(err)}
-                        tuples = append(tuples, append(tup[1:len(tup)], _unpack(v.Value)))
-                }
-                return nil, nil
-        })
+		for items.Advance() {
+			v := items.MustGet()
+			tup, err := doc.DocSS.Unpack(v.Key)
+			if err != nil {panic(err)}
+			tuples = append(tuples, append(tup[1:len(tup)], _unpack(v.Value)))
+		}
+		return nil, nil
+	})
 
-        return FromTuples(tuples)
+	return FromTuples(tuples)
 }
 
 func main() {
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
 	DocDemoDir, err := directory.CreateOrOpen(db, []string{"docdemo"}, nil)
 	if err != nil {panic(err)}
 
-        clear_subspace(db, DocDemoDir)
+	clear_subspace(db, DocDemoDir)
 
 	mydoc := Doc{DocDemoDir}
 
 	docdata, err := ioutil.ReadFile("./doctestjson.json")
 
 	id := mydoc.InsertDoc(db, docdata)
-        fmt.Println(mydoc.GetDoc(db, id), id)
+	fmt.Println(mydoc.GetDoc(db, id), id)
 
 }
-
-
-
-
-
-
-
-
-
diff --git a/recipes/go-recipes/graph.go b/recipes/go-recipes/graph.go
index 637449f31f..0c5e55deb7 100644
--- a/recipes/go-recipes/graph.go
+++ b/recipes/go-recipes/graph.go
@@ -21,10 +21,10 @@
 package main
 
 import (
-	"github.com/FoundationDB/fdb-go/fdb"
-	"github.com/FoundationDB/fdb-go/fdb/directory"
-	"github.com/FoundationDB/fdb-go/fdb/subspace"
-	"github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"log"
 	"fmt"
 )
@@ -116,7 +116,7 @@ func (graph *Graph) get_in_neighbors(trtr fdb.Transactor, node int) ([]int, erro
 }
 
 func main() {
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/indirect.go b/recipes/go-recipes/indirect.go
index d766cbe082..1af9fa4fea 100644
--- a/recipes/go-recipes/indirect.go
+++ b/recipes/go-recipes/indirect.go
@@ -21,20 +21,20 @@
 package main
 
 import (
-        "log"
-        "fmt"
-        "github.com/FoundationDB/fdb-go/fdb"
-        "github.com/FoundationDB/fdb-go/fdb/directory"
-        "github.com/FoundationDB/fdb-go/fdb/subspace"
-        "github.com/FoundationDB/fdb-go/fdb/tuple"
+	"log"
+	"fmt"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
-        _, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
-                tr.ClearRange(sub)
-                return nil, nil
-        })
-        return err
+	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.ClearRange(sub)
+		return nil, nil
+	})
+	return err
 }
 
 func print_subspace(trtr fdb.Transactor, sub subspace.Subspace) {
@@ -49,13 +49,13 @@ func print_subspace(trtr fdb.Transactor, sub subspace.Subspace) {
 }
 
 func _pack(t interface{}) []byte {
-        return tuple.Tuple{t}.Pack()
+	return tuple.Tuple{t}.Pack()
 }
 
 func _unpack(t []byte) tuple.Tuple {
-        i, e := tuple.Unpack(t)
-        if e != nil {return nil}
-        return i
+	i, e := tuple.Unpack(t)
+	if e != nil {return nil}
+	return i
 }
 
 type Workspace struct {
@@ -87,7 +87,7 @@ func (wrkspc Workspace) Session(foo func(directory.DirectorySubspace)) (err erro
 }
 
 func main() {
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/multi.go b/recipes/go-recipes/multi.go
index 80aa3f6d4f..8f857cbcea 100644
--- a/recipes/go-recipes/multi.go
+++ b/recipes/go-recipes/multi.go
@@ -24,10 +24,10 @@ import (
 	"fmt"
 	//"log"
 	//"time"
-	"github.com/FoundationDB/fdb-go/fdb"
-	"github.com/FoundationDB/fdb-go/fdb/directory"
-	"github.com/FoundationDB/fdb-go/fdb/subspace"
-	"github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 func clear_subspace(db fdb.Transactor, ss subspace.Subspace) {
@@ -70,21 +70,21 @@ func (multi MultiMap) MultiSubtract(trtr fdb.Transactor, index, value interface{
 }
 
 func (multi MultiMap) MultiGet(tr fdb.ReadTransactor, index int) (ret []interface{}, e error) {
-        _, e = tr.ReadTransact(func(tr fdb.ReadTransaction) (interface{}, error) {
-                pr, err := fdb.PrefixRange(multi.MapSS.Pack(tuple.Tuple{index}))
-                if err != nil {return nil, err}
-                kvs := tr.GetRange(pr, fdb.RangeOptions{0, -1, false}).GetSliceOrPanic()
-                ret := make([]interface{}, len(kvs))
-                i := 0
-                for _, kv := range kvs {
-                        temp, err := multi.MapSS.Unpack(kv.Key)
-                        if err != nil {return nil, err}
-                        ret[i] = temp[1]
-                        i++
-                }
-                return nil, nil
-        })
-        return
+	_, e = tr.ReadTransact(func(tr fdb.ReadTransaction) (interface{}, error) {
+		pr, err := fdb.PrefixRange(multi.MapSS.Pack(tuple.Tuple{index}))
+		if err != nil {return nil, err}
+		kvs := tr.GetRange(pr, fdb.RangeOptions{0, -1, false}).GetSliceOrPanic()
+		ret := make([]interface{}, len(kvs))
+		i := 0
+		for _, kv := range kvs {
+			temp, err := multi.MapSS.Unpack(kv.Key)
+			if err != nil {return nil, err}
+			ret[i] = temp[1]
+			i++
+		}
+		return nil, nil
+	})
+	return
 }
 
 func (multi MultiMap) MultiGetCounts(trtr fdb.Transactor, index interface{}) (map[interface{}]int, error) {
@@ -124,7 +124,7 @@ func (multi MultiMap) MultiIsElement(trtr fdb.Transactor, index, value interface
 
 func main() {
 
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
@@ -144,8 +144,3 @@ func main() {
 	fmt.Println(m.MultiIsElement(db, 1, 2))
 	fmt.Println(m.MultiGetCounts(db, 1))
 }
-
-
-
-
-
diff --git a/recipes/go-recipes/priority.go b/recipes/go-recipes/priority.go
index b1c0089946..447844860a 100644
--- a/recipes/go-recipes/priority.go
+++ b/recipes/go-recipes/priority.go
@@ -21,31 +21,31 @@
 package main
 
 import (
-        "log"
-        "fmt"
+	"log"
+	"fmt"
 	"math/rand"
-        "github.com/FoundationDB/fdb-go/fdb"
-        "github.com/FoundationDB/fdb-go/fdb/directory"
-        "github.com/FoundationDB/fdb-go/fdb/subspace"
-        "github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
-        _, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
-                tr.ClearRange(sub)
-                return nil, nil
-        })
-        return err
+	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.ClearRange(sub)
+		return nil, nil
+	})
+	return err
 }
 
 func _pack(t interface{}) []byte {
-        return tuple.Tuple{t}.Pack()
+	return tuple.Tuple{t}.Pack()
 }
 
 func _unpack(t []byte) tuple.Tuple {
-        i, e := tuple.Unpack(t)
-        if e != nil {return nil}
-        return i
+	i, e := tuple.Unpack(t)
+	if e != nil {return nil}
+	return i
 }
 
 type Priority struct {
@@ -100,7 +100,7 @@ func (prty Priority) Peek(trtr fdb.Transactor, max bool) interface{} {
 
 
 func main() {
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
@@ -131,13 +131,3 @@ func main() {
 	fmt.Println(p.Pop(db, false))
 	fmt.Println(p.Pop(db, false))
 }
-
-
-
-
-
-
-
-
-
-
diff --git a/recipes/go-recipes/queue.go b/recipes/go-recipes/queue.go
index 18df0a0738..b9c30a579d 100644
--- a/recipes/go-recipes/queue.go
+++ b/recipes/go-recipes/queue.go
@@ -21,10 +21,10 @@
 package main
 
 import (
-	"github.com/FoundationDB/fdb-go/fdb"
-	"github.com/FoundationDB/fdb-go/fdb/directory"
-	"github.com/FoundationDB/fdb-go/fdb/subspace"
-	"github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"log"
 	"fmt"
 )
@@ -101,7 +101,7 @@ func (q *Queue) FirstItem(trtr fdb.Transactor) (interface{}, error) {
 func main() {
 	fmt.Println("Queue Example Program")
 
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
 	db := fdb.MustOpenDefault()
 
diff --git a/recipes/go-recipes/table.go b/recipes/go-recipes/table.go
index 520dacf2f1..bdc5df86e7 100644
--- a/recipes/go-recipes/table.go
+++ b/recipes/go-recipes/table.go
@@ -23,18 +23,18 @@ package main
 import (
 	"log"
 	"fmt"
-	"github.com/FoundationDB/fdb-go/fdb"
-	"github.com/FoundationDB/fdb-go/fdb/directory"
-	"github.com/FoundationDB/fdb-go/fdb/subspace"
-	"github.com/FoundationDB/fdb-go/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 func clear_subspace(trtr fdb.Transactor, sub subspace.Subspace) error {
-        _, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
-                tr.ClearRange(sub)
-                return nil, nil
-        })
-        return err
+	_, err := trtr.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.ClearRange(sub)
+		return nil, nil
+	})
+	return err
 }
 
 func _pack(t interface{}) []byte {
@@ -53,7 +53,7 @@ type Table struct {
 
 func (tbl *Table) NewTable(ss subspace.Subspace) {
 	tbl.row = ss.Sub("row")
-        tbl.col = ss.Sub("col")
+	tbl.col = ss.Sub("col")
 }
 
 func (tbl Table) TableSetCell(trtr fdb.Transactor, row, column int, value interface{}) {
@@ -128,9 +128,9 @@ func (tbl Table) TableGetCol(tr fdb.ReadTransactor, col int) ([]interface{}, err
 }
 
 func main() {
-	fdb.MustAPIVersion(300)
+	fdb.MustAPIVersion(500)
 
-        db := fdb.MustOpenDefault()
+	db := fdb.MustOpenDefault()
 
 	TableDemoDir, err := directory.CreateOrOpen(db, []string{"Graph"}, nil)
 	if err != nil {log.Fatal(err)}
@@ -150,9 +150,3 @@ func main() {
 	g.TableSetRow(db, 1, "Hello", "World", "Again!", 1)
 	fmt.Println(g.TableGetRow(db, 1))
 }
-
-
-
-
-
-
diff --git a/tests/python_tests/python_performance.py b/tests/python_tests/python_performance.py
index d5609301c1..8595f5b816 100755
--- a/tests/python_tests/python_performance.py
+++ b/tests/python_tests/python_performance.py
@@ -47,6 +47,7 @@ class PythonPerformance(PythonTest):
         'serial_get' : 'Python API serial get throughput',
         'get_range' : 'Python API get_range throughput',
         'get_key' : 'Python API get_key throughput',
+        'get_single_key_range' : 'Python API get_single_key_range throughput',
         'alternating_get_set' : 'Python API alternating get and set throughput',
         'write_transaction' : 'Python API single-key transaction throughput',
     }
@@ -54,7 +55,7 @@ class PythonPerformance(PythonTest):
     def __init__(self, key_count=1000000, key_size=16, value_size=100):
         super(PythonPerformance, self).__init__()
         self.key_count = key_count
-        self.key_size = 16
+        self.key_size = key_size
         self.value_str = ''.join(['x' for i in range(value_size)])
 
     #Python Performance Tests (checks if functions run and yield correct results, gets performance indicators)
@@ -267,6 +268,17 @@ class PythonPerformance(PythonTest):
 
         return count / (time.time() - s)
 
+    @fdb.transactional
+    def run_get_single_key_range(self, tr, count=2000):
+        tr.options.set_retry_limit(5)
+        s = time.time()
+
+        for i in range(count):
+            index = random.randint(0, self.key_count)
+            list(tr[self.key(index):self.key(index+1)])
+
+        return count / (time.time() - s)
+
     @fdb.transactional
     def single_set(self, tr):
         key = self.random_key()
diff --git a/tests/python_tests/ryw_benchmark.py b/tests/python_tests/ryw_benchmark.py
index 8bb75e4796..d38887c438 100755
--- a/tests/python_tests/ryw_benchmark.py
+++ b/tests/python_tests/ryw_benchmark.py
@@ -42,8 +42,10 @@ class RYWBenchmark(PythonTest):
         'interleaved_sets_gets' : "RYW: interleaved sets and gets on a single key throughput",
     }
 
-    def __init__(self):
+    def __init__(self, key_count=10000, key_size=16):
         super(RYWBenchmark, self).__init__()
+        self.key_count = key_count
+        self.key_size = key_size
 
     def run_test(self):
         try:
@@ -61,6 +63,9 @@ class RYWBenchmark(PythonTest):
         except:
             self.result.add_error(self.get_error('Failed to complete all tests'))
 
+    def key(self, num):
+        return '%0*d' % (self.key_size, num)
+
     #Adds the stack trace to an error message
     def get_error(self, message):
         errorMessage = message + "\n" + traceback.format_exc()
@@ -102,44 +107,45 @@ class RYWBenchmark(PythonTest):
                 self.result.add_kpi(RYWBenchmark.tests[test], int(median), 'keys/s')
 
     def insert_data(self, tr):
+        del tr[:]
         for i in range(0, 10000):
-            tr[str(i)] = 'foo'
+            tr[self.key(i)] = 'foo'
 
     def run_get_single(self, tr, count=10000):
       start = time.time()
       for i in range(count):
-        tr.get("5001").wait()
+        tr.get(self.key(5001)).wait()
       return count / (time.time() - start)
 
     def run_get_many_sequential(self, tr, count=10000):
       start = time.time()
       for j in range(count):
-        tr.get(str(j)).wait()
+        tr.get(self.key(j)).wait()
       return count / (time.time() - start)
 
-    def run_get_range_basic(self, tr, count=1000):
+    def run_get_range_basic(self, tr, count=100):
       start = time.time()
       for i in range(count):
-        list(tr.get_range(str(0), str(10000)))
-      return 10000 * count / (time.time() - start)
+        list(tr.get_range(self.key(0), self.key(self.key_count)))
+      return self.key_count * count / (time.time() - start)
 
-    def run_single_clear_get_range(self, tr, count=1000):
-      for i in range(0, 10000, 2):
-        tr.clear(str(i))
+    def run_single_clear_get_range(self, tr, count=100):
+      for i in range(0, self.key_count, 2):
+        tr.clear(self.key(i))
       start = time.time()
       for i in range(0, count):
-        list(tr.get_range(str(0), str(10000)))
-      kpi = 10000 * count / (time.time() - start)
+        list(tr.get_range(self.key(0), self.key(self.key_count)))
+      kpi = self.key_count * count / 2 / (time.time() - start)
       self.insert_data(tr)
       return kpi
 
-    def run_clear_range_get_range(self, tr, count=1000):
-      for i in range(0, 10000, 4):
-        tr.clear_range(str(i), str(i+1))
+    def run_clear_range_get_range(self, tr, count=100):
+      for i in range(0, self.key_count, 4):
+        tr.clear_range(self.key(i), self.key(i+1))
       start = time.time()
       for i in range(0, count):
-        list(tr.get_range(str(0), str(10000)))
-      kpi = 10000 * count / (time.time() - start)
+        list(tr.get_range(self.key(0), self.key(self.key_count)))
+      kpi = self.key_count * count * 3 / 4 / (time.time() - start)
       self.insert_data(tr)
       return kpi