Merge branch 'master' into feature-redwood

# Conflicts:
#	fdbserver/fdbserver.vcxproj

Makefile

@@ -47,7 +47,7 @@ else ifeq ($(PLATFORM),Darwin)
   CXX := /usr/bin/clang
 
   CFLAGS += -mmacosx-version-min=10.7 -stdlib=libc++
-  CXXFLAGS += -std=c++11 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
+  CXXFLAGS += -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option
 
   .LIBPATTERNS := lib%.dylib lib%.a
 

bindings/bindingtester/known_testers.py

@@ -63,6 +63,6 @@ testers = {
     'java_async' :             Tester('java', _java_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION),
     'java_completable' :       Tester('java', _java_completable_cmd + 'StackTester', 2040, 500, MAX_API_VERSION),
     'java_completable_async' : Tester('java', _java_completable_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION),
-    'go' :                     Tester('go', _absolute_path('go/bin/_stacktester'), 63, 200, MAX_API_VERSION),
+    'go' :                     Tester('go', _absolute_path('go/build/bin/_stacktester'), 63, 200, MAX_API_VERSION),
     'flow' :                   Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION),
 }

bindings/c/local.mk

@@ -23,14 +23,18 @@
 fdb_c_CFLAGS := $(fdbclient_CFLAGS)
 fdb_c_LDFLAGS := $(fdbrpc_LDFLAGS)
 fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a
+fdb_c_tests_LIBS := -Llib -lfdb_c
+fdb_c_tests_HEADERS := -Ibindings/c
 
 ifeq ($(PLATFORM),linux)
   fdb_c_LIBS += lib/libstdc++.a -lm -lpthread -lrt -ldl
   fdb_c_LDFLAGS += -Wl,--version-script=bindings/c/fdb_c.map -static-libgcc -Wl,-z,nodelete
+  fdb_c_tests_LIBS += -lpthread
 endif
 
 ifeq ($(PLATFORM),osx)
   fdb_c_LDFLAGS += -lc++ -Xlinker -exported_symbols_list -Xlinker bindings/c/fdb_c.symbols
+  fdb_c_tests_LIBS += -lpthread
 
   lib/libfdb_c.dylib: bindings/c/fdb_c.symbols
 
@@ -74,3 +78,24 @@ fdb_c_BUILD_SOURCES += bindings/c/fdb_c.g.S
 bindings/c/foundationdb/fdb_c_options.g.h: bin/vexillographer.exe fdbclient/vexillographer/fdb.options $(ALL_MAKEFILES)
 	@echo "Building       $@"
 	@$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options c $@
+
+bin/fdb_c_performance_test: bindings/c/test/performance_test.c bindings/c/test/test.h fdb_c
+	@echo "Compiling      fdb_c_performance_test"
+	@$(CC) $(CFLAGS) $(fdb_c_tests_LIBS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c
+
+bin/fdb_c_ryw_benchmark: bindings/c/test/ryw_benchmark.c bindings/c/test/test.h fdb_c
+	@echo "Compiling      fdb_c_ryw_benchmark"
+	@$(CC) $(CFLAGS) $(fdb_c_tests_LIBS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/ryw_benchmark.c
+
+packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz: bin/fdb_c_performance_test bin/fdb_c_ryw_benchmark
+	@echo "Packaging      $@"
+	@rm -rf packages/fdb-c-tests-$(VERSION)-$(PLATFORM)
+	@mkdir -p packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin
+	@cp bin/fdb_c_performance_test packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin
+	@cp bin/fdb_c_ryw_benchmark packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin
+	@tar -C packages -czvf $@ fdb-c-tests-$(VERSION)-$(PLATFORM) > /dev/null
+	@rm -rf packages/fdb-c-tests-$(VERSION)-$(PLATFORM)
+
+fdb_c_tests: packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz
+
+packages: fdb_c_tests

bindings/c/test/performance_test.c

@@ -0,0 +1,623 @@
+/*
+ * performance_test.c
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test.h"
+#include <foundationdb/fdb_c.h>
+#include <foundationdb/fdb_c_options.g.h>
+
+#include <stdio.h>
+#include <pthread.h>
+
+pthread_t netThread;
+
+int numKeys = 1000000;
+int keySize = 16;
+uint8_t** keys = NULL;
+int valueSize = 100;
+uint8_t *valueStr = NULL;
+
+fdb_error_t waitError(FDBFuture *f) {
+	fdb_error_t blockError = fdb_future_block_until_ready(f);
+	if(!blockError) {
+		return fdb_future_get_error(f);
+	} else {
+		return blockError;
+	}
+}
+
+struct RunResult run(struct ResultSet *rs, FDBDatabase *db, struct RunResult (*func)(struct ResultSet*, FDBTransaction*)) {
+	FDBTransaction *tr = NULL;
+	checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
+	fdb_error_t e = fdb_database_create_transaction(db, &tr);
+	checkError(e, "create transaction", rs);
+
+	while(1) {
+		struct RunResult r = func(rs, tr);
+		e = r.e;
+		if(!e) {
+			FDBFuture *f = fdb_transaction_commit(tr);
+			e = waitError(f);
+			fdb_future_destroy(f);
+		}
+
+		if(e) {
+			FDBFuture *f = fdb_transaction_on_error(tr, e);
+			fdb_error_t retryE = waitError(f);
+			fdb_future_destroy(f);
+			if (retryE) {
+				return (struct RunResult) {0, retryE};
+			}
+		} else {
+			return r;
+		}
+	}
+
+	return RES(0, 4100); // internal_error ; we should never get here
+}
+
+int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
+	int numRuns = 25;
+	int *results = malloc(sizeof(int)*numRuns);
+	int i = 0;
+	for(; i < numRuns; ++i) {
+		struct RunResult res = run(rs, db, testFxn);
+		if(res.e) {
+			logError(res.e, kpiName, rs);
+			free(results);
+			return 0;
+		}
+		results[i] = res.res;
+		if(results[i] < 0) {
+			free(results);
+			return -1;
+		}
+	}
+
+	int result = median(results, numRuns);
+	free(results);
+
+	addKpi(rs, kpiName, result, "keys/s");
+
+	return result;
+}
+
+int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), FDBDatabase *db, struct ResultSet *rs, const char *kpiName) {
+	int numRuns = 25;
+	int *results = malloc(sizeof(int)*numRuns);
+	int i = 0;
+	for(; i < numRuns; ++i) {
+		struct RunResult res = testFxn(rs, db);
+		if(res.e) {
+			logError(res.e, kpiName, rs);
+			free(results);
+			return 0;
+		}
+		results[i] = res.res;
+		if(results[i] < 0) {
+			free(results);
+			return -1;
+		}
+	}
+
+	int result = median(results, numRuns);
+	free(results);
+
+	addKpi(rs, kpiName, result, "keys/s");
+
+	return result;
+}
+
+
+struct RunResult clearAll(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
+	return RES(0, 0);
+}
+
+uint32_t start = 0;
+uint32_t stop = 0;
+struct RunResult insertRange(struct ResultSet *rs, FDBTransaction *tr) {
+	int i;
+	for(i = start; i < stop; i++) {
+		fdb_transaction_set(tr, keys[i], keySize, valueStr, valueSize);
+	}
+	return RES(0, 0);
+}
+
+void insertData(struct ResultSet *rs, FDBDatabase *db) {
+	checkError(run(rs, db, &clearAll).e, "clearing database", rs);
+
+	// TODO: Do this asynchronously.
+	start = 0;
+	while(start < numKeys) {
+		stop = start + 1000;
+		if(stop > numKeys) stop = numKeys;
+		checkError(run(rs, db, &insertRange).e, "inserting data range", rs);
+		start = stop;
+	}
+}
+
+fdb_error_t setRetryLimit(struct ResultSet *rs, FDBTransaction *tr, uint64_t limit) {
+	return fdb_transaction_set_option(tr, FDB_TR_OPTION_RETRY_LIMIT, (const uint8_t*)&limit, sizeof(uint64_t));
+}
+
+uint32_t FUTURE_LATENCY_COUNT = 100000;
+const char *FUTURE_LATENCY_KPI = "C future throughput (local client)";
+struct RunResult futureLatency(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	FDBFuture *f = fdb_transaction_get_read_version(tr);
+	e = waitError(f);
+	fdb_future_destroy(f);
+	maybeLogError(e, "getting initial read version", rs);
+	if(e) return RES(0, e);
+
+	double start = getTime();
+	int i;
+	for(i = 0; i < FUTURE_LATENCY_COUNT; i++) {
+		FDBFuture *f = fdb_transaction_get_read_version(tr);
+		e = waitError(f);
+		fdb_future_destroy(f);
+		maybeLogError(e, "getting read version", rs);
+		if(e) return RES(0, e);
+	}
+	double end = getTime();
+
+	return RES(FUTURE_LATENCY_COUNT/(end - start), 0);
+}
+
+uint32_t CLEAR_COUNT = 100000;
+const char *CLEAR_KPI = "C clear throughput (local client)";
+struct RunResult clear(struct ResultSet *rs, FDBTransaction *tr) {
+	double start = getTime();
+	int i;
+	for(i = 0; i < CLEAR_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		fdb_transaction_clear(tr, keys[k], keySize);
+	}
+	double end = getTime();
+
+	fdb_transaction_reset(tr); // Don't actually clear things.
+	return RES(CLEAR_COUNT/(end - start), 0);
+}
+
+uint32_t CLEAR_RANGE_COUNT = 100000;
+const char *CLEAR_RANGE_KPI = "C clear range throughput (local client)";
+struct RunResult clearRange(struct ResultSet *rs, FDBTransaction *tr) {
+	double start = getTime();
+	int i;
+	for(i = 0; i < CLEAR_RANGE_COUNT; i++) {
+		int k = ((uint64_t)rand()) % (numKeys - 1);
+		fdb_transaction_clear_range(tr, keys[k], keySize, keys[k+1], keySize);
+	}
+	double end = getTime();
+
+	fdb_transaction_reset(tr); // Don't actually clear things.
+	return RES(CLEAR_RANGE_COUNT/(end - start), 0);
+}
+
+uint32_t SET_COUNT = 100000;
+const char *SET_KPI = "C set throughput (local client)";
+struct RunResult set(struct ResultSet *rs, FDBTransaction *tr) {
+	double start = getTime();
+	int i;
+	for(i = 0; i < SET_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
+	}
+	double end = getTime();
+
+	fdb_transaction_reset(tr); // Don't actually set things.
+	return RES(SET_COUNT/(end - start), 0);
+}
+
+uint32_t PARALLEL_GET_COUNT = 10000;
+const char *PARALLEL_GET_KPI = "C parallel get throughput (local client)";
+struct RunResult parallelGet(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * PARALLEL_GET_COUNT);
+
+	double start = getTime();
+
+	int i;
+	for(i = 0; i < PARALLEL_GET_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
+	}
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	for(i = 0; i < PARALLEL_GET_COUNT; i++) {
+		e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		fdb_future_destroy(futures[i]);
+	}
+
+	double end = getTime();
+
+	free(futures);
+	return RES(PARALLEL_GET_COUNT/(end - start), 0);
+}
+
+uint32_t ALTERNATING_GET_SET_COUNT = 2000;
+const char *ALTERNATING_GET_SET_KPI = "C alternating get set throughput (local client)";
+struct RunResult alternatingGetSet(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	FDBFuture **futures = (FDBFuture**)malloc((sizeof(FDBFuture*)) * ALTERNATING_GET_SET_COUNT);
+
+	double start = getTime();
+
+	int i;
+	for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
+		int k = ((uint64_t)rand()) % numKeys;
+		fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
+		futures[i] = fdb_transaction_get(tr, keys[k], keySize, 0);
+	}
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	for(i = 0; i < ALTERNATING_GET_SET_COUNT; i++) {
+		e = maybeLogError(fdb_future_block_until_ready(futures[i]), "waiting for get future", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(futures[i], &present, &outValue, &outValueLength), "getting future value", rs);
+		if(e) {
+			fdb_future_destroy(futures[i]);
+			return RES(0, e);
+		}
+
+		fdb_future_destroy(futures[i]);
+	}
+
+	double end = getTime();
+
+	free(futures);
+	return RES(ALTERNATING_GET_SET_COUNT/(end - start), 0);
+}
+
+uint32_t SERIAL_GET_COUNT = 2000;
+const char *SERIAL_GET_KPI = "C serial get throughput (local client)";
+struct RunResult serialGet(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	int i;
+	uint32_t *keyIndices = (uint32_t*)malloc((sizeof(uint32_t)) * SERIAL_GET_COUNT);
+
+	if(SERIAL_GET_COUNT > numKeys/2) {
+		for(i = 0; i < SERIAL_GET_COUNT; i++) {
+			keyIndices[i] = ((uint64_t)rand()) % numKeys;
+		}
+	} else {
+		for(i = 0; i < SERIAL_GET_COUNT; i++) {
+			while(1) {
+				// Yes, this is a linear scan. This happens outside
+				// the part we are measuring.
+				uint32_t index = ((uint64_t)rand()) % numKeys;
+				int j;
+				fdb_bool_t found = 0;
+				for(j = 0; j < i; j++) {
+					if(keyIndices[j] == index) {
+						found = 1;
+						break;
+					}
+				}
+
+				if(!found) {
+					keyIndices[i] = index;
+					break;
+				}
+			}
+		}
+	}
+
+	double start = getTime();
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	for(i = 0; i < SERIAL_GET_COUNT; i++) {
+		FDBFuture *f = fdb_transaction_get(tr, keys[keyIndices[i]], keySize, 0);
+		fdb_error_t e = maybeLogError(fdb_future_block_until_ready(f), "getting key in serial", rs);
+		if(e) {
+			free(keyIndices);
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
+		fdb_future_destroy(f);
+		if(e) {
+			free(keyIndices);
+			return RES(0, e);
+		}
+	}
+
+	double end = getTime();
+
+	free(keyIndices);
+	return RES(SERIAL_GET_COUNT/(end - start), 0);
+}
+
+uint32_t GET_RANGE_COUNT = 100000;
+const char *GET_RANGE_KPI = "C get range throughput (local client)";
+struct RunResult getRange(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	uint32_t startKey = ((uint64_t)rand()) % (numKeys - GET_RANGE_COUNT - 1);
+
+	double start = getTime();
+
+	const FDBKeyValue *outKv;
+	int outCount;
+	fdb_bool_t outMore = 1;
+	int totalOut = 0;
+	int iteration = 0;
+
+	FDBFuture *f = fdb_transaction_get_range(tr,
+		keys[startKey], keySize, 1, 0,
+		keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
+		0, 0,
+		FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
+
+	while(outMore) {
+		e = maybeLogError(fdb_future_block_until_ready(f), "getting range", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading range array", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		totalOut += outCount;
+
+		if(outMore) {
+			FDBFuture *f2 = fdb_transaction_get_range(tr,
+				outKv[outCount - 1].key, outKv[outCount - 1].key_length, 1, 1,
+				keys[startKey + GET_RANGE_COUNT], keySize, 1, 0,
+				0, 0,
+				FDB_STREAMING_MODE_WANT_ALL, ++iteration, 0, 0);
+			fdb_future_destroy(f);
+			f = f2;
+		}
+	}
+
+	if(totalOut != GET_RANGE_COUNT) {
+		char *msg = (char*)malloc((sizeof(char)) * 200);
+		sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT);
+		logError(4100, msg, rs);
+		free(msg);
+		fdb_future_destroy(f);
+		return RES(0, 4100);
+	}
+	if(outMore) {
+		logError(4100, "verifying no more in range", rs);
+		fdb_future_destroy(f);
+		return RES(0, 4100);
+	}
+	fdb_future_destroy(f);
+
+	double end = getTime();
+
+	return RES(GET_RANGE_COUNT/(end - start), 0);
+}
+
+uint32_t GET_KEY_COUNT = 2000;
+const char *GET_KEY_KPI = "C get key throughput (local client)";
+struct RunResult getKey(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	double start = getTime();
+
+	fdb_bool_t present;
+	uint8_t const *outValue;
+	int outValueLength;
+
+	int i;
+	for(i = 0; i < GET_KEY_COUNT; i++) {
+		int key = ((uint64_t)rand()) % numKeys;
+		int offset = (((uint64_t)rand()) % 21) - 10;
+		FDBFuture *f = fdb_transaction_get_key(tr, keys[key], keySize, 1, offset, 0);
+
+		e = maybeLogError(fdb_future_block_until_ready(f), "waiting for get key", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_value(f, &present, &outValue, &outValueLength), "getting future value", rs);
+		fdb_future_destroy(f);
+		if(e) {
+			return RES(0, e);
+		}
+	}
+
+	double end = getTime();
+
+	return RES(GET_KEY_COUNT/(end - start), 0);
+}
+
+uint32_t GET_SINGLE_KEY_RANGE_COUNT = 2000;
+const char *GET_SINGLE_KEY_RANGE_KPI = "C get_single_key_range throughput (local client)";
+struct RunResult getSingleKeyRange(struct ResultSet *rs, FDBTransaction *tr) {
+	fdb_error_t e = maybeLogError(setRetryLimit(rs, tr, 5), "setting retry limit", rs);
+	if(e) return RES(0, e);
+
+	double start = getTime();
+
+	const FDBKeyValue *outKv;
+	int outCount;
+	fdb_bool_t outMore;
+
+	int i;
+	for(i = 0; i < GET_SINGLE_KEY_RANGE_COUNT; i++) {
+		int key = ((uint64_t)rand()) % (numKeys - 1);
+		FDBFuture *f = fdb_transaction_get_range(tr,
+			keys[key], keySize, 1, 0,
+			keys[key + 1], keySize, 1, 0,
+			0, 0,
+			FDB_STREAMING_MODE_WANT_ALL, 1, 0, 0);
+
+		e = maybeLogError(fdb_future_block_until_ready(f), "waiting for single key range", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		e = maybeLogError(fdb_future_get_keyvalue_array(f, &outKv, &outCount, &outMore), "reading single key range array", rs);
+		if(e) {
+			fdb_future_destroy(f);
+			return RES(0, e);
+		}
+
+		if(outCount != 1) {
+			logError(4100, "non-1 number of keys returned in single key range read", rs);
+			fdb_future_destroy(f);
+			return RES(0, 4100);
+		}
+		if(outMore) {
+			logError(4100, "more keys to read in single key range read", rs);
+			fdb_future_destroy(f);
+			return RES(0, 4100);
+		}
+
+		fdb_future_destroy(f);
+	}
+
+	double end = getTime();
+
+	return RES(GET_SINGLE_KEY_RANGE_COUNT/(end - start), 0);
+}
+
+struct RunResult singleKey(struct ResultSet *rs, FDBTransaction *tr) {
+	int k = ((uint64_t)rand()) % numKeys;
+	fdb_transaction_set(tr, keys[k], keySize, valueStr, valueSize);
+	return RES(0, 0);
+}
+
+uint32_t WRITE_TRANSACTION_COUNT = 1000;
+const char *WRITE_TRANSACTION_KPI = "C write_transaction throughput (local client)";
+struct RunResult writeTransaction(struct ResultSet *rs, FDBDatabase *db) {
+	double start = getTime();
+
+	int i;
+	for(i = 0; i < WRITE_TRANSACTION_COUNT; i++) {
+		struct RunResult res = run(rs, db, &singleKey);
+		if(res.e) return res;
+	}
+
+	double end = getTime();
+
+	return RES(WRITE_TRANSACTION_COUNT/(end - start), 0);
+}
+
+void runTests(struct ResultSet *rs) {
+	FDBDatabase *db = openDatabase(rs, &netThread);
+
+	printf("Loading database...\n");
+	insertData(rs, db);
+
+	printf("future_latency\n");
+	runTest(&futureLatency, db, rs, FUTURE_LATENCY_KPI);
+
+	printf("clear\n");
+	runTest(&clear, db, rs, CLEAR_KPI);
+
+	printf("clear_range\n");
+	runTest(&clearRange, db, rs, CLEAR_RANGE_KPI);
+
+	printf("set\n");
+	runTest(&set, db, rs, SET_KPI);
+
+	printf("parallel_get\n");
+	runTest(&parallelGet, db, rs, PARALLEL_GET_KPI);
+
+	printf("alternating_get_set\n");
+	runTest(&alternatingGetSet, db, rs, ALTERNATING_GET_SET_KPI);
+
+	printf("serial_get\n");
+	runTest(&serialGet, db, rs, SERIAL_GET_KPI);
+
+	printf("get_range\n");
+	runTest(&getRange, db, rs, GET_RANGE_KPI);
+
+	printf("get_key\n");
+	runTest(&getKey, db, rs, GET_KEY_KPI);
+
+	printf("get_single_key_range\n");
+	runTest(&getSingleKeyRange, db, rs, GET_SINGLE_KEY_RANGE_KPI);
+
+	printf("write_transaction\n");
+	runTestDb(&writeTransaction, db, rs, WRITE_TRANSACTION_KPI);
+
+	fdb_database_destroy(db);
+	fdb_stop_network();
+}
+
+int main(int argc, char **argv) {
+	srand(time(NULL));
+	struct ResultSet *rs = newResultSet();
+	checkError(fdb_select_api_version(500), "select API version", rs);
+	printf("Running performance test at client version: %s\n", fdb_get_client_version());
+
+	valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize);
+	int i;
+	for(i = 0; i < valueSize; i++) {
+		valueStr[i] = (uint8_t)'x';
+	}
+
+	keys = generateKeys(numKeys, keySize);
+	runTests(rs);
+	writeResultSet(rs);
+
+	free(valueStr);
+	freeResultSet(rs);
+	freeKeys(keys, numKeys);
+
+	return 0;
+}

bindings/c/test/ryw_benchmark.c

@@ -0,0 +1,257 @@
+/*
+ * ryw_benchmark.c
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test.h"
+#include <foundationdb/fdb_c.h>
+#include <foundationdb/fdb_c_options.g.h>
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+
+pthread_t netThread;
+
+int numKeys = 10000;
+int keySize = 16;
+uint8_t** keys;
+
+void insertData(FDBTransaction *tr) {
+	fdb_transaction_clear_range(tr, (uint8_t*)"", 0, (uint8_t*)"\xff", 1);
+
+	uint8_t *v = (uint8_t*)"foo";
+	uint32_t i;
+	for(i = 0; i <= numKeys; ++i) {
+		fdb_transaction_set(tr, keys[i], keySize, v, 3);
+	}
+}
+
+int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), FDBTransaction *tr, struct ResultSet *rs, const char *kpiName) {
+	int numRuns = 25;
+	int *results = malloc(sizeof(int)*numRuns);
+	int i = 0;
+	for(; i < numRuns; ++i) {
+		results[i] = testFxn(tr, rs);
+		if(results[i] < 0) {
+			free(results);
+			return -1;
+		}
+	}
+
+	int result = median(results, numRuns);
+	free(results);
+
+	addKpi(rs, kpiName, result, "keys/s");
+
+	return result;
+}
+
+int getSingle(FDBTransaction *tr, struct ResultSet *rs) {
+	int present;
+	uint8_t const *value;
+	int length;
+	int i;
+
+	double start = getTime();
+	for(i = 0; i < numKeys; ++i) {
+		FDBFuture *f = fdb_transaction_get(tr, keys[5001], keySize, 0);
+		if(getError(fdb_future_block_until_ready(f), "GetSingle (block for get)", rs)) return -1;
+		if(getError(fdb_future_get_value(f, &present, &value, &length), "GetSingle (get result)", rs)) return -1;
+		fdb_future_destroy(f);
+	}
+	double end = getTime();
+
+	return numKeys / (end - start);
+}
+
+int getManySequential(FDBTransaction *tr, struct ResultSet *rs) {
+	int present;
+	uint8_t const *value;
+	int length;
+	int i;
+
+	double start = getTime();
+	for(i = 0; i < numKeys; ++i) {
+		FDBFuture *f = fdb_transaction_get(tr, keys[i], keySize, 0);
+		if(getError(fdb_future_block_until_ready(f), "GetManySequential (block for get)", rs)) return -1;
+		if(getError(fdb_future_get_value(f, &present, &value, &length), "GetManySequential (get result)", rs)) return -1;
+		fdb_future_destroy(f);
+	}
+	double end = getTime();
+
+	return numKeys / (end - start);
+}
+
+int getRangeBasic(FDBTransaction *tr, struct ResultSet *rs) {
+	int count;
+	const FDBKeyValue *kvs;
+	int more;
+	int i;
+
+	double start = getTime();
+	for(i = 0; i < 100; ++i) {
+		FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
+
+		if(getError(fdb_future_block_until_ready(f), "GetRangeBasic (block for get range)", rs)) return -1;
+		if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "GetRangeBasic (get range results)", rs)) return -1;
+
+		if(count != numKeys) {
+			fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
+			addError(rs, "GetRangeBasic bad count");
+			return -1;
+		}
+	}
+	double end = getTime();
+
+	return 100 * numKeys / (end - start);
+}
+
+int singleClearGetRange(FDBTransaction *tr, struct ResultSet *rs) {
+	int count;
+	const FDBKeyValue *kvs;
+	int more;
+	int i;
+
+	for(i = 0; i < numKeys; i+=2) {
+		fdb_transaction_clear(tr, keys[i], keySize);
+	}
+
+	double start = getTime();
+	for(i = 0; i < 100; ++i) {
+		FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
+
+		if(getError(fdb_future_block_until_ready(f), "SingleClearGetRange (block for get range)", rs)) return -1;
+		if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "SingleClearGetRange (get range results)", rs)) return -1;
+
+		fdb_future_destroy(f);
+
+		if(count != numKeys/2) {
+			fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys);
+			addError(rs, "SingleClearGetRange bad count");
+			return -1;
+		}
+	}
+	double end = getTime();
+
+	insertData(tr);
+	return 100 * numKeys / 2 / (end - start);
+}
+
+int clearRangeGetRange(FDBTransaction *tr, struct ResultSet *rs) {
+	int count;
+	const FDBKeyValue *kvs;
+	int more;
+	int i;
+
+	for(i = 0; i < numKeys; i+=4) {
+		fdb_transaction_clear_range(tr, keys[i], keySize, keys[i+1], keySize);
+	}
+
+	double start = getTime();
+	for(i = 0; i < 100; ++i) {
+		FDBFuture *f = fdb_transaction_get_range(tr, FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[0], keySize), FDB_KEYSEL_LAST_LESS_OR_EQUAL(keys[numKeys], keySize), numKeys, 0, 0, 1, 0, 0);
+
+		if(getError(fdb_future_block_until_ready(f), "ClearRangeGetRange (block for get range)", rs)) return -1;
+		if(getError(fdb_future_get_keyvalue_array(f, &kvs, &count, &more), "ClearRangeGetRange (get range results)", rs)) return -1;
+
+		fdb_future_destroy(f);
+
+		if(count != numKeys*3/4) {
+			fprintf(stderr, "Bad count %d (expected %d)\n", count, numKeys*3/4);
+			addError(rs, "ClearRangeGetRange bad count");
+			return -1;
+		}
+	}
+	double end = getTime();
+
+	insertData(tr);
+	return 100 * numKeys * 3 / 4 / (end - start);
+}
+
+int interleavedSetsGets(FDBTransaction *tr, struct ResultSet *rs) {
+	int present;
+	uint8_t const *value;
+	int length;
+	int i;
+
+	uint8_t *k = (uint8_t*)"foo";
+	uint8_t v[10];
+	int num = 1;
+
+	double start = getTime();
+	sprintf((char*)v, "%d", num);
+	fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
+
+	for(i = 0; i < 10000; ++i) {
+		FDBFuture *f = fdb_transaction_get(tr, k, 3, 0);
+		if(getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1;
+		if(getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1;
+		fdb_future_destroy(f);
+
+		sprintf((char*)v, "%d", ++num);
+		fdb_transaction_set(tr, k, 3, v, strlen((char*)v));
+	}
+	double end = getTime();
+
+	return 10000 / (end - start);
+}
+
+void runTests(struct ResultSet *rs) {
+	FDBDatabase *db = openDatabase(rs, &netThread);
+
+	FDBTransaction *tr;
+	checkError(fdb_database_create_transaction(db, &tr), "create transaction", rs);
+
+	FDBFuture *f = fdb_transaction_get_read_version(tr);
+	checkError(fdb_future_block_until_ready(f), "block for read version", rs);
+
+	int64_t version;
+	checkError(fdb_future_get_version(f, &version), "get version", rs);
+	fdb_future_destroy(f);
+
+	insertData(tr);
+
+	runTest(&getSingle, tr, rs, "C: get single cached value throughput");
+	runTest(&getManySequential, tr, rs, "C: get sequential cached values throughput");
+	runTest(&getRangeBasic, tr, rs, "C: get range cached values throughput");
+	runTest(&singleClearGetRange, tr, rs, "C: get range cached values with clears throughput");
+	runTest(&clearRangeGetRange, tr, rs, "C: get range cached values with clear ranges throughput");
+	runTest(&interleavedSetsGets, tr, rs, "C: interleaved sets and gets on a single key throughput");
+
+	fdb_database_destroy(db);
+	fdb_stop_network();
+}
+
+int main(int argc, char **argv) {
+	srand(time(NULL));
+	struct ResultSet *rs = newResultSet();
+	checkError(fdb_select_api_version(500), "select API version", rs);
+	printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version());
+
+	keys = generateKeys(numKeys, keySize);
+	runTests(rs);
+	writeResultSet(rs);
+	freeResultSet(rs);
+	freeKeys(keys, numKeys);
+
+	return 0;
+}
+

bindings/c/test/test.h

@@ -0,0 +1,257 @@
+/*
+ * test.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sys/time.h>
+#include <arpa/inet.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+#ifndef FDB_API_VERSION
+#define FDB_API_VERSION 500
+#endif
+
+#include <foundationdb/fdb_c.h>
+#include <foundationdb/fdb_c_options.g.h>
+
+double getTime() {
+	static struct timeval tv;
+	gettimeofday(&tv, NULL);
+	return tv.tv_usec/1000000.0 + tv.tv_sec;
+}
+
+void writeKey(uint8_t **dest, int key, int keySize) {
+	*dest = (uint8_t*)malloc((sizeof(uint8_t))*keySize);
+	sprintf((char*)*dest, "%0*d", keySize, key);
+}
+
+uint8_t **generateKeys(int numKeys, int keySize) {
+	uint8_t **keys = (uint8_t**)malloc(sizeof(uint8_t*)*(numKeys+1));
+
+	uint32_t i;
+	for(i = 0; i <= numKeys; ++i) {
+		writeKey(keys + i, i, keySize);
+	}
+
+	return keys;
+}
+void freeKeys(uint8_t **keys, int numKeys) {
+	uint32_t i;
+	for(i = 0; i < numKeys; i++) {
+		free(keys[i]);
+	}
+	free(keys);
+}
+
+int cmpfunc(const void* a, const void* b) {
+	return (*(int*)a - *(int*)b);
+}
+
+int median(int *values, int length) {
+	qsort(values, length, sizeof(int), cmpfunc);
+	return values[length/2];
+}
+
+struct RunResult {
+	int res;
+	fdb_error_t e;
+};
+#define RES(x, y) (struct RunResult) { x, y }
+
+struct Kpi {
+	const char *name;
+	int value;
+	const char *units;
+
+	struct Kpi *next;
+};
+
+struct Error {
+	char *message;
+
+	struct Error *next;
+};
+
+struct ResultSet {
+	struct Kpi *kpis;
+	struct Error *errors;
+};
+
+struct ResultSet* newResultSet() {
+	struct ResultSet *rs = malloc(sizeof(struct ResultSet));
+
+	rs->kpis = NULL;
+	rs->errors = NULL;
+
+	return rs;
+}
+
+void addKpi(struct ResultSet *rs, const char *name, int value, const char *units) {
+	struct Kpi *k = malloc(sizeof(struct Kpi));
+	k->name = name;
+	k->value = value;
+	k->units = units;
+	k->next = rs->kpis;
+	rs->kpis = k;
+}
+
+void addError(struct ResultSet *rs, const char *message) {
+	struct Error *e = malloc(sizeof(struct Error));
+	e->message = (char*)malloc(strlen(message)+1);
+	strcpy(e->message, message);
+	e->next = rs->errors;
+	rs->errors = e;
+}
+
+void writeResultSet(struct ResultSet *rs) {
+	uint64_t id = ((uint64_t)rand() << 32) + rand();
+	char name[100];
+	sprintf(name, "fdb-c_result-%llu.json", id);
+	FILE *fp = fopen(name, "w");
+	if(!fp) {
+		fprintf(stderr, "Could not open results file %s\n", name);
+		exit(1);
+	}
+
+	fprintf(fp, "{\n");
+	fprintf(fp, "\t\"kpis\": {\n");
+
+	struct Kpi *k = rs->kpis;
+	while(k != NULL) {
+		fprintf(fp, "\t\t\"%s\": { \"units\": \"%s\", \"value\": %d }", k->name, k->units, k->value);
+		if(k->next != NULL) {
+			fprintf(fp, ",");
+		}
+		fprintf(fp, "\n");
+		k = k->next;
+	}
+
+	fprintf(fp, "\t},\n");
+	fprintf(fp, "\t\"errors\": [\n");
+
+	struct Error *e = rs->errors;
+	while(e != NULL) {
+		fprintf(fp, "\t\t\"%s\"", e->message);
+		if(e->next != NULL) {
+			fprintf(fp, ",");
+		}
+		fprintf(fp, "\n");
+		e = e->next;
+	}
+
+	fprintf(fp, "\t]\n");
+	fprintf(fp, "}\n");
+
+	fclose(fp);
+}
+
+void freeResultSet(struct ResultSet *rs) {
+	struct Kpi *k = rs->kpis;
+	while(k != NULL) {
+		struct Kpi *next = k->next;
+		free(k);
+		k = next;
+	}
+
+	struct Error *e = rs->errors;
+	while(e != NULL) {
+		struct Error *next = e->next;
+		free(e->message);
+		free(e);
+		e = next;
+	}
+
+	free(rs);
+}
+
+fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	if(err) {
+		char *msg = (char*)malloc(strlen(context) + 100);
+		sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
+		fprintf(stderr, "%s\n", msg);
+		if(rs != NULL) {
+			addError(rs, msg);
+		}
+
+		free(msg);
+	}
+
+	return err;
+}
+
+void checkError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	if(getError(err, context, rs)) {
+		if(rs != NULL) {
+			writeResultSet(rs);
+			freeResultSet(rs);
+		}
+		exit(1);
+	}
+}
+
+fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	char *msg = (char*)malloc(strlen(context) + 100);
+	sprintf(msg, "Error in %s: %s", context, fdb_get_error(err));
+	fprintf(stderr, "%s\n", msg);
+	if(rs != NULL) {
+		addError(rs, msg);
+	}
+
+	free(msg);
+	return err;
+}
+
+fdb_error_t maybeLogError(fdb_error_t err, const char* context, struct ResultSet *rs) {
+	if(err && !fdb_error_predicate( FDB_ERROR_PREDICATE_RETRYABLE, err ) ) {
+		return logError(err, context, rs);
+	}
+	return err;
+}
+
+void* runNetwork() {
+	checkError(fdb_run_network(), "run network", NULL);
+	return NULL;
+}
+
+FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) {
+	checkError(fdb_setup_network(), "setup network", rs);
+	pthread_create(netThread, NULL, &runNetwork, NULL);
+
+	FDBFuture *f = fdb_create_cluster(NULL);
+	checkError(fdb_future_block_until_ready(f), "block for cluster", rs);
+
+	FDBCluster *cluster;
+	checkError(fdb_future_get_cluster(f, &cluster), "get cluster", rs);
+
+	fdb_future_destroy(f);
+
+	f = fdb_cluster_create_database(cluster, (uint8_t*)"DB", 2);
+	checkError(fdb_future_block_until_ready(f), "block for database", rs);
+
+	FDBDatabase *db;
+	checkError(fdb_future_get_database(f, &db), "get database", rs);
+
+	fdb_future_destroy(f);
+	fdb_cluster_destroy(cluster);
+
+	return db;
+}

bindings/flow/local.mk

@@ -22,5 +22,23 @@
 
 fdb_flow_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS)
 fdb_flow_LDFLAGS := -Llib -lfdb_c $(fdbrpc_LDFLAGS)
-fdb_flow_LIBS := lib/libfdbrpc.a
+fdb_flow_LIBS :=
 
+packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz: fdb_flow
+	@echo "Packaging      fdb_flow"
+	@rm -rf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+	@mkdir -p packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/flow packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/c/foundationdb
+	@cp lib/libfdb_flow.a packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib
+	@find bindings/flow -name '*.h' -not -name 'bindings/flow/tester/*' -exec cp {} packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/flow \;
+	@find bindings/c/foundationdb -name '*.h' -exec cp {} packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/c/foundationdb \;
+	@tar czf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz -C packages fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+	@rm -rf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)
+
+FDB_FLOW: packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz
+
+FDB_FLOW_clean:
+	@echo "Cleaning       fdb_flow package"
+	@rm -rf packages/fdb-flow-*.tar.gz
+
+packages: FDB_FLOW
+packages_clean: FDB_FLOW_clean

bindings/flow/tester/local.mk

@@ -22,7 +22,7 @@
 
 fdb_flow_tester_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS)
 fdb_flow_tester_LDFLAGS := -Llib $(fdbrpc_LDFLAGS) -lfdb_c
-fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libfdbrpc.a lib/libflow.a lib/libfdb_c.$(DLEXT)
+fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libflow.a lib/libfdb_c.$(DLEXT)
 
 fdb_flow_tester: lib/libfdb_c.$(DLEXT)
 	@mkdir -p bindings/flow/bin

bindings/go/Gopkg.lock

@@ -0,0 +1,9 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7"
+  solver-name = "gps-cdcl"
+  solver-version = 1

bindings/go/Gopkg.toml

@@ -0,0 +1,2 @@
+# The FoundationDB go bindings currently have no external golang dependencies outside of
+# the go standard library.

bindings/go/README.md

@@ -10,10 +10,20 @@ This package requires:
 
 Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-500.
 
-To install this package, in the top level of this repository run:
+To build this package, in the top level of this repository run:
 
     make fdb_go
 
+This will create binary packages for the appropriate platform within the "build" subdirectory of this folder.
+
+To install this package, you can run the "fdb-go-install.sh" script:
+
+    ./fdb-go-install.sh install 
+
+The "install" command of this script does not depend on the presence of the repo in general and will download the repository into
+your local go path. Running "localinstall" instead of "install" will use the local copy here (with a symlink) instead
+of downloading from the remote repository.
+
 Documentation
 -------------
 

bindings/go/fdb-go-install.sh

@@ -0,0 +1,304 @@
+#!/bin/bash -eu
+#
+# fdb-go-install.sh
+#
+# Installs the FoundationDB Go bindings for a client. This will download
+# the repository from the remote repo either into the go directory
+# with the appropriate semantic version. It will then build a few
+# generated files that need to be present for the go build to work.
+# At the end, it has some advice for flags to modify within your
+# go environment so that other packages may successfully use this
+# library.
+#
+
+DESTDIR="${DESTDIR:-}"
+FDBVER="${FDBVER:-5.0.1}"
+REMOTE="${REMOTE:-github.com}"
+FDBREPO="${FDBREPO:-apple/foundationdb}"
+
+status=0
+
+platform=$(uname)
+if [[ "${platform}" == "Darwin" ]] ; then
+    FDBLIBDIR="${FDBLIBDIR:-/usr/local/lib}"
+    libfdbc="libfdb_c.dylib"
+elif [[ "${platform}" == "Linux" ]] ; then
+    FDBLIBDIR="${FDBLIBDIR:-/usr/lib}"
+    libfdbc="libfdb_c.so"
+else
+    echo "Unsupported platform ${platform}".
+    echo "At the moment, only macOS and Linux are supported by this script."
+    let status="${status} + 1"
+fi
+
+filedir=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
+destdir=""
+
+function printUsage() {
+    echo "Usage: fdb-go-install.sh <cmd>"
+    echo
+    echo "cmd: One of the commands to run. The options are:"
+    echo "     install         Download the FDB go bindings and install them"
+    echo "     localinstall    Install a into the go path a local copy of the repo"
+    echo "     download        Download but do not prepare the FoundationDB bindings"
+    echo "     help            Print this help message and then quit"
+    echo
+    echo "Command Line Options:"
+    echo "     --fdbver <version>    FoundationDB semantic version (default is ${FDBVER})"
+    echo "     -d/--dest-dir <dest>  Local location for the repo (default is to place in go path)"
+    echo
+    echo "Environment Variable Options:"
+    echo "     REMOTE          Remote repository to download from (currently ${REMOTE})"
+    echo "     FDBREPO         Repository of FoundationDB library to download (currently ${FDBREPO})"
+    echo "     FDBLIBDIR       Directory within which should be the FoundationDB c library (currently ${FDBLIBDIR})"
+}
+
+function parseArgs() {
+    local status=0
+
+    if [[ "${#}" -lt 0 ]] ; then
+        printUsage
+        let status="${status} + 1"
+    else
+        operation="${1}"
+        shift
+        if [[ "${operation}" != "install" ]] && [[ "${operation}" != "localinstall" ]] && [[ "${operation}" != "download" ]] && [[ "${operation}" != "help" ]] ; then
+            echo "Unknown command: ${operation}"
+            printUsage
+            let status="${status} + 1"
+        fi
+    fi
+
+    while [[ "${#}" -gt 0 ]] && [[ "${status}" -eq 0 ]] ; do
+        local key="${1}"
+        case "${key}" in
+            --fdbver)
+            if [[ "${#}" -lt 2 ]] ; then
+                echo "No version specified with --fdbver flag"
+                printUsage
+                let status="${status} + 1"
+            else
+                FDBVER="${2}"
+            fi
+            shift
+            ;;
+
+            -d|--dest-dir)
+            if [[ "${#}" -lt 2 ]] ; then
+                echo "No destination specified with ${key} flag"
+                printUsage
+                let status="${status} + 1"
+            else
+                destdir="${2}"
+            fi
+            shift
+            ;;
+
+            *)
+            echo "Unrecognized argument ${key}"
+            printUsage
+            let status="${status} + 1"
+        esac
+        shift
+    done
+
+    return "${status}"
+}
+
+function checkBin() {
+    if [[ "${#}" -lt 1 ]] ; then
+        echo "Usage: checkBin <binary>"
+        return 1
+    else
+        if [[ -n $(which "${1}") ]] ; then
+            return 0
+        else
+            return 1
+        fi
+    fi
+}
+
+if [[ "${status}" -gt 0 ]] ; then
+    # We have already failed.
+    :
+elif [[ "${#}" -lt 1 ]] ; then
+    printUsage
+else
+    required_bins=( 'go' 'git' 'make' 'mono' )
+
+    missing_bins=()
+    for bin in "${required_bins[@]}" ; do
+        if ! checkBin "${bin}" ; then
+            missing_bins+=("${bin}")
+            let status="${status} + 1"
+        fi
+    done
+
+    if [[ "${status}" -gt 0 ]] ; then
+        echo "Missing binaries: ${missing_bins[*]}"
+    elif ! parseArgs ${@} ; then
+        let status="${status} + 1"
+    elif [[ "${operation}" == "help" ]] ; then
+        printUsage
+    else
+        # Add go-specific environment variables.
+        eval $(go env)
+
+        golibdir=$(dirname "${GOPATH}/src/${REMOTE}/${FDBREPO}")
+        if [[ -z "${destdir}" ]] ; then
+            if [[ "${operation}" == "localinstall" ]] ; then
+                # Assume its the local directory.
+                destdir=$(cd "${filedir}/../../.." && pwd)
+            else
+                destdir="${golibdir}"
+            fi
+        fi
+
+        if [[ ! -d "${destdir}" ]] ; then
+            cmd=( 'mkdir' '-p' "${destdir}" )
+            echo "${cmd[*]}"
+            if ! "${cmd[@]}" ; then
+                let status="${status} + 1"
+                echo "Could not create destination directory ${destdir}."
+            fi
+        fi
+
+        # Step 1: Make sure repository is present.
+
+        if [[ "${status}" -eq 0 ]] ; then
+            destdir=$( cd "${destdir}" && pwd ) # Get absolute path of destination dir.
+            fdbdir="${destdir}/foundation"
+
+            if [[ ! -d "${destdir}" ]] ; then
+                cmd=("mkdir" "-p" "${destdir}")
+                echo "${cmd[*]}"
+                if ! "${cmd[@]}" ; then
+                    echo "Could not create destination directory ${destdir}."
+                    let status="${status} + 1"
+                fi
+            fi
+        fi
+
+        if [[ "${operation}" == "localinstall" ]] ; then
+            # No download occurs in this case.
+            :
+        else
+            if [[ -d "${fdbdir}" ]] ; then
+                echo "Directory ${fdbdir} already exists ; checking out appropriate tag"
+                cmd1=( 'git' '-C' "${fdbdir}" 'fetch' 'origin' )
+                cmd2=( 'git' '-C' "${fdbdir}" 'checkout' "release-${FDBVER}" )
+
+                if ! echo "${cmd1[*]}" || ! "${cmd1[@]}" ; then
+                    let status="${status} + 1"
+                    echo "Could not pull latest changes from origin"
+                elif ! echo "${cmd2[*]}" ||  ! "${cmd2[@]}" ; then
+                    let status="${status} + 1"
+                    echo "Could not checkout tag release-${FDBVER}."
+                fi
+            else
+                echo "Downloading foundation repository into ${destdir}:"
+                cmd=( 'git' '-C' "${destdir}" 'clone' '--branch' "release-${FDBVER}" "git@${REMOTE}:${FDBREPO}.git" )
+
+                echo "${cmd[*]}"
+                if ! "${cmd[@]}" ; then
+                    let status="${status} + 1"
+                    echo "Could not download repository."
+                fi
+            fi
+        fi
+
+        # Step 2: Build generated things.
+
+        if [[ "${operation}" == "download" ]] ; then
+            # The generated files are not created under a strict download.
+            :
+        elif [[ "${status}" -eq 0 ]] ; then
+            echo "Building generated files."
+            cmd=( 'make' '-C' "${fdbdir}" 'bindings/c/foundationdb/fdb_c_options.g.h' )
+
+            echo "${cmd[*]}"
+            if ! "${cmd[@]}" ; then
+                let status="${status} + 1"
+                echo "Could not generate required c header"
+            else
+                infile="${fdbdir}/fdbclient/vexillographer/fdb.options"
+                outfile="${fdbdir}/bindings/go/src/fdb/generated.go"
+                cmd=( 'go' 'run' "${fdbdir}/bindings/go/src/_util/translate_fdb_options.go" )
+                echo "${cmd[*]} < ${infile} > ${outfile}"
+                if ! "${cmd[@]}" < "${infile}" > "${outfile}" ; then
+                    let status="${status} + 1"
+                    echo "Could not generate generated go file."
+                fi
+            fi
+        fi
+
+        # Step 3: Add to go path.
+
+        if [[ "${operation}" == "download" ]] ; then
+            # The files are not moved under a strict download.
+            :
+        elif [[ "${status}" -eq 0 ]] ; then
+            linkpath="${GOPATH}/src/${REMOTE}/${FDBREPO}"
+            if [[ "${linkpath}" == "${fdbdir}" ]] ; then
+                # Downloaded directly into go path. Skip making the link.
+                :
+            elif [[ -e "${linkpath}" ]] ; then
+                echo "Warning: link path (${linkpath}) already exists. Leaving in place."
+            else
+                dirpath=$(dirname "${linkpath}")
+                if [[ ! -d "${dirpath}" ]] ; then
+                    cmd=( 'mkdir' '-p' "${dirpath}" )
+                    echo "${cmd[*]}"
+                    if ! "${cmd[@]}" ; then
+                        let status="${status} + 1"
+                        echo "Could not create directory for link."
+                    fi
+                fi
+
+                if [[ "${status}" -eq 0 ]] ; then
+                    cmd=( 'ln' '-s' "${fdbdir}" "${linkpath}" )
+                    echo "${cmd[*]}"
+                    if ! "${cmd[@]}" ; then
+                        let status="${status} + 1"
+                        echo "Could not create link within go path."
+                    fi
+                fi
+            fi
+        fi
+
+        # Step 4: Build the binaries.
+
+        if [[ "${operation}" == "download" ]] ; then
+            # Do not install if only downloading
+            :
+        elif [[ "${status}" -eq 0 ]] ; then
+            cgo_cflags="-g -O2 -I${linkpath}/bindings/c"
+            cgo_ldflags="-g -O2 -L${FDBLIBDIR}"
+            fdb_go_path="${REMOTE}/${FDBREPO}/bindings/go/src"
+
+            if [[ ! -e "${FDBLIBDIR}/${libfdbc}" ]] ; then
+                # Just a warning. Don't fail script.
+                echo
+                echo "WARNING: The FoundationDB C library was not found within ${FDBLIBDIR}."
+                echo "Your installation may be incomplete."
+                echo
+            elif ! CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" go install "${fdb_go_path}/fdb" "${fdb_go_path}/fdb/tuple" "${fdb_go_path}/fdb/subspace" "${fdb_go_path}/fdb/directory" ; then
+                let status="${status} + 1"
+                echo "Could not build FoundationDB go libraries."
+            fi
+        fi
+
+        # Step 5: Explain CGO flags.
+
+        if [[ "${status}" -eq 0 && ("${operation}" == "localinstall" || "${operation}" == "install" ) ]] ; then
+            echo
+            echo "The FoundationDB go bindings were successfully installed."
+            echo "To build packages which use the go bindings, you will need to"
+            echo "set the following environment variables:"
+            echo "   CGO_CFLAGS=\"${cgo_cflags}\""
+            echo "   CGO_LDFLAGS=\"${cgo_ldflags}\""
+        fi
+    fi
+fi
+
+exit "${status}"

bindings/go/include.mk

@@ -21,7 +21,11 @@
 TARGETS += fdb_go fdb_go_tester
 CLEAN_TARGETS += fdb_go_clean fdb_go_tester_clean
 
-GOPATH := $(CURDIR)/bindings/go
+GOPATH := $(CURDIR)/bindings/go/build
+GO_IMPORT_PATH := github.com/apple/foundationdb/bindings/go/src
+GO_DEST := $(GOPATH)/src/$(GO_IMPORT_PATH)
+
+.PHONY: fdb_go fdb_go_path fdb_go_tester fdb_go_tester_clean godoc godoc_clean
 
 # We only override if the environment didn't set it (this is used by
 # the fdbwebsite documentation build process)
@@ -38,18 +42,23 @@ else
   $(error Not prepared to compile on platform $(PLATFORM))
 endif
 
-GO_PACKAGE_OUTDIR := $(GOPATH)/pkg/$(GOPLATFORM)
+GO_PACKAGE_OUTDIR := $(GOPATH)/pkg/$(GOPLATFORM)/$(GO_IMPORT_PATH)
 
 GO_PACKAGES := fdb fdb/tuple fdb/subspace fdb/directory
 GO_PACKAGE_OBJECTS := $(addprefix $(GO_PACKAGE_OUTDIR)/,$(GO_PACKAGES:=.a))
 
-GO_SRC := $(shell find $(GOPATH)/src -name '*.go')
+GO_SRC := $(shell find $(CURDIR)/bindings/go/src -name '*.go')
 
 fdb_go: $(GO_PACKAGE_OBJECTS) $(GO_SRC)
 
+fdb_go_path: $(GO_SRC)
+	@echo "Creating       fdb_go_path"
+	@mkdir -p $(GO_DEST)
+	@cp -r bindings/go/src/* $(GO_DEST)
+
 fdb_go_clean:
 	@echo "Cleaning       fdb_go"
-	@rm -rf $(GO_PACKAGE_OUTDIR)
+	@rm -rf $(GOPATH)
 
 fdb_go_tester: $(GOPATH)/bin/_stacktester
 
@@ -57,40 +66,40 @@ fdb_go_tester_clean:
 	@echo "Cleaning       fdb_go_tester"
 	@rm -rf $(GOPATH)/bin
 
-$(GOPATH)/bin/_stacktester: $(GO_SRC) $(GO_PACKAGE_OBJECTS) bindings/go/src/fdb/generated.go
+$(GOPATH)/bin/_stacktester: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OBJECTS) $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      $(basename $(notdir $@))"
-	@go install _stacktester
+	@go install $(GO_IMPORT_PATH)/_stacktester
 
-$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb/tuple"
-	@go install fdb/tuple
+	@go install $(GO_IMPORT_PATH)/fdb/tuple
 
-$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb/subspace"
-	@go install fdb/subspace
+	@go install $(GO_IMPORT_PATH)/fdb/subspace
 
-$(GO_PACKAGE_OUTDIR)/fdb/directory.a: $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb/directory.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb/directory"
-	@go install fdb/directory
+	@go install $(GO_IMPORT_PATH)/fdb/directory
 
-$(GO_PACKAGE_OUTDIR)/fdb.a: $(GO_SRC) bindings/go/src/fdb/generated.go
+$(GO_PACKAGE_OUTDIR)/fdb.a: fdb_go_path $(GO_SRC) $(GO_DEST)/fdb/generated.go
 	@echo "Compiling      fdb"
-	@go install fdb
+	@go install $(GO_IMPORT_PATH)/fdb
 
-bindings/go/src/fdb/generated.go: lib/libfdb_c.$(DLEXT) bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options
+$(GO_DEST)/fdb/generated.go: fdb_go_path lib/libfdb_c.$(DLEXT) bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options
 	@echo "Building       $@"
 	@go run bindings/go/src/_util/translate_fdb_options.go < fdbclient/vexillographer/fdb.options > $@
 
-godoc: $(GO_SRC)
+godoc: fdb_go_path $(GO_SRC)
 	@echo "Generating Go Documentation"
 	@rm -rf $(GODOC_DIR)/godoc
 	@mkdir -p $(GODOC_DIR)/godoc
 	@mkdir -p $(GODOC_DIR)/godoc/lib/godoc
-	@godoc -url "http://localhost:6060/pkg/fdb" > $(GODOC_DIR)/godoc/fdb.html
-	@godoc -url "http://localhost:6060/pkg/fdb/tuple" > $(GODOC_DIR)/godoc/fdb.tuple.html
-	@godoc -url "http://localhost:6060/pkg/fdb/subspace" > $(GODOC_DIR)/godoc/fdb.subspace.html
-	@godoc -url "http://localhost:6060/pkg/fdb/directory" > $(GODOC_DIR)/godoc/fdb.directory.html
-	@cp $(GOPATH)/godoc-resources/* $(GODOC_DIR)/godoc/lib/godoc
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb" > $(GODOC_DIR)/godoc/fdb.html
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb/tuple" > $(GODOC_DIR)/godoc/fdb.tuple.html
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb/subspace" > $(GODOC_DIR)/godoc/fdb.subspace.html
+	@godoc -url "pkg/$(GO_IMPORT_PATH)/fdb/directory" > $(GODOC_DIR)/godoc/fdb.directory.html
+	@cp $(CURDIR)/bindings/go/godoc-resources/* $(GODOC_DIR)/godoc/lib/godoc
 	@echo "Mangling paths in Go Documentation"
 	@(find $(GODOC_DIR)/godoc/ -name *.html -exec sed -i '' -e 's_/lib_lib_' {} \;)
 	@(sed -i -e 's_a href="tuple/"_a href="fdb.tuple.html"_' $(GODOC_DIR)/godoc/fdb.html)

bindings/go/src/_stacktester/directory.go

@@ -21,10 +21,10 @@
 package main
 
 import (
-	"fdb"
-	"fdb/tuple"
-	"fdb/subspace"
-	"fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
 	"strings"
 	"bytes"
 )
@@ -94,14 +94,14 @@ func (sm *StackMachine) maybePath() []string {
 }
 
 var createOps = map[string]bool {
-    "CREATE_SUBSPACE": true,
-    "CREATE_LAYER": true,
-    "CREATE_OR_OPEN": true,
-    "CREATE": true,
-    "OPEN": true,
-    "MOVE": true,
-    "MOVE_TO": true,
-    "OPEN_SUBSPACE": true,
+	"CREATE_SUBSPACE": true,
+	"CREATE_LAYER": true,
+	"CREATE_OR_OPEN": true,
+	"CREATE": true,
+	"OPEN": true,
+	"MOVE": true,
+	"MOVE_TO": true,
+	"OPEN_SUBSPACE": true,
 }
 
 func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, idx int, t fdb.Transactor, rt fdb.ReadTransactor) {

bindings/go/src/_stacktester/stacktester.go

@@ -24,8 +24,8 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/hex"
-	"fdb"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"log"
 	"fmt"
 	"os"

bindings/go/src/fdb/directory/allocator.go

@@ -23,8 +23,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	"encoding/binary"
 	"bytes"
 	"math/rand"

bindings/go/src/fdb/directory/directory.go

@@ -40,8 +40,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	"errors"
 )
 
@@ -140,15 +140,15 @@ type Directory interface {
 }
 
 func stringsEqual(a, b []string) bool {
-    if len(a) != len(b) {
-        return false
-    }
-    for i, v := range a {
-        if v != b[i] {
-            return false
-        }
-    }
-    return true
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
 }
 
 func moveTo(t fdb.Transactor, dl directoryLayer, path, newAbsolutePath []string) (DirectorySubspace, error) {

bindings/go/src/fdb/directory/directoryLayer.go

@@ -23,9 +23,9 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"encoding/binary"
 	"bytes"
 	"fmt"

bindings/go/src/fdb/directory/directoryPartition.go

@@ -23,9 +23,9 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 )
 
 type directoryPartition struct {

bindings/go/src/fdb/directory/directorySubspace.go

@@ -23,8 +23,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 )
 
 // DirectorySubspace represents a Directory that may also be used as a Subspace

bindings/go/src/fdb/directory/node.go

@@ -23,8 +23,8 @@
 package directory
 
 import (
-	"fdb"
-	"fdb/subspace"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	"bytes"
 )
 

bindings/go/src/fdb/doc.go

@@ -39,7 +39,7 @@ A basic interaction with the FoundationDB API is demonstrated below:
     package main
 
     import (
-        "github.com/apple/foundationdb/bindings/go/fdb"
+        "github.com/apple/foundationdb/bindings/go/src/fdb"
         "log"
         "fmt"
     )

bindings/go/src/fdb/errors.go

@@ -23,7 +23,7 @@
 package fdb
 
 /*
- #define FDB_API_VERSION 200
+ #define FDB_API_VERSION 500
  #include <foundationdb/fdb_c.h>
 */
 import "C"

bindings/go/src/fdb/fdb_test.go

@@ -23,7 +23,7 @@
 package fdb_test
 
 import (
-	"fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
 	"fmt"
 	"testing"
 )

bindings/go/src/fdb/subspace/subspace.go

@@ -33,8 +33,8 @@
 package subspace
 
 import (
-	"fdb"
-	"fdb/tuple"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
 	"bytes"
 	"errors"
 )

bindings/go/src/fdb/tuple/tuple.go

@@ -38,7 +38,7 @@ import (
 	"fmt"
 	"encoding/binary"
 	"bytes"
-	"fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
 )
 
 // A TupleElement is one of the types that may be encoded in FoundationDB

bindings/java/local.mk

@@ -144,6 +144,11 @@ define add_java_binding_targets
 	  @rm -r packages/jar$(1)_regular
 	  @cd bindings && jar uf $$(TOPDIR)/$$@ ../LICENSE
 
+    packages/fdb-java$(1)-$$(JARVER)-tests.jar: fdb_java$(1) versions.target
+	  @echo "Building       $$@"
+	  @rm -f $$@
+	  @cp $$(TOPDIR)/bindings/java/foundationdb-tests$(1).jar packages/fdb-java$(1)-$$(JARVER)-tests.jar
+
     packages/fdb-java$(1)-$$(JARVER)-sources.jar: $$(JAVA$(1)_GENERATED_SOURCES) versions.target
 	  @echo "Building       $$@"
 	  @rm -f $$@
@@ -165,7 +170,7 @@ define add_java_binding_targets
 	  @cd packages/bundle$(1)_regular && jar cf $(TOPDIR)/$$@ *
 	  @rm -rf packages/bundle$(1)_regular
 
-    fdb_java$(1)_release: packages/fdb-java$(1)-$$(JARVER)-bundle.jar
+    fdb_java$(1)_release: packages/fdb-java$(1)-$$(JARVER)-bundle.jar packages/fdb-java$(1)-$$(JARVER)-tests.jar
 
     fdb_java$(1)_release_clean:
 	  @echo "Cleaning       Java release"

bindings/java/src-completable/main/com/apple/cie/foundationdb/Database.java

@@ -44,8 +44,8 @@ public interface Database extends Disposable, TransactionContext {
 	 * Creates a {@link Transaction} that operates on this {@code Database}.<br>
 	 * <br>
 	 * Note: Java transactions automatically set the {@link TransactionOptions#setUsedDuringCommitProtectionDisable}
-	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after either
-	 *  {@link Transaction#reset} or {@link Transaction#onError} is called.
+	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after
+	 *  {@link Transaction#onError} is called.
 	 *
 	 * @return a newly created {@code Transaction} that reads from and writes to this {@code Database}.
 	 */

bindings/java/src-completable/main/com/apple/cie/foundationdb/async/AsyncIterator.java

@@ -61,7 +61,7 @@ public interface AsyncIterator<T> extends Iterator<T>, Disposable {
 	/**
 	 * Returns the next element in the sequence. This will not block if, since the
 	 *  last call to {@code next()}, {@link #onHasNext()} was called and the resulting
-	 *  <h1>FIXME!!!!</h1> has completed or the blocking call {@link #hasNext()} was called
+	 *  {@link CompletableFuture} has completed or the blocking call {@link #hasNext()} was called
 	 *  and has returned. It is legal, therefore, to make a call to {@code next()} without a
 	 *  preceding call to
 	 *  {@link #hasNext()} or {@link #onHasNext()}, but that invocation of {@code next()}

bindings/java/src-completable/main/com/apple/cie/foundationdb/async/package-info.java

@@ -19,7 +19,7 @@
  */
 
 /**
- * Provides additional constructs for asynchronous programming against Java's CompletableFutures.
+ * Provides additional constructs for asynchronous programming against Java's {@link java.util.concurrent.CompletableFuture CompletableFuture}s.
  *
  */
 package com.apple.cie.foundationdb.async;

bindings/java/src-completable/main/overview.html.in

@@ -24,10 +24,11 @@ and add it to your classpath.<br>
 <br>
 <h3>Getting started</h3>
 To start using FoundationDB from Java, create an instance of the 
-{@link FDB FoundationDB API interface} with the version of the
+{@link com.apple.cie.foundationdb.FDB FoundationDB API interface} with the version of the
 API that you want to use (this release of the FoundationDB Java API supports only version {@code 500}).
-With this API object you can then open {@link Cluster}s and
-{@link Database}s and start using {@link Transaction}s.
+With this API object you can then open {@link com.apple.cie.foundationdb.Cluster Cluster}s and
+{@link com.apple.cie.foundationdb.Database Database}s and start using
+{@link com.apple.cie.foundationdb.Transaction Transactions}s.
 Here we give an example. The example relies on a cluster file at the
 <a href="/documentation/api-general.html#default-cluster-file">default location</a> 
 for your platform and a running server.<br>
@@ -77,7 +78,7 @@ for information about how Tuples sort and can be used to efficiently model data.
 The {@link com.apple.cie.foundationdb.directory Directory API} is provided with the core
 Java API for FoundationDB. This layer is provided in some form in all official
 language bindings. The FoundationDB API provides directories as a tool for
-managing related {@link Subspace}s. Directories are a
+managing related {@link com.apple.cie.foundationdb.subspace.Subspace Subspace}s. Directories are a
 recommended approach for administering applications. Each application should 
 create or open at least one directory to manage its subspaces. Directories are 
 identified by hierarchical paths analogous to the paths in a Unix-like file system. 

bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java

@@ -0,0 +1,117 @@
+/*
+ * AbstractTester.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.Database;
+import com.apple.cie.foundationdb.FDB;
+
+import java.nio.charset.Charset;
+import java.util.Random;
+
+public abstract class AbstractTester {
+    public static final int API_VERSION = 500;
+    protected static final int NUM_RUNS = 25;
+    protected static final Charset ASCII = Charset.forName("ASCII");
+
+    protected TesterArgs args;
+    protected Random random;
+    protected TestResult result;
+    protected FDB fdb;
+
+    public AbstractTester() {
+        args = null;
+        random = new Random();
+        result = new TestResult(random);
+    }
+
+    public void runTest() {
+        Database db;
+
+        try {
+            db = fdb.open();
+        } catch (Exception e) {
+            result.addError(wrapAndPrintError(e, "fdb.open failed"));
+            return;
+        }
+
+        try {
+            testPerformance(db);
+        } catch (Exception e) {
+            result.addError(wrapAndPrintError(e, "Failed to complete all tests"));
+        }
+    }
+
+    public abstract void testPerformance(Database db);
+
+    public String multiVersionDescription() {
+        if (args == null) return "";
+
+        if (!args.useMultiversionApi()) {
+            return "multi-version API disabled";
+        } else if (args.useExternalClient()) {
+            if (args.putCallbacksOnExternalThread()) {
+                return "external client on external thread";
+            } else {
+                return "external client on main thread";
+            }
+        } else {
+            return "local client";
+        }
+    }
+
+    public void run(String[] argStrings) {
+        args = TesterArgs.parseArgs(argStrings);
+        if (args == null) return;
+
+        fdb = FDB.selectAPIVersion(API_VERSION);
+
+        // Validate argument combinations and set options.
+        if (!args.useMultiversionApi()) {
+            if (args.putCallbacksOnExternalThread() || args.useExternalClient()) {
+                throw new IllegalArgumentException("Invalid multi-version API argument combination");
+            }
+            fdb.options().setDisableMultiVersionClientApi();
+        }
+        if (args.putCallbacksOnExternalThread()) {
+            if (!args.useExternalClient()) {
+                throw new IllegalArgumentException("Cannot enable callbacks on external thread without using external client");
+            }
+            fdb.options().setCallbacksOnExternalThreads();
+        }
+        if (args.useExternalClient()) {
+            fdb.options().setDisableLocalClient();
+        }
+
+        try {
+            runTest();
+        } catch (Exception e) {
+            result.addError(e);
+        }
+
+        result.save(args.getOutputDirectory());
+    }
+
+    public RuntimeException wrapAndPrintError(Throwable t, String message) {
+        String errorMessage = message + ": " + t.getClass() + ": " + t.getMessage() + "\n";
+        t.printStackTrace();
+        return new RuntimeException(errorMessage, t);
+    }
+}

bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncPerformanceTester.java

@@ -0,0 +1,28 @@
+/*
+ * AsyncPerformanceTester.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+public class AsyncPerformanceTester {
+
+    public static void main(String[] args) {
+        System.out.println("Running Java async performance test on Java version " + System.getProperty("java.version"));
+    }
+}

bindings/java/src-completable/test/com/apple/cie/foundationdb/test/PerformanceTester.java

@@ -0,0 +1,408 @@
+/*
+ * PerformanceTester.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.Database;
+import com.apple.cie.foundationdb.KeySelector;
+import com.apple.cie.foundationdb.Transaction;
+import com.apple.cie.foundationdb.TransactionContext;
+import com.apple.cie.foundationdb.async.AsyncUtil;
+import com.apple.cie.foundationdb.tuple.ByteArrayUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+public class PerformanceTester extends AbstractTester {
+    private final int keyCount;
+    private final int keySize;
+    private final int valueSize;
+
+    private final String keyFormat;
+    private final byte[] valueBytes;
+
+    public static final int DEFAULT_KEY_COUNT = 10_000;
+    public static final int DEFAULT_KEY_SIZE = 16;
+    public static final int DEFAULT_VALUE_SIZE = 100;
+
+    private enum Tests {
+        FUTURE_LATENCY("Java Completable API future throughput"),
+        SET("Java Completable API set throughput"),
+        CLEAR("Java Completable API clear throughput"),
+        CLEAR_RANGE("Java Completable API clear_range throughput"),
+        PARALLEL_GET("Java Completable API parallel get throughput"),
+        SERIAL_GET("Java Completable API serial get throughput"),
+        GET_RANGE("Java Completable API get_range throughput"),
+        GET_KEY("Java Completable API get_key throughput"),
+        GET_SINGLE_KEY_RANGE("Java Completable API get_single_key_range throughput"),
+        ALTERNATING_GET_SET("Java Completable API alternating get and set throughput"),
+        WRITE_TRANSACTION("Java Completable API single-key transaction throughput");
+
+        private String kpi;
+        private Function<? super Database, ? extends Double> function;
+
+        Tests(String kpi) {
+            this.kpi = kpi;
+        }
+
+        public void setFunction(Function<?super Database, ? extends Double> function) {
+            this.function = function;
+        }
+
+        public Function<? super Database, ? extends Double> getFunction() {
+            return function;
+        }
+
+        public String getKpi() {
+            return kpi;
+        }
+    }
+
+    public PerformanceTester() {
+        this(DEFAULT_KEY_COUNT, DEFAULT_KEY_SIZE, DEFAULT_VALUE_SIZE);
+    }
+
+    public PerformanceTester(int keyCount, int keySize, int valueSize) {
+        super();
+        this.keyCount = keyCount;
+        this.keySize = keySize;
+        this.valueSize = valueSize;
+
+        keyFormat = "%0" + keySize + "d";
+
+        valueBytes = new byte[valueSize];
+        Arrays.fill(valueBytes, (byte)'x');
+
+        // Initialize tests.
+        Tests.FUTURE_LATENCY.setFunction(db -> futureLatency(db, 100_000));
+        Tests.SET.setFunction(db -> set(db, 100_000));
+        Tests.CLEAR.setFunction(db -> clear(db, 100_000));
+        Tests.CLEAR_RANGE.setFunction(db -> clearRange(db, 100_000));
+        Tests.PARALLEL_GET.setFunction(db -> parallelGet(db, 10_000));
+        Tests.SERIAL_GET.setFunction(db -> serialGet(db, 2_000));
+        Tests.GET_RANGE.setFunction(db -> getRange(db, 1_000));
+        Tests.GET_KEY.setFunction(db -> getKey(db, 2_000));
+        Tests.GET_SINGLE_KEY_RANGE.setFunction(db -> getSingleKeyRange(db, 2_000));
+        Tests.ALTERNATING_GET_SET.setFunction(db -> alternatingGetSet(db, 2_000));
+        Tests.WRITE_TRANSACTION.setFunction(db -> writeTransaction(db, 1_000));
+    }
+
+    @Override
+    public void testPerformance(Database db) {
+        insertData(db);
+
+        List<String> testsToRun;
+        if (args.getTestsToRun().isEmpty()) {
+            testsToRun = Arrays.stream(Tests.values()).map(Tests::name).map(String::toLowerCase).sorted().collect(Collectors.toList());
+        } else {
+            testsToRun = args.getTestsToRun();
+        }
+
+        for (String test : testsToRun) {
+            Tests testObj;
+            try {
+                testObj = Tests.valueOf(test.toUpperCase());
+            } catch (IllegalArgumentException e) {
+                result.addError(new IllegalArgumentException("Test " + test + " not implemented"));
+                continue;
+            }
+
+            Function<? super Database, ? extends Double> function = testObj.getFunction();
+
+            try {
+                Thread.sleep(5_000);
+            } catch (InterruptedException e) {
+                result.addError(wrapAndPrintError(e, "Interrupted while sleeping"));
+            }
+
+            System.out.println("Running test " + test);
+
+            List<Double> results = new ArrayList<>(NUM_RUNS);
+
+            for (int i = 0; i < NUM_RUNS; i++) {
+                try {
+                    results.add(function.apply(db));
+                } catch (Exception e) {
+                    result.addError(wrapAndPrintError(e, "Performance test failed: " + test));
+                    break;
+                }
+            }
+
+            if (results.size() == NUM_RUNS) {
+                Collections.sort(results);
+                result.addKpi(String.format("%s (%s)", testObj.getKpi(), multiVersionDescription()), results.get(results.size()/2).intValue(), "keys/s");
+            }
+        }
+    }
+
+    public void insertData(Database db) {
+        System.out.println("Loading database");
+
+        db.run(tr -> {
+            byte[] subspacePrefix = args.getSubspace().pack();
+            if (subspacePrefix.length == 0) {
+                // Clear user space.
+                tr.clear(new byte[0], new byte[]{(byte)0xff});
+            } else {
+                tr.clear(args.getSubspace().range());
+            }
+            return null;
+        });
+
+        int keysPerActor = 100_000 / (keySize + valueSize);
+        int numActors = (int)Math.ceil(keyCount*1.0/keysPerActor);
+
+        List<CompletableFuture<Void>> futures = IntStream.range(0, numActors).mapToObj(i -> {
+            int startKey = keysPerActor * i;
+            int endKey = (i + 1 == numActors) ? (keyCount) : (keysPerActor * (i+1));
+            return db.runAsync(tr -> {
+                IntStream.range(startKey, endKey).forEach(keyIndex -> tr.set(key(keyIndex), value(keyIndex)));
+                return CompletableFuture.completedFuture((Void)null);
+            });
+        }).collect(Collectors.toList());
+
+        try {
+            AsyncUtil.whenAll(futures).get();
+        } catch (InterruptedException | ExecutionException e) {
+            result.addError(wrapAndPrintError(e, "Data insertion failed"));
+        }
+
+        // Give the database time to re-balance
+        try {
+            Thread.sleep(15_000);
+        } catch (InterruptedException e) {
+            result.addError(wrapAndPrintError(e, "Interrupted while waiting for quiescence"));
+        }
+    }
+
+    public Double futureLatency(Database db, int count) {
+        return db.run(tr -> {
+            tr.options().setRetryLimit(5);
+            tr.getReadVersion().join();
+
+            long start = System.nanoTime();
+
+            for (int i = 0; i < count; i++) {
+                tr.getReadVersion().join();
+            }
+
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double clear(Database db, int count) {
+        Transaction tr = db.createTransaction();
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.clear(randomKey());
+        }
+        long end = System.nanoTime();
+        tr.cancel();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double clearRange(Database db, int count) {
+        Transaction tr = db.createTransaction();
+
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            int keyIndex = randomKeyIndex();
+            tr.clear(key(keyIndex), key(keyIndex+1));
+        }
+        long end = System.nanoTime();
+        tr.cancel();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double set(Database db, int count) {
+        Transaction tr = db.createTransaction();
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            int keyIndex = randomKeyIndex();
+            tr.set(key(keyIndex), value(keyIndex));
+        }
+        long end = System.nanoTime();
+        tr.cancel();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double parallelGet(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            long start = System.nanoTime();
+
+            List<CompletableFuture<byte[]>> futures = IntStream.range(0, count)
+                    .mapToObj(ignore -> tr.get(randomKey()))
+                    .collect(Collectors.toList());
+            AsyncUtil.whenAll(futures).join();
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double alternatingGetSet(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            long start = System.nanoTime();
+
+            List<CompletableFuture<byte[]>> futures = IntStream.range(0, count)
+                    .mapToObj(ignore -> {
+                        int keyIndex = randomKeyIndex();
+                        byte[] keyBytes = key(keyIndex);
+                        byte[] valBytes = value(keyIndex);
+
+                        tr.set(keyBytes, valBytes);
+                        return tr.get(keyBytes);
+                    }).collect(Collectors.toList());
+            AsyncUtil.whenAll(futures).join();
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double serialGet(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+
+            List<byte[]> keys;
+            if (count > keyCount/2) {
+                keys = Stream.generate(this::randomKey).limit(count).collect(Collectors.toList());
+            } else {
+                Set<Integer> keySet = new HashSet<>();
+                while (keySet.size() < count) {
+                    keySet.add(randomKeyIndex());
+                }
+                keys = keySet.stream().map(this::key).collect(Collectors.toList());
+            }
+
+            long start = System.nanoTime();
+            for (byte[] key : keys) {
+                tr.get(key).join();
+            }
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double getRange(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            int startIndex = random.nextInt(keyCount - count);
+
+            long start = System.nanoTime();
+            tr.getRange(key(startIndex), key(startIndex+count)).asList().join();
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double getKey(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+
+            long start = System.nanoTime();
+            for (int i = 0; i < count; i++) {
+                tr.getKey(new KeySelector(randomKey(), true, random.nextInt(20) - 10)).join();
+            }
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double getSingleKeyRange(TransactionContext tcx, int count) {
+        return tcx.run(tr -> {
+            tr.options().setRetryLimit(5);
+            long start = System.nanoTime();
+            for (int i = 0; i < count; i++) {
+                int keyIndex = randomKeyIndex();
+                tr.getRange(key(keyIndex), key(keyIndex + 1)).asList().join();
+            }
+            long end = System.nanoTime();
+
+            return count*1_000_000_000.0/(end - start);
+        });
+    }
+
+    public Double writeTransaction(TransactionContext tcx, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tcx.run(tr -> {
+                int keyIndex = randomKeyIndex();
+                tr.set(key(keyIndex), value(keyIndex));
+                return null;
+            });
+        }
+        long end = System.nanoTime();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public byte[] key(int i) {
+        return ByteArrayUtil.join(args.getSubspace().pack(), String.format(keyFormat, i).getBytes(ASCII));
+    }
+
+    public int randomKeyIndex() {
+        return random.nextInt(keyCount);
+    }
+
+    public byte[] randomKey() {
+        return key(randomKeyIndex());
+    }
+
+    public byte[] value(int key) {
+        return valueBytes;
+    }
+
+    public static void main(String[] args) {
+        System.out.println("Running Java performance test on Java version " + System.getProperty("java.version"));
+        try {
+            new PerformanceTester().run(args);
+        } catch (IllegalArgumentException e) {
+            System.out.println("Could not run test due to malformed arguments.");
+            System.out.println(e.getMessage());
+            System.exit(1);
+        } catch (Exception e) {
+            System.out.println("Fatal error encountered during run: " + e);
+            e.printStackTrace();
+            System.exit(2);
+        }
+    }
+
+}

bindings/java/src-completable/test/com/apple/cie/foundationdb/test/RYWBenchmark.java

@@ -0,0 +1,233 @@
+/*
+ * RYWBenchmark.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.Database;
+import com.apple.cie.foundationdb.Transaction;
+import com.apple.cie.foundationdb.tuple.ByteArrayUtil;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class RYWBenchmark extends AbstractTester {
+    private int keyCount;
+
+    public static final int DEFAULT_KEY_COUNT = 10_000;
+    public static final int DEFAULT_KEY_SIZE = 16;
+
+    private final String keyFormat;
+
+    private enum Tests {
+        GET_SINGLE("RYW Java Completable: get single cached value throughput"),
+        GET_MANY_SEQUENTIAL("RYW Java Completable: get sequential cached value throughput"),
+        GET_RANGE_BASIC("RYW Java Completable: get range cached values throughput"),
+        SINGLE_CLEAR_GET_RANGE("RYW Java Completable: get range cached values with clears throughput"),
+        CLEAR_RANGE_GET_RANGE("RYW Java Completable: get range cached values with clear ranges throughput"),
+        INTERLEAVED_SETS_GETS("RYW Java Completable: interleaved sets and gets on a single key throughput");
+
+        private String kpi;
+        private Function<? super Transaction, ? extends Double> function;
+
+        Tests(String kpi) {
+            this.kpi = kpi;
+        }
+
+        public void setFunction(Function<?super Transaction, ? extends Double> function) {
+            this.function = function;
+        }
+
+        public Function<? super Transaction, ? extends Double> getFunction() {
+            return function;
+        }
+
+        public String getKpi() {
+            return kpi;
+        }
+    }
+
+    public RYWBenchmark() {
+        this(DEFAULT_KEY_COUNT, DEFAULT_KEY_SIZE);
+    }
+
+    public RYWBenchmark(int keyCount, int keySize) {
+        super();
+        this.keyCount = keyCount;
+
+        keyFormat = "%0" + keySize + "d";
+
+        Tests.GET_SINGLE.setFunction(tr -> getSingle(tr, 10_000));
+        Tests.GET_MANY_SEQUENTIAL.setFunction(tr -> getManySequential(tr, 10_000));
+        Tests.GET_RANGE_BASIC.setFunction(tr -> getRangeBasic(tr, 1_000));
+        Tests.SINGLE_CLEAR_GET_RANGE.setFunction(tr -> singleClearGetRange(tr, 1_000));
+        Tests.CLEAR_RANGE_GET_RANGE.setFunction(tr -> clearRangeGetRange(tr, 1_000));
+        Tests.INTERLEAVED_SETS_GETS.setFunction(tr -> interleavedSetsGets(tr, 10_000));
+    }
+
+    @Override
+    public void testPerformance(Database db) {
+        Transaction tr = db.createTransaction();
+        insertData(tr);
+
+        List<String> testsToRun;
+        if (args.getTestsToRun().isEmpty()) {
+            testsToRun = Arrays.stream(Tests.values()).map(Tests::name).map(String::toLowerCase).sorted().collect(Collectors.toList());
+        } else {
+            testsToRun = args.getTestsToRun();
+        }
+
+        for (String test : testsToRun) {
+            Tests testObj;
+            try {
+                testObj = Tests.valueOf(test.toUpperCase());
+            } catch (IllegalArgumentException e) {
+                result.addError(new IllegalArgumentException("Test " + test + " not implemented"));
+                continue;
+            }
+
+            Function<? super Transaction, ? extends Double> function = testObj.getFunction();
+
+            try {
+                Thread.sleep(5_000);
+            } catch (InterruptedException e) {
+                result.addError(wrapAndPrintError(e, "Interrupted while sleeping"));
+            }
+
+            System.out.println("Running test " + test);
+
+            List<Double> results = new ArrayList<>(NUM_RUNS);
+
+            for (int i = 0; i < NUM_RUNS; i++) {
+                try {
+                    results.add(function.apply(tr));
+                } catch (Exception e) {
+                    result.addError(wrapAndPrintError(e, "Performance test failed: " + test));
+                    break;
+                }
+            }
+
+            if (results.size() == NUM_RUNS) {
+                Collections.sort(results);
+                result.addKpi(String.format("%s", testObj.getKpi()), results.get(results.size() / 2).intValue(), "keys/s");
+            }
+        }
+
+        tr.cancel();
+    }
+
+    public Double getSingle(Transaction tr, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.get(key(5001)).join();
+        }
+        long end = System.nanoTime();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double getManySequential(Transaction tr, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.get(key(i)).join();
+        }
+        long end = System.nanoTime();
+
+        return count*1_000_000_000.0/(end - start);
+    }
+
+    public Double getRangeBasic(Transaction tr, int count) {
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+           tr.getRange(key(0), key(keyCount)).asList().join();
+        }
+        long end = System.nanoTime();
+
+        return count * 1_000_000_000.0 * keyCount/(end - start);
+    }
+
+    public Double singleClearGetRange(Transaction tr, int count) {
+        for (int i = 0; i < keyCount; i += 2) {
+            tr.clear(("" + i).getBytes(ASCII));
+        }
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.getRange(key(0), key(keyCount)).asList().join();
+        }
+        long end = System.nanoTime();
+
+        Double kpi = count * 1_000_000_000.0 * keyCount / 2 / (end - start);
+        insertData(tr);
+        return kpi;
+    }
+
+    public Double clearRangeGetRange(Transaction tr, int count) {
+        for (int i = 0; i < keyCount; i += 4) {
+            tr.clear(key(i), key(i+1));
+        }
+        long start = System.nanoTime();
+        for (int i = 0; i < count; i++) {
+            tr.getRange(key(0), key(keyCount)).asList().join();
+        }
+        long end = System.nanoTime();
+
+        Double kpi = count * 1_000_000_000.0 * keyCount * 3 / 4 / (end - start);
+        insertData(tr);
+        return kpi;
+    }
+
+    public Double interleavedSetsGets(Transaction tr, int count) {
+        long start = System.nanoTime();
+        byte[] keyBytes = "foo".getBytes(ASCII);
+        tr.set(keyBytes, "1".getBytes(ASCII));
+        for (int i = 0; i < count; i++) {
+            int old = Integer.parseInt(new String(tr.get(keyBytes).join(), ASCII));
+            tr.set(keyBytes, ("" + (old + 1)).getBytes(ASCII));
+        }
+        long end = System.nanoTime();
+
+        return count * 1_000_000_000.0/(end - start);
+    }
+
+    public void insertData(Transaction tr) {
+        tr.clear(new byte[0], new byte[]{(byte)0xff}); // Clear user space.
+        for (int i = 0; i < keyCount; i++) {
+            tr.set(key(i), "foo".getBytes(ASCII));
+        }
+    }
+
+    public byte[] key(int i) {
+        return ByteArrayUtil.join(args.getSubspace().pack(), String.format(keyFormat, i).getBytes(ASCII));
+    }
+
+    public static void main(String[] args) {
+        System.out.println("Running Java RYW benchmark on Java version " + System.getProperty("java.version"));
+        try {
+            new RYWBenchmark().run(args);
+        } catch (IllegalArgumentException e) {
+            System.out.println("Could not run test due to malformed arguments.");
+            System.exit(1);
+        } catch (Exception e) {
+            System.out.println("Fatal error encountered during run: " + e);
+            e.printStackTrace();
+            System.exit(2);
+        }
+    }
+}

bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TestResult.java

@@ -0,0 +1,151 @@
+/*
+ * TestResult.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+
+public class TestResult {
+    private long id;
+    private Map<String,Map<String,Object>> kpis;
+    private List<Throwable> errors;
+
+    public TestResult(Random r) {
+        id = Math.abs(r.nextLong());
+        kpis = new TreeMap<String,Map<String,Object>>(); // Tree map because we will have to print this out.
+        errors = new ArrayList<Throwable>();
+    }
+
+    public void addKpi(String name, Number value, String units) {
+        TreeMap<String,Object> kpi = new TreeMap<String,Object>();
+        kpi.put("value", value);
+        kpi.put("units", units);
+        kpis.put(name, kpi);
+    }
+
+    public void addError(Throwable t) {
+        errors.add(t);
+    }
+
+    public void save(String directory) {
+        String file = "javacompletableresult-" + id + ".json";
+        if(directory.length() > 0) {
+            file = directory + "/" + file;
+        }
+
+        // TODO: Should we use a really JSON library?
+
+        StringBuilder outputBuilder = new StringBuilder();
+        outputBuilder.append('{');
+
+        // Add KPIs:
+        outputBuilder.append("\"kpis\": {");
+        boolean firstKpi = true;
+        for (Map.Entry<String,Map<String,Object>> kpi : kpis.entrySet()) {
+            if (firstKpi) {
+                firstKpi = false;
+            } else {
+                outputBuilder.append(", ");
+            }
+
+            outputBuilder.append("\"");
+            outputBuilder.append(kpi.getKey());
+            outputBuilder.append("\": {");
+
+            boolean firstEntry = true;
+
+            for (Map.Entry<String,Object> entry : kpi.getValue().entrySet()) {
+                if (firstEntry) {
+                    firstEntry = false;
+                } else {
+                    outputBuilder.append(", ");
+                }
+
+                outputBuilder.append("\"");
+                outputBuilder.append(entry.getKey());
+                outputBuilder.append("\": ");
+
+                Object value = entry.getValue();
+                if (value instanceof String) {
+                    outputBuilder.append("\"");
+                    outputBuilder.append((String)value);
+                    outputBuilder.append("\"");
+                } else {
+                    outputBuilder.append(value.toString());
+                }
+            }
+
+            outputBuilder.append("}");
+        }
+        outputBuilder.append("}, ");
+
+        // Add errors:
+        outputBuilder.append("\"errors\":[");
+        boolean firstError = true;
+        for (Throwable t : errors) {
+            if (firstError) {
+                firstError = false;
+            } else {
+                outputBuilder.append(", ");
+            }
+
+            StringBuilder msgBuilder = new StringBuilder();
+            msgBuilder.append(t.getClass().toString());
+            msgBuilder.append(": ");
+            msgBuilder.append(t.getMessage()); // Escaping quotes. Yeah, this won't work in the general case....
+            StackTraceElement[] stackTraceElements = t.getStackTrace();
+            for (StackTraceElement element : stackTraceElements) {
+                msgBuilder.append("\n    ");
+                msgBuilder.append(element.toString());
+            }
+            outputBuilder.append('"');
+            outputBuilder.append(msgBuilder.toString()
+                    .replace("\\", "\\\\")
+                    .replace("\"", "\\\"")
+                    .replace("\t", "\\t")
+                    .replace("\r", "\\r")
+                    .replace("\n", "\\n")
+                    .replace("\f", "\\f")
+                    .replace("\b", "\\b")
+            );
+            outputBuilder.append('"');
+        }
+        outputBuilder.append("]");
+
+        outputBuilder.append('}');
+
+        BufferedWriter writer = null;
+        try  {
+            writer = new BufferedWriter(new FileWriter(file));
+            writer.write(outputBuilder.toString());
+            writer.close();
+        } catch (IOException e) {
+            System.out.println("Could not write results to file " + file);
+            throw new RuntimeException("Could not save results: " + e.getMessage(), e);
+        }
+    }
+}

bindings/java/src-completable/test/com/apple/cie/foundationdb/test/TesterArgs.java

@@ -0,0 +1,153 @@
+/*
+ * TesterArgs.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.cie.foundationdb.test;
+
+import com.apple.cie.foundationdb.subspace.Subspace;
+import com.apple.cie.foundationdb.tuple.Tuple;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class TesterArgs {
+    private String outputDirectory;
+    private boolean multiversionApi;
+    private boolean callbacksOnExternalThread;
+    private boolean externalClient;
+    private Subspace subspace;
+    private List<String> testsToRun;
+
+    private TesterArgs(String outputDirectory, boolean multiversionApi, boolean callbacksOnExternalThread, boolean externalClient, Subspace subspace, List<String> testsToRun) {
+        this.outputDirectory = outputDirectory;
+        this.multiversionApi = multiversionApi;
+        this.callbacksOnExternalThread = callbacksOnExternalThread;
+        this.externalClient = externalClient;
+        this.subspace = subspace;
+        this.testsToRun = testsToRun;
+    }
+
+    public static void printUsage() {
+        String usage = "Arguments:  [-o/--output-directory DIR] [--disable-multiversion-api] [--enable-callbacks-on-external-threads] [--use-external-client] [--tests-to-run TEST [TEST ...]] [-h/--help]\n"
+                     + "\n"
+                     + "Arguments:\n"
+                     + "   -o/--output-directory DIR                Directory to store JSON output. If not set, the current directory is used.\n"
+                     + "   --disable-multiversion-api               Disables the multi-version client API\n"
+                     + "   --enable-callbacks-on-external-threads   Allows callbacks to be called on threads created by the client library.\n"
+                     + "   --use-external-client                    Connect to the server using an external client.\n"
+                     + "   --tests-to-run TEST [TEST ...]           List of test names to run.\n"
+                     + "   -h/--help                                Print this help message and then quit.\n";
+
+        System.out.print(usage);
+    }
+
+    /**
+     * Parses the argument strings into a <code>TesterArgs</code> instance.
+     * This will return <code>null</code> if the args include an argument telling
+     * it to print the help message and it will throw an {@link IllegalArgumentException}
+     * if it can't parse the arguments.
+     *
+     * @param args command-line args
+     * @return built instance or <code>null</code>
+     * @throws IllegalArgumentException if the arguments can't be parsed
+     */
+    public static TesterArgs parseArgs(String[] args) {
+        String outputDirectory = "";
+        boolean multiversionApi = true;
+        boolean callbacksOnExternalThread = false;
+        boolean externalClient = false;
+        Subspace subspace = new Subspace();
+        List<String> testsToRun = new ArrayList<String>();
+
+        for (int i = 0; i < args.length; i++) {
+            String arg = args[i];
+            if (arg.equals("-o") || arg.equals("--output-directory")) {
+                if (i + 1 < args.length) {
+                    outputDirectory = args[++i];
+                } else {
+                    System.out.println("No output directory specified for argument " + arg + "\n");
+                    printUsage();
+                    throw new IllegalArgumentException("No output directory specified for argument " + arg);
+                }
+            } else if (arg.equals("--subspace")) {
+                if (i + 1 < args.length) {
+                    subspace = new Subspace(Tuple.from(args[++i]));
+                } else {
+                    System.out.println("No subspace specified for argument " + args + "\n");
+                    printUsage();
+                    throw new IllegalArgumentException("Not subspace specified for argument " + arg);
+                }
+            } else if (arg.equals("--disable-multiversion-api")) {
+                multiversionApi = false;
+            } else if (arg.equals("--enable-callbacks-on-external-threads")) {
+                callbacksOnExternalThread = true;
+            } else if (arg.equals("--use-external-client")) {
+                externalClient = true;
+            } else if (arg.equals("--tests-to-run")) {
+                if (i + 1 < args.length && args[i + 1].charAt(0) != '-') {
+                    int j;
+                    for (j = i + 1; j < args.length && args[j].charAt(0) != '-'; j++) {
+                        testsToRun.add(args[j]);
+                    }
+                    i = j;
+                } else {
+                    System.out.println("No tests specified with argument " + arg + "\n");
+                    printUsage();
+                    throw new IllegalArgumentException("No tests specified with argument " + arg);
+                }
+            } else if (arg.equals("-h") || arg.equals("--help")) {
+                printUsage();
+                return null;
+            } else {
+                System.out.println("Unknown argument " + arg + "\n");
+                printUsage();
+                throw new IllegalArgumentException("Unknown argument " + arg);
+            }
+        }
+
+        return new TesterArgs(outputDirectory, multiversionApi, callbacksOnExternalThread, externalClient, subspace, testsToRun);
+    }
+
+    // Accessors.
+
+    public String getOutputDirectory() {
+        return outputDirectory;
+    }
+
+    public boolean useMultiversionApi() {
+        return multiversionApi;
+    }
+
+    public boolean putCallbacksOnExternalThread() {
+        return callbacksOnExternalThread;
+    }
+
+    public boolean useExternalClient() {
+        return externalClient;
+    }
+
+    public Subspace getSubspace() {
+        return subspace;
+    }
+
+    public List<String> getTestsToRun() {
+        return testsToRun;
+    }
+}

bindings/java/src/main/com/apple/cie/foundationdb/Database.java

@@ -46,8 +46,8 @@ public interface Database extends Disposable, TransactionContext {
 	 * Creates a {@link Transaction} that operates on this {@code Database}.<br>
 	 * <br>
 	 * Note: Java transactions automatically set the {@link TransactionOptions#setUsedDuringCommitProtectionDisable}
-	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after either
-	 *  {@link Transaction#reset} or {@link Transaction#onError} is called.
+	 *  option. This is because the Java bindings disallow use of {@code Transaction} objects after
+	 *  {@link Transaction#onError} is called.
 	 *
 	 * @return a newly created {@code Transaction} that reads from and writes to this {@code Database}.
 	 */

bindings/java/src/main/overview.html.in

@@ -24,11 +24,12 @@ and add it to your classpath.<br>
 <br>
 <h3>Getting started</h3>
 To start using FoundationDB from Java, create an instance of the 
-{@link FDB FoundationDB API interface} with the version of the
+{@link com.apple.cie.foundationdb.FDB FoundationDB API interface} with the version of the
 API that you want to use (this release of the FoundationDB Java API supports only version {@code 500}).
-With this API object you can then open {@link Cluster}s and
-{@link Database}s and start using {@link Transaction}s.
-Here we give an example. The example relies on a cluster file at the
+With this API object you can then open {@link com.apple.cie.foundationdb.Cluster}s and
+{@link com.apple.cie.foundationdb.Database}s and start using
+{@link com.apple.cie.foundationdb.Transaction}s. Here we give an example. The example relies on a
+cluster file at the
 <a href="/documentation/api-general.html#default-cluster-file">default location</a> 
 for your platform and a running server.<br>
 <br>
@@ -77,7 +78,7 @@ for information about how Tuples sort and can be used to efficiently model data.
 The {@link com.apple.cie.foundationdb.directory Directory API} is provided with the core
 Java API for FoundationDB. This layer is provided in some form in all official
 language bindings. The FoundationDB API provides directories as a tool for
-managing related {@link Subspace}s. Directories are a
+managing related {@link com.apple.cie.foundationdb.subspace.Subspace Subspace}s. Directories are a
 recommended approach for administering applications. Each application should 
 create or open at least one directory to manage its subspaces. Directories are 
 identified by hierarchical paths analogous to the paths in a Unix-like file system. 
@@ -87,12 +88,12 @@ for the corresponding subspace. In effect, directories provide a level of indire
 for access to subspaces.
 <br>
 <h3>{@link com.apple.cie.foundationdb.async.Future Future}s and asynchronous operation</h3>
-Asynchronous FoundationDB operations return {@link Future}s.
-A {@link Future} can be used in a blocking way using the
-{@link Future#get() get()} method or in a
+Asynchronous FoundationDB operations return {@link com.apple.cie.foundationdb.async.Future Future}s.
+A {@link com.apple.cie.foundationdb.async.Future Future} can be used in a blocking way using the
+{@link com.apple.cie.foundationdb.async.Future#get() get()} method or in a
 fully-asynchronous way using the 
-{@link Future#map(Function) map()} and
-{@link Future#flatMap(Function) flatMap()}
+{@link com.apple.cie.foundationdb.async.Future#map(Function) map()} and
+{@link com.apple.cie.foundationdb.async.Future#flatMap(Function) flatMap()}
 methods. Generally, the blocking style is more straightforward and the asynchronous style
 is more efficient. Mixing the two styles correctly can be tricky, so consider choosing
 one or the other. See the {@linkplain com.apple.cie.foundationdb.async async Package documentation}

bindings/nodejs/binding.gyp

@@ -2,7 +2,7 @@
   'targets': [
     {
       'target_name': 'fdblib',
-      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp' ],
+      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp', 'src/FdbUtil.cpp' ],
       'include_dirs': ['../c'],
       'conditions': [
         ['OS=="linux"', {

bindings/nodejs/binding.gyp.npmsrc

@@ -2,7 +2,7 @@
   'targets': [
     {
       'target_name': 'fdblib',
-      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp' ],
+      'sources': [ 'src/FdbV8Wrapper.cpp', 'src/Database.cpp', 'src/Transaction.cpp', 'src/Cluster.cpp', 'src/FdbError.cpp', 'src/FdbOptions.cpp', 'src/FdbOptions.g.cpp', 'src/FdbUtil.cpp' ],
       'conditions': [
         ['OS=="linux"', {
           'link_settings': { 'libraries': ['-lfdb_c'] },

bindings/nodejs/fdb_node.target

@@ -136,12 +136,14 @@ copy "$(TargetPath)" "modules\$(NodeVersionDir)\fdblib.node"
     <ClCompile Include="src\Cluster.cpp" />
     <ClCompile Include="src\FdbV8Wrapper.cpp" />
     <ClCompile Include="src\FdbOptions.cpp" />
+    <ClCompile Include="src\FdbUtil.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="src\Cluster.h" />
     <ClInclude Include="src\Database.h" />
     <ClInclude Include="src\FdbError.h" />
     <ClInclude Include="src\FdbOptions.h" />
+    <ClInclude Include="src\FdbUtil.h" />
     <ClInclude Include="src\NodeCallback.h" />
     <ClInclude Include="src\Transaction.h" />
     <ClInclude Include="src\Version.h" />

bindings/nodejs/include.mk

@@ -46,7 +46,7 @@ bindings/nodejs/fdb_node.stamp: bindings/nodejs/src/FdbOptions.g.cpp bindings/no
 	for ver in $(NODE_VERSIONS); do \
 		MMVER=`echo $$ver | sed -e 's,\., ,g' | awk '{print $$1 "." $$2}'` && \
 		mkdir modules/$$MMVER && \
-		node-gyp configure --target=$$ver && \
+		node-gyp configure --dist-url=https://nodejs.org/dist --target=$$ver && \
 		node-gyp -v build && \
 		cp build/Release/fdblib.node modules/$${MMVER} ; \
 	done
@@ -67,6 +67,7 @@ bindings/nodejs/package.json: bindings/nodejs/package.json.in $(ALL_MAKEFILES) v
 	@m4 -DVERSION=$(NPMVER) $< > $@
 	@echo "Updating       Node dependencies"
 	@cd bindings/nodejs && \
+	npm config set registry "https://registry.npmjs.org/" && \
 	npm update
 
 fdb_node_npm: fdb_node versions.target bindings/nodejs/README.md bindings/nodejs/lib/*.js bindings/nodejs/src/* bindings/nodejs/binding.gyp LICENSE

bindings/nodejs/lib/tuple.js

@@ -24,6 +24,7 @@
 var assert = require('assert');
 var buffer = require('./bufferConversion');
 var fdbUtil = require('./fdbUtil');
+var fdb = require('./fdbModule');
 var FDBError = require('./error');
 
 var sizeLimits = new Array(8);
@@ -83,7 +84,7 @@ function Float(value) {
 			return this.rawData;
 		} else {
 			var buf = new Buffer(4);
-			buf.writeFloatBE(this.value, 0);
+			buf.writeFloatBE(fdb.toFloat(this.value), 0);
 			return buf;
 		}
 	};
@@ -230,7 +231,7 @@ function encode(item, buf, pos) {
 		if (isNaN(item.value) && item.rawData !== undefined) {
 			item.rawData.copy(outBuf, 1, 0, 4);
 		} else {
-			outBuf.writeFloatBE(item.value, 1);
+			outBuf.writeFloatBE(fdb.toFloat(item.value), 1);
 		}
 		adjustFloat(outBuf, 1, true);
 		return outBuf;

bindings/nodejs/src/FdbUtil.cpp

@@ -0,0 +1,42 @@
+/*
+ * FdbUtil.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <node.h>
+#include "FdbUtil.h"
+
+using namespace v8;
+
+Handle<Value> ToFloat(const Arguments &args) {
+    HandleScope scope;
+
+    if (args.Length() != 1) {
+        return ThrowException(Exception::TypeError(String::NewSymbol("Wrong number of arguments (must be exactly 1)")));
+    }
+
+    if (!args[0]->IsNumber()) {
+        return ThrowException(Exception::TypeError(String::NewSymbol("Argument is not a Number")));
+    }
+
+    float value = (float)args[0]->NumberValue();
+    Handle<Value> jsValue = Number::New(value);
+
+    return scope.Close(jsValue);
+}

bindings/nodejs/src/FdbUtil.h

@@ -0,0 +1,29 @@
+/*
+ * FdbUtil.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef FDB_NODE_FDB_UTIL_H
+#define FDB_NODE_FDB_UTIL_H
+
+#include <node.h>
+
+v8::Handle<v8::Value> ToFloat(const v8::Arguments &args);
+
+#endif

bindings/nodejs/src/FdbV8Wrapper.cpp

@@ -33,6 +33,7 @@
 #include "Version.h"
 #include "FdbError.h"
 #include "FdbOptions.h"
+#include "FdbUtil.h"
 
 uv_thread_t fdbThread;
 
@@ -140,6 +141,7 @@ void init(Handle<Object> target){
 	target->Set(String::NewSymbol("options"), FdbOptions::CreateOptions(FdbOptions::NetworkOption));
 	target->Set(String::NewSymbol("streamingMode"), FdbOptions::CreateEnum(FdbOptions::StreamingMode));
 	target->Set(String::NewSymbol("atomic"), FdbOptions::CreateOptions(FdbOptions::MutationType));
+	target->Set(String::NewSymbol("toFloat"), FunctionTemplate::New(ToFloat)->GetFunction());
 }
 
 #if NODE_VERSION_AT_LEAST(0, 8, 0)

bindings/nodejs/tests/tuple_test.js

@@ -18,7 +18,8 @@
  * limitations under the License.
  */
 
-var fdb = require('../lib/fdb.js').apiVersion(200);
+var fdb = require('../lib/fdb.js').apiVersion(500);
+var fdbModule = require('../lib/fdbModule.js');
 
 console.log(fdb.tuple.pack([-Math.pow(2,53)]));
 console.log(fdb.tuple.pack([-Math.pow(2,53)+1]));
@@ -76,3 +77,25 @@ tuples = [
 ];
 tuples.sort(fdb.tuple.compare);
 console.log(tuples);
+
+// Float overruns.
+const floats = [ 2.037036e90, -2.037036e90, 4.9090935e-91, -4.9090935e-91, 2.345624805922133125e14, -2.345624805922133125e14 ];
+for (var i = 0; i < floats.length; i++) {
+    var f = floats[i];
+    console.log(f + " -> " + fdb.tuple.Float.fromBytes((new fdb.tuple.Float(f)).toBytes()).value);
+}
+
+// Float type errors.
+try {
+    console.log((new fdb.tuple.Float("asdf")).toBytes());
+} catch (e) {
+    console.log("Caught!");
+    console.log(e);
+}
+
+try {
+    console.log(fdbModule.toFloat(3.14, 2.718));
+} catch (e) {
+    console.log("Caught!");
+    console.log(e);
+}

design/tuple.md

@@ -0,0 +1,196 @@
+# FDB Tuple layer typecodes
+
+This document is intended to be the system of record for the allocation of typecodes in the Tuple layer. The source code isn’t good enough because a typecode might be added to one language (or by a customer) before another.
+
+Status: Standard means that all of our language bindings implement this typecode
+Status: Reserved means that this typecode is not yet used in our standard language bindings, but may be in use by third party bindings or specific applications
+Status: Deprecated means that a previous layer used this type, but issues with that type code have led us to mark this type code as not to be used.
+
+
+### **Null Value**
+
+Typecode: 0x00  
+Length: 0 bytes  
+Status: Standard
+
+### **Byte String**
+
+Typecode: 0x01  
+Length: Variable (terminated by` [\x00]![\xff]`)  
+Encoding: `b'\x01' + value.replace(b'\x00', b'\x00\xFF') + b'\x00'`  
+Test case: `pack(“foo\x00bar”) == b'\x01foo\x00\xffbar\x00'`  
+Status: Standard
+
+In other words, byte strings are null terminated with null values occurring in the string escaped in an order-preserving way.
+
+### **Unicode String**
+
+Typecode: 0x02  
+Length: Variable (terminated by [\x00]![\xff])  
+Encoding: `b'\x02' + value.encode('utf-8').replace(b'\x00', b'\x00\xFF') + b'\x00'`  
+Test case: `pack( u"F\u00d4O\u0000bar" ) == b'\x02F\xc3\x94O\x00\xffbar\x00'`  
+Status: Standard
+
+This is the same way that byte strings are encoded, but first, the unicode string is encoded in UTF-8.
+
+### **(DEBRECATED) Nested Tuple**
+
+Typecodes: 0x03-0x04  
+Length: Variable (terminated by 0x04 type code)  
+Status: Deprecated  
+
+This encoding was used by a few layers. However, it had ordering problems when one tuple was a prefix of another and the type of the first element in the longer tuple was either null or a byte string. For an example, consider the empty tuple and the tuple containing only null. In the old scheme, the empty tuple would be encoded as `\x03\x04` while the tuple containing only null would be encoded as `\x03\x00\x04`, so the second tuple would sort first based on their bytes, which is incorrect semantically.
+
+### **Nested Tuple**
+
+Typecodes: 0x05  
+Length: Variable (terminated by `[\x00]![\xff]` at beginning of nested element)  
+Encoding: `b'\x05' + ''.join(map(lambda x: b'\x00\xff' if x is None else pack(x), value)) + b'\x00'`  
+Test case: `pack( (“foo\x00bar”, None, ()) ) == b'\x05\x01foo\x00\xffbar\x00\x00\xff\x05\x00\x00'`  
+Status: Standard
+
+The list is ended with a 0x00 byte. Nulls within the tuple are encoded as `\x00\xff`. There is no other null escaping. In particular, 0x00 bytes that are within the nested types can be left as-is as they are passed over when decoding the interior types. To show how this fixes the bug in the previous version of nested tuples, the empty tuple is now encoded as `\x05\x00` while the tuple containing only null is encoded as `\x05\x00\xff\x00`, so the first tuple will sort first.
+
+### **Negative arbitrary-precision Integer**
+
+Typecodes: 0x0a, 0x0b  
+Encoding: Not defined yet  
+Status: Reserved; 0x0b used in Python and Java
+
+These typecodes are reserved for encoding integers larger than 8 bytes. Presumably the type code would be followed by some encoding of the length, followed by the big endian one’s complement number. Reserving two typecodes for each of positive and negative numbers is probably overkill, but until there’s a design in place we might as well not use them. In the Python and Java implementations, 0x0b stores negative numbers which are expressed with between 9 and 255 bytes. The first byte following the type code (0x0b) is a single byte expressing the number of bytes in the integer (with its bits flipped to preserve order), followed by that number of bytes representing the number in big endian order in one's complement.
+
+### **Integer**
+
+Typecodes: 0x0c - 0x1c  
+ 0x0c is an 8 byte negative number  
+ 0x13 is a 1 byte negative number  
+ 0x14 is a zero  
+ 0x15 is a 1 byte positive number  
+ 0x1c is an 8 byte positive number  
+Length: Depends on typecode (0-8 bytes)  
+Encoding: positive numbers are big endian  
+ negative numbers are big endian one’s complement (so -1 is 0x13 0xfe)  
+Test case: `pack( -5551212 ) == b'\x11\xabK\x93'`  
+Status: Standard
+
+There is some variation in the ability of language bindings to encode and decode values at the outside of the possible range, because of different native representations of integers. 
+
+### **Positive arbitrary-precision Integer**
+
+Typecodes: 0x1d, 0x1e  
+Encoding: Not defined yet  
+Status: Reserved; 0x1d used in Python and Java
+
+These typecodes are reserved for encoding integers larger than 8 bytes. Presumably the type code would be followed by some encoding of the length, followed by the big endian one’s complement number. Reserving two typecodes for each of positive and negative numbers is probably overkill, but until there’s a design in place we might as well not use them. In the Python and Java implementations, 0x1d stores positive numbers which are expressed with between 9 and 255 bytes. The first byte following the type code (0x1d) is a single byte expressing the number of bytes in the integer, followed by that number of bytes representing the number in big endian order.
+
+### **IEEE Binary Floating Point**
+
+Typecodes:   
+ 0x20 - float (32 bits)  
+ 0x21 - double (64 bits)  
+ 0x22 - long double (80 bits)  
+Length: 4 - 10 bytes  
+Test case: `pack( -42f ) == b'=\xd7\xff\xff'`  
+Encoding: Big-endian IEEE binary representation, followed by the following transformation:  
+```python
+ if ord(rep[0])&0x80: # Check sign bit
+    # Flip all bits, this is easier in most other languages!
+    return "".join( chr(0xff^ord(r)) for r in rep )
+ else:
+    # Flip just the sign bit
+    return chr(0x80^ord(rep[0])) + rep[1:]
+```
+Status: Standard (float and double) ; Reserved (long double)
+
+The binary representation should not be assumed to be canonicalized (as to multiple representations of NaN, for example) by a reader. This order sorts all numbers in the following way:
+
+* All negative NaN values with order determined by mantissa bits (which are semantically meaningless)
+* Negative inifinity
+* All real numbers in the standard order (except that -0.0 < 0.0)
+* Positive infinity
+* All positive NaN values with order determined by mantissa bits
+
+This should be equivalent to the standard IEEE total ordering.
+
+### **Arbitrary-precision Decimal**
+
+Typecodes: 0x23, 0x24  
+Length: Arbitrary  
+Encoding: Scale followed by arbitrary precision integer  
+Status: Reserved
+
+This encoding format has been used by layers. Note that this encoding makes almost no guarantees about ordering properties of tuple-encoded values and should thus generally be avoided.
+
+### **(DEPRECATED) True Value**
+
+Typecode: 0x25  
+Length: 0 bytes  
+Status: Deprecated
+
+### **False Value**
+
+Typecode: 0x26  
+Length: 0 bytes  
+Status: Standard
+
+### **True Value**
+
+Typecode: 0x27  
+Length: 0 bytes  
+Status: Standard
+
+Note that false will sort before true with the given encoding.
+
+### **RFC 4122 UUID**
+
+Typecode: 0x30  
+Length: 16 bytes  
+Encoding: Network byte order as defined in the rfc: [_http://www.ietf.org/rfc/rfc4122.txt_](http://www.ietf.org/rfc/rfc4122.txt)  
+Status: Standard
+
+This is equivalent to the unsigned byte ordering of the UUID bytes in big-endian order.
+
+### **64 bit identifier**
+
+Typecode: 0x31  
+Length: 8 bytes  
+Encoding: Big endian unsigned 8-byte integer (typically random or perhaps semi-sequential)  
+Status: Reserved
+
+There’s definitely some question of whether this deserves to be separated from a plain old 64 bit integer, but a separate type was desired in one of the third-party bindings. This type has not been ported over to the first-party bindings.
+
+### **80 Bit versionstamp**
+
+Typecode: 0x32  
+Length: 10 bytes  
+Encoding: Big endian 10-byte integer. First/high 8 bytes are a database version, next two are batch version.  
+Status: Reserved
+
+### **96 Bit Versionstamp**
+
+Typecode: 0x33  
+Length: 12 bytes  
+Encoding: Big endian 12-byte integer. First/high 8 bytes are a database version, next two are batch version, next two are ordering within transaction.  
+Status: Reserved
+
+The two versionstamp typecodes are reserved for future work adding compatibility between the tuple layer and versionstamp operations. Note that the first 80 bits of the 96 bit versionstamp are the same as the contents of the 80 bit versionstamp, and they correspond to what the `SET_VERSIONSTAMP_KEY` mutation will write into a database key , i.e., the first 8 bytes are a big-endian, unsigned version corresponding to the commit version of a transaction, and the next to bytes are a big-endian, unsigned batch number ordering transactions are committed at the same version. The final two bytes of the 96 bit versionstamp are written by the client and should order writes within a single transaction, thereby providing a global order for all versions.
+
+### **User type codes**
+
+Typecode: 0x40 - 0x4f  
+Length: Variable (user defined)  
+Encoding: User defined  
+Status: Reserved
+
+These type codes may be used by third party extenders without coordinating with us. If used in shipping software, the software should use the directory layer and specify a specific layer name when opening its directories to eliminate the possibility of conflicts.
+
+The only way in which future official, otherwise backward-compatible versions of the tuple layer would be expected to use these type codes is to implement some kind of actual extensibility point for this purpose - they will not be used for standard types.
+
+### **Escape Character**
+
+Typecode: 0xff  
+Length: N/A  
+Encoding: N/A  
+Status: Reserved
+
+This type code is not used for anything. However, several of the other tuple types depend on this type code not being used as a type code for other types in order to correctly escape bytes in an order-preserving way. Therefore, it would be a Very Bad Idea™ for future development to start using this code for anything else.

fdbbackup/backup.actor.cpp

@@ -23,6 +23,7 @@
 #include "flow/serialize.h"
 #include "flow/IRandom.h"
 #include "flow/genericactors.actor.h"
+#include "flow/SignalSafeUnwind.h"
 
 #include "fdbclient/FDBTypes.h"
 #include "fdbclient/BackupAgent.h"
@@ -1820,6 +1821,7 @@ extern uint8_t *g_extra_memory;
 
 int main(int argc, char* argv[]) {
 	platformInit();
+	initSignalSafeUnwind();
 
 	int	status = FDB_EXIT_SUCCESS;
 

fdbcli/fdbcli.actor.cpp

@@ -30,6 +30,7 @@
 #include "fdbclient/FDBOptions.g.h"
 
 #include "flow/DeterministicRandom.h"
+#include "flow/SignalSafeUnwind.h"
 #include "fdbrpc/TLSConnection.h"
 #include "fdbrpc/Platform.h"
 
@@ -436,9 +437,9 @@ void initHelp() {
 		"clear a range of keys from the database",
 		"All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK);
 	helpMap["configure"] = CommandHelp(
-		"configure [new] <single|double|triple|three_data_hall|three_datacenter|fast_recovery_double|fast_recovery_triple|ssd|memory|proxies=<PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
+		"configure [new] <single|double|triple|three_data_hall|three_datacenter|ssd|memory|proxies=<PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
 		"change database configuration",
-		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n  fast_recovery_double - two copies of data on the storage servers, three copies of the data on the logs, non-copying recovery if one log is missing.\n  fast_recovery_triple - three copies of data on the storage servers, four copies of the data on the logs, non-copying recovery if one log is missing.\n\nStorage engine:\n  ssd - B-Tree storage engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
+		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
 	helpMap["coordinators"] = CommandHelp(
 		"coordinators auto|<ADDRESS>+ [description=new_cluster_description]",
 		"change cluster coordinators or description",
@@ -504,6 +505,7 @@ void initHelp() {
 		"If no addresses are specified, populates the list of processes which can be killed. Processes cannot be killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is unresponsive.\n\nFor each IP:port pair in <ADDRESS>*, attempt to kill the specified process.");
 
 	hiddenCommands.insert("expensive_data_check");
+	hiddenCommands.insert("datadistribution");
 }
 
 void printVersion() {
@@ -1672,7 +1674,18 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 			state double worstFreeSpaceRatio = 1.0;
 			try {
 				for (auto proc : processesMap.obj()){
+					bool storageServer = false;
 					StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
+					for (StatusObjectReader role : rolesArray) {
+						if (role["role"].get_str() == "storage") {
+							storageServer = true;
+							break;
+						}
+					}
+					// Skip non-storage servers in free space calculation
+					if (!storageServer)
+						continue;
+
 					StatusObjectReader process(proc.second);
 					std::string addrStr;
 					if (!process.get("address", addrStr)) {
@@ -1681,6 +1694,9 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 					}
 					NetworkAddress addr = NetworkAddress::parse(addrStr);
 					bool excluded = (process.has("excluded") && process.last().get_bool()) || addressExcluded(exclusions, addr);
+					ssTotalCount++;
+					if (excluded)
+						ssExcludedCount++;
 
 					if(!excluded) {
 						StatusObjectReader disk;
@@ -1703,15 +1719,6 @@ ACTOR Future<bool> exclude( Database db, std::vector<StringRef> tokens, Referenc
 
 						worstFreeSpaceRatio = std::min(worstFreeSpaceRatio, double(free_bytes)/total_bytes);
 					}
-
-					for (StatusObjectReader role : rolesArray) {
-						if (role["role"].get_str() == "storage") {
-							if (excluded)
-								ssExcludedCount++;
-							ssTotalCount++;
-							break;
-						}
-					}
 				}
 			}
 			catch (...)  // std::exception
@@ -1895,7 +1902,7 @@ void onoff_generator(const char* text, const char *line, std::vector<std::string
 }
 
 void configure_generator(const char* text, const char *line, std::vector<std::string>& lc) {
-	const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "fast_recovery_double", "fast_recovery_triple", "ssd", "ssd-1", "ssd-2", "memory", "proxies=", "logs=", "resolvers=", NULL};
+	const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "proxies=", "logs=", "resolvers=", NULL};
 	array_generator(text, line, opts, lc);
 }
 
@@ -2210,35 +2217,45 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 			state UID randomID = g_random->randomUniqueID();
 			TraceEvent(SevInfo, "CLICommandLog", randomID).detail("command", printable(StringRef(line)));
 
-			bool err, partial;
-			state std::vector<std::vector<StringRef>> parsed = parseLine(line, err, partial);
-			if (err) {
-				LogCommand(line, randomID, "ERROR: malformed escape sequence");
-				is_error = true;
-				continue;
-			}
-			if (partial) {
-				LogCommand(line, randomID, "ERROR: unterminated quote");
-				is_error = true;
-				continue;
+			bool malformed, partial;
+			state std::vector<std::vector<StringRef>> parsed = parseLine(line, malformed, partial);
+			if (malformed) LogCommand(line, randomID, "ERROR: malformed escape sequence");
+			if (partial) LogCommand(line, randomID, "ERROR: unterminated quote");
+			if (malformed || partial) {
+				if (parsed.size() > 0) {
+					// Denote via a special token that the command was a parse failure.
+					auto& last_command = parsed.back();
+					last_command.insert(last_command.begin(), StringRef((const uint8_t*)"parse_error", strlen("parse_error")));
+				}
 			}
 
 			state bool multi = parsed.size() > 1;
+			is_error = false;
 
 			state std::vector<std::vector<StringRef>>::iterator iter;
 			for (iter = parsed.begin(); iter != parsed.end(); ++iter) {
 				state std::vector<StringRef> tokens = *iter;
 
-				if (opt.exec.present() && is_error) {
+				if (is_error) {
 					printf("WARNING: the previous command failed, the remaining commands will not be executed.\n");
-					return 1;
+					break;
 				}
 
-				is_error = false;
-
 				if (!tokens.size())
 					continue;
 
+				if (tokencmp(tokens[0], "parse_error")) {
+					printf("ERROR: Command failed to completely parse.\n");
+					if (tokens.size() > 1) {
+						printf("ERROR: Not running partial or malformed command:");
+						for (auto t = tokens.begin() + 1; t != tokens.end(); ++t)
+							printf(" %s", formatStringRef(*t, true).c_str());
+						printf("\n");
+					}
+					is_error = true;
+					continue;
+				}
+
 				if (multi) {
 					printf(">>>");
 					for (auto t = tokens.begin(); t != tokens.end(); ++t)
@@ -2717,6 +2734,25 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "datadistribution")) {
+					if (tokens.size() != 2) {
+						printf("Usage: datadistribution <on|off>\n");
+						is_error = true;
+					} else {
+						if(tokencmp(tokens[1], "on")) {
+							int _ = wait(setDDMode(db, 1));
+							printf("Data distribution is enabled\n");
+						} else if(tokencmp(tokens[1], "off")) {
+							int _ = wait(setDDMode(db, 0));
+							printf("Data distribution is disabled\n");
+						} else {
+							printf("Usage: datadistribution <on|off>\n");
+							is_error = true;
+						}
+					}
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "option")) {
 					if (tokens.size() == 2 || tokens.size() > 4) {
 						printUsage(tokens[0]);
@@ -2841,6 +2877,7 @@ ACTOR Future<Void> timeExit(double duration) {
 
 int main(int argc, char **argv) {
 	platformInit();
+	initSignalSafeUnwind();
 	Error::init();
 
 	registerCrashHandler();

fdbclient/BackupAgent.h

@@ -25,7 +25,7 @@
 #include "flow/flow.h"
 #include "NativeAPI.h"
 #include "TaskBucket.h"
-#include "flow/Notified.h"
+#include "Notified.h"
 #include <fdbrpc/IAsyncFile.h>
 #include "KeyBackedTypes.h"
 #include <ctime>

fdbclient/FailureMonitorClient.actor.cpp

@@ -22,12 +22,20 @@
 #include "fdbrpc/FailureMonitor.h"
 #include "ClusterInterface.h"
 
+struct FailureMonitorClientState : ReferenceCounted<FailureMonitorClientState> {
+	std::set<NetworkAddress> knownAddrs;
+	double serverFailedTimeout;
+
+	FailureMonitorClientState() {
+		serverFailedTimeout = CLIENT_KNOBS->FAILURE_TIMEOUT_DELAY;
+	}
+};
+
 ACTOR Future<Void> failureMonitorClientLoop( 
 	SimpleFailureMonitor* monitor, 
 	ClusterInterface controller,
-	double* pServerFailedTimeout,
-	bool trackMyStatus,
-	std::set<NetworkAddress>* knownAddrs)
+	Reference<FailureMonitorClientState> fmState,
+	bool trackMyStatus)
 {
 	state Version version = 0;
 	state Future<FailureMonitoringReply> request = Never();
@@ -37,7 +45,7 @@ ACTOR Future<Void> failureMonitorClientLoop(
 	state double waitfor = 0;
 
 	monitor->setStatus(controller.failureMonitoring.getEndpoint().address, FailureStatus(false));
-	knownAddrs->insert( controller.failureMonitoring.getEndpoint().address );
+	fmState->knownAddrs.insert( controller.failureMonitoring.getEndpoint().address );
 
 	//The cluster controller's address (controller.failureMonitoring.getEndpoint().address) is treated specially because we can declare that it is down independently
 	//of the response from the cluster controller. It still needs to be in knownAddrs in case the cluster controller changes, so the next cluster controller resets its state
@@ -51,14 +59,14 @@ ACTOR Future<Void> failureMonitorClientLoop(
 					requestTimeout = Never();
 					if (reply.allOthersFailed) {
 						// Reset all systems *not* mentioned in the reply to the default (failed) state
-						knownAddrs->erase( controller.failureMonitoring.getEndpoint().address );
+						fmState->knownAddrs.erase( controller.failureMonitoring.getEndpoint().address );
 						std::set<NetworkAddress> changedAddresses;
 						for(int c=0; c<reply.changes.size(); c++)
 							changedAddresses.insert( reply.changes[c].address );
-						for(auto it : *knownAddrs)
+						for(auto it : fmState->knownAddrs)
 							if (!changedAddresses.count( it ))
 								monitor->setStatus( it, FailureStatus() );
-						knownAddrs->clear();
+						fmState->knownAddrs.clear();
 					} else {
 						ASSERT( version != 0 );
 					}
@@ -66,20 +74,20 @@ ACTOR Future<Void> failureMonitorClientLoop(
 					if( monitor->getState( controller.failureMonitoring.getEndpoint() ).isFailed() )
 						TraceEvent("FailureMonitoringServerUp").detail("OldServer",controller.id());
 					monitor->setStatus( controller.failureMonitoring.getEndpoint().address, FailureStatus(false) );
-					knownAddrs->insert( controller.failureMonitoring.getEndpoint().address );
+					fmState->knownAddrs.insert( controller.failureMonitoring.getEndpoint().address );
 
 					//if (version != reply.failureInformationVersion)
 					//	printf("Client '%s': update from %lld to %lld (%d changes, aof=%d)\n", g_network->getLocalAddress().toString().c_str(), version, reply.failureInformationVersion, reply.changes.size(), reply.allOthersFailed);
 
 					version = reply.failureInformationVersion;
-					*pServerFailedTimeout = reply.considerServerFailedTimeoutMS * .001;
+					fmState->serverFailedTimeout = reply.considerServerFailedTimeoutMS * .001;
 					for(int c=0; c<reply.changes.size(); c++) {
 						//printf("Client '%s': status of '%s' is now '%s'\n", g_network->getLocalAddress().toString().c_str(), reply.changes[c].address.toString().c_str(), reply.changes[c].status.failed ? "Failed" : "OK");
 						monitor->setStatus( reply.changes[c].address, reply.changes[c].status );
 						if (reply.changes[c].status != FailureStatus())
-							knownAddrs->insert( reply.changes[c].address );
+							fmState->knownAddrs.insert( reply.changes[c].address );
 						else
-							knownAddrs->erase( reply.changes[c].address );
+							fmState->knownAddrs.erase( reply.changes[c].address );
 						ASSERT( reply.changes[c].address != controller.failureMonitoring.getEndpoint().address || !reply.changes[c].status.failed );
 					}
 					before = now();
@@ -91,7 +99,7 @@ ACTOR Future<Void> failureMonitorClientLoop(
 					requestTimeout = Never();
 					TraceEvent(SevWarn, "FailureMonitoringServerDown").detail("OldServerID",controller.id());
 					monitor->setStatus( controller.failureMonitoring.getEndpoint().address, FailureStatus(true) );
-					knownAddrs->erase( controller.failureMonitoring.getEndpoint().address );
+					fmState->knownAddrs.erase( controller.failureMonitoring.getEndpoint().address );
 				}
 				when( Void _ = wait( nextRequest ) ) {
 					g_network->setCurrentTask(TaskDefaultDelay);
@@ -111,7 +119,7 @@ ACTOR Future<Void> failureMonitorClientLoop(
 						req.senderStatus = FailureStatus(false);
 					request = controller.failureMonitoring.getReply( req, TaskFailureMonitor );
 					if(!controller.failureMonitoring.getEndpoint().isLocal())
-						requestTimeout = delay( *pServerFailedTimeout, TaskFailureMonitor );
+						requestTimeout = delay( fmState->serverFailedTimeout, TaskFailureMonitor );
 				}
 			}
 		}
@@ -125,11 +133,10 @@ ACTOR Future<Void> failureMonitorClientLoop(
 
 ACTOR Future<Void> failureMonitorClient( Reference<AsyncVar<Optional<struct ClusterInterface>>> ci, bool trackMyStatus ) {
 	state SimpleFailureMonitor* monitor = static_cast<SimpleFailureMonitor*>( &IFailureMonitor::failureMonitor() );
-	state std::set<NetworkAddress> knownAddrs;
-	state double serverFailedTimeout = CLIENT_KNOBS->FAILURE_TIMEOUT_DELAY;
+	state Reference<FailureMonitorClientState> fmState = Reference<FailureMonitorClientState>(new FailureMonitorClientState()); 
 
 	loop {
-		state Future<Void> client = ci->get().present() ? failureMonitorClientLoop(monitor, ci->get().get(), &serverFailedTimeout, trackMyStatus, &knownAddrs) : Void();
+		state Future<Void> client = ci->get().present() ? failureMonitorClientLoop(monitor, ci->get().get(), fmState, trackMyStatus) : Void();
 		Void _ = wait( ci->onChange() );
 	}
 }

fdbclient/ManagementAPI.actor.cpp

@@ -87,7 +87,7 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 		return out;
 	}
 
-	std::string redundancy, log_replicas, log_recovery_anti_quorum, dc="1", minDC="1";
+	std::string redundancy, log_replicas;
 	IRepPolicyRef storagePolicy;
 	IRepPolicyRef tLogPolicy;
 
@@ -95,36 +95,23 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 	if (mode == "single") {
 		redundancy="1";
 		log_replicas="1";
-		log_recovery_anti_quorum="0";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyOne());
 
-	} else if(mode == "double") {
+	} else if(mode == "double" || mode == "fast_recovery_double") {
 		redundancy="2";
 		log_replicas="2";
-		log_recovery_anti_quorum="0";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
-	} else if(mode == "triple") {
+	} else if(mode == "triple" || mode == "fast_recovery_triple") {
 		redundancy="3";
 		log_replicas="3";
-		log_recovery_anti_quorum="0";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
-	} else if(mode == "fast_recovery_double") {
-		redundancy="2";
-		log_replicas="3";
-		log_recovery_anti_quorum="1";
-		storagePolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
-		tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
-	} else if(mode == "fast_recovery_triple") {
-		redundancy="3";
-		log_replicas="4";
-		log_recovery_anti_quorum="1";
-		storagePolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
-		tLogPolicy = IRepPolicyRef(new PolicyAcross(4, "zoneid", IRepPolicyRef(new PolicyOne())));
 	} else if(mode == "two_datacenter") {
-		redundancy="3"; log_replicas="3"; log_recovery_anti_quorum="0"; dc="2"; minDC="1";
+		redundancy="3";
+		log_replicas="3";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
 	} else if(mode == "three_datacenter") {
-		redundancy="3"; log_replicas="3"; log_recovery_anti_quorum="0"; dc="3"; minDC="2";
+		redundancy="3";
+		log_replicas="3";
 		storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAnd({
 			IRepPolicyRef(new PolicyAcross(3, "dcid", IRepPolicyRef(new PolicyOne()))),
 			IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())))
@@ -132,7 +119,6 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 	} else if(mode == "three_data_hall") {
 		redundancy="3";
 		log_replicas="4";
-		log_recovery_anti_quorum="0";
 		storagePolicy = IRepPolicyRef(new PolicyAcross(3, "data_hall", IRepPolicyRef(new PolicyOne())));
 		tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "data_hall",
 			IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))
@@ -144,9 +130,6 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
 			out[p+"storage_quorum"] = redundancy;
 		out[p+"log_replicas"] = log_replicas;
 		out[p+"log_anti_quorum"] = "0";
-		out[p+"log_recovery_anti_quorum"] = log_recovery_anti_quorum;
-		out[p+"replica_datacenters"] = dc;
-		out[p+"min_replica_datacenters"] = minDC;
 
 		BinaryWriter policyWriter(IncludeVersion());
 		serializeReplicationPolicy(policyWriter, storagePolicy);
@@ -214,9 +197,7 @@ ConfigurationResult::Type buildConfiguration( std::string const& configMode, std
 bool isCompleteConfiguration( std::map<std::string, std::string> const& options ) {
 	std::string p = configKeysPrefix.toString();
 
-	return options.count( p+"min_replica_datacenters" ) == 1 &&
-			options.count( p+"replica_datacenters" ) == 1 &&
-			options.count( p+"log_replicas" ) == 1 &&
+	return 	options.count( p+"log_replicas" ) == 1 &&
 			options.count( p+"log_anti_quorum" ) == 1 &&
 			options.count( p+"storage_quorum" ) == 1 &&
 			options.count( p+"storage_replicas" ) == 1 &&
@@ -307,10 +288,10 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 		result.auto_replication = "double";
 		storage_replication = 2;
 		log_replication = 2;
-	} else if( result.old_replication == "double" ) {
+	} else if( result.old_replication == "double" || result.old_replication == "fast_recovery_double" ) {
 		storage_replication = 2;
 		log_replication = 2;
-	} else if( result.old_replication == "triple" ) {
+	} else if( result.old_replication == "triple" || result.old_replication == "fast_recovery_triple" ) {
 		storage_replication = 3;
 		log_replication = 3;
 	} else if( result.old_replication == "two_datacenter" ) {
@@ -319,12 +300,6 @@ ConfigureAutoResult parseConfig( StatusObject const& status ) {
 	} else if( result.old_replication == "three_datacenter" ) {
 		storage_replication = 3;
 		log_replication = 3;
-	} else if( result.old_replication == "fast_recovery_double" ) {
-		storage_replication = 2;
-		log_replication = 3;
-	} else if( result.old_replication == "fast_recovery_triple" ) {
-		storage_replication = 3;
-		log_replication = 4;
 	} else
 		return ConfigureAutoResult();
 
@@ -705,7 +680,9 @@ ACTOR Future<CoordinatorsResult::Type> changeQuorum( Database cx, Reference<IQuo
 
 			if(g_network->isSimulated()) {
 				for(int i = 0; i < (desiredCoordinators.size()/2)+1; i++) {
-					g_simulator.protectedAddresses.insert( desiredCoordinators[i] );
+					auto address = NetworkAddress(desiredCoordinators[i].ip,desiredCoordinators[i].port,true,false);
+					g_simulator.protectedAddresses.insert(address);
+					TraceEvent("ProtectCoordinator").detail("Address", address).backtrace();
 				}
 			}
 
@@ -1067,6 +1044,39 @@ ACTOR Future<vector<AddressExclusion>> getExcludedServers( Database cx ) {
 	}
 }
 
+ACTOR Future<int> setDDMode( Database cx, int mode ) {
+	state Transaction tr(cx);
+	state int oldMode = -1;
+	state BinaryWriter wr(Unversioned());
+	wr << mode;
+
+	loop {
+		try {
+			Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
+			if (oldMode < 0) {
+				oldMode = 1;
+				if (old.present()) {
+					BinaryReader rd(old.get(), Unversioned());
+					rd >> oldMode;
+				}
+			}
+			if (!mode) {
+				BinaryWriter wrMyOwner(Unversioned());
+				wrMyOwner << dataDistributionModeLock;
+				tr.set( moveKeysLockOwnerKey, wrMyOwner.toStringRef() );
+			}
+
+			tr.set( dataDistributionModeKey, wr.toStringRef() );
+
+			Void _ = wait( tr.commit() );
+			return oldMode;
+		} catch (Error& e) {
+			TraceEvent("setDDModeRetrying").error(e);
+			Void _ = wait (tr.onError(e));
+		}
+	}
+}
+
 ACTOR Future<Void> waitForExcludedServers( Database cx, vector<AddressExclusion> excl ) {
 	state std::set<AddressExclusion> exclusions( excl.begin(), excl.end() );
 

fdbclient/ManagementAPI.h

@@ -153,6 +153,8 @@ Future<Void> unlockDatabase( Database const& cx, UID const& id );
 Future<Void> checkDatabaseLock( Transaction* const& tr, UID const& id );
 Future<Void> checkDatabaseLock( Reference<ReadYourWritesTransaction> const& tr, UID const& id );
 
+Future<int> setDDMode( Database const& cx, int const& mode );
+
 // Gets the cluster connection string
 Future<std::vector<NetworkAddress>> getCoordinators( Database const& cx );
 #endif

fdbclient/MasterProxyInterface.h

@@ -33,7 +33,7 @@ struct MasterProxyInterface {
 	RequestStream< struct CommitTransactionRequest > commit;
 	RequestStream< struct GetReadVersionRequest > getConsistentReadVersion;  // Returns a version which (1) is committed, and (2) is >= the latest version reported committed (by a commit response) when this request was sent
 															     //   (at some point between when this request is sent and when its response is received, the latest version reported committed)
-	RequestStream< ReplyPromise<vector<StorageServerInterface>> > getKeyServersLocations;
+	RequestStream< ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> > getKeyServersLocations;
 	RequestStream< struct GetStorageServerRejoinInfoRequest > getStorageServerRejoinInfo;
 
 	RequestStream<ReplyPromise<Void>> waitFailure;

fdbclient/MetricLogger.h

@@ -0,0 +1,25 @@
+/*
+ * MetricLogger.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "NativeAPI.h"
+
+Future<Void> runMetrics( Future<Database> const& fcx, Key const& metricsPrefix );

fdbclient/NativeAPI.actor.cpp

@@ -602,7 +602,7 @@ Reference<LocationInfo> DatabaseContext::setCachedLocation( const KeyRangeRef& k
 		attempts++;
 		auto r = locationCache.randomRange();
 		Key begin = r.begin(), end = r.end();  // insert invalidates r, so can't be passed a mere reference into it
-		if( begin >= keyServersPrefix )
+		if( begin >= keyServersPrefix && attempts > maxEvictionAttempts / 2)
 			continue;
 		locationCache.insert( KeyRangeRef(begin, end), Reference<LocationInfo>() );
 	}
@@ -875,7 +875,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) {
 	if (!networkOptions.logClientInfo.present())
 		networkOptions.logClientInfo = true;
 
-	g_network = newNet2(NetworkAddress(), false, useMetrics);
+	g_network = newNet2(NetworkAddress(), false, useMetrics || networkOptions.traceDirectory.present());
 	FlowTransport::createInstance(transportId);
 	Net2FileSystem::newFileSystem();
 
@@ -1102,22 +1102,33 @@ ACTOR Future< pair<KeyRange,Reference<LocationInfo>> > getKeyLocation( Database
 
 	state vector<StorageServerInterface> serverInterfaces;
 	state KeyRangeRef range;
-
+	
 	// We assume that not only /FF/keyServers but /FF/serverList is present on the keyServersLocations since we now need both of them to terminate our search. Currently this is guaranteed because nothing after /FF/keyServers is split.
 	if ( ( key.startsWith( serverListPrefix) && (!isBackward || key.size() > serverListPrefix.size()) ) ||
 		( key.startsWith( keyServersPrefix ) && (!isBackward || key.size() > keyServersPrefix.size()) )) {
 		if( info.debugID.present() )
-			g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.Before");
+			g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.Before");	
 		loop {
 			choose {
 				when ( Void _ = wait( cx->onMasterProxiesChanged() ) ) {}
-				when ( vector<StorageServerInterface> s = wait( loadBalance( cx->getMasterProxies(), &MasterProxyInterface::getKeyServersLocations, ReplyPromise<vector<StorageServerInterface>>(), info.taskID ) ) ) {
+				when ( vector<pair<KeyRangeRef, vector<StorageServerInterface>>> keyServersShards = wait( loadBalance( cx->getMasterProxies(), &MasterProxyInterface::getKeyServersLocations, ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>>(), info.taskID ) ) ) {
 					if( info.debugID.present() )
 						g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.After");
-					ASSERT( s.size() );  // There should always be storage servers, except on version 0 which should not get to this function
-					range = KeyRangeRef( keyServersPrefix, allKeys.end );
-					serverInterfaces = s;
-					break;
+					ASSERT( keyServersShards.size() );  // There should always be storage servers, except on version 0 which should not get to this function
+
+					Reference<LocationInfo> cachedLocation;
+					for (pair<KeyRangeRef, vector<StorageServerInterface>> keyServersShard : keyServersShards) {
+						auto locationInfo = cx->setCachedLocation(keyServersShard.first, keyServersShard.second);
+
+						if (isBackward ? (keyServersShard.first.begin < key && keyServersShard.first.end >= key) : keyServersShard.first.contains(key)) {
+							range = keyServersShard.first;
+							cachedLocation = locationInfo;
+						}
+					}
+
+					ASSERT(isBackward ? (range.begin < key && range.end >= key) : range.contains(key));
+
+					return make_pair(range, cachedLocation);
 				}
 			}
 		}
@@ -1654,6 +1665,15 @@ Future<Key> resolveKey( Database const& cx, KeySelector const& key, Version cons
 ACTOR Future<Standalone<RangeResultRef>> getRangeFallback( Database cx, Version version,
 	KeySelector begin, KeySelector end, GetRangeLimits limits, bool reverse, TransactionInfo info )
 {
+	if(version == latestVersion) {
+		state Transaction transaction(cx);
+		transaction.setOption(FDBTransactionOptions::CAUSAL_READ_RISKY);
+		transaction.setOption(FDBTransactionOptions::LOCK_AWARE);
+		transaction.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		Version ver = wait( transaction.getReadVersion() );
+		version = ver;
+	}
+
 	Future<Key> fb = resolveKey(cx, begin, version, info);
 	state Future<Key> fe = resolveKey(cx, end, version, info);
 
@@ -1849,15 +1869,8 @@ ACTOR Future<Standalone<RangeResultRef>> getRange( Database cx, Future<Version>
 				cx->invalidateCache( beginServer.second );
 
 				if (e.code() == error_code_wrong_shard_server) {
-					if (version == latestVersion) {
-						// latestVersion queries are only for keyServersPrefix/*, which shard is guaranteed not to split,
-						//   so we should always be able to use the fast path--try again
-						TEST(true); //Latest version retry fast path
-						TraceEvent("LatestVersionRetryFastPath").detail("KeyBegin", printable(begin.getKey())).detail("KeyEnd", printable(end.getKey()));
-					} else {
 						Standalone<RangeResultRef> result = wait( getRangeFallback(cx, version, originalBegin, originalEnd, originalLimits, reverse, info ) );
 						return result;
-					}
 				}
 
 				Void _ = wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID));
@@ -2424,6 +2437,7 @@ ACTOR static Future<Void> commitDummyTransaction( Database cx, KeyRange range, T
 			tr.info.taskID = info.taskID;
 			tr.setOption( FDBTransactionOptions::ACCESS_SYSTEM_KEYS );
 			tr.setOption( FDBTransactionOptions::CAUSAL_WRITE_RISKY );
+			tr.setOption( FDBTransactionOptions::LOCK_AWARE );
 			tr.addReadConflictRange(range);
 			tr.addWriteConflictRange(range);
 			Void _ = wait( tr.commit() );

fdbclient/Notified.h

@@ -18,11 +18,11 @@
  * limitations under the License.
  */
 
-#ifndef FLOW_NOTIFIED_H
-#define FLOW_NOTIFIED_H
+#ifndef FDBCLIENT_NOTIFIED_H
+#define FDBCLIENT_NOTIFIED_H
 #pragma once
 
-#include "fdbclient/FDBTypes.h"
+#include "FDBTypes.h"
 #include "flow/TDMetric.actor.h"
 
 struct NotifiedVersion {
@@ -78,4 +78,4 @@ private:
 	VersionMetricHandle val;
 };
 
-#endif
+#endif

fdbclient/SystemData.cpp

@@ -33,7 +33,8 @@ const KeyRef afterAllKeys = LiteralStringRef("\xff\xff\x00");
 const KeyRangeRef keyServersKeys( LiteralStringRef("\xff/keyServers/"), LiteralStringRef("\xff/keyServers0") );
 const KeyRef keyServersPrefix = keyServersKeys.begin;
 const KeyRef keyServersEnd = keyServersKeys.end;
-const KeyRef keyServersKeyServersKey = LiteralStringRef("\xff/keyServers/\xff/keyServers/");
+const KeyRangeRef keyServersKeyServersKeys ( LiteralStringRef("\xff/keyServers/\xff/keyServers/"), LiteralStringRef("\xff/keyServers/\xff/keyServers0"));
+const KeyRef keyServersKeyServersKey = keyServersKeyServersKeys.begin;
 
 const Key keyServersKey( const KeyRef& k ) {
 	return k.withPrefix( keyServersPrefix );

fdbclient/SystemData.h

@@ -34,7 +34,7 @@ extern const KeyRangeRef allKeys; // '' to systemKeys.end
 extern const KeyRef afterAllKeys;
 
 //    "\xff/keyServers/[[begin]]" := "[[vector<serverID>, vector<serverID>]]"
-extern const KeyRangeRef keyServersKeys;
+extern const KeyRangeRef keyServersKeys, keyServersKeyServersKeys;
 extern const KeyRef keyServersPrefix, keyServersEnd, keyServersKeyServersKey;
 const Key keyServersKey( const KeyRef& k );
 const KeyRef keyServersKey( const KeyRef& k, Arena& arena );

fdbclient/fdbclient.vcxproj

@@ -38,6 +38,8 @@
       <EnableCompile Condition="'$(Configuration)|$(Platform)'=='Debug|X64'">false</EnableCompile>
       <EnableCompile Condition="'$(Configuration)|$(Platform)'=='Release|X64'">false</EnableCompile>
     </ActorCompiler>
+    <ClInclude Include="MetricLogger.h" />
+    <ActorCompiler Include="MetricLogger.actor.cpp" />
     <ClInclude Include="FailureMonitorClient.h" />
     <ClInclude Include="FDBOptions.g.h" />
     <ClInclude Include="FDBOptions.h" />
@@ -57,6 +59,7 @@
     <ClInclude Include="MultiVersionTransaction.h" />
     <ClInclude Include="MutationList.h" />
     <ClInclude Include="NativeAPI.h" />
+    <ClInclude Include="Notified.h" />
     <ClInclude Include="ReadYourWrites.h" />
     <ActorCompiler Include="RunTransaction.actor.h" />
     <ClInclude Include="RYWIterator.h" />
@@ -201,4 +204,4 @@
   <Target Name="MyPreCompileSteps" AfterTargets="CLCompile">
     <Exec Command="&quot;$(SolutionDir)bin\$(Configuration)\coveragetool.exe&quot; &quot;$(OutDir)coverage.$(TargetName).xml&quot; @(ActorCompiler -> '%(RelativeDir)%(Filename)%(Extension)', ' ') @(CLInclude -> '%(RelativeDir)%(Filename)%(Extension)', ' ') @(CLCompile -> '%(RelativeDir)%(Filename)%(Extension)', ' ')" />
   </Target>
-</Project>
+</Project>

fdbclient/vexillographer/fdb.options

@@ -94,7 +94,7 @@ description is not currently required but encouraged.
     <Option name="disable_client_statistics_logging" code="70"
             description="Disables logging of client statistics, such as sampled transaction activity." />
     <Option name="enable_slow_task_profiling" code="71"
-            description="Enables slow task profiling. Requires trace logging to be enabled." />
+            description="Enables debugging feature to perform slow task profiling. Requires trace logging to be enabled. WARNING: this feature is not recommended for use in production." />
     <Option name="supported_client_versions" code="1000"
             paramType="String" paramDescription="[release version],[source version],[protocol version];..."
             description="This option is set automatically to communicate the list of supported clients to the active client."

fdbmonitor/fdbmonitor.cpp

@@ -253,7 +253,8 @@ public:
 	uint32_t restart_delay_reset_interval;
 	double last_start;
 	bool quiet;
-	bool delete_wd40_env;
+	//bool delete_wd40_env;
+	const char *delete_envvars;
 	bool deconfigured;
 	bool kill_on_configuration_change;
 
@@ -261,7 +262,7 @@ public:
 	int pipes[2][2];
 
 	Command() : argv(NULL) { }
-	Command(const CSimpleIni& ini, std::string _section, uint64_t id, fdb_fd_set fds, int* maxfd) : section(_section), argv(NULL), quiet(false), delete_wd40_env(false), fds(fds), deconfigured(false), kill_on_configuration_change(true) {
+	Command(const CSimpleIni& ini, std::string _section, uint64_t id, fdb_fd_set fds, int* maxfd) : section(_section), argv(NULL), quiet(false), delete_envvars(NULL), fds(fds), deconfigured(false), kill_on_configuration_change(true) {
 		char _ssection[strlen(section.c_str()) + 22];
 		snprintf(_ssection, strlen(section.c_str()) + 22, "%s.%llu", section.c_str(), id);
 		ssection = _ssection;
@@ -351,10 +352,8 @@ public:
 		if (q && !strcmp(q, "true"))
 			quiet = true;
 
-		const char* dwe = get_value_multi(ini, "delete_wd40_env", ssection.c_str(), section.c_str(), "general", NULL);
-		if(dwe && !strcmp(dwe, "true")) {
-			delete_wd40_env = true;
-		}
+		const char* del_env = get_value_multi(ini, "delete_envvars", ssection.c_str(), section.c_str(), "general", NULL);
+		delete_envvars = del_env;
 
 		const char* kocc = get_value_multi(ini, "kill_on_configuration_change", ssection.c_str(), section.c_str(), "general", NULL);
 		if(kocc && strcmp(kocc, "true")) {
@@ -373,7 +372,7 @@ public:
 
 		for (auto i : keys) {
 			if (!strcmp(i.pItem, "command") || !strcmp(i.pItem, "restart_delay") || !strcmp(i.pItem, "initial_restart_delay") || !strcmp(i.pItem, "restart_backoff") ||
-				!strcmp(i.pItem, "restart_delay_reset_interval") || !strcmp(i.pItem, "disable_lifecycle_logging") || !strcmp(i.pItem, "delete_wd40_env") ||
+				!strcmp(i.pItem, "restart_delay_reset_interval") || !strcmp(i.pItem, "disable_lifecycle_logging") || !strcmp(i.pItem, "delete_envvars") ||
 				!strcmp(i.pItem, "kill_on_configuration_change"))
 			{
 				continue;
@@ -408,7 +407,7 @@ public:
 	}
 	void update(const Command& other) {
 		quiet = other.quiet;
-		delete_wd40_env = other.delete_wd40_env;
+		delete_envvars = other.delete_envvars;
 		initial_restart_delay = other.initial_restart_delay;
 		max_restart_delay = other.max_restart_delay;
 		restart_backoff = other.restart_backoff;
@@ -474,12 +473,21 @@ void start_process(Command* cmd, uint64_t id, uid_t uid, gid_t gid, int delay, s
 		signal(SIGINT, SIG_DFL);
 		signal(SIGTERM, SIG_DFL);
 
-		if(cmd->delete_wd40_env) {
-			/* remove WD40 environment variables */
-			if(unsetenv("WD40_BV") || unsetenv("WD40_IS_MY_DADDY") || unsetenv("CONF_BUILD_VERSION")) {
-				log_err("unsetenv", errno, "Failed to remove parent environment variables");
-				exit(1);
-			}
+		if(cmd->delete_envvars != NULL && std::strlen(cmd->delete_envvars) > 0) {
+			std::string vars(cmd->delete_envvars);
+			size_t start = 0;
+			do {
+				size_t bound = vars.find(" ", start);
+				std::string var = vars.substr(start, bound - start);
+				log_msg(LOG_INFO, "Deleting parent environment variable: \"%s\"\n", var.c_str());
+				if(unsetenv(var.c_str())) {
+					log_err("unsetenv", errno, "Failed to remove parent environment variable: %s\n", var.c_str());
+					exit(1);
+				}
+				start = bound;
+				while(vars[start] == ' ')
+					start++;
+			} while(start <= vars.length());
 		}
 
 		dup2( cmd->pipes[0][1], fileno(stdout) );

fdbrpc/AsyncFileKAIO.actor.h

@@ -268,6 +268,7 @@ public:
 		}
 
 		lastFileSize = nextFileSize = size;
+
 		return Void();
 	}
 
@@ -503,7 +504,7 @@ private:
 		}
 
 		void setIOTimeout(double timeout) {
-			ioTimeout = timeout;
+			ioTimeout = fabs(timeout);
 			timeoutWarnOnly = timeout < 0;
 		}
 
@@ -550,6 +551,7 @@ private:
 	static Context ctx;
 
 	explicit AsyncFileKAIO(int fd, int flags, std::string const& filename) : fd(fd), flags(flags), filename(filename), failed(false) {
+
 		if( !g_network->isSimulated() ) {
 			countFileLogicalWrites.init(LiteralStringRef("AsyncFile.CountFileLogicalWrites"), filename);
 			countFileLogicalReads.init( LiteralStringRef("AsyncFile.CountFileLogicalReads"), filename);

fdbrpc/AsyncFileNonDurable.actor.h

@@ -213,8 +213,9 @@ public:
 			//If we are in the process of deleting a file, we can't let someone else modify it at the same time.  We therefore block the creation of new files until deletion is complete
 			state std::map<std::string, Future<Void>>::iterator deletedFile = filesBeingDeleted.find(filename);
 			if(deletedFile != filesBeingDeleted.end()) {
-				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete").detail("Filename", filename);
+				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete1").detail("Filename", filename);
 				Void _ = wait( deletedFile->second || shutdown );
+				//TraceEvent("AsyncFileNonDurableOpenWaitOnDelete2").detail("Filename", filename);
 				if(shutdown.isReady())
 					throw io_error().asInjectedFault();
 			}
@@ -425,7 +426,7 @@ private:
 			std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(offset, length, true, writeEnded);
 
 			if(BUGGIFY_WITH_PROB(0.001))
-				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY));
+				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture());
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, length) || self->killed.getFuture());
 
@@ -547,7 +548,7 @@ private:
 			std::vector<Future<Void>> priorModifications = self->getModificationsAndInsert(size, -1, true, truncateEnded);
 
 			if(BUGGIFY_WITH_PROB(0.001))
-				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY));
+				priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture());
 			else
 				priorModifications.push_back(waitUntilDiskReady(self->diskParameters, 0) || self->killed.getFuture());
 
@@ -711,35 +712,44 @@ private:
 
 	//Finishes all outstanding actors on an AsyncFileNonDurable and then deletes it
 	ACTOR Future<Void> deleteFile(AsyncFileNonDurable *self) {
-		//We must run on the main thread (instead of a SQLite coroutine).  We don't want to signal any promises from a coroutine, so we switch at the beginning
-		//of this ACTOR
-		Void _ = wait(self->returnToMainThread());
+		state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+		state int currentTaskID = g_network->getCurrentTask();
+		state std::string filename = self->filename;
 
-		//Make sure all writes have gone through.
-		Promise<bool> startSyncPromise = self->startSyncPromise;
-		self->startSyncPromise = Promise<bool>();
-		startSyncPromise.send(true);
+		Void _ = wait( g_simulator.onMachine( currentProcess ) );
+		try {
+			//Make sure all writes have gone through.
+			Promise<bool> startSyncPromise = self->startSyncPromise;
+			self->startSyncPromise = Promise<bool>();
+			startSyncPromise.send(true);
 
-		std::vector<Future<Void>> outstandingModifications;
+			std::vector<Future<Void>> outstandingModifications;
 
-		for(auto itr = self->pendingModifications.ranges().begin(); itr != self->pendingModifications.ranges().end(); ++itr)
-			if(itr->value().isValid() && !itr->value().isReady())
-				outstandingModifications.push_back(itr->value());
+			for(auto itr = self->pendingModifications.ranges().begin(); itr != self->pendingModifications.ranges().end(); ++itr)
+				if(itr->value().isValid() && !itr->value().isReady())
+					outstandingModifications.push_back(itr->value());
 
-		//Ignore errors here so that all modifications can finish
-		Void _ = wait(waitForAllReady(outstandingModifications));
+			//Ignore errors here so that all modifications can finish
+			Void _ = wait(waitForAllReady(outstandingModifications));
 
-		//Make sure we aren't in the process of killing the file
-		if(self->killed.isSet())
-			Void _ = wait(self->killComplete.getFuture());
+			//Make sure we aren't in the process of killing the file
+			if(self->killed.isSet())
+				Void _ = wait(self->killComplete.getFuture());
 
-		//Remove this file from the filesBeingDeleted map so that new files can be created with this filename
-		g_simulator.getMachineByNetworkAddress( self->openedAddress )->closingFiles.erase(self->getFilename());
-		AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
-		//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
+			//Remove this file from the filesBeingDeleted map so that new files can be created with this filename
+			g_simulator.getMachineByNetworkAddress( self->openedAddress )->closingFiles.erase(self->getFilename());
+			g_simulator.getMachineByNetworkAddress( self->openedAddress )->deletingFiles.erase(self->getFilename());
+			AsyncFileNonDurable::filesBeingDeleted.erase(self->filename);
+			//TraceEvent("AsyncFileNonDurable_FinishDelete", self->id).detail("Filename", self->filename);
 
-		delete self;
-		return Void();
+			delete self;
+			Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+			return Void();
+		} catch( Error &e ) {
+			state Error err = e;
+			Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+			throw err;
+		}
 	}
 };
 

fdbrpc/AsyncFileWriteChecker.cpp

@@ -0,0 +1,24 @@
+/*
+ * AsyncFileWriteChecker.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "AsyncFileWriteChecker.h"
+
+int AsyncFileWriteChecker::checksumHistoryBudget;
+int AsyncFileWriteChecker::checksumHistoryPageSize = 4096;

fdbrpc/AsyncFileWriteChecker.h

@@ -0,0 +1,151 @@
+/*
+ * AsyncFileWriteChecker.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "IAsyncFile.h"
+
+#if VALGRIND
+#include <memcheck.h>
+#endif
+
+class AsyncFileWriteChecker : public IAsyncFile, public ReferenceCounted<AsyncFileWriteChecker> {
+public:
+	void addref() { ReferenceCounted<AsyncFileWriteChecker>::addref(); }
+	void delref() { ReferenceCounted<AsyncFileWriteChecker>::delref(); }
+
+	// For read() and write(), the data buffer must remain valid until the future is ready
+	Future<int> read( void* data, int length, int64_t offset ) {
+		return map(m_f->read(data, length, offset), [=](int r) { updateChecksumHistory(false, offset, length, (uint8_t *)data); return r; });
+	}
+	Future<Void> readZeroCopy( void** data, int* length, int64_t offset ) {
+		return map(m_f->readZeroCopy(data, length, offset), [=](Void r) { updateChecksumHistory(false, offset, *length, (uint8_t *)data); return r; });
+	}
+
+	Future<Void> write( void const* data, int length, int64_t offset ) {
+		updateChecksumHistory(true, offset, length, (uint8_t *)data);
+		return m_f->write(data, length, offset);
+	}
+
+	Future<Void> truncate( int64_t size )  {
+		return map(m_f->truncate(size), [=](Void r) {
+			// Truncate the page checksum history if it is in use
+			if( (size / checksumHistoryPageSize) < checksumHistory.size() ) {
+				int oldCapacity = checksumHistory.capacity();
+				checksumHistory.resize(size / checksumHistoryPageSize);
+				checksumHistoryBudget -= (checksumHistory.capacity() - oldCapacity);
+			}
+			return r;
+		});
+	}
+
+	Future<Void> sync() { return m_f->sync(); }
+	Future<Void> flush() { return m_f->flush(); }
+	Future<int64_t> size() { return m_f->size(); }
+	std::string getFilename() { return m_f->getFilename(); }
+	void releaseZeroCopy( void* data, int length, int64_t offset )  { return m_f->releaseZeroCopy(data, length, offset); }
+	int64_t debugFD() { return m_f->debugFD(); }
+
+	AsyncFileWriteChecker(Reference<IAsyncFile> f) : m_f(f) {
+		// Initialize the static history budget the first time (and only the first time) a file is opened.
+		static int _ = checksumHistoryBudget = FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY;
+
+		// Adjust the budget by the initial capacity of history, which should be 0 but maybe not for some implementations.
+		checksumHistoryBudget -= checksumHistory.capacity();
+	}
+
+
+	virtual ~AsyncFileWriteChecker() {
+		checksumHistoryBudget += checksumHistory.capacity();
+	}
+
+private:
+	Reference<IAsyncFile> m_f;
+
+	struct WriteInfo {
+		WriteInfo() : checksum(0), timestamp(0) {}
+		uint32_t checksum;
+		uint32_t timestamp;
+	};
+
+	std::vector<WriteInfo> checksumHistory;
+	// This is the most page checksum history blocks we will use across all files.
+	static int checksumHistoryBudget;
+	static int checksumHistoryPageSize;
+
+	// Update or check checksum(s) in history for any full pages covered by this operation
+	void updateChecksumHistory(bool write, int64_t offset, int len, uint8_t *buf) {
+		// Check or set each full block in the the range
+		int page = offset / checksumHistoryPageSize;            // First page number
+		int slack = offset % checksumHistoryPageSize;           // Bytes after most recent page boundary
+		uint8_t *start = buf;                                   // Position in buffer to start checking from
+		// If offset is not page-aligned, move to next page and adjust start
+		if(slack != 0) {
+			++page;
+			start += (checksumHistoryPageSize - slack);
+		}
+		int pageEnd = (offset + len) / checksumHistoryPageSize; // Last page plus 1
+
+		// Make sure history is large enough or limit pageEnd
+		if(checksumHistory.size() < pageEnd) {
+			if(checksumHistoryBudget > 0) {
+				// Resize history and update budget based on capacity change
+				auto initialCapacity = checksumHistory.capacity();
+				checksumHistory.resize(checksumHistory.size() + std::min<int>(checksumHistoryBudget, pageEnd - checksumHistory.size()));
+				checksumHistoryBudget -= (checksumHistory.capacity() - initialCapacity);
+			}
+
+			// Limit pageEnd to end of history, which works whether or not all of the desired
+			// history slots were allocated.
+			pageEnd = checksumHistory.size();
+		}
+
+		while(page < pageEnd) {
+			uint32_t checksum = hashlittle(start, checksumHistoryPageSize, 0xab12fd93);
+			WriteInfo &history = checksumHistory[page];
+			//printf("%d %d %u %u\n", write, page, checksum, history.checksum);
+
+#if VALGRIND
+			// It's possible we'll read or write a page where not all of the data is defined, but the checksum of the page is still valid
+			VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE(&checksum, sizeof(uint32_t));
+#endif
+
+			// For writes, just update the stored sum
+			if(write) {
+				history.timestamp = (uint32_t)now();
+				history.checksum = checksum;
+			}
+			else {
+				if(history.checksum != 0 && history.checksum != checksum) {
+					// For reads, verify the stored sum if it is not 0.  If it fails, clear it.
+					TraceEvent (SevError, "AsyncFileLostWriteDetected")
+						.detail("Filename", m_f->getFilename())
+						.detail("PageNumber", page)
+						.detail("ChecksumOfPage", checksum)
+						.detail("ChecksumHistory", history.checksum)
+						.detail("LastWriteTime", history.timestamp)
+						.error(checksum_failed());
+					history.checksum = 0;
+				}
+			}
+
+			start += checksumHistoryPageSize;
+			++page;
+		}
+	}
+};

fdbrpc/FlowTransport.actor.cpp

@@ -137,7 +137,7 @@ class TransportData {
 public:
 	TransportData(uint64_t transportId) 
 	  : endpointNotFoundReceiver(endpoints),
-	    pingReceiver(endpoints),
+		pingReceiver(endpoints),
 		warnAlwaysForLargePacket(true),
 		lastIncompatibleMessage(0),
 		transportId(transportId)
@@ -204,7 +204,9 @@ struct ConnectPacket {
 static_assert( sizeof(ConnectPacket) == CONNECT_PACKET_V2_SIZE, "ConnectPacket packed incorrectly" );
 #pragma pack( pop )
 
-static Future<Void> connectionReader( TransportData* const& transport, Reference<IConnection> const& conn, bool const& isOutgoing, Promise<NetworkAddress> const& onPeerAddress );
+static Future<Void> connectionReader( TransportData* const& transport, Reference<IConnection> const& conn, Peer* const& peer, Promise<Peer*> const& onConnected );
+
+static PacketID sendPacket( TransportData* self, ISerializeSource const& what, const Endpoint& destination, bool reliable );
 
 struct Peer : NonCopyable {
 	// FIXME: Peers don't die!
@@ -215,12 +217,14 @@ struct Peer : NonCopyable {
 	ReliablePacketList reliable;
 	AsyncTrigger dataToSend;  // Triggered when unsent.empty() becomes false
 	Future<Void> connect;
+	AsyncVar<bool> incompatibleDataRead;
+	bool compatible;
 	bool outgoingConnectionIdle;  // We don't actually have a connection open and aren't trying to open one because we don't have anything to send
 	double lastConnectTime;
 	double reconnectionDelay;
 
 	explicit Peer( TransportData* transport, NetworkAddress const& destination, bool doConnect = true ) 
-		: transport(transport), destination(destination), outgoingConnectionIdle(!doConnect), lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME)
+		: transport(transport), destination(destination), outgoingConnectionIdle(!doConnect), lastConnectTime(0.0), reconnectionDelay(FLOW_KNOBS->INITIAL_RECONNECTION_TIME), compatible(true)
 	{
 		if(doConnect) {
 			connect = connectionKeeper(this);
@@ -293,8 +297,7 @@ struct Peer : NonCopyable {
 		}
 	}
 
-	ACTOR static Future<Void> connectionMonitor( Peer* peer ) {
-
+	ACTOR static Future<Void> connectionMonitor( Peer *peer ) {
 		state RequestStream< ReplyPromise<Void> > remotePing( Endpoint( peer->destination, WLTOKEN_PING_PACKET ) );
 
 		loop {
@@ -305,9 +308,11 @@ struct Peer : NonCopyable {
 			state ReplyPromise<Void> reply;
 			FlowTransport::transport().sendUnreliable( SerializeSource<ReplyPromise<Void>>(reply), remotePing.getEndpoint() );
 
+			peer->incompatibleDataRead.set(false);
 			choose {
 				when (Void _ = wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) { TraceEvent("ConnectionTimeout").detail("WithAddr", peer->destination); throw connection_failed(); }
 				when (Void _ = wait( reply.getFuture() )) {}
+				when (Void _ = wait( peer->incompatibleDataRead.onChange())) {}
 			}
 		}
 	}
@@ -364,18 +369,17 @@ struct Peer : NonCopyable {
 					Reference<IConnection> _conn = wait( timeout( INetworkConnections::net()->connect(self->destination), FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT, Reference<IConnection>() ) );
 					if (_conn) {
 						conn = _conn;
-						TraceEvent("ConnEstablishedTo", conn->getDebugID()).detail("PeerAddr", self->destination);
+						TraceEvent("ConnectionExchangingConnectPacket", conn->getDebugID()).detail("PeerAddr", self->destination);
 						self->prependConnectPacket();
 					} else {
-						TraceEvent("ConnTimedOut", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination);
+						TraceEvent("ConnectionTimedOut", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination);
 						throw connection_failed();
 					}
 
-					reader = connectionReader( self->transport, conn, true, Promise<NetworkAddress>() );
+					reader = connectionReader( self->transport, conn, self, Promise<Peer*>());
 				} else {
 					self->outgoingConnectionIdle = false;
 				}
-				self->transport->countConnEstablished++;
 
 				Void _ = wait( connectionWriter( self, conn ) || reader || connectionMonitor(self) );
 
@@ -389,12 +393,17 @@ struct Peer : NonCopyable {
 				self->discardUnreliablePackets();
 				reader = Future<Void>();
 				bool ok = e.code() == error_code_connection_failed || e.code() == error_code_actor_cancelled || ( g_network->isSimulated() && e.code() == error_code_checksum_failed );
-				TraceEvent(ok ? SevInfo : SevError, "ConnectionClosed", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination).error(e, true);
 
-				if (ok)
-					self->transport->countConnClosedWithoutError++;
-				else
-					self->transport->countConnClosedWithError++;
+				if(self->compatible) {
+					TraceEvent(ok ? SevInfo : SevError, "ConnectionClosed", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination).error(e, true);
+					if (ok)
+						self->transport->countConnClosedWithoutError++;
+					else
+						self->transport->countConnClosedWithError++;
+				}
+				else {
+					TraceEvent(ok ? SevInfo : SevError, "IncompatibleConnectionClosed", conn ? conn->getDebugID() : UID()).detail("PeerAddr", self->destination).error(e, true);
+				}
 
 				if (conn) {
 					conn->close();
@@ -408,8 +417,6 @@ struct Peer : NonCopyable {
 	}
 };
 
-static PacketID sendPacket( TransportData* self, ISerializeSource const& what, const Endpoint& destination, bool reliable );
-
 ACTOR static void deliver( TransportData* self, Endpoint destination, ArenaReader reader, bool inReadSocket ) {
 	int priority = self->endpoints.getPriority(destination.token);
 	if (priority < TaskReadSocket || !inReadSocket) {
@@ -449,7 +456,7 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
 	uint8_t* p = unprocessed_begin;
 
 	bool checksumEnabled = true;
-	if (!g_network->isSimulated() && transport->localAddress.isTLS() && peerAddress.isTLS()) {
+	if (transport->localAddress.isTLS() || peerAddress.isTLS()) {
 		checksumEnabled = false;
 	}
 
@@ -533,7 +540,9 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
 ACTOR static Future<Void> connectionReader(
 		TransportData* transport,
 		Reference<IConnection> conn, 
-		bool isOutgoing, Promise<NetworkAddress> onPeerAddress ) {
+		Peer *peer,
+		Promise<Peer*> onConnected) 
+{
 	// This actor exists whenever there is an open or opening connection, whether incoming or outgoing
 	// For incoming connections conn is set and peer is initially NULL; for outgoing connections it is the reverse
 
@@ -542,12 +551,14 @@ ACTOR static Future<Void> connectionReader(
 	state uint8_t* unprocessed_end = NULL;
 	state uint8_t* buffer_end = NULL;
 	state bool expectConnectPacket = true;
+	state bool compatible = false;
 	state NetworkAddress peerAddress;
 	state uint64_t peerProtocolVersion = 0;
 
 	peerAddress = conn->getPeerAddress();
-	if (!isOutgoing)
+	if (peer == nullptr) { 
 		ASSERT( !peerAddress.isPublic() );
+	}
 
 	loop {
 		loop {
@@ -592,7 +603,8 @@ ACTOR static Future<Void> connectionReader(
 									.detail("LocalVersion", currentProtocolVersion)
 									.detail("RejectedVersion", p->protocolVersion)
 									.detail("VersionMask", compatibleProtocolVersionMask)
-									.detail("Peer", p->canonicalRemotePort ? NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort ) : conn->getPeerAddress());
+									.detail("Peer", p->canonicalRemotePort ? NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort ) : conn->getPeerAddress())
+									.detail("ConnectionId", connectionId);
 								transport->lastIncompatibleMessage = now();
 							}
 							if(!transport->incompatiblePeers.count(addr)) {
@@ -601,7 +613,20 @@ ACTOR static Future<Void> connectionReader(
 						} else if(connectionId > 1) {
 							transport->multiVersionConnections[connectionId] = now() + FLOW_KNOBS->CONNECTION_ID_TIMEOUT;
 						}
-						throw incompatible_protocol_version();
+
+						compatible = false;
+						if(p->protocolVersion < 0x0FDB00A551000000LL) {
+							// Older versions expected us to hang up. It may work even if we don't hang up here, but it's safer to keep the old behavior.
+							throw incompatible_protocol_version();
+						}
+					}
+					else {
+						compatible = true;
+						TraceEvent("ConnectionEstablished", conn->getDebugID())
+							.detail("Peer", conn->getPeerAddress())
+							.detail("ConnectionId", connectionId);
+
+						transport->countConnEstablished++;
 					}
 
 					if(connectionId > 1) {
@@ -611,21 +636,29 @@ ACTOR static Future<Void> connectionReader(
 					expectConnectPacket = false;
 
 					peerProtocolVersion = p->protocolVersion;
-					if (isOutgoing) {
+					if (peer != nullptr) {
 						// Outgoing connection; port information should be what we expect
 						TraceEvent("ConnectedOutgoing").detail("PeerAddr", NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort ) );
+						peer->compatible = compatible;
 						ASSERT( p->canonicalRemotePort == peerAddress.port );
 					} else {
 						if (p->canonicalRemotePort) {
 							peerAddress = NetworkAddress( p->canonicalRemoteIp, p->canonicalRemotePort, true, peerAddress.isTLS() );
 						}
-						onPeerAddress.send( peerAddress );
+						peer = transport->getPeer(peerAddress);
+						peer->compatible = compatible;
+						onConnected.send( peer );
 						Void _ = wait( delay(0) );  // Check for cancellation
 					}
 				}
 			}
-			if (!expectConnectPacket)
+			if (compatible) {
 				scanPackets( transport, unprocessed_begin, unprocessed_end, arena, peerAddress, peerProtocolVersion );
+			}
+			else if(!expectConnectPacket) {
+				unprocessed_begin = unprocessed_end;
+				peer->incompatibleDataRead.set(true);
+			}
 
 			if (readWillBlock)
 				break;
@@ -640,12 +673,11 @@ ACTOR static Future<Void> connectionReader(
 
 ACTOR static Future<Void> connectionIncoming( TransportData* self, Reference<IConnection> conn ) {
 	try {
-		state Promise<NetworkAddress> onPeerAddress;
-		state Future<Void> reader = connectionReader( self, conn, false, onPeerAddress );
+		state Promise<Peer*> onConnected;
+		state Future<Void> reader = connectionReader( self, conn, nullptr, onConnected );
 		choose {
 			when( Void _ = wait( reader ) ) { ASSERT(false); return Void(); }
-			when( NetworkAddress pa = wait( onPeerAddress.getFuture() ) ) {
-				Peer* p = self->getPeer( pa, false );
+			when( Peer *p = wait( onConnected.getFuture() ) ) {
 				p->onIncomingConnection( conn, reader );
 			}
 			when( Void _ = wait( delayJittered(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT) ) ) {
@@ -777,7 +809,7 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 		return (PacketID)NULL;
 	} else {
 		bool checksumEnabled = true;
-		if (!g_network->isSimulated() && self->localAddress.isTLS() && destination.address.isTLS()) {
+		if (self->localAddress.isTLS() || destination.address.isTLS()) {
 			checksumEnabled = false;
 		}
 
@@ -785,9 +817,9 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
 
 		Peer* peer = self->getPeer(destination.address);
 
-		// If there isn't an open connection or public address, we can't send
-		if (peer->outgoingConnectionIdle && !destination.address.isPublic()) {
-			TEST(true);  // Can't send to private address without an open connection
+		// If there isn't an open connection, a public address, or the peer isn't compatible, we can't send
+		if ((peer->outgoingConnectionIdle && !destination.address.isPublic()) || (!peer->compatible && destination.token != WLTOKEN_PING_PACKET)) {
+			TEST(true);  // Can't send to private address without a compatible open connection
 			return (PacketID)NULL;
 		}
 

fdbrpc/IAsyncFile.actor.cpp

@@ -27,25 +27,36 @@
 
 IAsyncFile::IAsyncFile(){};
 
-ACTOR static Future<Void> incrementalDeleteHelper( std::string filename, int64_t truncateAmt, double interval ){
+ACTOR static Future<Void> incrementalDeleteHelper( std::string filename, bool mustBeDurable, int64_t truncateAmt, double interval ) {
+	state Reference<IAsyncFile> file;
+	state int64_t remainingFileSize;
+	state bool exists = fileExists(filename);
 
-	state Reference<IAsyncFile> f = wait(
-		IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READWRITE, 0));
-	state int64_t filesize = wait(f->size());
-	state int64_t i = filesize;
+	if(exists) {
+		Reference<IAsyncFile> f = wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_UNCACHED, 0));
+		file = f;
 
-	Void _ = wait(IAsyncFileSystem::filesystem()->deleteFile(filename, true));
-	for( ;i > 0; i -= truncateAmt ){
-		Void _ = wait(f->truncate(i));
-		Void _ = wait(f->sync());
-		Void _ = wait(delay(interval));
+		int64_t fileSize = wait(file->size());
+		remainingFileSize = fileSize;
 	}
+
+	Void _ = wait(IAsyncFileSystem::filesystem()->deleteFile(filename, mustBeDurable));
+
+	if(exists) {
+		for( ; remainingFileSize > 0; remainingFileSize -= truncateAmt ){
+			Void _ = wait(file->truncate(remainingFileSize));
+			Void _ = wait(file->sync());
+			Void _ = wait(delay(interval));
+		}
+	}
+
 	return Void();
 }
 
-Future<Void> IAsyncFile::incrementalDelete( std::string filename){
+Future<Void> IAsyncFile::incrementalDelete( std::string filename, bool mustBeDurable ) {
 	return uncancellable(incrementalDeleteHelper(
 		filename,
+		mustBeDurable,
 		FLOW_KNOBS->INCREMENTAL_DELETE_TRUNCATE_AMOUNT,
 		FLOW_KNOBS->INCREMENTAL_DELETE_INTERVAL));
 }
@@ -63,6 +74,6 @@ TEST_CASE( "fileio/incrementalDelete" ) {
 	Void _ = wait(f->truncate(fileSize));
 	//close the file by deleting the reference
 	f.clear();
-	Void _ = wait(IAsyncFile::incrementalDelete(filename));
+	Void _ = wait(IAsyncFile::incrementalDelete(filename, true));
 	return Void();
 }

fdbrpc/IAsyncFile.h

@@ -57,8 +57,10 @@ public:
 	virtual Future<Void> flush() { return Void();  }      // Sends previous writes to the OS if they have been buffered in memory, but does not make them power safe
 	virtual Future<int64_t> size() = 0;
 	virtual std::string getFilename() = 0;
-	//start an actor to truncate the file repeatedly so that the operating system doesn't delete it all at once
-	static Future<Void> incrementalDelete( std::string filename);
+
+	// Unlinks a file and then deletes it slowly by truncating the file repeatedly.
+	// If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.
+	static Future<Void> incrementalDelete( std::string filename, bool mustBeDurable );
 
 	// Attempt to read the *length bytes at offset without copying.  If successful, a pointer to the
 	//   requested bytes is written to *data, and the number of bytes successfully read is

fdbrpc/ITLSPlugin.h

@@ -120,7 +120,7 @@ struct ITLSPolicy {
 // remaining arguments must be pairs of (const char*); the first of
 // each pair must be a valid XML attribute name, and the second a
 // valid XML attribute value. The final parameter must be NULL.
-typedef void (*ITLSLogFunc)(const char* event, void* uid, bool is_error, ...);
+typedef void (*ITLSLogFunc)(const char* event, void* uid, int is_error, ...);
 
 struct ITLSPlugin {
 	virtual void addref() = 0;

fdbrpc/Net2FileSystem.cpp

@@ -37,6 +37,7 @@
 #include "AsyncFileKAIO.actor.h"
 #include "flow/AsioReactor.h"
 #include "flow/Platform.h"
+#include "AsyncFileWriteChecker.h"
 
 // Opens a file for asynchronous I/O
 Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename, int64_t flags, int64_t mode )
@@ -54,12 +55,17 @@ Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename
 	if ( (flags & IAsyncFile::OPEN_EXCLUSIVE) ) ASSERT( flags & IAsyncFile::OPEN_CREATE );
 	if (!(flags & IAsyncFile::OPEN_UNCACHED))
 		return AsyncFileCached::open(filename, flags, mode);
+
+	Future<Reference<IAsyncFile>> f;
 #ifdef __linux__
 	if ( (flags & IAsyncFile::OPEN_UNBUFFERED) && !(flags & IAsyncFile::OPEN_NO_AIO) )
-		return AsyncFileKAIO::open(filename, flags, mode, NULL);
+		f = AsyncFileKAIO::open(filename, flags, mode, NULL);
+	else
 #endif
-
-	return Net2AsyncFile::open(filename, flags, mode, static_cast<boost::asio::io_service*> ((void*) g_network->global(INetwork::enASIOService)));
+	f = Net2AsyncFile::open(filename, flags, mode, static_cast<boost::asio::io_service*> ((void*) g_network->global(INetwork::enASIOService)));
+	if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
+		f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+	return f;
 }
 
 // Deletes the given file.  If mustBeDurable, returns only when the file is guaranteed to be deleted even after a power failure.

fdbrpc/ReplicationPolicy.cpp

@@ -60,7 +60,7 @@ bool IReplicationPolicy::validateFull(
 		}
 		else if (validate(fromServers->getGroupEntries(), fromServers)) {
 			if (g_replicationdebug > 2) {
-				printf("Error: Validated unsolved policy with all%5lu servers\n", fromServers->size());
+				printf("Error: Validated unsolved policy with all%5d servers\n", fromServers->size());
 			}
 			valid = false;
 		}
@@ -194,7 +194,7 @@ bool PolicyAcross::validate(
 				count ++;
 			}
 			else if (g_replicationdebug > 4) {
-				printf("Across invalid solution:%5lu key: %-7s value: (%3lu) %-10s policy: %-10s => %s\n", itValid.second.size(), _attribKey.c_str(), itValid.first._id, fromServers->valueText(itValid.first).c_str(), _policy->name().c_str(), _policy->info().c_str());
+				printf("Across invalid solution:%5lu key: %-7s value: (%3d) %-10s policy: %-10s => %s\n", itValid.second.size(), _attribKey.c_str(), itValid.first._id, fromServers->valueText(itValid.first).c_str(), _policy->name().c_str(), _policy->info().c_str());
 				if (g_replicationdebug > 5) {
 					for (auto& entry : itValid.second) {
 						printf("   entry: %s\n", fromServers->getEntryInfo(entry).c_str());

fdbrpc/TLSConnection.actor.cpp

@@ -273,7 +273,7 @@ Reference<ITLSPolicy> TLSOptions::get_policy() {
 	return policy;
 }
 
-static void TLSConnectionLogFunc( const char* event, void* uid_ptr, bool is_error, ... ) {
+static void TLSConnectionLogFunc( const char* event, void* uid_ptr, int is_error, ... ) {
 	UID uid;
 
 	if ( uid_ptr )

fdbrpc/fdbrpc.vcxproj

@@ -30,6 +30,7 @@
     <ClCompile Include="libb64\cdecode.c" />
     <ClCompile Include="md5\md5.c" />
     <ClCompile Include="Platform.cpp" />
+    <ClCompile Include="AsyncFileWriteChecker.cpp" />
     <ClCompile Include="libcoroutine\Common.c" />
     <ClCompile Include="libcoroutine\Coro.c" />
     <ClCompile Include="Locality.cpp" />

fdbrpc/sim2.actor.cpp

@@ -31,6 +31,7 @@
 #include "fdbclient/FDBTypes.h"
 #include "fdbrpc/Replication.h"
 #include "fdbrpc/ReplicationUtils.h"
+#include "AsyncFileWriteChecker.h"
 
 
 using std::min;
@@ -49,14 +50,13 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l
 			h2 = p->fault_injection_r;
 
 		if (h1 < p->fault_injection_p1*std::numeric_limits<uint32_t>::max()) {
-			TEST(true);
-			TEST(error_code == error_code_io_timeout);
-			TEST(error_code == error_code_io_error);
-			TEST(error_code == error_code_platform_error);
+			TEST(true);                                     // A fault was injected
+			TEST(error_code == error_code_io_timeout);      // An io timeout was injected
+			TEST(error_code == error_code_io_error);        // An io error was injected
+			TEST(error_code == error_code_platform_error);  // A platform error was injected.
 			TraceEvent(SevWarn, "FaultInjected").detail("Context", context).detail("File", file).detail("Line", line).detail("ErrorCode", error_code);
 			if(error_code == error_code_io_timeout) {
 				g_network->setGlobal(INetwork::enASIOTimedOut, (flowGlobalType)true);
-				g_pSimulator->getCurrentProcess()->io_timeout_injected = true;
 			}
 			return true;
 		}
@@ -65,6 +65,29 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l
 	return false;
 }
 
+void ISimulator::displayWorkers() const
+{
+	std::map<std::string, std::vector<ISimulator::ProcessInfo*>>	zoneMap;
+
+	// Create a map of zone Id
+	for (auto processInfo : getAllProcesses()) {
+		std::string dataHall = processInfo->locality.dataHallId().present() ? processInfo->locality.dataHallId().get().printable() : "[unset]";
+		std::string zoneId = processInfo->locality.zoneId().present() ? processInfo->locality.zoneId().get().printable() : "[unset]";
+		zoneMap[format("%-8s  %s", dataHall.c_str(), zoneId.c_str())].push_back(processInfo);
+	}
+
+	printf("DataHall  ZoneId\n");
+	printf("                  Address   Name      Class        Excluded Failed Rebooting Role                                              DataFolder\n");
+	for (auto& zoneRecord : zoneMap) {
+		printf("\n%s\n", zoneRecord.first.c_str());
+		for (auto& processInfo : zoneRecord.second) {
+			printf("                  %9s %-10s%-13s%-8s %-6s %-9s %-48s %-40s\n",
+			processInfo->address.toString().c_str(), processInfo->name, processInfo->startingClass.toString().c_str(), (processInfo->excluded ? "True" : "False"), (processInfo->failed ? "True" : "False"), (processInfo->rebooting ? "True" : "False"), getRoles(processInfo->address).c_str(), processInfo->dataFolder);
+		}
+	}
+
+	return;
+}
 
 namespace std {
 template<>
@@ -863,16 +886,29 @@ public:
 		// This is a _rudimentary_ simulation of the untrustworthiness of non-durable deletes and the possibility of
 		// rebooting during a durable one.  It isn't perfect: for example, on real filesystems testing
 		// for the existence of a non-durably deleted file BEFORE a reboot will show that it apparently doesn't exist.
-		g_simulator.getCurrentProcess()->machine->openFiles.erase(filename);
+		if(g_simulator.getCurrentProcess()->machine->openFiles.count(filename)) {
+			g_simulator.getCurrentProcess()->machine->openFiles.erase(filename);
+			g_simulator.getCurrentProcess()->machine->deletingFiles.insert(filename);
+		}
 		if ( mustBeDurable || g_random->random01() < 0.5 ) {
-			Void _ = wait( ::delay(0.05 * g_random->random01()) );
-			if (!self->getCurrentProcess()->rebooting) {
-				auto f = IAsyncFileSystem::filesystem(self->net2)->deleteFile(filename, false);
-				ASSERT( f.isReady() );
+			state ISimulator::ProcessInfo* currentProcess = g_simulator.getCurrentProcess();
+			state int currentTaskID = g_network->getCurrentTask();
+			Void _ = wait( g_simulator.onMachine( currentProcess ) );
+			try {
 				Void _ = wait( ::delay(0.05 * g_random->random01()) );
-				TEST( true );  // Simulated durable delete
+				if (!currentProcess->rebooting) {
+					auto f = IAsyncFileSystem::filesystem(self->net2)->deleteFile(filename, false);
+					ASSERT( f.isReady() );
+					Void _ = wait( ::delay(0.05 * g_random->random01()) );
+					TEST( true );  // Simulated durable delete
+				}
+				Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+				return Void();
+			} catch( Error &e ) {
+				state Error err = e;
+				Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
+				throw err;
 			}
-			return Void();
 		} else {
 			TEST( true );  // Simulated non-durable delete
 			return Void();
@@ -946,11 +982,13 @@ public:
 		machine.processes.push_back(m);
 		currentlyRebootingProcesses.erase(address);
 		addressMap[ m->address ] = m;
+		m->excluded = g_simulator.isExcluded(address);
 
 		m->setGlobal(enTDMetrics, (flowGlobalType) &m->tdmetrics);
 		m->setGlobal(enNetworkConnections, (flowGlobalType) m->network);
+		m->setGlobal(enASIOTimedOut, (flowGlobalType) false);
 
-		TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId());
+		TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId()).detail("Excluded", m->excluded);
 
 		// FIXME: Sometimes, connections to/from this process will explicitly close
 
@@ -962,14 +1000,17 @@ public:
 
 		for (auto processInfo : getAllProcesses()) {
 			// Add non-test processes (ie. datahall is not be set for test processes)
-			if (processInfo->startingClass != ProcessClass::TesterClass) {
-				// Do not kill protected processes
-				if (protectedAddresses.count(processInfo->address))
-					processesLeft.push_back(processInfo);
-				else if (processInfo->isAvailable())
+			if (processInfo->isAvailableClass()) {
+				// Ignore excluded machines
+				if (processInfo->excluded)
+					;
+				// Mark all of the unavailable as dead
+				else if (!processInfo->isAvailable())
+					processesDead.push_back(processInfo);
+				else if (protectedAddresses.count(processInfo->address))
 					processesLeft.push_back(processInfo);
 				else
-					processesDead.push_back(processInfo);
+					processesLeft.push_back(processInfo);
 			}
 		}
 		return canKillProcesses(processesLeft, processesDead, KillInstantly, NULL);
@@ -979,16 +1020,20 @@ public:
 	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const
 	{
 		bool	canSurvive = true;
+		int		nQuorum = ((desiredCoordinators+1)/2)*2-1;
+
 		KillType	newKt = kt;
 		if ((kt == KillInstantly) || (kt == InjectFaults) || (kt == RebootAndDelete) || (kt == RebootProcessAndDelete))
 		{
 			LocalityGroup	processesLeft, processesDead;
 			std::vector<LocalityData>	localitiesDead, localitiesLeft, badCombo;
+			std::set<Optional<Standalone<StringRef>>>	uniqueMachines;
 			ASSERT(storagePolicy);
 			ASSERT(tLogPolicy);
 			for (auto processInfo : availableProcesses) {
 				processesLeft.add(processInfo->locality);
 				localitiesLeft.push_back(processInfo->locality);
+				uniqueMachines.insert(processInfo->locality.machineId());
 			}
 			for (auto processInfo : deadProcesses) {
 				processesDead.add(processInfo->locality);
@@ -1014,17 +1059,22 @@ public:
 			}
 			// Reboot and Delete if remaining machines do NOT fulfill policies
 			else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (!processesLeft.validate(tLogPolicy))) {
-				auto newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
+				newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
 				canSurvive = false;
 				TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("tLogPolicy", tLogPolicy->info()).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("RemainingZones", ::describeZones(localitiesLeft)).detail("RemainingDataHalls", ::describeDataHalls(localitiesLeft)).detail("Reason", "tLogPolicy does not validates against remaining processes.");
 			}
 			else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (!processesLeft.validate(storagePolicy))) {
-				auto newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
+				newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
 				canSurvive = false;
 				TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("storagePolicy", storagePolicy->info()).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("RemainingZones", ::describeZones(localitiesLeft)).detail("RemainingDataHalls", ::describeDataHalls(localitiesLeft)).detail("Reason", "storagePolicy does not validates against remaining processes.");
 			}
+			else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (nQuorum > uniqueMachines.size())) {
+				newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
+				canSurvive = false;
+				TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("storagePolicy", storagePolicy->info()).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("RemainingZones", ::describeZones(localitiesLeft)).detail("RemainingDataHalls", ::describeDataHalls(localitiesLeft)).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
+			}
 			else {
-				TraceEvent("CanSurviveKills").detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("DeadZones", ::describeZones(localitiesDead)).detail("DeadDataHalls", ::describeDataHalls(localitiesDead)).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
+				TraceEvent("CanSurviveKills").detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("DeadZones", ::describeZones(localitiesDead)).detail("DeadDataHalls", ::describeDataHalls(localitiesDead)).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("ZonesLeft", ::describeZones(localitiesLeft)).detail("DataHallsLeft", ::describeDataHalls(localitiesLeft)).detail("ValidateRemaining", processesLeft.validate(tLogPolicy));
 			}
 		}
 		if (newKillType) *newKillType = newKt;
@@ -1032,7 +1082,7 @@ public:
 	}
 
 	virtual void destroyProcess( ISimulator::ProcessInfo *p ) {
-		TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("zoneId", p->locality.zoneId()).backtrace();
+		TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("zoneId", p->locality.zoneId());
 		currentlyRebootingProcesses.insert(std::pair<NetworkAddress, ProcessInfo*>(p->address, p));
 		std::vector<ProcessInfo*>& processes = machines[ p->locality.zoneId().get() ].processes;
 		if( p != processes.back() ) {
@@ -1048,12 +1098,12 @@ public:
 		TEST( kt == InjectFaults ); // Simulated machine was killed with faults
 
 		if (kt == KillInstantly) {
-			TraceEvent(SevWarn, "FailMachine").detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).backtrace();
+			TraceEvent(SevWarn, "FailMachine", machine->locality.zoneId()).detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).detail("Protected", protectedAddresses.count(machine->address)).backtrace();
 			// This will remove all the "tracked" messages that came from the machine being killed
 			latestEventCache.clear();
 			machine->failed = true;
 		} else if (kt == InjectFaults) {
-			TraceEvent(SevWarn, "FaultMachine").detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).backtrace();
+			TraceEvent(SevWarn, "FaultMachine", machine->locality.zoneId()).detail("Name", machine->name).detail("Address", machine->address).detailext("ZoneId", machine->locality.zoneId()).detail("Process", describe(*machine)).detail("Rebooting", machine->rebooting).detail("Protected", protectedAddresses.count(machine->address)).backtrace();
 			should_inject_fault = simulator_should_inject_fault;
 			machine->fault_injection_r = g_random->randomUniqueID().first();
 			machine->fault_injection_p1 = 0.1;
@@ -1064,8 +1114,10 @@ public:
 		ASSERT(!protectedAddresses.count(machine->address) || machine->rebooting);
 	}
 	virtual void rebootProcess( ProcessInfo* process, KillType kt ) {
-		if( kt == RebootProcessAndDelete && protectedAddresses.count(process->address) )
+		if( kt == RebootProcessAndDelete && protectedAddresses.count(process->address) ) {
+			TraceEvent("RebootChanged").detail("ZoneId", process->locality.describeZone()).detail("KillType", RebootProcess).detail("OrigKillType", kt).detail("Reason", "Protected process");
 			kt = RebootProcess;
+		}
 		doReboot( process, kt );
 	}
 	virtual void rebootProcess(Optional<Standalone<StringRef>> zoneId, bool allProcesses ) {
@@ -1105,11 +1157,12 @@ public:
 		auto ktOrig = kt;
 		if (killIsSafe) ASSERT( kt == ISimulator::RebootAndDelete );  // Only types of "safe" kill supported so far
 
-		TEST(true); // Trying to killing a machine 
+		TEST(true); // Trying to killing a machine
 		TEST(kt == KillInstantly); // Trying to kill instantly
 		TEST(kt == InjectFaults);  // Trying to kill by injecting faults
 
 		if(speedUpSimulation && !forceKill) {
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detailext("ZoneId", zoneId).detail("Reason", "Unforced kill within speedy simulation.").backtrace();
 			return false;
 		}
 
@@ -1124,20 +1177,37 @@ public:
 				processesOnMachine++;
 		}
 
+		// Do nothing, if no processes to kill
+		if (processesOnMachine == 0) {
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detailext("ZoneId", zoneId).detail("Reason", "The target had no processes running.").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
+			return false;
+		}
+
 		// Check if machine can be removed, if requested
 		if ((kt == KillInstantly) || (kt == InjectFaults) || (kt == RebootAndDelete) || (kt == RebootProcessAndDelete))
 		{
 			std::vector<ProcessInfo*>	processesLeft, processesDead;
+			int	protectedWorker = 0, unavailable = 0, excluded = 0;
 
 			for (auto machineRec : machines) {
 				for (auto processInfo : machineRec.second.processes) {
 					// Add non-test processes (ie. datahall is not be set for test processes)
-					if (processInfo->startingClass != ProcessClass::TesterClass) {
-						if (protectedAddresses.count(processInfo->address))
-							processesLeft.push_back(processInfo);
-						else if (processInfo->isAvailable() && (machineRec.second.zoneId != zoneId)) {
-							processesLeft.push_back(processInfo);
+					if (processInfo->isAvailableClass()) {
+						// Do not include any excluded machines
+						if (processInfo->excluded) {
+							processesDead.push_back(processInfo);
+							excluded ++;
 						}
+						else if (!processInfo->isAvailable()) {
+							processesDead.push_back(processInfo);
+							unavailable ++;
+						}
+						else if (protectedAddresses.count(processInfo->address)) {
+							processesLeft.push_back(processInfo);
+							protectedWorker ++;
+						}
+						else if (machineRec.second.zoneId != zoneId)
+							processesLeft.push_back(processInfo);
 						// Add processes from dead machines and datacenter machines to dead group
 						else
 							processesDead.push_back(processInfo);
@@ -1148,16 +1218,25 @@ public:
 				if ((kt != Reboot) && (!killIsSafe)) {
 					kt = Reboot;
 				}
-				TraceEvent("ChangedKillMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
+				TraceEvent("ChangedKillMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("Protected", protectedWorker).detail("Unavailable", unavailable).detail("Excluded", excluded).detail("ProtectedTotal", protectedAddresses.size()).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
 			}
 			else if ((kt == KillInstantly) || (kt == InjectFaults)) {
 				TraceEvent("DeadMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
 				for (auto process : processesLeft) {
 					TraceEvent("DeadMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
 				}
+				for (auto process : processesDead) {
+					TraceEvent("DeadMachineVictims", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", describe(*process));
+				}
 			}
 			else {
 				TraceEvent("ClearMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info());
+				for (auto process : processesLeft) {
+					TraceEvent("ClearMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
+				}
+				for (auto process : processesDead) {
+					TraceEvent("ClearMachineVictims", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("VictimProcess", describe(*process));
+				}
 			}
 		}
 
@@ -1166,21 +1245,30 @@ public:
 		// Check if any processes on machine are rebooting
 		if( processesOnMachine != processesPerMachine && kt >= RebootAndDelete ) {
 			TEST(true); //Attempted reboot, but the target did not have all of its processes running
-			TraceEvent(SevWarn, "AbortedReboot", zoneId).detailext("ZoneId", zoneId).detail("Reason", "The target did not have all of its processes running.").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
+			return false;
+		}
+
+		// Check if any processes on machine are rebooting
+		if ( processesOnMachine != processesPerMachine) {
+			TEST(true); //Attempted reboot, but the target did not have all of its processes running
+			TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace();
 			return false;
 		}
 
 		TraceEvent("KillMachine", zoneId).detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KilledMachines", killedMachines).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig).detail("killIsSafe", killIsSafe);
 		if (kt < RebootAndDelete ) {
+			if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr)
+				killProcess_internal( machines[zoneId].machineProcess, kt );
 			for (auto& process : machines[zoneId].processes) {
-				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString());
+				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()).detail("failed", process->failed).detail("excluded", process->excluded).detail("rebooting", process->rebooting);
 				if (process->startingClass != ProcessClass::TesterClass)
 					killProcess_internal( process, kt );
 			}
 		}
 		else if ( kt == Reboot || killIsSafe) {
 			for (auto& process : machines[zoneId].processes) {
-				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString());
+				TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()).detail("failed", process->failed).detail("excluded", process->excluded).detail("rebooting", process->rebooting);
 				if (process->startingClass != ProcessClass::TesterClass)
 					doReboot(process, kt );
 			}
@@ -1196,13 +1284,16 @@ public:
 		int	dcProcesses = 0;
 
 		// Switch to a reboot, if anything protected on machine
-		for (auto& process : processes) {
-			auto processDcId = process->locality.dcId();
-			auto processZoneId = process->locality.zoneId();
+		for (auto& procRecord : processes) {
+			auto processDcId = procRecord->locality.dcId();
+			auto processZoneId = procRecord->locality.zoneId();
 			ASSERT(processZoneId.present());
 			if (processDcId.present() && (processDcId == dcId)) {
-				if (protectedAddresses.count(process->address))
+				if ((kt != Reboot) && (protectedAddresses.count(procRecord->address))) {
 					kt = Reboot;
+					TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig)
+						.detail("Reason", "Datacenter has protected process").detail("ProcessAddress", procRecord->address).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("excluded", procRecord->excluded).detail("Process", describe(*procRecord));
+				}
 				datacenterZones[processZoneId.get()] ++;
 				dcProcesses ++;
 			}
@@ -1215,13 +1306,17 @@ public:
 			for (auto machineRec : machines) {
 				for (auto processInfo : machineRec.second.processes) {
 					// Add non-test processes (ie. datahall is not be set for test processes)
-					if (processInfo->startingClass != ProcessClass::TesterClass) {
-						// Do not kill protected processes
-						if (protectedAddresses.count(processInfo->address))
+					if (processInfo->isAvailableClass()) {
+						// Mark all of the unavailable as dead
+						if (processInfo->excluded)
+							processesDead.push_back(processInfo);
+						else if (!processInfo->isAvailable())
+							processesDead.push_back(processInfo);
+						else if (protectedAddresses.count(processInfo->address))
 							processesLeft.push_back(processInfo);
-						else if (processInfo->isAvailable() && (datacenterZones.find(machineRec.second.zoneId) == datacenterZones.end())) {
+						// Keep all not in the datacenter zones
+						else if (datacenterZones.find(machineRec.second.zoneId) == datacenterZones.end())
 							processesLeft.push_back(processInfo);
-						}
 						else
 							processesDead.push_back(processInfo);
 					}
@@ -1229,7 +1324,7 @@ public:
 			}
 
 			if (!canKillProcesses(processesLeft, processesDead, kt, &kt)) {
-				TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", ktOrig).detail("NewKillType", kt);
+				TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig);
 			}
 			else {
 				TraceEvent("DeadDataCenter").detailext("DataCenter", dcId).detail("KillType", kt).detail("DcZones", datacenterZones.size()).detail("DcProcesses", dcProcesses).detail("ProcessesDead", processesDead.size()).detail("ProcessesLeft", processesLeft.size()).detail("tLogPolicy", storagePolicy->info()).detail("storagePolicy", storagePolicy->info());
@@ -1244,10 +1339,13 @@ public:
 			.detail("DcZones", datacenterZones.size())
 			.detail("DcProcesses", dcProcesses)
 			.detailext("DCID", dcId)
-			.detail("KillType", kt);
+			.detail("KillType", kt)
+			.detail("OrigKillType", ktOrig);
 
 		for (auto& datacenterZone : datacenterZones)
-			killMachine( datacenterZone.first, kt, (kt == RebootAndDelete), true);
+		killMachine( datacenterZone.first, kt, (kt == RebootAndDelete), true);
+// ahm  If above doesn't work, go conservative
+//	killMachine( datacenterZone.first, kt, false, true);
 	}
 	virtual void clogInterface( uint32_t ip, double seconds, ClogMode mode = ClogDefault ) {
 		if (mode == ClogDefault) {
@@ -1391,6 +1489,7 @@ public:
 	std::map<Optional<Standalone<StringRef>>, MachineInfo > machines;
 	std::map<NetworkAddress, ProcessInfo*> addressMap;
 	std::map<ProcessInfo*, Promise<Void>> filesDeadMap;
+	std::set<AddressExclusion>	exclusionSet;
 
 	//tasks is guarded by ISimulator::mutex
 	std::priority_queue<Task, std::vector<Task>> tasks;
@@ -1424,6 +1523,9 @@ static double networkLatency() {
 }
 
 ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) {
+	TraceEvent("RebootingProcessAttempt").detailext("ZoneId", p->locality.zoneId()).detail("KillType", kt).detail("Process", p->toString()).detail("startingClass", p->startingClass.toString()).detail("failed", p->failed).detail("excluded", p->excluded).detail("rebooting", p->rebooting).detail("TaskDefaultDelay", TaskDefaultDelay);
+//	ASSERT(p->failed); //ahm
+
 	Void _ = wait( g_sim2.delay( 0, TaskDefaultDelay, p ) ); // Switch to the machine in question
 
 	try {
@@ -1436,7 +1538,7 @@ ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) {
 
 		if( p->rebooting )
 			return;
-		TraceEvent("RebootingMachine").detail("KillType", kt).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId()).detailext("DataHall", p->locality.dataHallId()).detail("Locality", p->locality.toString());
+		TraceEvent("RebootingProcess").detail("KillType", kt).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId()).detailext("DataHall", p->locality.dataHallId()).detail("Locality", p->locality.toString()).detail("failed", p->failed).detail("excluded", p->excluded).backtrace();
 		p->rebooting = true;
 		p->shutdownSignal.send( kt );
 	} catch (Error& e) {
@@ -1503,14 +1605,20 @@ Future< Reference<class IAsyncFile> > Sim2FileSystem::open( std::string filename
 				actualFilename = filename + ".part";
 				auto partFile = machineCache.find(actualFilename);
 				if(partFile != machineCache.end()) {
-					return AsyncFileDetachable::open(partFile->second);
+					Future<Reference<IAsyncFile>> f = AsyncFileDetachable::open(partFile->second);
+					if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
+						f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+					return f;
 				}
 			}
 			//Simulated disk parameters are shared by the AsyncFileNonDurable and the underlying SimpleFile.  This way, they can both keep up with the time to start the next operation
 			Reference<DiskParameters> diskParameters(new DiskParameters(FLOW_KNOBS->SIM_DISK_IOPS, FLOW_KNOBS->SIM_DISK_BANDWIDTH));
 			machineCache[actualFilename] = AsyncFileNonDurable::open(filename, actualFilename, SimpleFile::open(filename, flags, mode, diskParameters, false), diskParameters);
 		}
-		return AsyncFileDetachable::open( machineCache[actualFilename] );
+		Future<Reference<IAsyncFile>> f = AsyncFileDetachable::open( machineCache[actualFilename] );
+		if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0)
+			f = map(f, [=](Reference<IAsyncFile> r) { return Reference<IAsyncFile>(new AsyncFileWriteChecker(r)); });
+		return f;
 	}
 	else
 		return AsyncFileCached::open(filename, flags, mode);

fdbrpc/simulator.h

@@ -64,27 +64,41 @@ public:
 
 		uint64_t fault_injection_r;
 		double fault_injection_p1, fault_injection_p2;
-		bool io_timeout_injected;
 
 		ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddress address,
 					INetworkConnections *net, const char* dataFolder, const char* coordinationFolder )
 			: name(name), locality(locality), startingClass(startingClass), address(address), dataFolder(dataFolder),
 				network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0),
 				rebooting(false), fault_injection_p1(0), fault_injection_p2(0),
-				fault_injection_r(0), machine(0), io_timeout_injected(false)
-		{}
+				fault_injection_r(0), machine(0) {}
 
 		Future<KillType> onShutdown() { return shutdownSignal.getFuture(); }
 
 		bool isReliable() const { return !failed && fault_injection_p1 == 0 && fault_injection_p2 == 0; }
 		bool isAvailable() const { return !excluded && isReliable(); }
 
+		// Returns true if the class represents an acceptable worker
+		bool isAvailableClass() const {
+			switch (startingClass._class) {
+				case ProcessClass::UnsetClass: return true;
+				case ProcessClass::StorageClass: return true;
+				case ProcessClass::TransactionClass: return true;
+				case ProcessClass::ResolutionClass: return false;
+				case ProcessClass::ProxyClass: return false;
+				case ProcessClass::MasterClass: return false;
+				case ProcessClass::TesterClass: return false;
+				case ProcessClass::StatelessClass: return false;
+				case ProcessClass::LogClass: return true;
+				default: return false;
+			}
+		}
+
 		inline flowGlobalType global(int id) { return (globals.size() > id) ? globals[id] : NULL; };
 		inline void setGlobal(size_t id, flowGlobalType v) { globals.resize(std::max(globals.size(),id+1)); globals[id] = v; };
 
 		std::string toString() const {
-			return format("name: %s  address: %d.%d.%d.%d:%d  zone: %s  datahall: %s  class: %s  coord: %s data: %s",
-			name, (address.ip>>24)&0xff, (address.ip>>16)&0xff, (address.ip>>8)&0xff, address.ip&0xff, address.port, (locality.zoneId().present() ? locality.zoneId().get().printable().c_str() : "[unset]"), (locality.dataHallId().present() ? locality.dataHallId().get().printable().c_str() : "[unset]"), startingClass.toString().c_str(), coordinationFolder, dataFolder); }
+			return format("name: %s  address: %d.%d.%d.%d:%d  zone: %s  datahall: %s  class: %s  coord: %s data: %s  excluded: %d",
+			name, (address.ip>>24)&0xff, (address.ip>>16)&0xff, (address.ip>>8)&0xff, address.ip&0xff, address.port, (locality.zoneId().present() ? locality.zoneId().get().printable().c_str() : "[unset]"), (locality.dataHallId().present() ? locality.dataHallId().get().printable().c_str() : "[unset]"), startingClass.toString().c_str(), coordinationFolder, dataFolder, excluded); }
 
 		// Members not for external use
 		Promise<KillType> shutdownSignal;
@@ -94,6 +108,7 @@ public:
 		ProcessInfo* machineProcess;
 		std::vector<ProcessInfo*> processes;
 		std::map<std::string, Future<Reference<IAsyncFile>>> openFiles;
+		std::set<std::string> deletingFiles;
 		std::set<std::string> closingFiles;
 		Optional<Standalone<StringRef>>	zoneId;
 
@@ -134,6 +149,84 @@ public:
 	//virtual KillType getMachineKillState( UID zoneID ) = 0;
 	virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const = 0;
 	virtual bool isAvailable() const = 0;
+	virtual void displayWorkers() const;
+
+	virtual void addRole(NetworkAddress const& address, std::string const& role) {
+		roleAddresses[address][role] ++;
+		TraceEvent("RoleAdd").detail("Address", address).detail("Role", role).detail("Roles", roleAddresses[address].size()).detail("Value", roleAddresses[address][role]);
+	}
+
+	virtual void removeRole(NetworkAddress const& address, std::string const& role) {
+		auto addressIt = roleAddresses.find(address);
+		if (addressIt != roleAddresses.end()) {
+			auto rolesIt = addressIt->second.find(role);
+			if (rolesIt != addressIt->second.end()) {
+				if (rolesIt->second > 1) {
+					rolesIt->second --;
+					TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", addressIt->second.size()).detail("Value", rolesIt->second).detail("Result", "Decremented Role");
+				}
+				else {
+					addressIt->second.erase(rolesIt);
+					if (addressIt->second.size()) {
+						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", addressIt->second.size()).detail("Value", 0).detail("Result", "Removed Role");
+					}
+					else {
+						roleAddresses.erase(addressIt);
+						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", 0).detail("Value", 0).detail("Result", "Removed Address");
+					}
+				}
+			}
+			else {
+				TraceEvent(SevWarn,"RoleRemove").detail("Address", address).detail("Role", role).detail("Result", "Role Missing");
+			}
+		}
+		else {
+			TraceEvent(SevWarn,"RoleRemove").detail("Address", address).detail("Role", role).detail("Result", "Address Missing");
+		}
+	}
+
+	virtual std::string getRoles(NetworkAddress const& address, bool skipWorkers = true) const {
+		auto addressIt = roleAddresses.find(address);
+		std::string roleText;
+		if (addressIt != roleAddresses.end()) {
+			for (auto& roleIt : addressIt->second) {
+				if ((!skipWorkers) || (roleIt.first != "Worker"))
+					roleText += roleIt.first + ((roleIt.second > 1) ? format("-%d ", roleIt.second) : " ");
+			}
+		}
+		if (roleText.empty())
+				roleText = "[unset]";
+		return roleText;
+	}
+
+	virtual void excludeAddress(NetworkAddress const& address) {
+		excludedAddresses[address]++;
+		TraceEvent("ExcludeAddress").detail("Address", address).detail("Value", excludedAddresses[address]);
+	}
+
+	virtual void includeAddress(NetworkAddress const& address) {
+		auto addressIt = excludedAddresses.find(address);
+		if (addressIt != excludedAddresses.end()) {
+			if (addressIt->second > 1) {
+				addressIt->second --;
+				TraceEvent("IncludeAddress").detail("Address", address).detail("Value", addressIt->second).detail("Result", "Decremented");
+			}
+			else {
+				excludedAddresses.erase(addressIt);
+				TraceEvent("IncludeAddress").detail("Address", address).detail("Value", 0).detail("Result", "Removed");
+			}
+		}
+		else {
+			TraceEvent(SevWarn,"IncludeAddress").detail("Address", address).detail("Result", "Missing");
+		}
+	}
+	virtual void includeAllAddresses() {
+		TraceEvent("IncludeAddressAll").detail("AddressTotal", excludedAddresses.size());
+		excludedAddresses.clear();
+	}
+	virtual bool isExcluded(NetworkAddress const& address) const {
+		return excludedAddresses.find(address) != excludedAddresses.end();
+	}
 
 	virtual void disableSwapToMachine(Optional<Standalone<StringRef>> zoneId ) {
 		swapsDisabled.insert(zoneId);
@@ -201,6 +294,8 @@ protected:
 
 private:
 	std::set<Optional<Standalone<StringRef>>> swapsDisabled;
+	std::map<NetworkAddress, int> excludedAddresses;
+	std::map<NetworkAddress, std::map<std::string, int>> roleAddresses;
 	bool allSwapsDisabled;
 };
 

fdbserver/ApplyMetadataMutation.h

@@ -25,10 +25,10 @@
 #include "fdbclient/MutationList.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/BackupAgent.h"
+#include "fdbclient/Notified.h"
 #include "IKeyValueStore.h"
 #include "LogSystem.h"
 #include "LogProtocolMessage.h"
-#include "flow/Notified.h"
 
 static bool isMetadataMutation(MutationRef const& m) {
 	// FIXME: This is conservative - not everything in system keyspace is necessarily processed by applyMetadataMutations

fdbserver/ClusterController.actor.cpp

@@ -218,6 +218,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 		std::vector<LocalityData>							unavailableLocals;
 		LocalitySetRef																					logServerSet;
 		LocalityMap<std::pair<WorkerInterface, ProcessClass>>*	logServerMap;
+		UID 		functionId = g_nondeterministic_random->randomUniqueID();
 		bool		bCompleted = false;
 
 		logServerSet = Reference<LocalitySet>(new LocalityMap<std::pair<WorkerInterface, ProcessClass>>());
@@ -230,7 +231,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			}
 			else {
 				if (it.second.interf.locality.dataHallId().present())
-					TraceEvent(SevWarn,"GWFTADNotAvailable", id)
+					TraceEvent(SevWarn,"GWFTADNotAvailable", functionId)
 						.detail("Fitness", fitness)
 						.detailext("Zone", it.second.interf.locality.zoneId())
 						.detailext("DataHall", it.second.interf.locality.dataHallId())
@@ -243,7 +244,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 						.detail("Locality", it.second.interf.locality.toString())
 						.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 				unavailableLocals.push_back(it.second.interf.locality);
 			}
 		}
@@ -258,12 +260,13 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 				logServerMap->add(worker.first.locality, &worker);
 			}
 			if (logServerSet->size() < conf.tLogReplicationFactor) {
-				TraceEvent(SevWarn,"GWFTADTooFew", id)
+				TraceEvent(SevWarn,"GWFTADTooFew", functionId)
 					.detail("Fitness", fitness)
 					.detail("Processes", logServerSet->size())
 					.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 					.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-					.detail("DesiredLogs", conf.getDesiredLogs());
+					.detail("DesiredLogs", conf.getDesiredLogs())
+					.detail("InterfaceId", id);
 			}
 			else if (logServerSet->size() <= conf.getDesiredLogs()) {
 				ASSERT(conf.tLogPolicy);
@@ -275,12 +278,13 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 					break;
 				}
 				else {
-					TraceEvent(SevWarn,"GWFTADNotAcceptable", id)
+					TraceEvent(SevWarn,"GWFTADNotAcceptable", functionId)
 						.detail("Fitness", fitness)
 						.detail("Processes", logServerSet->size())
 						.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 				}
 			}
 			// Try to select the desired size, if larger
@@ -300,7 +304,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 						results.push_back(*object);
 						tLocalities.push_back(object->first.locality);
 					}
-					TraceEvent("GWFTADBestResults", id)
+					TraceEvent("GWFTADBestResults", functionId)
 						.detail("Fitness", fitness)
 						.detail("Processes", logServerSet->size())
 						.detail("BestCount", bestSet.size())
@@ -308,17 +312,19 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 						.detail("BestDataHalls", ::describeDataHalls(tLocalities))
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
 						.detail("TotalResults", results.size())
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 					bCompleted = true;
 					break;
 				}
 				else {
-					TraceEvent(SevWarn,"GWFTADNoBest", id)
+					TraceEvent(SevWarn,"GWFTADNoBest", functionId)
 						.detail("Fitness", fitness)
 						.detail("Processes", logServerSet->size())
 						.detail("tLogReplicationFactor", conf.tLogReplicationFactor)
 						.detail("tLogPolicy", conf.tLogPolicy ? conf.tLogPolicy->info() : "[unset]")
-						.detail("DesiredLogs", conf.getDesiredLogs());
+						.detail("DesiredLogs", conf.getDesiredLogs())
+						.detail("InterfaceId", id);
 				}
 			}
 		}
@@ -331,7 +337,7 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 					tLocalities.push_back(object->first.locality);
 				}
 
-				TraceEvent(SevWarn, "GetTLogTeamFailed")
+				TraceEvent(SevWarn, "GetTLogTeamFailed", functionId)
 					.detail("Policy", conf.tLogPolicy->info())
 					.detail("Processes", logServerSet->size())
 					.detail("Workers", id_worker.size())
@@ -344,7 +350,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 					.detail("DesiredLogs", conf.getDesiredLogs())
 					.detail("RatingTests",SERVER_KNOBS->POLICY_RATING_TESTS)
 					.detail("checkStable", checkStable)
-					.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS).backtrace();
+					.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS)
+					.detail("InterfaceId", id).backtrace();
 
 			// Free the set
 			logServerSet->clear();
@@ -356,14 +363,25 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			id_used[result.first.locality.processId()]++;
 		}
 
-		TraceEvent("GetTLogTeamDone")
+		TraceEvent("GetTLogTeamDone", functionId)
 			.detail("Completed", bCompleted).detail("Policy", conf.tLogPolicy->info())
 			.detail("Results", results.size()).detail("Processes", logServerSet->size())
 			.detail("Workers", id_worker.size())
 			.detail("Replication", conf.tLogReplicationFactor)
 			.detail("Desired", conf.getDesiredLogs())
 			.detail("RatingTests",SERVER_KNOBS->POLICY_RATING_TESTS)
-			.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS);
+			.detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS)
+			.detail("InterfaceId", id);
+
+		for (auto& result : results) {
+			TraceEvent("GetTLogTeamWorker", functionId)
+				.detail("Class", result.second.toString())
+				.detail("Address", result.first.address())
+				.detailext("Zone", result.first.locality.zoneId())
+				.detailext("DataHall", result.first.locality.dataHallId())
+				.detail("isExcludedServer", conf.isExcludedServer(result.first.address()))
+				.detail("isAvailable", IFailureMonitor::failureMonitor().getState(result.first.storage.getEndpoint()).isAvailable());
+		}
 
 		// Free the set
 		logServerSet->clear();
@@ -452,28 +470,23 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 		ProcessClass::Fitness resolverFit;
 		int proxyCount;
 		int resolverCount;
-		int datacenters;
 
-		InDatacenterFitness( ProcessClass::Fitness proxyFit, ProcessClass::Fitness resolverFit, int proxyCount, int resolverCount, int datacenters)
-			: proxyFit(proxyFit), resolverFit(resolverFit), proxyCount(proxyCount), resolverCount(resolverCount), datacenters(datacenters) {}
+		InDatacenterFitness( ProcessClass::Fitness proxyFit, ProcessClass::Fitness resolverFit, int proxyCount, int resolverCount)
+			: proxyFit(proxyFit), resolverFit(resolverFit), proxyCount(proxyCount), resolverCount(resolverCount) {}
 
-		InDatacenterFitness() : proxyFit( ProcessClass::NeverAssign ), resolverFit( ProcessClass::NeverAssign ), datacenters(10000000) {}
+		InDatacenterFitness() : proxyFit( ProcessClass::NeverAssign ), resolverFit( ProcessClass::NeverAssign ) {}
 
 		InDatacenterFitness( vector<std::pair<WorkerInterface, ProcessClass>> proxies, vector<std::pair<WorkerInterface, ProcessClass>> resolvers ) {
-			std::set<Optional<Standalone<StringRef>>> dcs;
 			proxyFit = ProcessClass::BestFit;
 			resolverFit = ProcessClass::BestFit;
 			for(auto it: proxies) {
-				dcs.insert(it.first.locality.dcId());
 				proxyFit = std::max(proxyFit, it.second.machineClassFitness( ProcessClass::Proxy ));
 			}
 			for(auto it: resolvers) {
-				dcs.insert(it.first.locality.dcId());
 				resolverFit = std::max(resolverFit, it.second.machineClassFitness( ProcessClass::Resolver ));
 			}
 			proxyCount = proxies.size();
 			resolverCount = resolvers.size();
-			datacenters = dcs.size();
 		}
 
 		InDatacenterFitness( vector<MasterProxyInterface> proxies, vector<ResolverInterface> resolvers, vector<ProcessClass> proxyClasses, vector<ProcessClass> resolverClasses ) {
@@ -491,12 +504,9 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 
 			proxyCount = proxies.size();
 			resolverCount = resolvers.size();
-			datacenters = dcs.size();
 		}
 
 		bool operator < (InDatacenterFitness const& r) const {
-			if(datacenters != r.datacenters) return datacenters < r.datacenters;
-
 			int lmax = std::max(resolverFit,proxyFit);
 			int lmin = std::min(resolverFit,proxyFit);
 			int rmax = std::max(r.resolverFit,r.proxyFit);
@@ -508,17 +518,16 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			return resolverCount > r.resolverCount;
 		}
 
-		bool operator == (InDatacenterFitness const& r) const { return proxyFit == r.proxyFit && resolverFit == r.resolverFit && datacenters == r.datacenters && proxyCount == r.proxyCount && resolverCount == r.resolverCount; }
+		bool operator == (InDatacenterFitness const& r) const { return proxyFit == r.proxyFit && resolverFit == r.resolverFit && proxyCount == r.proxyCount && resolverCount == r.resolverCount; }
 	};
 
 	struct AcrossDatacenterFitness {
 		ProcessClass::Fitness tlogFit;
 		int tlogCount;
-		int datacenters;
 
-		AcrossDatacenterFitness( ProcessClass::Fitness tlogFit, int tlogCount, int datacenters ) : tlogFit(tlogFit), tlogCount(tlogCount), datacenters(datacenters) {}
+		AcrossDatacenterFitness( ProcessClass::Fitness tlogFit, int tlogCount) : tlogFit(tlogFit), tlogCount(tlogCount) {}
 
-		AcrossDatacenterFitness() : tlogFit( ProcessClass::NeverAssign ), datacenters(0), tlogCount(0) {}
+		AcrossDatacenterFitness() : tlogFit( ProcessClass::NeverAssign ), tlogCount(0) {}
 
 		AcrossDatacenterFitness( vector<std::pair<WorkerInterface, ProcessClass>> tlogs ) {
 			std::set<Optional<Standalone<StringRef>>> dcs;
@@ -527,7 +536,6 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 				dcs.insert(it.first.locality.dcId());
 				tlogFit = std::max(tlogFit, it.second.machineClassFitness( ProcessClass::TLog ));
 			}
-			datacenters = dcs.size();
 			tlogCount = tlogs.size();
 		}
 
@@ -539,17 +547,15 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 				dcs.insert(tlogs[i].interf().locality.dcId());
 				tlogFit = std::max(tlogFit, processClasses[i].machineClassFitness( ProcessClass::TLog ));
 			}
-			datacenters = dcs.size();
 			tlogCount = tlogs.size();
 		}
 
 		bool operator < (AcrossDatacenterFitness const& r) const {
 			if(tlogFit != r.tlogFit) return tlogFit < r.tlogFit;
-			if(tlogCount != r.tlogCount) return tlogCount > r.tlogCount;
-			return datacenters > r.datacenters;
+			return tlogCount > r.tlogCount;
 		}
 
-		bool operator == (AcrossDatacenterFitness const& r) const { return datacenters == r.datacenters && tlogFit == r.tlogFit && tlogCount == r.tlogCount; }
+		bool operator == (AcrossDatacenterFitness const& r) const { return tlogFit == r.tlogFit && tlogCount == r.tlogCount; }
 	};
 
 	std::set<Optional<Standalone<StringRef>>> getDatacenters( DatabaseConfiguration const& conf, bool checkStable = false ) {
@@ -613,8 +619,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 			.detail("desiredResolvers", req.configuration.getDesiredResolvers()).detail("actualResolvers", result.resolvers.size());
 
 		if( now() - startTime < SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY &&
-			( AcrossDatacenterFitness(tlogs) > AcrossDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs(), req.configuration.minDataCenters) ||
-			bestFitness > InDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_PROXY_FITNESS, (ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, req.configuration.getDesiredProxies(), req.configuration.getDesiredResolvers(), 1) ) ) {
+			( AcrossDatacenterFitness(tlogs) > AcrossDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs()) ||
+			bestFitness > InDatacenterFitness((ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_PROXY_FITNESS, (ProcessClass::Fitness)SERVER_KNOBS->EXPECTED_RESOLVER_FITNESS, req.configuration.getDesiredProxies(), req.configuration.getDesiredResolvers()) ) ) {
 			throw operation_failed();
 		}
 
@@ -697,10 +703,8 @@ std::vector<std::pair<WorkerInterface, ProcessClass>> getWorkersForTlogsAcrossDa
 		if(oldInFit < newInFit) return false;
 		if(oldMasterFit > newMasterFit || oldAcrossFit > newAcrossFit || oldInFit > newInFit) {
 			TraceEvent("BetterMasterExists", id).detail("oldMasterFit", oldMasterFit).detail("newMasterFit", newMasterFit)
-				.detail("oldAcrossFitD", oldAcrossFit.datacenters).detail("newAcrossFitD", newAcrossFit.datacenters)
 				.detail("oldAcrossFitC", oldAcrossFit.tlogCount).detail("newAcrossFitC", newAcrossFit.tlogCount)
 				.detail("oldAcrossFitT", oldAcrossFit.tlogFit).detail("newAcrossFitT", newAcrossFit.tlogFit)
-				.detail("oldInFitD", oldInFit.datacenters).detail("newInFitD", newInFit.datacenters)
 				.detail("oldInFitP", oldInFit.proxyFit).detail("newInFitP", newInFit.proxyFit)
 				.detail("oldInFitR", oldInFit.resolverFit).detail("newInFitR", newInFit.resolverFit)
 				.detail("oldInFitPC", oldInFit.proxyCount).detail("newInFitPC", newInFit.proxyCount)

fdbserver/DataDistribution.actor.cpp

@@ -438,7 +438,6 @@ struct DDTeamCollection {
 	Database cx;
 	UID masterId;
 	int teamSize;
-	int minDataCenters, desiredDataCenters;
 	IRepPolicyRef replicationPolicy;
 	KeyValueStoreType storeType;
 
@@ -481,15 +480,14 @@ struct DDTeamCollection {
 		PromiseStream<RelocateShard> const& output,
 		Reference<ShardsAffectedByTeamFailure> const& shardsAffectedByTeamFailure,
 		int teamSize,
-		int minDataCenters,
-		int desiredDataCenters,
 		IRepPolicyRef replicationPolicy,
 		KeyValueStoreType storeType,
-		PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > const& serverChanges )
+		PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > const& serverChanges,
+		Future<Void> readyToStart )
 		:cx(cx), masterId(masterId), lock(lock), output(output), shardsAffectedByTeamFailure(shardsAffectedByTeamFailure), doBuildTeams( true ), teamBuilder( Void() ),
-		 teamSize( teamSize ), minDataCenters( minDataCenters ), desiredDataCenters( desiredDataCenters ), replicationPolicy(replicationPolicy), storeType( storeType ), serverChanges(serverChanges),
+		 teamSize( teamSize ), replicationPolicy(replicationPolicy), storeType( storeType ), serverChanges(serverChanges),
 		 initialFailureReactionDelay( delay( BUGGIFY ? 0 : SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskDataDistribution  ) ), healthyTeamCount( 0 ),
-		 initializationDoneActor(logOnCompletion(initialFailureReactionDelay, this)), optimalTeamCount( 0 ), recruitingStream(0), restartRecruiting( SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY ),
+		 initializationDoneActor(logOnCompletion(readyToStart && initialFailureReactionDelay, this)), optimalTeamCount( 0 ), recruitingStream(0), restartRecruiting( SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY ),
 		 unhealthyServers(0)
 	{
 		TraceEvent("DDTrackerStarting", masterId)
@@ -958,7 +956,6 @@ struct DDTeamCollection {
 		int serverCount = 0;
 		int uniqueDataCenters = 0;
 		int uniqueMachines = 0;
-		std::set<Optional<Standalone<StringRef>>> dataCenters;
 		std::set<Optional<Standalone<StringRef>>> machines;
 
 		for(auto i = self->server_info.begin(); i != self->server_info.end(); ++i) {
@@ -966,15 +963,12 @@ struct DDTeamCollection {
 				++serverCount;
 				LocalityData& serverLocation = i->second->lastKnownInterface.locality;
 				machines.insert( serverLocation.zoneId() );
-				// Only add the datacenter if it's set or we don't care (less than two datacenters targeted)
-				if( serverLocation.dcId().present() || self->desiredDataCenters < 2 )
-					dataCenters.insert( serverLocation.dcId() );
 			}
 		}
 		uniqueMachines = machines.size();
 
 		// If there are too few machines to even build teams or there are too few represented datacenters, build no new teams
-		if( uniqueMachines >= self->teamSize && dataCenters.size() >= self->minDataCenters ) {
+		if( uniqueMachines >= self->teamSize ) {
 			desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER*serverCount;
 
 			// Count only properly sized teams against the desired number of teams. This is to prevent "emergency" merged teams (see MoveKeys)
@@ -989,7 +983,7 @@ struct DDTeamCollection {
 			}
 
 			TraceEvent("BuildTeamsBegin", self->masterId).detail("DesiredTeams", desiredTeams).detail("UniqueMachines", uniqueMachines)
-				.detail("TeamSize", self->teamSize).detail("Servers", serverCount).detail("DataCenters", dataCenters.size())
+				.detail("TeamSize", self->teamSize).detail("Servers", serverCount)
 				.detail("CurrentTrackedTeams", self->teams.size()).detail("TeamCount", teamCount);
 
 			if( desiredTeams > teamCount ) {
@@ -1767,14 +1761,13 @@ ACTOR Future<Void> dataDistributionTeamCollection(
 	Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure,
 	MoveKeysLock lock,
 	PromiseStream<RelocateShard> output,
-	UID masterId, int teamSize, int minDataCenters, int desiredDataCenters,
+	UID masterId, int teamSize,
 	IRepPolicyRef replicationPolicy,
 	KeyValueStoreType storeType,
 	PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges,
 	Future<Void> readyToStart )
 {
-	state DDTeamCollection self( cx, masterId, lock, output, shardsAffectedByTeamFailure, teamSize, minDataCenters,
-		desiredDataCenters, replicationPolicy, storeType, serverChanges );
+	state DDTeamCollection self( cx, masterId, lock, output, shardsAffectedByTeamFailure, teamSize, replicationPolicy, storeType, serverChanges, readyToStart );
 
 	state Future<Void> loggingTrigger = Void();
 	state PromiseStream<Void> serverRemoved;
@@ -1885,54 +1878,6 @@ ACTOR Future<bool> isDataDistributionEnabled( Database cx ) {
 	}
 }
 
-ACTOR Future<int> disableDataDistribution( Database cx ) {
-	state Transaction tr(cx);
-	state int oldMode = -1;
-	state BinaryWriter wr(Unversioned());
-	wr << 0;
-
-	loop {
-		try {
-			Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
-			if (oldMode < 0) {
-				oldMode = 1;
-				if (old.present()) {
-					BinaryReader rd(old.get(), Unversioned());
-					rd >> oldMode;
-				}
-			}
-			// SOMEDAY: Write a wrapper in MoveKeys.h
-			BinaryWriter wrMyOwner(Unversioned()); wrMyOwner << dataDistributionModeLock;
-			tr.set( moveKeysLockOwnerKey, wrMyOwner.toStringRef() );
-			tr.set( dataDistributionModeKey, wr.toStringRef() );
-
-			Void _ = wait( tr.commit() );
-			return oldMode;
-		} catch (Error& e) {
-			TraceEvent("disableDDModeRetrying").error(e);
-			Void _ = wait ( tr.onError(e) );
-		}
-	}
-}
-
-ACTOR Future<Void> enableDataDistribution( Database cx, int mode ) {
-	state Transaction tr(cx);
-	state BinaryWriter wr(Unversioned());
-	wr << mode;
-
-	loop {
-		try {
-			Optional<Value> old = wait( tr.get( dataDistributionModeKey ) );
-			tr.set( dataDistributionModeKey, wr.toStringRef() );
-			Void _ = wait( tr.commit() );
-			return Void();
-		} catch (Error& e) {
-			TraceEvent("enableDDModeRetrying").error(e);
-			Void _ = wait( tr.onError(e) );
-		}
-	}
-}
-
 //Ensures that the serverKeys key space is properly coalesced
 //This method is only used for testing and is not implemented in a manner that is safe for large databases
 ACTOR Future<Void> debugCheckCoalescing(Database cx) {
@@ -2120,8 +2065,7 @@ ACTOR Future<Void> dataDistribution(
 			actors.push_back( popOldTags( cx, logSystem, recoveryCommitVersion) );
 			actors.push_back( reportErrorsExcept( dataDistributionTracker( initData, cx, shardsAffectedByTeamFailure, output, getShardMetrics, getAverageShardBytes.getFuture(), readyToStart, mi.id() ), "DDTracker", mi.id(), &normalDDQueueErrors() ) );
 			actors.push_back( reportErrorsExcept( dataDistributionQueue( cx, output, getShardMetrics, tci, shardsAffectedByTeamFailure, lock, getAverageShardBytes, mi, configuration.storageTeamSize, configuration.durableStorageQuorum, lastLimited ), "DDQueue", mi.id(), &normalDDQueueErrors() ) );
-			actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( initData, tci, cx, db, shardsAffectedByTeamFailure, lock, output, mi.id(), configuration.storageTeamSize, configuration.minDataCenters,
-				configuration.desiredDataCenters, configuration.storagePolicy, configuration.storageServerStoreType, serverChanges, readyToStart.getFuture() ), "DDTeamCollection", mi.id(), &normalDDQueueErrors() ) );
+			actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( initData, tci, cx, db, shardsAffectedByTeamFailure, lock, output, mi.id(), configuration.storageTeamSize, configuration.storagePolicy, configuration.storageServerStoreType, serverChanges, readyToStart.getFuture() ), "DDTeamCollection", mi.id(), &normalDDQueueErrors() ) );
 
 			Void _ = wait( waitForAll( actors ) );
 			return Void();
@@ -2153,11 +2097,10 @@ DDTeamCollection* testTeamCollection(int teamSize, IRepPolicyRef policy, int pro
 		PromiseStream<RelocateShard>(),
 		Reference<ShardsAffectedByTeamFailure>(new ShardsAffectedByTeamFailure()),
 		teamSize,
-		-1,
-		-1,
 		policy,
 		KeyValueStoreType(),
-		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>()
+		PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>(),
+		Future<Void>(Void())
 	);
 
 	for(int id = 1; id <= processCount; id++) {

fdbserver/DataDistribution.h

@@ -210,7 +210,4 @@ struct ShardSizeBounds {
 ShardSizeBounds getShardSizeBounds(KeyRangeRef shard, int64_t maxShardSize);
 
 //Determines the maximum shard size based on the size of the database
-int64_t getMaxShardSize( double dbSizeEstimate );
-
-Future<Void> enableDataDistribution( Database const& cx, int const& mode );
-Future<int> disableDataDistribution( Database const& cx );
+int64_t getMaxShardSize( double dbSizeEstimate );

fdbserver/DataDistributionTracker.actor.cpp

@@ -376,15 +376,6 @@ Future<Void> shardMerger(
 	TEST(true);  // shard to be merged
 	ASSERT( keys.begin > allKeys.begin );
 
-	// We must not merge the keyServers shard
-	if (keys.begin == keyServersPrefix) {
-		TraceEvent(SevError, "LastShardMerge", self->masterId)
-			.detail("ShardKeyBegin", printable(keys.begin))
-			.detail("ShardKeyEnd", printable(keys.end))
-			.detail("TrackerID", trackerId);
-		ASSERT(false);
-	}
-
 	// This will merge shards both before and after "this" shard in keyspace.
 	int shardsMerged = 1;
 	bool forwardComplete = false;
@@ -394,7 +385,7 @@ Future<Void> shardMerger(
 	loop {
 		Optional<StorageMetrics> newMetrics;
 		if( !forwardComplete ) {
-			if( nextIter->range().end == keyServersPrefix ) {
+			if( nextIter->range().end == allKeys.end ) {
 				forwardComplete = true;
 				continue;
 			}
@@ -610,14 +601,6 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker *self,
 	state int lastBegin = -1;
 	state vector<UID> last;
 
-	//The ending shard does not have a shardTracker, so instead just track the size of the shard
-	Reference<AsyncVar<Optional<StorageMetrics>>> endShardSize( new AsyncVar<Optional<StorageMetrics>>() );
-	KeyRangeRef endShardRange( keyServersPrefix, allKeys.end );
-	ShardTrackedData endShardData;
-	endShardData.stats = endShardSize;
-	endShardData.trackBytes = trackShardBytes( self, endShardRange, endShardSize, g_random->randomUniqueID(), false );
-	self->shards.insert( endShardRange, endShardData );
-
 	state int s;
 	for(s=0; s<initData->shards.size(); s++) {
 		state InitialDataDistribution::Team src = initData->shards[s].value.first;
@@ -637,8 +620,7 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker *self,
 
 			if (lastBegin >= 0) {
 				state KeyRangeRef keys( initData->shards[lastBegin].begin, initData->shards[s].begin );
-				if (keys.begin < keyServersPrefix) // disallow spliting of keyServers shard
-					restartShardTrackers( self, keys );
+				restartShardTrackers( self, keys );
 				shardsAffectedByTeamFailure->defineShard( keys );
 				shardsAffectedByTeamFailure->moveShard( keys, last );
 			}
@@ -648,7 +630,7 @@ ACTOR Future<Void> trackInitialShards(DataDistributionTracker *self,
 		Void _ = wait( yield( TaskDataDistribution ) );
 	}
 
-	Future<Void> initialSize = changeSizes( self, KeyRangeRef(allKeys.begin, keyServersPrefix), 0 );
+	Future<Void> initialSize = changeSizes( self, KeyRangeRef(allKeys.begin, allKeys.end), 0 );
 	self->readyToStart.send(Void());
 	Void _ = wait( initialSize );
 	self->maxShardSizeUpdater = updateMaxShardSize( self->cx->dbName, self->dbSizeEstimate, self->maxShardSize );

fdbserver/DatabaseConfiguration.cpp

@@ -29,7 +29,6 @@ DatabaseConfiguration::DatabaseConfiguration()
 void DatabaseConfiguration::resetInternal() {
 	// does NOT reset rawConfiguration
 	initialized = false;
-	minDataCenters = desiredDataCenters = -1;
 	masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = durableStorageQuorum = storageTeamSize = -1;
 	tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
 	autoMasterProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES;
@@ -56,9 +55,6 @@ void DatabaseConfiguration::setDefaultReplicationPolicy() {
 
 bool DatabaseConfiguration::isValid() const {
 	return initialized &&
-		minDataCenters >= 1 &&
-		desiredDataCenters >= 1 &&
-		minDataCenters <= desiredDataCenters &&
 		tLogWriteAntiQuorum >= 0 &&
 		tLogReplicationFactor >= 1 &&
 		durableStorageQuorum >= 1 &&
@@ -81,19 +77,18 @@ std::map<std::string, std::string> DatabaseConfiguration::toMap() const {
 	std::map<std::string, std::string> result;
 
 	if( initialized ) {
-		if( tLogReplicationFactor == durableStorageQuorum &&
-			durableStorageQuorum == storageTeamSize &&
+		if( durableStorageQuorum == storageTeamSize &&
 			tLogWriteAntiQuorum == 0 ) {
-			if( durableStorageQuorum == 1 && desiredDataCenters == 1 && minDataCenters == 1 )
+			if( tLogReplicationFactor == 1 && durableStorageQuorum == 1 )
 				result["redundancy_mode"] = "single";
-			else if( durableStorageQuorum == 2 && desiredDataCenters == 1 && minDataCenters == 1 )
+			else if( tLogReplicationFactor == 2 && durableStorageQuorum == 2 )
 				result["redundancy_mode"] = "double";
-			else if( durableStorageQuorum == 3 && desiredDataCenters == 1 && minDataCenters == 1 )
+			else if( tLogReplicationFactor == 3 && durableStorageQuorum == 3 )
 				result["redundancy_mode"] = "triple";
-			else if( durableStorageQuorum == 3 && desiredDataCenters == 2 && minDataCenters == 1 )
-				result["redundancy_mode"] = "two_datacenter";
-			else if( durableStorageQuorum == 3 && desiredDataCenters == 3 && minDataCenters == 2 )
-				result["redundancy_mode"] = "three_datacenter";
+			else if( tLogReplicationFactor == 3 && durableStorageQuorum == 2 )
+				result["redundancy_mode"] = "fast_recovery_double";
+			else if( tLogReplicationFactor == 4 && durableStorageQuorum == 3 )
+				result["redundancy_mode"] = "fast_recovery_triple";
 			else
 				result["redundancy_mode"] = "custom";
 		} else
@@ -129,8 +124,6 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
 	int type;
 
 	if (ck == LiteralStringRef("initialized")) initialized = true;
-	else if (ck == LiteralStringRef("min_replica_datacenters")) parse(&minDataCenters, value);
-	else if (ck == LiteralStringRef("replica_datacenters")) parse(&desiredDataCenters, value);
 	else if (ck == LiteralStringRef("proxies")) parse(&masterProxyCount, value);
 	else if (ck == LiteralStringRef("resolvers")) parse(&resolverCount, value);
 	else if (ck == LiteralStringRef("logs")) parse(&desiredTLogCount, value);

fdbserver/DatabaseConfiguration.h

@@ -43,14 +43,11 @@ struct DatabaseConfiguration {
 	std::string toString() const;
 	std::map<std::string, std::string> toMap() const;
 
-	// SOMEDAY: think about changing desiredDataCenters to minDataCenters
 	// SOMEDAY: think about changing storageTeamSize to durableStorageQuorum
-	int32_t minMachinesRequired() const { return std::max(tLogReplicationFactor, std::max(storageTeamSize, desiredDataCenters)); }
+	int32_t minMachinesRequired() const { return std::max(tLogReplicationFactor, storageTeamSize); }
 	int32_t maxMachineFailuresTolerated() const { return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, durableStorageQuorum - 1); }
 
 	// Redundancy Levels
-	int32_t minDataCenters;
-	int32_t desiredDataCenters;
 	IRepPolicyRef storagePolicy;
 
 	// MasterProxy Servers

fdbserver/DiskQueue.actor.cpp

@@ -298,11 +298,11 @@ public:
 				.detail("File0Size", self->files[0].size).detail("File1Size", self->files[1].size)
 				.detail("File0Name", self->files[0].dbgFilename).detail("SyncedFiles", syncFiles.size());*/
 
-			committed.send(Void());
 			if(g_random->random01() < 0.01) {
 				//occasionally delete all the ready future in the AndFuture
 				self->lastCommit.cleanup();
 			}
+			committed.send(Void());
 		} catch (Error& e) {
 			delete pageMem;
 			TEST(true);  // push error
@@ -405,8 +405,8 @@ public:
 				TraceEvent("DiskQueueShutdownDeleting", self->dbgid)
 					.detail("File0", self->filename(0))
 					.detail("File1", self->filename(1));
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename(0), false ) );
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename(1), true ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename(0), false ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename(1), true ) );
 			}
 			TraceEvent("DiskQueueShutdownComplete", self->dbgid)
 				.detail("DeleteFiles", deleteFiles)
@@ -419,8 +419,8 @@ public:
 		}
 
 		if( error.code() != error_code_actor_cancelled ) {
-			if (!self->stopped.isSet()) self->stopped.send(Void());
-			if (!self->error.isSet()) self->error.send(Never());
+			if (self->stopped.canBeSet()) self->stopped.send(Void());
+			if (self->error.canBeSet()) self->error.send(Never());
 			delete self;
 		}
 	}

fdbserver/KeyValueStoreMemory.actor.cpp

@@ -22,8 +22,8 @@
 #include "IKeyValueStore.h"
 #include "IDiskQueue.h"
 #include "flow/IndexedSet.h"
-#include "flow/Notified.h"
 #include "flow/ActorCollection.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
 
 #define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1)
@@ -705,4 +705,4 @@ IKeyValueStore* keyValueStoreMemory( std::string const& basename, UID logID, int
 
 IKeyValueStore* keyValueStoreLogSystem( class IDiskQueue* queue, UID logID, int64_t memoryLimit, bool disableSnapshot ) {
 	return new KeyValueStoreMemory( queue, logID, memoryLimit, disableSnapshot );
-}
+}

fdbserver/KeyValueStoreSQLite.actor.cpp

@@ -65,9 +65,7 @@ struct SpringCleaningStats {
 };
 
 struct PageChecksumCodec {
-	PageChecksumCodec(std::string const &filename, int verifyChecksumHistorySize) : pageSize(0), reserveSize(0), filename(filename), silent(false) {
-		checksumHistory.resize(verifyChecksumHistorySize);
-	}
+	PageChecksumCodec(std::string const &filename) : pageSize(0), reserveSize(0), filename(filename), silent(false) {}
 
 	int pageSize;
 	int reserveSize;
@@ -80,7 +78,6 @@ struct PageChecksumCodec {
 		uint32_t part2;
 		std::string toString() { return format("0x%08x%08x", part1, part2); }
 	};
-	std::vector<std::pair<Pgno, SumType>> checksumHistory;
 
 	// Calculates and then either stores or verifies a checksum.
 	// The checksum is read/stored at the end of the page buffer.
@@ -118,33 +115,6 @@ struct PageChecksumCodec {
 			return false;
 		}
 
-		// Update or check sum in history if the history buffer isn't empty and if we're not in a simulated injected fault situation
-		if(!checksumHistory.empty() &&
-			(!g_network->isSimulated() || (!g_simulator.getCurrentProcess()->fault_injection_p1 && !g_simulator.getCurrentProcess()->rebooting))
-		) {
-			auto &bucket = checksumHistory[pageNumber % checksumHistory.size()];
-			if(write) {
-				// For writes, put this pagenumber and sum into the bucket
-				bucket.first = pageNumber;
-				bucket.second = *sumOut;
-			}
-			else {
-				// For reads, see if the bucket has the right page number, if so then verify sum
-				if(bucket.first == pageNumber && bucket.second != *pSumInPage) {
-					TraceEvent (SevError, "SQLitePageChecksumDetectedLostWrite")
-						.detail("CodecPageSize", pageSize)
-						.detail("CodecReserveSize", reserveSize)
-						.detail("Filename", filename)
-						.detail("PageNumber", pageNumber)
-						.detail("PageSize", pageLen)
-						.detail("ChecksumInPage", pSumInPage->toString())
-						.detail("ChecksumHistory", bucket.second.toString())
-						.error(checksum_failed());
-					return false;
-				}
-			}
-		}
-
 		return true;
 	}
 
@@ -246,7 +216,7 @@ struct SQLiteDB : NonCopyable {
 				ASSERT(false);
 			}
 			// Always start with a new pager codec with default options.
-			pPagerCodec = new PageChecksumCodec(filename, SERVER_KNOBS->SQLITE_PAGER_CHECKSUM_HISTORY);
+			pPagerCodec = new PageChecksumCodec(filename);
 			sqlite3BtreePagerSetCodec(btree, PageChecksumCodec::codec, PageChecksumCodec::sizeChange, PageChecksumCodec::free, pPagerCodec);
 		}
 	}
@@ -257,7 +227,7 @@ struct SQLiteDB : NonCopyable {
 			// Our exceptions don't propagate through sqlite, so we don't know for sure if the error that caused this was
 			// an injected fault.  Assume that if fault injection is happening, this is an injected fault.
 			Error err = io_error();
-			if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))
+			if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->machine->machineProcess->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))
 				err = err.asInjectedFault();
 
 			if (db)
@@ -1853,8 +1823,8 @@ private:
 			self->logging.cancel();
 			Void _ = wait( self->readThreads->stop() && self->writeThread->stop() );
 			if (deleteOnClose) {
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename, true ) );
-				Void _ = wait( IAsyncFileSystem::filesystem()->deleteFile( self->filename + "-wal", false ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename, true ) );
+				Void _ = wait( IAsyncFile::incrementalDelete( self->filename + "-wal", false ) );
 			}
 		} catch (Error& e) {
 			TraceEvent(SevError, "KVDoCloseError", self->logID)

fdbserver/Knobs.cpp

@@ -48,7 +48,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE,                     1e5 );
 	init( MAX_MESSAGE_SIZE,            std::max<int>(LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE, 1e5 + 2e4 + 1) + 8 ); // VALUE_SIZE_LIMIT + SYSTEM_KEY_SIZE_LIMIT + 9 bytes (4 bytes for length, 4 bytes for sequence number, and 1 byte for mutation type)
 	init( TLOG_MESSAGE_BLOCK_BYTES,                             10e6 );
-	init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR,      double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) );
+	init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR,      double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) ); //1.0121466709838096006362758832473
 	init( PEEK_TRACKER_EXPIRATION_TIME,                          600 ); if( randomize && BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = 0.1;
 	init( PARALLEL_GET_MORE_REQUESTS,                             32 ); if( randomize && BUGGIFY ) PARALLEL_GET_MORE_REQUESTS = 2;
 	init( MAX_QUEUE_COMMIT_BYTES,                               15e6 ); if( randomize && BUGGIFY ) MAX_QUEUE_COMMIT_BYTES = 5000;
@@ -57,7 +57,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( MAX_VERSIONS_IN_FLIGHT,                          100000000 );
 	init( VERSIONS_PER_SECOND,                               1000000 );
 	init( MAX_READ_TRANSACTION_LIFE_VERSIONS,      5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_READ_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 0.1 * VERSIONS_PER_SECOND); else if( randomize && BUGGIFY ) MAX_READ_TRANSACTION_LIFE_VERSIONS = 10 * VERSIONS_PER_SECOND;
-	init( MAX_WRITE_TRANSACTION_LIFE_VERSIONS,     5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_WRITE_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 0.5 * VERSIONS_PER_SECOND);
+	init( MAX_WRITE_TRANSACTION_LIFE_VERSIONS,     5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_WRITE_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 1 * VERSIONS_PER_SECOND);
 	init( MAX_COMMIT_BATCH_INTERVAL,                             0.5 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 2.0; // Each master proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly
 
 	// Data distribution queue
@@ -163,7 +163,6 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( DISK_METRIC_LOGGING_INTERVAL,                          5.0 );
 	init( SOFT_HEAP_LIMIT,                                     300e6 );
 
-	init( SQLITE_PAGER_CHECKSUM_HISTORY,                           0 );
 	init( SQLITE_PAGE_SCAN_ERROR_LIMIT,                        10000 );
 	init( SQLITE_BTREE_PAGE_USABLE,                          4096 - 8);  // pageSize - reserveSize for page checksum
 

fdbserver/Knobs.h

@@ -137,7 +137,6 @@ public:
 	int SQLITE_FRAGMENT_PRIMARY_PAGE_USABLE;
 	int SQLITE_FRAGMENT_OVERFLOW_PAGE_USABLE;
 	double SQLITE_FRAGMENT_MIN_SAVINGS;
-	int SQLITE_PAGER_CHECKSUM_HISTORY;
 
 	// KeyValueStoreSqlite spring cleaning
 	double CLEANING_INTERVAL;

fdbserver/LogSystem.h

@@ -115,6 +115,7 @@ struct ILogSystem {
 		bool parallelGetMore;
 		int sequence;
 		Deque<Future<TLogPeekReply>> futureResults;
+		Future<Void> interfaceChanged;
 
 		ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf, Tag tag, Version begin, Version end, bool returnIfBlocked, bool parallelGetMore );
 

fdbserver/LogSystemPeekCursor.actor.cpp

@@ -119,6 +119,10 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
 		throw internal_error();
 	}
 
+	if(!self->interfaceChanged.isValid()) {
+		self->interfaceChanged = self->interf->onChange();
+	}
+
 	loop {
 		try {
 			while(self->futureResults.size() < SERVER_KNOBS->PARALLEL_GET_MORE_REQUESTS && self->interf->get().present()) {
@@ -139,7 +143,9 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
 					//TraceEvent("SPC_getMoreB", self->randomID).detail("has", self->hasMessage()).detail("end", res.end).detail("popped", res.popped.present() ? res.popped.get() : 0);
 					return Void();
 				}
-				when( Void _ = wait( self->interf->onChange() ) ) {
+				when( Void _ = wait( self->interfaceChanged ) ) {
+					self->interfaceChanged = self->interf->onChange();
+					self->randomID = g_random->randomUniqueID();
 					self->sequence = 0;
 					self->futureResults.clear();
 				}
@@ -150,6 +156,7 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
 				return Void();
 			} else if(e.code() == error_code_timed_out) {
 				TraceEvent("PeekCursorTimedOut", self->randomID);
+				self->interfaceChanged = self->interf->onChange();
 				self->randomID = g_random->randomUniqueID();
 				self->sequence = 0;
 				self->futureResults.clear();

fdbserver/MasterProxyServer.actor.cpp

@@ -31,9 +31,9 @@
 #include "LogSystemDiskQueueAdapter.h"
 #include "IKeyValueStore.h"
 #include "fdbclient/SystemData.h"
-#include "flow/Notified.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbrpc/batcher.actor.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "ConflictSet.h"
 #include "flow/Stats.h"
@@ -1001,17 +1001,49 @@ ACTOR static Future<Void> readRequestServer(
 	TraceEvent("ProxyReadyForReads", proxy.id());
 
 	loop choose{
-		when(ReplyPromise<vector<StorageServerInterface>> req = waitNext(proxy.getKeyServersLocations.getFuture())) {
-			// SOMEDAY: keep ssis around?
-			vector<UID> src, dest;
-			decodeKeyServersValue(commitData->txnStateStore->readValue(keyServersKeyServersKey).get().get(), src, dest);
-			vector<StorageServerInterface> ssis;
-			ssis.reserve(src.size());
-			for (auto const& id : src) {
-				ssis.push_back(decodeServerListValue(commitData->txnStateStore->readValue(serverListKeyFor(id)).get().get()));
+		when(ReplyPromise<vector<pair<KeyRangeRef, vector<StorageServerInterface>>>> req = waitNext(proxy.getKeyServersLocations.getFuture())) {
+			Standalone<VectorRef<KeyValueRef>> keyServersBegin = commitData->txnStateStore->readRange(KeyRangeRef(allKeys.begin, keyServersKeyServersKeys.begin), -1).get();
+			Standalone<VectorRef<KeyValueRef>> keyServersEnd = commitData->txnStateStore->readRange(KeyRangeRef(keyServersKeyServersKeys.end, allKeys.end), 2).get();
+			Standalone<VectorRef<KeyValueRef>> keyServersShardBoundaries = commitData->txnStateStore->readRange(KeyRangeRef(keyServersBegin[0].key, keyServersEnd[1].key)).get();
+
+			Standalone<VectorRef<KeyValueRef>> serverListBegin = commitData->txnStateStore->readRange(KeyRangeRef(allKeys.begin, keyServersKey(serverListKeys.begin)), -1).get();
+			Standalone<VectorRef<KeyValueRef>> serverListEnd = commitData->txnStateStore->readRange(KeyRangeRef(keyServersKey(serverListKeys.end), allKeys.end), 2).get();
+			Standalone<VectorRef<KeyValueRef>> serverListShardBoundaries = commitData->txnStateStore->readRange(KeyRangeRef(serverListBegin[0].key, serverListEnd[1].key)).get();
+
+			bool ignoreFirstServerListShard = false;
+			if (keyServersShardBoundaries.back().key > serverListShardBoundaries.front().key)
+				ignoreFirstServerListShard = true;
+
+			// shards include all keyServers and serverLists information
+			vector<pair<KeyRangeRef, vector<StorageServerInterface>>> shards;
+			int reserveSize = keyServersShardBoundaries.size() + serverListShardBoundaries.size() - 2 - (ignoreFirstServerListShard ? 1 : 0);
+			shards.reserve(reserveSize);
+
+			for (int i = 0; i < keyServersShardBoundaries.size() - 1; i++) {
+				vector<UID> src, dest;
+				decodeKeyServersValue(keyServersShardBoundaries[i].value, src, dest);
+				vector<StorageServerInterface> ssis;
+				ssis.reserve(src.size());
+				for (auto const& id : src) {
+					ssis.push_back(decodeServerListValue(commitData->txnStateStore->readValue(serverListKeyFor(id)).get().get()));
+				}
+
+				shards.push_back(std::make_pair(KeyRangeRef(keyServersShardBoundaries[i].key.removePrefix(keyServersPrefix), keyServersShardBoundaries[i + 1].key.removePrefix(keyServersPrefix)), ssis));
 			}
 
-			req.send(ssis);
+			for (int i = ignoreFirstServerListShard ? 1 : 0 ; i < serverListShardBoundaries.size() - 1; i++) {
+				vector<UID> src, dest;
+				decodeKeyServersValue(serverListShardBoundaries[i].value, src, dest);
+				vector<StorageServerInterface> ssis;
+				ssis.reserve(src.size());
+				for (auto const& id : src) {
+					ssis.push_back(decodeServerListValue(commitData->txnStateStore->readValue(serverListKeyFor(id)).get().get()));
+				}
+
+				shards.push_back(std::make_pair(KeyRangeRef(serverListShardBoundaries[i].key.removePrefix(keyServersPrefix), serverListShardBoundaries[i + 1].key.removePrefix(keyServersPrefix)), ssis));
+			}
+
+			req.send(shards);
 		}
 		when(GetStorageServerRejoinInfoRequest req = waitNext(proxy.getStorageServerRejoinInfo.getFuture())) {
 			if (commitData->txnStateStore->readValue(serverListKeyFor(req.id)).get().present()) {

fdbserver/MoveKeys.actor.cpp

@@ -828,8 +828,7 @@ void seedShardServers(
 
 	// We have to set this range in two blocks, because the master tracking of "keyServersLocations" depends on a change to a specific
 	//   key (keyServersKeyServersKey)
-	krmSetPreviouslyEmptyRange( tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), keyServersPrefix), keyServersValue( serverIds ), Value() );
-	krmSetPreviouslyEmptyRange( tr, arena, keyServersPrefix, KeyRangeRef(keyServersPrefix, allKeys.end), keyServersValue( serverIds ), Value() );
+	krmSetPreviouslyEmptyRange( tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), keyServersValue( serverIds ), Value() );
 
 	for(int s=0; s<servers.size(); s++)
 		krmSetPreviouslyEmptyRange( tr, arena, serverKeysPrefixFor( servers[s].id() ), allKeys, serverKeysTrue, serverKeysFalse );

fdbserver/OldTLogServer.actor.cpp

@@ -23,11 +23,11 @@
 #include "flow/Stats.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/NativeAPI.h"
+#include "fdbclient/Notified.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/SystemData.h"
 #include "WorkerInterface.h"
 #include "TLogInterface.h"
-#include "flow/Notified.h"
 #include "Knobs.h"
 #include "IKeyValueStore.h"
 #include "flow/ActorCollection.h"
@@ -1448,4 +1448,4 @@ namespace oldTLog {
 		Void _ = wait( tLogStart( &self, LogSystemConfig(), Version(0), Version(0), std::vector<Tag>(), true, tli, ReplyPromise<TLogInterface>(), recoveryCount ) || removed );
 		throw internal_error();  // tLogStart doesn't return without an error
 	}
-}
+}

fdbserver/Orderer.actor.h

@@ -27,8 +27,8 @@
 #elif !defined(FDBSERVER_ORDERER_ACTOR_H)
 	#define FDBSERVER_ORDERER_ACTOR_H
 
+#include "fdbclient/Notified.h"
 #include "flow/actorcompiler.h"
-#include "flow/Notified.h"
 
 template <class Seq>
 class Orderer {
@@ -71,4 +71,4 @@ private:
 	Promise<Void> shutdown;  // Never set, only broken on destruction
 };
 
-#endif
+#endif

fdbserver/QuietDatabase.actor.cpp

@@ -139,6 +139,7 @@ ACTOR Future<vector<StorageServerInterface>> getStorageServers( Database cx, boo
 	state Transaction tr( cx );
 	if (use_system_priority)
 		tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+		tr.setOption(FDBTransactionOptions::LOCK_AWARE);
 	loop {
 		try {
 			Standalone<RangeResultRef> serverList = wait( tr.getRange( serverListKeys, CLIENT_KNOBS->TOO_MANY ) );

fdbserver/SimulatedCluster.actor.cpp

@@ -198,15 +198,16 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 		bool runBackupAgents)
 {
 	state ISimulator::ProcessInfo *simProcess = g_simulator.getCurrentProcess();
-	state int cycles =0;
+	state UID randomId = g_nondeterministic_random->randomUniqueID();
+	state int cycles = 0;
 
 	loop {
 		auto waitTime = SERVER_KNOBS->MIN_REBOOT_TIME + (SERVER_KNOBS->MAX_REBOOT_TIME - SERVER_KNOBS->MIN_REBOOT_TIME) * g_random->random01();
 		cycles ++;
-		TraceEvent("SimulatedFDBDWait").detail("Cycles", cycles)
-			.detail("ProcessAddress", NetworkAddress(ip, port, true, false))
+		TraceEvent("SimulatedFDBDPreWait").detail("Cycles", cycles).detail("RandomId", randomId)
+			.detail("Address", NetworkAddress(ip, port, true, false))
 			.detailext("ZoneId", localities.zoneId())
-			.detail("waitTime", waitTime);
+			.detail("waitTime", waitTime).detail("Port", port);
 
 		Void _ = wait( delay( waitTime ) );
 
@@ -215,12 +216,13 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 		state Future<ISimulator::KillType> onShutdown = process->onShutdown();
 
 		try {
-			TraceEvent("SimulatedRebooterStarting", localities.zoneId()).detail("Cycles", cycles)
+			TraceEvent("SimulatedRebooterStarting", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
 				.detailext("ZoneId", localities.zoneId())
 				.detailext("DataHall", localities.dataHallId())
-				.detail("ProcessAddress", process->address.toString())
+				.detail("Address", process->address.toString())
+				.detail("Excluded", process->excluded)
 				.detail("UsingSSL", useSSL);
-			TraceEvent("ProgramStart").detail("Cycles", cycles)
+			TraceEvent("ProgramStart").detail("Cycles", cycles).detail("RandomId", randomId)
 				.detail("SourceVersion", getHGVersion())
 				.detail("Version", FDB_VT_VERSION)
 				.detail("PackageName", FDB_VT_PACKAGE_NAME)
@@ -246,7 +248,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 			} catch (Error& e) {
 				// If in simulation, if we make it here with an error other than io_timeout but enASIOTimedOut is set then somewhere an io_timeout was converted to a different error.
 				if(g_network->isSimulated() && e.code() != error_code_io_timeout && (bool)g_network->global(INetwork::enASIOTimedOut))
-					TraceEvent(SevError, "IOTimeoutErrorSuppressed").detail("ErrorCode", e.code()).backtrace();
+					TraceEvent(SevError, "IOTimeoutErrorSuppressed").detail("ErrorCode", e.code()).detail("RandomId", randomId).backtrace();
 
 				if (onShutdown.isReady() && onShutdown.isError()) throw onShutdown.getError();
 				if(e.code() != error_code_actor_cancelled)
@@ -255,15 +257,16 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 				TraceEvent(e.code() == error_code_actor_cancelled || e.code() == error_code_file_not_found || destructed ? SevInfo : SevError, "SimulatedFDBDTerminated", localities.zoneId()).error(e, true);
 			}
 
-			TraceEvent("SimulatedFDBDDone", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDDone", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
+				.detail("Excluded", process->excluded)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", onShutdown.isReady() ? onShutdown.get() : ISimulator::None);
 
 			if (!onShutdown.isReady())
 				onShutdown = ISimulator::InjectFaults;
 		} catch (Error& e) {
-			TraceEvent(destructed ? SevInfo : SevError, "SimulatedFDBDRebooterError", localities.zoneId()).error(e, true);
+			TraceEvent(destructed ? SevInfo : SevError, "SimulatedFDBDRebooterError", localities.zoneId()).detail("RandomId", randomId).error(e, true);
 			onShutdown = e;
 		}
 
@@ -273,6 +276,11 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 			process->rebooting = true;
 			process->shutdownSignal.send(ISimulator::None);
 		}
+		TraceEvent("SimulatedFDBDWait", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+			.detail("Address", process->address)
+			.detail("Excluded", process->excluded)
+			.detail("Rebooting", process->rebooting)
+			.detailext("ZoneId", localities.zoneId());
 		Void _ = wait( g_simulator.onProcess( simProcess ) );
 
 		Void _ = wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY));  // One last chance for the process to clean up?
@@ -280,22 +288,24 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 		g_simulator.destroyProcess( process );  // Leak memory here; the process may be used in other parts of the simulation
 
 		auto shutdownResult = onShutdown.get();
-		TraceEvent("SimulatedFDBDShutdown", localities.zoneId()).detail("Cycles", cycles)
-			.detail("ProcessAddress", process->address)
+		TraceEvent("SimulatedFDBDShutdown", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+			.detail("Address", process->address)
+			.detail("Excluded", process->excluded)
 			.detailext("ZoneId", localities.zoneId())
 			.detail("KillType", shutdownResult);
 
 		if( shutdownResult < ISimulator::RebootProcessAndDelete ) {
-			TraceEvent("SimulatedFDBDLowerReboot", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDLowerReboot", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
+				.detail("Excluded", process->excluded)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", shutdownResult);
 			return onShutdown.get();
 		}
 
 		if( onShutdown.get() == ISimulator::RebootProcessAndDelete ) {
-			TraceEvent("SimulatedFDBDRebootAndDelete", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDRebootAndDelete", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", shutdownResult);
 			*coordFolder = joinPath(baseFolder, g_random->randomUniqueID().toString());
@@ -311,8 +321,8 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
 			}
 		}
 		else {
-			TraceEvent("SimulatedFDBDJustRepeat", localities.zoneId()).detail("Cycles", cycles)
-				.detail("ProcessAddress", process->address)
+			TraceEvent("SimulatedFDBDJustRepeat", localities.zoneId()).detail("Cycles", cycles).detail("RandomId", randomId)
+				.detail("Address", process->address)
 				.detailext("ZoneId", localities.zoneId())
 				.detail("KillType", shutdownResult);
 		}
@@ -346,6 +356,7 @@ ACTOR Future<Void> simulatedMachine(
 	state int bootCount = 0;
 	state std::vector<std::string> myFolders;
 	state std::vector<std::string> coordFolders;
+	state UID randomId = g_nondeterministic_random->randomUniqueID();
 
 	try {
 		CSimpleIni ini;
@@ -382,6 +393,7 @@ ACTOR Future<Void> simulatedMachine(
 				std::string path = joinPath(myFolders[i], "fdb.cluster");
 				Reference<ClusterConnectionFile> clusterFile(useSeedFile ? new ClusterConnectionFile(path, connStr.toString()) : new ClusterConnectionFile(path));
 				processes.push_back(simulatedFDBDRebooter(clusterFile, ips[i], sslEnabled, i + 1, localities, processClass, &myFolders[i], &coordFolders[i], baseFolder, connStr, useSeedFile, runBackupAgents));
+				TraceEvent("SimulatedMachineProcess", randomId).detail("Address", NetworkAddress(ips[i], i+1, true, false)).detailext("ZoneId", localities.zoneId()).detailext("DataHall", localities.dataHallId()).detail("Folder", myFolders[i]);
 			}
 
 			TEST( bootCount >= 1 ); // Simulated machine rebooted
@@ -389,7 +401,7 @@ ACTOR Future<Void> simulatedMachine(
 			TEST( bootCount >= 3 ); // Simulated machine rebooted three times
 			++bootCount;
 
-			TraceEvent("SimulatedMachineStart")
+			TraceEvent("SimulatedMachineStart", randomId)
 				.detail("Folder0", myFolders[0])
 				.detail("CFolder0", coordFolders[0])
 				.detail("MachineIPs", toIPVectorString(ips))
@@ -405,7 +417,7 @@ ACTOR Future<Void> simulatedMachine(
 
 			Void _ = wait( waitForAll( processes ) );
 
-			TraceEvent("SimulatedMachineRebootStart")
+			TraceEvent("SimulatedMachineRebootStart", randomId)
 				.detail("Folder0", myFolders[0])
 				.detail("CFolder0", coordFolders[0])
 				.detail("MachineIPs", toIPVectorString(ips))
@@ -437,7 +449,12 @@ ACTOR Future<Void> simulatedMachine(
 				ASSERT( it.second.isReady() && !it.second.isError() );
 			}
 
-			TraceEvent("SimulatedMachineRebootAfterKills")
+			for( auto it : g_simulator.getMachineById(localities.zoneId())->deletingFiles ) {
+				filenames.insert( it );
+				closingStr += it + ", ";
+			}
+
+			TraceEvent("SimulatedMachineRebootAfterKills", randomId)
 				.detail("Folder0", myFolders[0])
 				.detail("CFolder0", coordFolders[0])
 				.detail("MachineIPs", toIPVectorString(ips))
@@ -466,12 +483,12 @@ ACTOR Future<Void> simulatedMachine(
 						openFiles += *it + ", ";
 						i++;
 					}
-					TraceEvent("MachineFilesOpen").detail("PAddr", toIPVectorString(ips)).detail("OpenFiles", openFiles);
+					TraceEvent("MachineFilesOpen", randomId).detail("PAddr", toIPVectorString(ips)).detail("OpenFiles", openFiles);
 				} else
 					break;
 
 				if( shutdownDelayCount++ >= 50 ) {  // Worker doesn't shut down instantly on reboot
-					TraceEvent(SevError, "SimulatedFDBDFilesCheck")
+					TraceEvent(SevError, "SimulatedFDBDFilesCheck", randomId)
 						.detail("PAddrs", toIPVectorString(ips))
 						.detailext("ZoneId", localities.zoneId())
 						.detailext("DataHall", localities.dataHallId());
@@ -482,8 +499,8 @@ ACTOR Future<Void> simulatedMachine(
 				backoff = std::min( backoff + 1.0, 6.0 );
 			}
 
-			TraceEvent("SimulatedFDBDFilesClosed")
-				.detail("ProcessAddress", toIPVectorString(ips))
+			TraceEvent("SimulatedFDBDFilesClosed", randomId)
+				.detail("Address", toIPVectorString(ips))
 				.detailext("ZoneId", localities.zoneId())
 				.detailext("DataHall", localities.dataHallId());
 
@@ -505,7 +522,7 @@ ACTOR Future<Void> simulatedMachine(
 
 			auto rebootTime = g_random->random01() * MACHINE_REBOOT_TIME;
 
-			TraceEvent("SimulatedMachineShutdown")
+			TraceEvent("SimulatedMachineShutdown", randomId)
 				.detail("Swap", swap)
 				.detail("KillType", killType)
 				.detail("RebootTime", rebootTime)
@@ -525,7 +542,7 @@ ACTOR Future<Void> simulatedMachine(
 
 				if( myFolders != toRebootFrom ) {
 					TEST( true ); // Simulated machine swapped data folders
-					TraceEvent("SimulatedMachineFolderSwap")
+					TraceEvent("SimulatedMachineFolderSwap", randomId)
 						.detail("OldFolder0", myFolders[0]).detail("NewFolder0", toRebootFrom[0])
 						.detail("MachineIPs", toIPVectorString(ips));
 				}
@@ -643,93 +660,157 @@ ACTOR Future<Void> restartSimulatedSystem(vector<Future<Void>> *systemActors, st
 	return Void();
 }
 
-std::string randomConfiguration( int physicalDatacenters ) {
-	int r = std::min(g_random->randomInt(0, 6), 3);
-//	r = 1; //ahm
+struct SimulationConfig {
+	explicit SimulationConfig(int extraDB);
+	int extraDB;
 
-	// See also random configuration choices in ConfigureDatabase workload
+	DatabaseConfiguration db;
 
-	std::string startingConfig = "new";
-	if (r == 0) {
+	void set_config(std::string config);
+
+	// Simulation layout
+	int datacenters;
+	int machine_count;  // Total, not per DC.
+	int processes_per_machine;
+	int coordinators;
+
+	std::string toString();
+
+private:
+	void generateNormalConfig();
+};
+
+SimulationConfig::SimulationConfig(int extraDB) : extraDB(extraDB) {
+	generateNormalConfig();
+}
+
+void SimulationConfig::set_config(std::string config) {
+	// The only mechanism we have for turning "single" into what single means
+	// is buildConfiguration()... :/
+	std::map<std::string, std::string> hack_map;
+	ASSERT( buildConfiguration(config, hack_map) );
+	for(auto kv : hack_map) db.set( kv.first, kv.second );
+}
+
+StringRef StringRefOf(const char* s) {
+  return StringRef((uint8_t*)s, strlen(s));
+}
+
+void SimulationConfig::generateNormalConfig() {
+	set_config("new");
+	datacenters = g_random->randomInt( 1, 4 );
+	if (g_random->random01() < 0.25) db.desiredTLogCount = g_random->randomInt(1,7);
+	if (g_random->random01() < 0.25) db.masterProxyCount = g_random->randomInt(1,7);
+	if (g_random->random01() < 0.25) db.resolverCount = g_random->randomInt(1,7);
+	if (g_random->random01() < 0.5) {
+		set_config("ssd");
+	} else {
+		set_config("memory");
+	}
+
+	int replication_type = std::min(g_random->randomInt( 1, 6 ), 3);
+	//replication_type = 1;  //ahm
+	switch (replication_type) {
+	case 0: {
 		TEST( true );  // Simulated cluster using custom redundancy mode
-		int storage_replicas = g_random->randomInt(1,5);
-		startingConfig += " storage_replicas:=" + format("%d", storage_replicas);
-		startingConfig += " storage_quorum:=" + format("%d", storage_replicas);
-		int log_replicas = g_random->randomInt(1,5);
-		startingConfig += " log_replicas:=" + format("%d", log_replicas);
-		int log_anti_quorum = g_random->randomInt(0, log_replicas);
-		startingConfig += " log_anti_quorum:=" + format("%d", log_anti_quorum);
-		startingConfig += " replica_datacenters:=1";
-		startingConfig += " min_replica_datacenters:=1";
+		int storage_servers = g_random->randomInt(1,5);
+		int replication_factor = g_random->randomInt(1,5);
+		int anti_quorum = g_random->randomInt(0, db.tLogReplicationFactor);
+		// Go through buildConfiguration, as it sets tLogPolicy/storagePolicy.
+		set_config(format("storage_replicas:=%d storage_quorum:=%d "
+		                  "log_replicas:=%d log_anti_quorum:=%1 "
+		                  "replica_datacenters:=1 min_replica_datacenters:=1",
+		                  storage_servers, storage_servers,
+		                  replication_factor, anti_quorum));
+		break;
 	}
-	else if (r == 1) {
+	case 1: {
 		TEST( true );  // Simulated cluster running in single redundancy mode
-		startingConfig += " single";
+		set_config("single");
+		break;
 	}
-	else if( r == 2 ) {
+	case 2: {
 		TEST( true );  // Simulated cluster running in double redundancy mode
-		startingConfig += " double";
+		set_config("double");
+		break;
 	}
-	else if( r == 3 ) {
-		if( physicalDatacenters == 1 ) {
+	case 3: {
+		if( datacenters == 1 ) {
 			TEST( true );  // Simulated cluster running in triple redundancy mode
-			startingConfig += " triple";
+			set_config("triple");
 		}
-		else if( physicalDatacenters == 2 ) {
+		else if( datacenters == 2 ) {
 			TEST( true );  // Simulated cluster running in 2 datacenter mode
-			startingConfig += " two_datacenter";
+			set_config("two_datacenter");
 		}
-		else if( physicalDatacenters == 3 ) {
+		else if( datacenters == 3 ) {
 			TEST( true );  // Simulated cluster running in 3 data-hall mode
-			startingConfig += " three_data_hall";
+			set_config("three_data_hall");
 		}
 		else {
 			ASSERT( false );
 		}
+		break;
+	}
+	default:
+		ASSERT(false);  // Programmer forgot to adjust cases.
 	}
 
-	if (g_random->random01() < 0.25) startingConfig += " logs=" + format("%d", g_random->randomInt(1,7));
-	if (g_random->random01() < 0.25) startingConfig += " proxies=" + format("%d", g_random->randomInt(1,7));
-	if (g_random->random01() < 0.25) startingConfig += " resolvers=" + format("%d", g_random->randomInt(1,7));
+	machine_count = g_random->randomInt( std::max( 2+datacenters, db.minMachinesRequired() ), extraDB ? 6 : 10 );
+	processes_per_machine = g_random->randomInt(1, (extraDB ? 14 : 28)/machine_count + 2 );
+	coordinators = BUGGIFY ? g_random->randomInt(1, machine_count+1) : std::min( machine_count, db.maxMachineFailuresTolerated()*2 + 1 );
+}
 
-	startingConfig += g_random->random01() < 0.5 ? " ssd" : " memory";
-	return startingConfig;
+std::string SimulationConfig::toString() {
+	std::stringstream config;
+	std::map<std::string, std::string>&& dbconfig = db.toMap();
+	config << "new";
+
+	if (dbconfig["redundancy_mode"] != "custom") {
+		config << " " << dbconfig["redundancy_mode"];
+	} else {
+		config << " " << "log_replicas:=" << db.tLogReplicationFactor;
+		config << " " << "log_anti_quorum:=" << db.tLogWriteAntiQuorum;
+		config << " " << "storage_replicas:=" << db.storageTeamSize;
+		config << " " << "storage_quorum:=" << db.durableStorageQuorum;
+	}
+
+	config << " logs=" << db.getDesiredLogs();
+	config << " proxies=" << db.getDesiredProxies();
+	config << " resolvers=" << db.getDesiredResolvers();
+
+	config << " " << dbconfig["storage_engine"];
+	return config.str();
 }
 
 void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseFolder,
 							int* pTesterCount, Optional<ClusterConnectionString> *pConnString,
 							Standalone<StringRef> *pStartingConfiguration, int extraDB)
 {
-	int dataCenters = g_random->randomInt( 1, 4 );
-
 	// SOMEDAY: this does not test multi-interface configurations
-	std::string startingConfigString = randomConfiguration(dataCenters);
-	std::map<std::string,std::string> startingConfigMap;
-	ASSERT( buildConfiguration( startingConfigString, startingConfigMap ) == ConfigurationResult::SUCCESS );
+	SimulationConfig simconfig(extraDB);
+	std::string startingConfigString = simconfig.toString();
 
-	DatabaseConfiguration startingConfig;
-	for(auto kv : startingConfigMap) startingConfig.set( kv.first, kv.second );
-	g_simulator.storagePolicy = startingConfig.storagePolicy;
-	g_simulator.tLogPolicy = startingConfig.tLogPolicy;
-	g_simulator.tLogWriteAntiQuorum = startingConfig.tLogWriteAntiQuorum;
+	g_simulator.storagePolicy = simconfig.db.storagePolicy;
+	g_simulator.tLogPolicy = simconfig.db.tLogPolicy;
+	g_simulator.tLogWriteAntiQuorum = simconfig.db.tLogWriteAntiQuorum;
 	ASSERT(g_simulator.storagePolicy);
 	ASSERT(g_simulator.tLogPolicy);
 	TraceEvent("simulatorConfig").detail("tLogPolicy", g_simulator.tLogPolicy->info()).detail("storagePolicy", g_simulator.storagePolicy->info()).detail("tLogWriteAntiQuorum", g_simulator.tLogWriteAntiQuorum).detail("ConfigString", startingConfigString);
 
-	int machineCount = g_random->randomInt( std::max( 2+dataCenters, startingConfig.minMachinesRequired() ), extraDB ? 6 : 10 );
+	const int dataCenters = simconfig.datacenters;
+	const int machineCount = simconfig.machine_count;
+	const int coordinatorCount = simconfig.coordinators;
+	const int processesPerMachine = simconfig.processes_per_machine;
 
 	// half the time, when we have more than 4 machines that are not the first in their dataCenter, assign classes
 	bool assignClasses = machineCount - dataCenters > 4 && g_random->random01() < 0.5;
-	int processesPerMachine = g_random->randomInt(1, (extraDB ? 14 : 28)/machineCount + 2 );
 
 	// Use SSL half the time
 	bool sslEnabled = g_random->random01() < 0.05;
 	TEST( sslEnabled ); // SSL enabled
 	TEST( !sslEnabled ); // SSL disabled
 
-	// Pick coordination processes.
-	int coordinatorCount = BUGGIFY ? g_random->randomInt(1, machineCount+1) : std::min( machineCount, startingConfig.maxMachineFailuresTolerated()*2 + 1 );
-
 	vector<NetworkAddress> coordinatorAddresses;
 	for( int dc = 0; dc < dataCenters; dc++ ) {
 		int machines = machineCount / dataCenters + (dc < machineCount % dataCenters); // add remainder of machines to first datacenter
@@ -744,8 +825,8 @@ void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseF
 
 	g_random->randomShuffle(coordinatorAddresses);
 	for(int i = 0; i < (coordinatorAddresses.size()/2)+1; i++) {
-		TraceEvent("ProtectMachine").detail("Address", coordinatorAddresses[i]).detail("Coordinators", coordinatorAddresses.size()).backtrace();
-		g_simulator.protectedAddresses.insert(coordinatorAddresses[i]);
+		TraceEvent("ProtectCoordinator").detail("Address", coordinatorAddresses[i]).detail("Coordinators", describe(coordinatorAddresses)).backtrace();
+		g_simulator.protectedAddresses.insert(NetworkAddress(coordinatorAddresses[i].ip,coordinatorAddresses[i].port,true,false));
 	}
 	g_random->randomShuffle(coordinatorAddresses);
 
@@ -812,12 +893,12 @@ void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseF
 	}
 
 	g_simulator.desiredCoordinators = coordinatorCount;
-	g_simulator.killableMachines = startingConfig.maxMachineFailuresTolerated();
-	g_simulator.neededDatacenters = startingConfig.minDataCenters;
-	g_simulator.killableDatacenters = startingConfig.minDataCenters - 1;
+	g_simulator.killableMachines = simconfig.db.maxMachineFailuresTolerated();
+	g_simulator.neededDatacenters = 1;
+	g_simulator.killableDatacenters = 0;
 	g_simulator.physicalDatacenters = dataCenters;
 	g_simulator.maxCoordinatorsInDatacenter = ((coordinatorCount-1)/dataCenters) + 1;
-	g_simulator.machinesNeededForProgress = startingConfig.minMachinesRequired() + nonVersatileMachines;
+	g_simulator.machinesNeededForProgress = simconfig.db.minMachinesRequired() + nonVersatileMachines;
 	g_simulator.processesPerMachine = processesPerMachine;
 
 	TraceEvent("SetupSimulatorSettings")