From bc0832bb55ade6fe49c2604e3f7304e82edaf3e3 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 15 Aug 2018 10:11:08 -0700
Subject: [PATCH 01/37] Deprecate TLS_plugin option. Don't remove deprecated
 methods from Go bindings, since that breaks compatibility with old API
 versions.

---
 .../go/src/_util/translate_fdb_options.go     | 19 ++----
 bindings/go/src/fdb/generated.go              | 63 +++++++++++++++----
 fdbclient/NativeAPI.actor.cpp                 |  1 -
 fdbclient/vexillographer/fdb.options          |  2 +-
 4 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/bindings/go/src/_util/translate_fdb_options.go b/bindings/go/src/_util/translate_fdb_options.go
index 2f96d5e491..37d64af6c6 100644
--- a/bindings/go/src/_util/translate_fdb_options.go
+++ b/bindings/go/src/_util/translate_fdb_options.go
@@ -30,8 +30,6 @@ import (
 	"log"
 	"os"
 	"strings"
-	"unicode"
-	"unicode/utf8"
 )
 
 type Option struct {
@@ -114,23 +112,14 @@ func translateName(old string) string {
 	return strings.Replace(strings.Title(strings.Replace(old, "_", " ", -1)), " ", "", -1)
 }
 
-func lowerFirst(s string) string {
-	if s == "" {
-		return ""
-	}
-	r, n := utf8.DecodeRuneInString(s)
-	return string(unicode.ToLower(r)) + s[n:]
-}
-
 func writeMutation(opt Option) {
-	desc := lowerFirst(opt.Description)
 	tname := translateName(opt.Name)
 	fmt.Printf(`
-// %s %s
+// %s
 func (t Transaction) %s(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, %d)
 }
-`, tname, desc, tname, opt.Code)
+`, opt.Description, tname, opt.Code)
 }
 
 func writeEnum(scope Scope, opt Option, delta int) {
@@ -207,7 +196,7 @@ func int64ToBytes(i int64) ([]byte, error) {
 			receiver := scope.Name + "s"
 
 			for _, opt := range scope.Option {
-				if opt.Description != "Deprecated" && !opt.Hidden { // Eww
+				if !opt.Hidden {
 					writeOpt(receiver, opt)
 				}
 			}
@@ -216,7 +205,7 @@ func int64ToBytes(i int64) ([]byte, error) {
 
 		if scope.Name == "MutationType" {
 			for _, opt := range scope.Option {
-				if opt.Description != "Deprecated" && !opt.Hidden { // Eww
+				if !opt.Hidden {
 					writeMutation(opt)
 				}
 			}
diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go
index a173142e23..c42757d677 100644
--- a/bindings/go/src/fdb/generated.go
+++ b/bindings/go/src/fdb/generated.go
@@ -42,6 +42,20 @@ func int64ToBytes(i int64) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
+// Deprecated
+//
+// Parameter: IP:PORT
+func (o NetworkOptions) SetLocalAddress(param string) error {
+	return o.setOpt(10, []byte(param))
+}
+
+// Deprecated
+//
+// Parameter: path to cluster file
+func (o NetworkOptions) SetClusterFile(param string) error {
+	return o.setOpt(20, []byte(param))
+}
+
 // Enables trace output to a file in a directory of the clients choosing
 //
 // Parameter: path to output directory (or NULL for current working directory)
@@ -85,7 +99,7 @@ func (o NetworkOptions) SetKnob(param string) error {
 	return o.setOpt(40, []byte(param))
 }
 
-// Set the TLS plugin to load. This option, if used, must be set before any other TLS options
+// Deprecated
 //
 // Parameter: file path or linker-resolved name
 func (o NetworkOptions) SetTLSPlugin(param string) error {
@@ -280,6 +294,11 @@ func (o TransactionOptions) SetReadYourWritesDisable() error {
 	return o.setOpt(51, nil)
 }
 
+// Deprecated
+func (o TransactionOptions) SetReadAheadDisable() error {
+	return o.setOpt(52, nil)
+}
+
 // Not yet implemented.
 func (o TransactionOptions) SetDurabilityDatacenter() error {
 	return o.setOpt(110, nil)
@@ -290,6 +309,11 @@ func (o TransactionOptions) SetDurabilityRisky() error {
 	return o.setOpt(120, nil)
 }
 
+// Deprecated
+func (o TransactionOptions) SetDurabilityDevNullIsWebScale() error {
+	return o.setOpt(130, nil)
+}
+
 // Specifies that this transaction should be treated as highest priority and that lower priority transactions should block behind this one. Use is discouraged outside of low-level tools
 func (o TransactionOptions) SetPrioritySystemImmediate() error {
 	return o.setOpt(200, nil)
@@ -431,57 +455,72 @@ const (
 	StreamingModeSerial StreamingMode = 5
 )
 
-// Add performs an addition of little-endian integers. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The integers to be added must be stored in a little-endian representation.  They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to ``param`` and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow.
+// Performs an addition of little-endian integers. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The integers to be added must be stored in a little-endian representation.  They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to ``param`` and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow.
 func (t Transaction) Add(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 2)
 }
 
-// BitAnd performs a bitwise ``and`` operation.  If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
+// Deprecated
+func (t Transaction) And(key KeyConvertible, param []byte) {
+	t.atomicOp(key.FDBKey(), param, 6)
+}
+
+// Performs a bitwise ``and`` operation.  If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
 func (t Transaction) BitAnd(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 6)
 }
 
-// BitOr performs a bitwise ``or`` operation.  If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
+// Deprecated
+func (t Transaction) Or(key KeyConvertible, param []byte) {
+	t.atomicOp(key.FDBKey(), param, 7)
+}
+
+// Performs a bitwise ``or`` operation.  If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
 func (t Transaction) BitOr(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 7)
 }
 
-// BitXor performs a bitwise ``xor`` operation.  If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
+// Deprecated
+func (t Transaction) Xor(key KeyConvertible, param []byte) {
+	t.atomicOp(key.FDBKey(), param, 8)
+}
+
+// Performs a bitwise ``xor`` operation.  If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``.
 func (t Transaction) BitXor(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 8)
 }
 
-// AppendIfFits appends ``param`` to the end of the existing value already in the database at the given key (or creates the key and sets the value to ``param`` if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size.
+// Appends ``param`` to the end of the existing value already in the database at the given key (or creates the key and sets the value to ``param`` if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size.
 func (t Transaction) AppendIfFits(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 9)
 }
 
-// Max performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The larger of the two values is then stored in the database.
+// Performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The larger of the two values is then stored in the database.
 func (t Transaction) Max(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 12)
 }
 
-// Min performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database.
+// Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes.  If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database.
 func (t Transaction) Min(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 13)
 }
 
-// SetVersionstampedKey transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes.
+// Transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes.
 func (t Transaction) SetVersionstampedKey(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 14)
 }
 
-// SetVersionstampedValue transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset.
+// Transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset.
 func (t Transaction) SetVersionstampedValue(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 15)
 }
 
-// ByteMin performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the smaller of the two values is then stored in the database.
+// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the smaller of the two values is then stored in the database.
 func (t Transaction) ByteMin(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 16)
 }
 
-// ByteMax performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database.
+// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database.
 func (t Transaction) ByteMax(key KeyConvertible, param []byte) {
 	t.atomicOp(key.FDBKey(), param, 17)
 }
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index a73225bfb1..5e09b1a4db 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -789,7 +789,6 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 		}
 		case FDBNetworkOptions::TLS_PLUGIN:
 			validateOptionValue(value, true);
-			initTLSOptions();
 			break;
 		case FDBNetworkOptions::TLS_CERT_PATH:
 			validateOptionValue(value, true);
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 33ee913ba9..89b07c125a 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -53,7 +53,7 @@ description is not currently required but encouraged.
             description="Set internal tuning or debugging knobs"/>
     <Option name="TLS_plugin" code="41"
             paramType="String" paramDescription="file path or linker-resolved name"
-            description="Set the TLS plugin to load. This option, if used, must be set before any other TLS options" />
+            description="Deprecated" />
     <Option name="TLS_cert_bytes" code="42"
             paramType="Bytes" paramDescription="certificates"
             description="Set the certificate chain" />

From d8ea3dbf9a70be54a7d83639c8d8fc059b0c26eb Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Thu, 16 Aug 2018 17:34:59 -0700
Subject: [PATCH 02/37] Added the ability to configure a cluster from a JSON
 file

---
 fdbcli/fdbcli.actor.cpp                      |  81 ++-
 fdbclient/ManagementAPI.actor.cpp            | 134 ++++
 fdbclient/ManagementAPI.h                    |   4 +
 fdbclient/Schemas.cpp                        | 636 +++++++++++++++++++
 fdbclient/Schemas.h                          |  32 +
 fdbclient/fdbclient.vcxproj                  |   2 +
 fdbserver/workloads/StatusWorkload.actor.cpp | 150 +----
 tests/fast/SidebandWithStatus.txt            |   1 -
 tests/rare/LargeApiCorrectnessStatus.txt     |   1 -
 tests/slow/DDBalanceAndRemoveStatus.txt      |   1 -
 10 files changed, 899 insertions(+), 143 deletions(-)
 create mode 100644 fdbclient/Schemas.cpp
 create mode 100644 fdbclient/Schemas.h

diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index f3e05c13e7..f389cc878d 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -27,6 +27,7 @@
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/ClusterInterface.h"
 #include "fdbclient/ManagementAPI.h"
+#include "fdbclient/Schemas.h"
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/FDBOptions.g.h"
 
@@ -443,8 +444,12 @@ void initHelp() {
 		"All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK);
 	helpMap["configure"] = CommandHelp(
 		"configure [new] <single|double|triple|three_data_hall|three_datacenter|ssd|memory|proxies=<PROXIES>|logs=<LOGS>|resolvers=<RESOLVERS>>*",
-		"change database configuration",
+		"change the database configuration",
 		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
+	helpMap["fileconfigure"] = CommandHelp(
+		"fileconfigure <FILENAME>",
+		"change the database configuration from a file",
+		"Load a JSON document from the provided file, and change the database configuration to match the contents of the JSON document. The format should be the same as the value of the \"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\". Add \"create\":\"new\" to initialize a new database.");
 	helpMap["coordinators"] = CommandHelp(
 		"coordinators auto|<ADDRESS>+ [description=new_cluster_description]",
 		"change cluster coordinators or description",
@@ -1595,6 +1600,69 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
 	return ret;
 }
 
+ACTOR Future<bool> fileConfigure(Database db, std::string filePath) {
+	std::string contents(readFileBytes(filePath, 100000));
+	json_spirit::mValue config;
+	if(!json_spirit::read_string( contents, config )) {
+		printf("ERROR: Invalid JSON\n");
+		return true;
+	}
+	StatusObject configJSON = config.get_obj();
+
+	json_spirit::mValue schema;
+	json_spirit::read_string( configurationSchema.toString(), schema );
+	if( !schemaMatch(schema.get_obj(), configJSON) ) {
+		return true;
+	}
+
+	std::string configString;
+	for(auto kv : configJSON) {
+		if(!configString.empty()) {
+			configString += " ";
+		}
+		if( kv.second.type() == json_spirit::int_type ) {
+			configString += kv.first + ":=" + format("%d", kv.second.get_int());
+		} else if( kv.second.type() == json_spirit::str_type ) {
+			configString += kv.second.get_str();
+		} else if( kv.second.type() == json_spirit::array_type ) {
+			configString += kv.first + "=" + json_spirit::write_string(json_spirit::mValue(kv.second.get_array()), json_spirit::Output_options::none);
+		} else {
+			printUsage(LiteralStringRef("fileconfigure"));
+			return true;
+		}
+	}
+	ConfigurationResult::Type result = wait( makeInterruptable( changeConfig(db, configString) ) );
+	// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
+	// there are various results specific to changeConfig() that we need to report:
+	bool ret;
+	switch(result) {
+	case ConfigurationResult::NO_OPTIONS_PROVIDED:
+	case ConfigurationResult::CONFLICTING_OPTIONS:
+	case ConfigurationResult::UNKNOWN_OPTION:
+	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
+		printUsage(LiteralStringRef("fileconfigure"));
+		ret = true;
+		break;
+
+	case ConfigurationResult::DATABASE_ALREADY_CREATED:
+		printf("ERROR: Database already exists! To change the configuration, remove \"create\":\"new\"\n");
+		ret=true;
+		break;
+	case ConfigurationResult::DATABASE_CREATED:
+		printf("Database created\n");
+		ret=false;
+		break;
+	case ConfigurationResult::SUCCESS:
+		printf("Configuration changed\n");
+		ret=false;
+		break;
+	default:
+		ASSERT(false);
+		ret=true;
+	};
+	return ret;
+}
+
 // FIXME: Factor address parsing from coordinators, include, exclude
 
 ACTOR Future<bool> coordinators( Database db, std::vector<StringRef> tokens, bool isClusterTLS ) {
@@ -2457,6 +2525,17 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 					continue;
 				}
 
+				if (tokencmp(tokens[0], "fileconfigure")) {
+					if (tokens.size() != 2) {
+						printUsage(tokens[0]);
+						is_error = true;
+					} else {
+						bool err = wait( fileConfigure( db, tokens[1].toString() ) );
+						if (err) is_error = true;
+					}
+					continue;
+				}
+
 				if (tokencmp(tokens[0], "coordinators")) {
 					auto cs = ClusterConnectionFile( ccf->getFilename() ).getConnectionString();
 					if (tokens.size() < 2) {
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index ba35a33d2d..c7641ce6f7 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -1385,6 +1385,140 @@ ACTOR Future<Void> forceRecovery (Reference<ClusterConnectionFile> clusterFile)
 	}
 }
 
+json_spirit::Value_type normJSONType(json_spirit::Value_type type) {
+	if (type == json_spirit::int_type)
+		return json_spirit::real_type;
+	return type;
+}
+
+void schemaCoverage( std::string const& spath, bool covered ) {
+	static std::set<std::string> coveredSchemaPaths;
+	if (coveredSchemaPaths.insert(spath).second) {
+		TraceEvent ev(SevInfo, "CodeCoverage");
+		ev.detail("File", "documentation/StatusSchema.json/" + spath).detail("Line", 0);
+		if (!covered)
+			ev.detail("Covered", 0);
+	}
+}
+
+bool schemaMatch( StatusObject const schema, StatusObject const result, Severity sev, bool printErrors, bool checkCoverage, std::string path, std::string schema_path ) {
+	// Returns true if everything in `result` is permitted by `schema`
+
+	// Really this should recurse on "values" rather than "objects"?
+
+	bool ok = true;
+
+	try {
+		for(auto& rkv : result) {
+			auto& key = rkv.first;
+			auto& rv = rkv.second;
+			std::string kpath = path + "." + key;
+			std::string spath = schema_path + "." + key;
+
+			if(checkCoverage) schemaCoverage(spath);
+
+			if (!schema.count(key)) {
+				if(printErrors) printf("ERROR: Unknown key `%s'\n", kpath.c_str());
+				TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaPath", spath);
+				ok = false;
+				continue;
+			}
+			auto& sv = schema.at(key);
+
+			if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$enum")) {
+				auto& enum_values = sv.get_obj().at("$enum").get_array();
+
+				bool any_match = false;
+				for(auto& enum_item : enum_values)
+					if (enum_item == rv) {
+						any_match = true;
+						if(checkCoverage) schemaCoverage(spath + ".$enum." + enum_item.get_str());
+						break;
+					}
+				if (!any_match) {
+					if(printErrors) printf("ERROR: Unknown value `%s' for key `%s'\n", json_spirit::write_string(rv).c_str(), kpath.c_str());
+					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaEnumItems", enum_values.size()).detail("Value", json_spirit::write_string(rv));
+					if(checkCoverage) schemaCoverage(spath + ".$enum." + json_spirit::write_string(rv));
+					ok = false;
+				}
+			} else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) {
+				if (rv.type() != json_spirit::obj_type) {
+					if(printErrors) printf("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str());
+					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
+					ok = false;
+					continue;
+				}
+				if(sv.get_obj().at("$map").type() != json_spirit::obj_type) {
+					continue;
+				}
+				auto& schema_obj = sv.get_obj().at("$map").get_obj();
+				auto& value_obj = rv.get_obj();
+
+				if(checkCoverage) schemaCoverage(spath + ".$map");
+
+				for(auto& value_pair : value_obj) {
+					auto vpath = kpath + "[" + value_pair.first + "]";
+					auto upath = spath + ".$map";
+					if (value_pair.second.type() != json_spirit::obj_type) {
+						if(printErrors) printf("ERROR: Expected an object for `%s'\n", vpath.c_str());
+						TraceEvent(sev, "SchemaMismatch").detail("Path", vpath).detail("ValueType", value_pair.second.type());
+						ok = false;
+						continue;
+					}
+					if (!schemaMatch(schema_obj, value_pair.second.get_obj(), sev, printErrors, checkCoverage, vpath, upath))
+						ok = false;
+				}
+			} else {
+				// The schema entry isn't an operator, so it asserts a type and (depending on the type) recursive schema definition
+				if (normJSONType(sv.type()) != normJSONType(rv.type())) {
+					if(printErrors) printf("ERROR: Incorrect value type for key `%s'\n", kpath.c_str());
+					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
+					ok = false;
+					continue;
+				}
+				if (rv.type() == json_spirit::array_type) {
+					auto& value_array = rv.get_array();
+					auto& schema_array = sv.get_array();
+					if (!schema_array.size()) {
+						// An empty schema array means that the value array is required to be empty
+						if (value_array.size()) {
+							if(printErrors) printf("ERROR: Expected an empty array for key `%s'\n", kpath.c_str());
+							TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaSize", schema_array.size()).detail("ValueSize", value_array.size());
+							ok = false;
+							continue;
+						}
+					} else if (schema_array.size() == 1 && schema_array[0].type() == json_spirit::obj_type) {
+						// A one item schema array means that all items in the value must match the first item in the schema
+						auto& schema_obj = schema_array[0].get_obj();
+						int index = 0;
+						for(auto &value_item : value_array) {
+							if (value_item.type() != json_spirit::obj_type) {
+								if(printErrors) printf("ERROR: Expected all array elements to be objects for key `%s'\n", kpath.c_str());
+								TraceEvent(sev, "SchemaMismatch").detail("Path", kpath + format("[%d]",index)).detail("ValueType", value_item.type());
+								ok = false;
+								continue;
+							}
+							if (!schemaMatch(schema_obj, value_item.get_obj(), sev, printErrors, checkCoverage, kpath + format("[%d]", index), spath + "[0]"))
+								ok = false;
+							index++;
+						}
+					} else
+						ASSERT(false);  // Schema doesn't make sense
+				} else if (rv.type() == json_spirit::obj_type) {
+					auto& schema_obj = sv.get_obj();
+					auto& value_obj = rv.get_obj();
+					if (!schemaMatch(schema_obj, value_obj, sev, printErrors, checkCoverage, kpath, spath))
+						ok = false;
+				}
+			}
+		}
+		return ok;
+	} catch (std::exception& e) {
+		TraceEvent(SevError, "SchemaMatchException").detail("What", e.what()).detail("Path", path).detail("SchemaPath", schema_path);
+		throw unknown_error();
+	}
+}
+
 TEST_CASE("ManagementAPI/AutoQuorumChange/checkLocality") {
 	Void _ = wait(Future<Void>(Void()));
 
diff --git a/fdbclient/ManagementAPI.h b/fdbclient/ManagementAPI.h
index a55213ddee..b588f1d008 100644
--- a/fdbclient/ManagementAPI.h
+++ b/fdbclient/ManagementAPI.h
@@ -166,4 +166,8 @@ Future<Void> forceRecovery (Reference<ClusterConnectionFile> const& clusterFile)
 
 // Gets the cluster connection string
 Future<std::vector<NetworkAddress>> getCoordinators( Database const& cx );
+
+void schemaCoverage( std::string const& spath, bool covered=true );
+bool schemaMatch( StatusObject const schema, StatusObject const result, Severity sev=SevError, bool printErrors=true, bool checkCoverage=false, std::string path = std::string(), std::string schema_path = std::string() );
+
 #endif
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
new file mode 100644
index 0000000000..f33415f7d8
--- /dev/null
+++ b/fdbclient/Schemas.cpp
@@ -0,0 +1,636 @@
+/*
+ * Schemas.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Schemas.h"
+
+const KeyRef statusSchema = LiteralStringRef(R"statusSchema(
+{
+   "cluster":{
+      "layers":{
+         "_valid":true,
+         "_error":"some error description"
+      },
+      "processes":{
+         "$map":{
+            "version":"3.0.0",
+            "machine_id":"0ccb4e0feddb5583010f6b77d9d10ece",
+            "locality":{
+                "$map":"value"
+            },
+            "class_source":{
+               "$enum":[
+                  "command_line",
+                  "configure_auto",
+                  "set_class"
+               ]
+            },
+            "class_type":{
+               "$enum":[
+                  "unset",
+                  "storage",
+                  "transaction",
+                  "resolution",
+                  "proxy",
+                  "master",
+                  "test"
+               ]
+            },
+            "roles":[
+               {
+                  "query_queue_max":0,
+                  "input_bytes":{
+                     "hz":0.0,
+                     "counter":0,
+                     "roughness":0.0
+                  },
+                  "stored_bytes":12341234,
+                  "kvstore_used_bytes":12341234,
+                  "kvstore_available_bytes":12341234,
+                  "kvstore_free_bytes":12341234,
+                  "kvstore_total_bytes":12341234,
+                  "durable_bytes":{
+                     "hz":0.0,
+                     "counter":0,
+                     "roughness":0.0
+                  },
+                  "queue_disk_used_bytes":12341234,
+                  "queue_disk_available_bytes":12341234,
+                  "queue_disk_free_bytes":12341234,
+                  "queue_disk_total_bytes":12341234,
+                  "role":{
+                     "$enum":[
+                        "master",
+                        "proxy",
+                        "log",
+                        "storage",
+                        "resolver",
+                        "cluster_controller"
+                     ]
+                  },
+                  "data_version":12341234,
+                  "data_lag": {
+                     "seconds":5.0,
+                     "versions":12341234
+                  },
+                  "id":"eb84471d68c12d1d26f692a50000003f",
+                  "finished_queries":{
+                     "hz":0.0,
+                     "counter":0,
+                     "roughness":0.0
+                  }
+               }
+            ],
+            "command_line":"-r simulation",
+            "memory":{
+               "available_bytes":0,
+               "limit_bytes":0,
+               "unused_allocated_memory":0,
+               "used_bytes":0
+            },
+            "messages":[
+               {
+                  "time":12345.12312,
+                  "type":"x",
+                  "name":{
+                     "$enum":[
+                        "file_open_error",
+                        "incorrect_cluster_file_contents",
+                        "process_error",
+                        "io_error",
+                        "io_timeout",
+                        "platform_error",
+                        "storage_server_lagging",
+                        "(other FDB error messages)"
+                     ]
+                  },
+                  "raw_log_message":"<stuff/>",
+                  "description":"abc"
+               }
+            ],
+            "fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece",
+            "excluded":false,
+            "address":"1.2.3.4:1234",
+            "disk":{
+               "free_bytes":3451233456234,
+               "reads":{
+                  "hz":0.0,
+                  "counter":0,
+                  "sectors":0
+               },
+               "busy":0.0,
+               "writes":{
+                  "hz":0.0,
+                  "counter":0,
+                  "sectors":0
+               },
+               "total_bytes":123412341234
+            },
+            "uptime_seconds":1234.2345,
+            "cpu":{
+               "usage_cores":0.0
+            },
+            "network":{
+               "current_connections":0,
+               "connections_established":{
+                   "hz":0.0
+               },
+               "connections_closed":{
+                   "hz":0.0
+               },
+               "connection_errors":{
+                   "hz":0.0
+               },
+               "megabits_sent":{
+                  "hz":0.0
+               },
+               "megabits_received":{
+                  "hz":0.0
+               }
+            }
+         }
+      },
+      "old_logs":[
+         {
+            "logs":[
+               {
+                  "id":"7f8d623d0cb9966e",
+                  "healthy":true,
+                  "address":"1.2.3.4:1234"
+               }
+            ],
+            "log_replication_factor":3,
+            "log_write_anti_quorum":0,
+            "log_fault_tolerance":2,
+            "remote_log_replication_factor":3,
+            "remote_log_fault_tolerance":2,
+            "satellite_log_replication_factor":3,
+            "satellite_log_write_anti_quorum":0,
+            "satellite_log_fault_tolerance":2
+         }
+      ],
+      "fault_tolerance":{
+         "max_machine_failures_without_losing_availability":0,
+         "max_machine_failures_without_losing_data":0
+      },
+      "qos":{
+         "worst_queue_bytes_log_server":460,
+         "performance_limited_by":{
+            "reason_server_id":"7f8d623d0cb9966e",
+			"reason_id":0,
+            "name":{
+               "$enum":[
+                  "workload",
+                  "storage_server_write_queue_size",
+                  "storage_server_write_bandwidth_mvcc",
+                  "storage_server_readable_behind",
+                  "log_server_mvcc_write_bandwidth",
+                  "log_server_write_queue",
+                  "storage_server_min_free_space",
+                  "storage_server_min_free_space_ratio",
+                  "log_server_min_free_space",
+                  "log_server_min_free_space_ratio"
+               ]
+            },
+            "description":"The database is not being saturated by the workload."
+         },
+		 "transactions_per_second_limit":0,
+		 "released_transactions_per_second":0,
+		 "limiting_queue_bytes_storage_server":0,
+         "worst_queue_bytes_storage_server":0,
+		 "limiting_version_lag_storage_server":0,
+		 "worst_version_lag_storage_server":0
+      },
+      "incompatible_connections":[
+
+      ],
+      "datacenter_version_difference":0,
+      "database_available":true,
+      "database_locked":false,
+      "generation":2,
+      "latency_probe":{
+         "read_seconds":7,
+		 "immediate_priority_transaction_start_seconds":0.0,
+		 "batch_priority_transaction_start_seconds":0.0,
+         "transaction_start_seconds":0.0,
+         "commit_seconds":0.02
+      },
+      "clients":{
+         "count":1,
+         "supported_versions":[
+             {
+                 "client_version":"3.0.0",
+                 "connected_clients":[
+                     {
+                         "address":"127.0.0.1:9898",
+                         "log_group":"default"
+                     }
+                 ],
+                 "count" : 1,
+                 "protocol_version" : "fdb00a400050001",
+                 "source_version" : "9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"
+             }
+         ]
+      },
+      "messages":[
+         {
+            "reasons":[
+               {
+                  "description":"Blah."
+               }
+            ],
+            "unreachable_processes":[
+               {
+                  "address":"1.2.3.4:1234"
+               }
+            ],
+            "name":{
+               "$enum":[
+                  "unreachable_master_worker",
+                  "unreadable_configuration",
+                  "full_replication_timeout",
+                  "client_issues",
+                  "unreachable_processes",
+                  "immediate_priority_transaction_start_probe_timeout",
+                  "batch_priority_transaction_start_probe_timeout",
+                  "transaction_start_probe_timeout",
+                  "read_probe_timeout",
+                  "commit_probe_timeout",
+                  "storage_servers_error",
+                  "status_incomplete",
+                  "layer_status_incomplete",
+                  "database_availability_timeout"
+               ]
+            },
+            "issues":[
+               {
+                  "name":{
+                     "$enum":[
+                        "incorrect_cluster_file_contents"
+                     ]
+                  },
+                  "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."
+               }
+            ],
+            "description":"abc"
+         }
+      ],
+      "recovery_state":{
+         "required_resolvers":1,
+         "required_proxies":1,
+         "name":{
+            "$enum":[
+               "reading_coordinated_state",
+               "locking_coordinated_state",
+               "locking_old_transaction_servers",
+               "reading_transaction_system_state",
+               "configuration_missing",
+               "configuration_never_created",
+               "configuration_invalid",
+               "recruiting_transaction_servers",
+               "initializing_transaction_servers",
+               "recovery_transaction",
+               "writing_coordinated_state",
+               "accepting_commits",
+               "all_logs_recruited",
+               "storage_recovered",
+               "fully_recovered"
+            ]
+         },
+         "required_logs":3,
+         "missing_logs":"7f8d623d0cb9966e",
+         "description":"Recovery complete."
+      },
+      "workload":{
+         "operations":{
+            "writes":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            },
+            "reads":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            }
+         },
+         "bytes":{
+            "written":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            },
+            "read":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            }
+         },
+         "keys":{
+            "read":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            }
+         },
+         "transactions":{
+            "started":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            },
+            "conflicted":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            },
+            "committed":{
+               "hz":0.0,
+               "counter":0,
+               "roughness":0.0
+            }
+         }
+      },
+      "cluster_controller_timestamp":1415650089,
+      "protocol_version":"fdb00a400050001",
+      "connection_string":"a:a@127.0.0.1:4000",
+      "full_replication":true,
+      "configuration":{
+         "log_anti_quorum":0,
+         "log_replicas":2,
+         "log_replication_policy":"(zoneid^3x1)",
+         "redundancy_mode":{
+         "$enum":[
+             "single",
+             "double",
+             "triple",
+             "three_datacenter",
+             "three_datacenter_fallback",
+             "three_data_hall"
+         ]},
+         "regions":[{
+         "datacenters":[{
+             "id":"mr",
+             "priority":1,
+             "satellite":1
+         }],
+         "satellite_redundancy_mode":{
+         "$enum":[
+             "one_satellite_single",
+             "one_satellite_double",
+             "one_satellite_triple",
+             "two_satellite_safe",
+             "two_satellite_fast"
+         ]},
+         "satellite_log_replicas":1,
+         "satellite_usable_dcs":1,
+         "satellite_anti_quorum":0,
+         "satellite_log_policy":"(zoneid^3x1)",
+         "satellite_logs":2
+         }],
+         "remote_redundancy_mode":{
+         "$enum":[
+             "remote_default",
+             "remote_single",
+             "remote_double",
+             "remote_triple",
+             "remote_three_data_hall"
+         ]},
+         "remote_log_replicas":3,
+         "remote_logs":5,
+         "log_routers":10,
+         "usable_regions":1,
+         "repopulate_anti_quorum":1,
+         "storage_replicas":1,
+         "resolvers":1,
+         "storage_replication_policy":"(zoneid^3x1)",
+         "logs":2,
+         "storage_engine":{
+         "$enum":[
+             "ssd",
+             "ssd-1",
+             "ssd-2",
+             "memory"
+         ]},
+         "coordinators_count":1,
+         "excluded_servers":[
+            {
+               "address":"10.0.4.1"
+            }
+         ],
+         "auto_proxies":3,
+         "auto_resolvers":1,
+         "auto_logs":3,
+         "proxies":5
+      },
+      "data":{
+         "least_operating_space_bytes_log_server":0,
+         "average_partition_size_bytes":0,
+         "state":{
+            "healthy":true,
+            "min_replicas_remaining":0,
+            "name":{
+               "$enum":[
+                  "initializing",
+                  "missing_data",
+                  "healing",
+                  "healthy_repartitioning",
+                  "healthy_removing_server",
+                  "healthy_rebalancing",
+                  "healthy"
+               ]
+            },
+            "description":""
+         },
+         "least_operating_space_ratio_storage_server":0.1,
+         "max_machine_failures_without_losing_availability":0,
+         "total_disk_used_bytes":0,
+         "total_kv_size_bytes":0,
+         "partitions_count":2,
+         "moving_data":{
+            "total_written_bytes":0,
+            "in_flight_bytes":0,
+            "in_queue_bytes":0,
+            "highest_priority":0
+         },
+         "team_trackers":[
+            {
+                "primary":true,
+                "in_flight_bytes":0,
+                "unhealthy_servers":0,
+                "state":{
+                    "healthy":true,
+                    "min_replicas_remaining":0,
+                    "name":{
+                       "$enum":[
+                          "initializing",
+                          "missing_data",
+                          "healing",
+                          "healthy_repartitioning",
+                          "healthy_removing_server",
+                          "healthy_rebalancing",
+                          "healthy"
+                       ]
+                    },
+                    "description":""
+                }
+            }
+         ],
+         "least_operating_space_bytes_storage_server":0,
+         "max_machine_failures_without_losing_data":0
+      },
+      "machines":{
+         "$map":{
+            "network":{
+               "megabits_sent":{
+                  "hz":0.0
+               },
+               "megabits_received":{
+                  "hz":0.0
+               },
+               "tcp_segments_retransmitted":{
+                  "hz":0.0
+               }
+            },
+            "memory":{
+               "free_bytes":0,
+               "committed_bytes":0,
+               "total_bytes":0
+            },
+            "contributing_workers":4,
+            "datacenter_id":"6344abf1813eb05b",
+            "excluded":false,
+            "address":"1.2.3.4",
+            "machine_id":"6344abf1813eb05b",
+            "locality":{
+                "$map":"value"
+            },
+            "cpu":{
+               "logical_core_utilization":0.4
+            }
+         }
+      }
+   },
+   "client":{
+      "coordinators":{
+         "coordinators":[
+            {
+               "reachable":true,
+               "address":"127.0.0.1:4701"
+            }
+         ],
+         "quorum_reachable":true
+      },
+      "database_status":{
+         "available":true,
+         "healthy":true
+      },
+      "messages":[
+         {
+            "name":{
+               "$enum":[
+                  "inconsistent_cluster_file",
+                  "unreachable_cluster_controller",
+                  "no_cluster_controller",
+                  "status_incomplete_client",
+                  "status_incomplete_coordinators",
+                  "status_incomplete_error",
+                  "status_incomplete_timeout",
+                  "status_incomplete_cluster",
+                  "quorum_not_reachable"
+               ]
+            },
+            "description":"The cluster file is not up to date."
+         }
+      ],
+      "timestamp":1415650089,
+      "cluster_file":{
+         "path":"/etc/foundationdb/fdb.cluster",
+         "up_to_date":true
+      }
+   }
+})statusSchema");
+
+const KeyRef configurationSchema = LiteralStringRef(R"configSchema(
+{
+    "create":{
+    "$enum":[
+        "new"
+    ]},
+	"log_anti_quorum":0,
+    "log_replicas":2,
+    "log_replication_policy":"(zoneid^3x1)",
+    "redundancy_mode":{
+    "$enum":[
+        "single",
+        "double",
+        "triple",
+        "three_datacenter",
+        "three_datacenter_fallback",
+        "three_data_hall"
+    ]},
+    "regions":[{
+    "datacenters":[{
+        "id":"mr",
+        "priority":1,
+        "satellite":1
+    }],
+    "satellite_redundancy_mode":{
+    "$enum":[
+        "one_satellite_single",
+        "one_satellite_double",
+        "one_satellite_triple",
+        "two_satellite_safe",
+        "two_satellite_fast"
+    ]},
+    "satellite_log_replicas":1,
+    "satellite_usable_dcs":1,
+    "satellite_anti_quorum":0,
+    "satellite_log_policy":"(zoneid^3x1)",
+    "satellite_logs":2
+    }],
+    "remote_redundancy_mode":{
+    "$enum":[
+        "remote_default",
+        "remote_single",
+        "remote_double",
+        "remote_triple",
+        "remote_three_data_hall"
+    ]},
+    "remote_log_replicas":3,
+    "remote_logs":5,
+    "log_routers":10,
+    "usable_regions":1,
+    "repopulate_anti_quorum":1,
+    "storage_replicas":1,
+    "resolvers":1,
+    "storage_replication_policy":"(zoneid^3x1)",
+    "logs":2,
+    "storage_engine":{
+    "$enum":[
+        "ssd",
+        "ssd-1",
+        "ssd-2",
+        "memory"
+    ]},
+    "auto_proxies":3,
+    "auto_resolvers":1,
+    "auto_logs":3,
+    "proxies":5
+})configSchema");
diff --git a/fdbclient/Schemas.h b/fdbclient/Schemas.h
new file mode 100644
index 0000000000..5cdce5390b
--- /dev/null
+++ b/fdbclient/Schemas.h
@@ -0,0 +1,32 @@
+/*
+ * Schemas.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_SCHEMAS_H
+#define FDBCLIENT_SCHEMAS_H
+
+#pragma once
+
+#include "flow/flow.h"
+#include "FDBTypes.h"
+
+extern const KeyRef statusSchema;
+extern const KeyRef configurationSchema;
+
+#endif /* FDBCLIENT_SCHEMAS_H */
diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj
index ceae4f38c3..8ea99302e1 100755
--- a/fdbclient/fdbclient.vcxproj
+++ b/fdbclient/fdbclient.vcxproj
@@ -65,6 +65,7 @@
     <ClInclude Include="ReadYourWrites.h" />
     <ActorCompiler Include="RunTransaction.actor.h" />
     <ClInclude Include="RYWIterator.h" />
+    <ClInclude Include="Schemas.h" />
     <ClInclude Include="SnapshotCache.h" />
     <ClInclude Include="Status.h" />
     <ClInclude Include="StatusClient.h" />
@@ -97,6 +98,7 @@
     <ActorCompiler Include="MultiVersionTransaction.actor.cpp" />
     <ClCompile Include="RYWIterator.cpp" />
     <ActorCompiler Include="StatusClient.actor.cpp" />
+    <ClCompile Include="Schemas.cpp" />
     <ClCompile Include="SystemData.cpp" />
     <ActorCompiler Include="ThreadSafeTransaction.actor.cpp" />
     <ActorCompiler Include="TaskBucket.actor.cpp" />
diff --git a/fdbserver/workloads/StatusWorkload.actor.cpp b/fdbserver/workloads/StatusWorkload.actor.cpp
index 83e80471cf..e2bd6566c9 100644
--- a/fdbserver/workloads/StatusWorkload.actor.cpp
+++ b/fdbserver/workloads/StatusWorkload.actor.cpp
@@ -24,6 +24,8 @@
 #include "workloads.h"
 #include "fdbclient/StatusClient.h"
 #include "flow/UnitTest.h"
+#include "fdbclient/Schemas.h"
+#include "fdbclient/ManagementAPI.h"
 
 extern bool noUnseed;
 
@@ -31,7 +33,7 @@ struct StatusWorkload : TestWorkload {
 	double testDuration, requestsPerSecond;
 
 	PerfIntCounter requests, replies, errors, totalSize;
-	Optional<StatusObject> statusSchema;
+	Optional<StatusObject> parsedSchema;
 
 	StatusWorkload(WorkloadContext const& wcx)
 		: TestWorkload(wcx),
@@ -39,14 +41,14 @@ struct StatusWorkload : TestWorkload {
 	{
 		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
 		requestsPerSecond = getOption(options, LiteralStringRef("requestsPerSecond"), 0.5);
-		auto statusSchemaStr = getOption(options, LiteralStringRef("schema"), StringRef());
+		auto statusSchemaStr = getOption(options, LiteralStringRef("schema"), statusSchema);
 		if (statusSchemaStr.size()) {
 			json_spirit::mValue schema;
 			json_spirit::read_string( statusSchemaStr.toString(), schema );
-			statusSchema = schema.get_obj();
+			parsedSchema = schema.get_obj();
 
 			// This is sort of a hack, but generate code coverage *requirements* for everything in schema
-			schemaCoverageRequirements(statusSchema.get());
+			schemaCoverageRequirements(parsedSchema.get());
 		}
 
 		noUnseed = true;
@@ -80,30 +82,19 @@ struct StatusWorkload : TestWorkload {
 		m.push_back(errors.getMetric());
 	}
 
-	template <bool Covered=true>
-	static void schemaCoverage( std::string const& spath ) {
-		static std::set<std::string> coveredSchemaPaths;
-		if (coveredSchemaPaths.insert(spath).second) {
-			TraceEvent ev(SevInfo, "CodeCoverage");
-			ev.detail("File", "documentation/StatusSchema.json/" + spath).detail("Line", 0);
-			if (!Covered)
-				ev.detail("Covered", 0);
-		}
-	}
-
 	static void schemaCoverageRequirements( StatusObject const& schema, std::string schema_path = std::string() ) {
 		try {
 			for(auto& skv : schema) {
 				std::string spath = schema_path + "." + skv.first;
 
-				schemaCoverage<false>(spath);
+				schemaCoverage(spath, false);
 
 				if (skv.second.type() == json_spirit::array_type && skv.second.get_array().size()) {
 					schemaCoverageRequirements( skv.second.get_array()[0].get_obj(), spath + "[0]" );
 				} else if (skv.second.type() == json_spirit::obj_type) {
 					if (skv.second.get_obj().count("$enum")) {
 						for(auto& enum_item : skv.second.get_obj().at("$enum").get_array())
-							schemaCoverage<false>(spath + ".$enum." + enum_item.get_str());
+							schemaCoverage(spath + ".$enum." + enum_item.get_str(), false);
 					} else
 						schemaCoverageRequirements( skv.second.get_obj(), spath );
 				}
@@ -117,125 +108,6 @@ struct StatusWorkload : TestWorkload {
 		}
 	}
 
-	static json_spirit::Value_type normJSONType(json_spirit::Value_type type) {
-		if (type == json_spirit::int_type)
-			return json_spirit::real_type;
-		return type;
-	}
-
-	static bool schemaMatch( StatusObject const schema, StatusObject const result, Severity sev=SevError, std::string path = std::string(), std::string schema_path = std::string() ) {
-		// Returns true if everything in `result` is permitted by `schema`
-
-		// Really this should recurse on "values" rather than "objects"?
-
-		bool ok = true;
-
-		try {
-			for(auto& rkv : result) {
-				auto& key = rkv.first;
-				auto& rv = rkv.second;
-				std::string kpath = path + "." + key;
-				std::string spath = schema_path + "." + key;
-
-				schemaCoverage(spath);
-
-				if (!schema.count(key)) {
-					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaPath", spath);
-					ok = false;
-					continue;
-				}
-				auto& sv = schema.at(key);
-
-				if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$enum")) {
-					auto& enum_values = sv.get_obj().at("$enum").get_array();
-
-					bool any_match = false;
-					for(auto& enum_item : enum_values)
-						if (enum_item == rv) {
-							any_match = true;
-							schemaCoverage(spath + ".$enum." + enum_item.get_str());
-							break;
-						}
-					if (!any_match) {
-						TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaEnumItems", enum_values.size()).detail("Value", json_spirit::write_string(rv));
-						schemaCoverage(spath + ".$enum." + json_spirit::write_string(rv));
-						ok = false;
-					}
-				} else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) {
-					if (rv.type() != json_spirit::obj_type) {
-						TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
-						ok = false;
-						continue;
-					}
-					
-					if(sv.get_obj().at("$map").type() != json_spirit::obj_type) {
-						continue;
-					}
-					
-					auto& schema_obj = sv.get_obj().at("$map").get_obj();
-					auto& value_obj = rv.get_obj();
-
-					schemaCoverage(spath + ".$map");
-
-					for(auto& value_pair : value_obj) {
-						auto vpath = kpath + "[" + value_pair.first + "]";
-						auto upath = spath + ".$map";
-						if (value_pair.second.type() != json_spirit::obj_type) {
-							TraceEvent(sev, "SchemaMismatch").detail("Path", vpath).detail("ValueType", value_pair.second.type());
-							ok = false;
-							continue;
-						}
-						if (!schemaMatch(schema_obj, value_pair.second.get_obj(), sev, vpath, upath))
-							ok = false;
-					}
-				} else {
-					// The schema entry isn't an operator, so it asserts a type and (depending on the type) recursive schema definition
-					if (normJSONType(sv.type()) != normJSONType(rv.type())) {
-						TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
-						ok = false;
-						continue;
-					}
-					if (rv.type() == json_spirit::array_type) {
-						auto& value_array = rv.get_array();
-						auto& schema_array = sv.get_array();
-						if (!schema_array.size()) {
-							// An empty schema array means that the value array is required to be empty
-							if (value_array.size()) {
-								TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaSize", schema_array.size()).detail("ValueSize", value_array.size());
-								ok = false;
-								continue;
-							}
-						} else if (schema_array.size() == 1 && schema_array[0].type() == json_spirit::obj_type) {
-							// A one item schema array means that all items in the value must match the first item in the schema
-							auto& schema_obj = schema_array[0].get_obj();
-							int index = 0;
-							for(auto &value_item : value_array) {
-								if (value_item.type() != json_spirit::obj_type) {
-									TraceEvent(sev, "SchemaMismatch").detail("Path", kpath + format("[%d]",index)).detail("ValueType", value_item.type());
-									ok = false;
-									continue;
-								}
-								if (!schemaMatch(schema_obj, value_item.get_obj(), sev, kpath + format("[%d]", index), spath + "[0]"))
-									ok = false;
-								index++;
-							}
-						} else
-							ASSERT(false);  // Schema doesn't make sense
-					} else if (rv.type() == json_spirit::obj_type) {
-						auto& schema_obj = sv.get_obj();
-						auto& value_obj = rv.get_obj();
-						if (!schemaMatch(schema_obj, value_obj, sev, kpath, spath))
-							ok = false;
-					}
-				}
-			}
-			return ok;
-		} catch (std::exception& e) {
-			TraceEvent(SevError, "SchemaMatchException").detail("What", e.what()).detail("Path", path).detail("SchemaPath", schema_path);
-			throw unknown_error();
-		}
-	}
-
 	ACTOR Future<Void> fetcher(Reference<ClusterConnectionFile> connFile, StatusWorkload *self) {
 		state double lastTime = now();
 
@@ -252,7 +124,7 @@ struct StatusWorkload : TestWorkload {
 				self->totalSize += br.getLength();
 				TraceEvent("StatusWorkloadReply").detail("ReplySize", br.getLength()).detail("Latency", now() - issued);//.detail("Reply", json_spirit::write_string(json_spirit::mValue(result)));
 
-				if (self->statusSchema.present() && !schemaMatch(self->statusSchema.get(), result) )
+				if (self->parsedSchema.present() && !schemaMatch(self->parsedSchema.get(), result, SevError, false, true) )
 					TraceEvent(SevError, "StatusWorkloadValidationFailed").detail("JSON", json_spirit::write_string(json_spirit::mValue(result)));
 			}
 			catch (Error& e) {
@@ -277,7 +149,7 @@ TEST_CASE("fdbserver/status/schema/basic") {
 		json_spirit::mValue test;
 		json_spirit::read_string( t, test );
 		TraceEvent("SchemaMatch").detail("Schema", json_spirit::write_string(schema)).detail("Value", json_spirit::write_string(test)).detail("Expect", expect_ok);
-		ASSERT( expect_ok == StatusWorkload::schemaMatch(schema.get_obj(), test.get_obj(), expect_ok ? SevError : SevInfo) );
+		ASSERT( expect_ok == schemaMatch(schema.get_obj(), test.get_obj(), expect_ok ? SevError : SevInfo, false, true) );
 	};
 	check(true, "{}");
 	check(true, "{\"apple\":4}");
@@ -293,4 +165,4 @@ TEST_CASE("fdbserver/status/schema/basic") {
 	check(false, "{\"mapped\":{\"item1\":{\"x\":false},\"item2\":{\"y\":1}}}");
 
 	return Void();
-}
\ No newline at end of file
+}
diff --git a/tests/fast/SidebandWithStatus.txt b/tests/fast/SidebandWithStatus.txt
index 38dcdcaa36..2f3fed0b34 100644
--- a/tests/fast/SidebandWithStatus.txt
+++ b/tests/fast/SidebandWithStatus.txt
@@ -5,7 +5,6 @@ testTitle=CloggedCausalConsistencyTest
 
     testName=Status
     testDuration=30.0
-    schema={"cluster":{"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"database_locked":false,"full_replication":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","accepting_commits","all_logs_recruited","storage_recovered","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"connection_string":"a:a@127.0.0.1:4000","machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"layers":{"_valid":true,"_error":"some error description"},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"generation":2,"protocol_version":"fdb00a400050001","datacenter_version_difference":0,"configuration":{"logs":2,"log_replicas":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"excluded_servers":[{"address":"10.0.4.1"}],"remote_logs":5,"log_anti_quorum":0,"storage_replicas":1,"coordinators_count":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"usable_regions":1,"redundancy_mode":"single","auto_logs":3,"proxies":5,"resolvers":1,"log_replication_policy":"(zoneid^3x1)","remote_redundancy_mode":"remote_single","repopulate_anti_quorum":1,"remote_log_replicas":3,"log_routers":10,"storage_replication_policy":"(zoneid^3x1)","auto_proxies":3,"auto_resolvers":1},"data":{"team_trackers":[{"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"unhealthy_servers":0,"primary":true,"in_flight_bytes":0}],"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"highest_priority":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}],"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_lag":{"seconds":5.0,"versions":12341234},"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"unused_allocated_memory":0,"limit_bytes":0,"used_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"incompatible_connections":[],"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"cluster_controller_timestamp":1415650089,"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
 
     testName=RandomClogging
     testDuration=30.0
diff --git a/tests/rare/LargeApiCorrectnessStatus.txt b/tests/rare/LargeApiCorrectnessStatus.txt
index 0cf6149f39..070c8a6219 100644
--- a/tests/rare/LargeApiCorrectnessStatus.txt
+++ b/tests/rare/LargeApiCorrectnessStatus.txt
@@ -24,4 +24,3 @@ testTitle=ApiCorrectnessTest
 
     testName=Status
     testDuration=30.0
-	schema={"cluster":{"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"database_locked":false,"full_replication":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","accepting_commits","all_logs_recruited","storage_recovered","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"connection_string":"a:a@127.0.0.1:4000","machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"layers":{"_valid":true,"_error":"some error description"},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"generation":2,"protocol_version":"fdb00a400050001","datacenter_version_difference":0,"configuration":{"logs":2,"log_replicas":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"excluded_servers":[{"address":"10.0.4.1"}],"remote_logs":5,"log_anti_quorum":0,"storage_replicas":1,"coordinators_count":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"usable_regions":1,"redundancy_mode":"single","auto_logs":3,"proxies":5,"resolvers":1,"log_replication_policy":"(zoneid^3x1)","remote_redundancy_mode":"remote_single","repopulate_anti_quorum":1,"remote_log_replicas":3,"log_routers":10,"storage_replication_policy":"(zoneid^3x1)","auto_proxies":3,"auto_resolvers":1},"data":{"team_trackers":[{"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"unhealthy_servers":0,"primary":true,"in_flight_bytes":0}],"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"highest_priority":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}],"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_lag":{"seconds":5.0,"versions":12341234},"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"unused_allocated_memory":0,"limit_bytes":0,"used_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"incompatible_connections":[],"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"cluster_controller_timestamp":1415650089,"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
diff --git a/tests/slow/DDBalanceAndRemoveStatus.txt b/tests/slow/DDBalanceAndRemoveStatus.txt
index eebafa8623..a59a47308f 100644
--- a/tests/slow/DDBalanceAndRemoveStatus.txt
+++ b/tests/slow/DDBalanceAndRemoveStatus.txt
@@ -43,4 +43,3 @@ testTitle=DDBalance_test
 
     testName=Status
     testDuration=30.0
-    schema={"cluster":{"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"database_locked":false,"full_replication":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","accepting_commits","all_logs_recruited","storage_recovered","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"connection_string":"a:a@127.0.0.1:4000","machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"layers":{"_valid":true,"_error":"some error description"},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"generation":2,"protocol_version":"fdb00a400050001","datacenter_version_difference":0,"configuration":{"logs":2,"log_replicas":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"excluded_servers":[{"address":"10.0.4.1"}],"remote_logs":5,"log_anti_quorum":0,"storage_replicas":1,"coordinators_count":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"usable_regions":1,"redundancy_mode":"single","auto_logs":3,"proxies":5,"resolvers":1,"log_replication_policy":"(zoneid^3x1)","remote_redundancy_mode":"remote_single","repopulate_anti_quorum":1,"remote_log_replicas":3,"log_routers":10,"storage_replication_policy":"(zoneid^3x1)","auto_proxies":3,"auto_resolvers":1},"data":{"team_trackers":[{"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"unhealthy_servers":0,"primary":true,"in_flight_bytes":0}],"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"highest_priority":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}],"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_lag":{"seconds":5.0,"versions":12341234},"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"unused_allocated_memory":0,"limit_bytes":0,"used_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"incompatible_connections":[],"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"cluster_controller_timestamp":1415650089,"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}

From d6582b82d166da19795ad1d7199dd1d5f74056ca Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 22 Aug 2018 10:38:07 -0700
Subject: [PATCH 03/37] Allow setting flow knobs in backup.

---
 fdbbackup/backup.actor.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 6c52dc371d..0d18c5f207 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2620,13 +2620,19 @@ int main(int argc, char* argv[]) {
 			commandLine += argv[a];
 		}
 
+		delete FLOW_KNOBS;
+		FlowKnobs* flowKnobs = new FlowKnobs(true);
+		FLOW_KNOBS = flowKnobs;
+
 		delete CLIENT_KNOBS;
 		ClientKnobs* clientKnobs = new ClientKnobs(true);
 		CLIENT_KNOBS = clientKnobs;
 
 		for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
 			try {
-				if (!clientKnobs->setKnob( k->first, k->second )) {
+				if (!const_cast<FlowKnobs*>(FLOW_KNOBS)->setKnob( k->first, k->second ) &&
+					!const_cast<ClientKnobs*>(CLIENT_KNOBS)->setKnob( k->first, k->second )) 
+				{
 					fprintf(stderr, "Unrecognized knob option '%s'\n", k->first.c_str());
 					return FDB_EXIT_ERROR;
 				}

From 5ddbd70d002545961fe356f87148db53bf0f0d3b Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 22 Aug 2018 10:38:56 -0700
Subject: [PATCH 04/37] Add release note for backup flow knob support.

---
 documentation/sphinx/source/release-notes.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 10358b62e6..f2bbfbd3e4 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -2,7 +2,7 @@
 Release Notes
 #############
 
-6.0.8
+6.0.9
 =====
 
 Features
@@ -48,6 +48,7 @@ Fixes
 * Sometimes a minority of coordinators would fail to converge after a new leader was elected. [6.0.6] `(PR #700) <https://github.com/apple/foundationdb/pull/700>`_
 * Calling status too many times in a 5 second interval caused the cluster controller to pause for a few seconds. [6.0.7] `(PR #711) <https://github.com/apple/foundationdb/pull/711>`_
 * Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.8] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_
+* Backup and DR didn't allow setting certain knobs. [6.0.9] `(Issue #715) <https://github.com/apple/foundationdb/issues/715>`_
 
 Status
 ------

From ac336c632a09c0fde5e87febeb4d0fc60a81baaf Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 22 Aug 2018 10:40:55 -0700
Subject: [PATCH 05/37] WATCH_TIMEOUT knob was always buggified

---
 fdbclient/Knobs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp
index b208302dae..1288922b99 100644
--- a/fdbclient/Knobs.cpp
+++ b/fdbclient/Knobs.cpp
@@ -77,7 +77,7 @@ ClientKnobs::ClientKnobs(bool randomize) {
 	init( WATCH_POLLING_TIME,                      1.0 ); if( randomize && BUGGIFY ) WATCH_POLLING_TIME = 5.0;
 	init( NO_RECENT_UPDATES_DURATION,             20.0 ); if( randomize && BUGGIFY ) NO_RECENT_UPDATES_DURATION = 0.1;
 	init( FAST_WATCH_TIMEOUT,                     20.0 ); if( randomize && BUGGIFY ) FAST_WATCH_TIMEOUT = 1.0;
-	init( WATCH_TIMEOUT,                         900.0 ); if( randomize ) WATCH_TIMEOUT = 20.0;
+	init( WATCH_TIMEOUT,                         900.0 ); if( randomize && BUGGIFY ) WATCH_TIMEOUT = 20.0;
 
 	// Core
 	init( CORE_VERSIONSPERSECOND,		           1e6 );

From 441e23903cefc69ebef5ef2705955cd8d81b4daa Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 22 Aug 2018 10:47:43 -0700
Subject: [PATCH 06/37] Add release note for watch knob fix.

---
 documentation/sphinx/source/release-notes.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 10358b62e6..ebb42eec83 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -48,6 +48,7 @@ Fixes
 * Sometimes a minority of coordinators would fail to converge after a new leader was elected. [6.0.6] `(PR #700) <https://github.com/apple/foundationdb/pull/700>`_
 * Calling status too many times in a 5 second interval caused the cluster controller to pause for a few seconds. [6.0.7] `(PR #711) <https://github.com/apple/foundationdb/pull/711>`_
 * Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.8] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_
+* Watches polled the server much more frequently than intended. [6.0.9] `(PR #728) <https://github.com/apple/foundationdb/pull/728>`_
 
 Status
 ------

From 89dd9cc4a3ceff543dddd19a40bbdfa57e3629e2 Mon Sep 17 00:00:00 2001
From: Steve Atherton <stevea@apple.com>
Date: Wed, 22 Aug 2018 15:09:12 -0700
Subject: [PATCH 07/37] Cherry-pick pull request #717 to release-6.0

Which contains:
* Improve TLS cert refresh logging.
* Loading a mismatching cert shouldn't prevent TLS connections.
* Initialize the cached copy of ca/cert/key data.
* Open certificates as uncached, which means they can be write-protected.
---
 fdbrpc/TLSConnection.actor.cpp | 61 +++++++++++++++++++++++++---------
 1 file changed, 45 insertions(+), 16 deletions(-)

diff --git a/fdbrpc/TLSConnection.actor.cpp b/fdbrpc/TLSConnection.actor.cpp
index 95468dee23..7c2eb93298 100644
--- a/fdbrpc/TLSConnection.actor.cpp
+++ b/fdbrpc/TLSConnection.actor.cpp
@@ -231,6 +231,7 @@ void TLSOptions::set_ca_data(std::string const& ca_data) {
 		init_plugin();
 
 	TraceEvent("TLSConnectionSettingCAData").detail("CADataSize", ca_data.size());
+	policyInfo.ca_contents = Standalone<StringRef>(ca_data);
 	if (!policyVerifyPeersSet.get()->set_ca_data((const uint8_t*)&ca_data[0], ca_data.size()))
 		throw tls_error();
 	if (!policyVerifyPeersNotSet.get()->set_ca_data((const uint8_t*)&ca_data[0], ca_data.size()))
@@ -244,6 +245,7 @@ void TLSOptions::set_cert_data( std::string const& cert_data ) {
 		init_plugin();
 
 	TraceEvent("TLSConnectionSettingCertData").detail("CertDataSize", cert_data.size());
+	policyInfo.cert_contents = Standalone<StringRef>(cert_data);
 	if ( !policyVerifyPeersSet.get()->set_cert_data( (const uint8_t*)&cert_data[0], cert_data.size() ) )
 		throw tls_error();
 	if (!policyVerifyPeersNotSet.get()->set_cert_data((const uint8_t*)&cert_data[0], cert_data.size()))
@@ -273,6 +275,7 @@ void TLSOptions::set_key_data( std::string const& key_data ) {
 		init_plugin();
 	const char *passphrase = policyInfo.keyPassword.empty() ? NULL : policyInfo.keyPassword.c_str();
 	TraceEvent("TLSConnectionSettingKeyData").detail("KeyDataSize", key_data.size());
+	policyInfo.key_contents = Standalone<StringRef>(key_data);
 	if ( !policyVerifyPeersSet.get()->set_key_data( (const uint8_t*)&key_data[0], key_data.size(), passphrase) )
 		throw tls_error();
 	if (!policyVerifyPeersNotSet.get()->set_key_data((const uint8_t*)&key_data[0], key_data.size(), passphrase))
@@ -310,7 +313,7 @@ void TLSOptions::register_network() {
 }
 
 ACTOR static Future<ErrorOr<Standalone<StringRef>>> readEntireFile( std::string filename ) {
-	state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READONLY, 0));
+	state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0));
 	state int64_t filesize = wait(file->size());
 	state Standalone<StringRef> buf = makeString(filesize);
 	int rc = wait(file->read(mutateString(buf), filesize, 0));
@@ -347,6 +350,7 @@ ACTOR static Future<Void> reloadConfigurationOnChange( TLSOptions::PolicyInfo *p
 		}
 		Void _ = wait(delay(1.0));
 	}
+	state int mismatches = 0;
 	state AsyncVar<Standalone<StringRef>> ca_var;
 	state AsyncVar<Standalone<StringRef>> key_var;
 	state AsyncVar<Standalone<StringRef>> cert_var;
@@ -359,14 +363,30 @@ ACTOR static Future<Void> reloadConfigurationOnChange( TLSOptions::PolicyInfo *p
 		state Future<Void> key_changed = key_var.onChange();
 		state Future<Void> cert_changed = cert_var.onChange();
 		Void _ = wait( ca_changed || key_changed || cert_changed );
-		if (ca_changed.isReady()) pci->ca_contents = ca_var.get();
-		if (key_changed.isReady()) pci->key_contents = key_var.get();
-		if (cert_changed.isReady()) pci->cert_contents = cert_var.get();
-		try {
-			Reference<ITLSPolicy> verifypeers = Reference<ITLSPolicy>(plugin->create_policy());
-			verifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size());
-			verifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str());
-			verifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size());
+		if (ca_changed.isReady()) {
+			TraceEvent(SevInfo, "TLSRefreshCAChanged").detail("path", pci->ca_path).detail("length", ca_var.get().size());
+			pci->ca_contents = ca_var.get();
+		}
+		if (key_changed.isReady()) {
+			TraceEvent(SevInfo, "TLSRefreshKeyChanged").detail("path", pci->key_path).detail("length", key_var.get().size());
+			pci->key_contents = key_var.get();
+		}
+		if (cert_changed.isReady()) {
+			TraceEvent(SevInfo, "TLSRefreshCertChanged").detail("path", pci->cert_path).detail("length", cert_var.get().size());
+			pci->cert_contents = cert_var.get();
+		}
+		bool rc = true;
+		Reference<ITLSPolicy> verifypeers = Reference<ITLSPolicy>(plugin->create_policy());
+		Reference<ITLSPolicy> noverifypeers = Reference<ITLSPolicy>(plugin->create_policy());
+		loop {
+			// Don't actually loop.  We're just using loop/break as a `goto err`.
+			// This loop always ends with an unconditional break.
+			rc = verifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size());
+			if (!rc) break;
+			rc = verifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str());
+			if (!rc) break;
+			rc = verifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size());
+			if (!rc) break;
 			{
 				std::unique_ptr<const uint8_t *[]> verify_peers_arr(new const uint8_t*[pci->verify_peers.size()]);
 				std::unique_ptr<int[]> verify_peers_len(new int[pci->verify_peers.size()]);
@@ -374,18 +394,27 @@ ACTOR static Future<Void> reloadConfigurationOnChange( TLSOptions::PolicyInfo *p
 					verify_peers_arr[i] = (const uint8_t *)&pci->verify_peers[i][0];
 					verify_peers_len[i] = pci->verify_peers[i].size();
 				}
-				verifypeers->set_verify_peers(pci->verify_peers.size(), verify_peers_arr.get(), verify_peers_len.get());
+				rc = verifypeers->set_verify_peers(pci->verify_peers.size(), verify_peers_arr.get(), verify_peers_len.get());
+				if (!rc) break;
 			}
-			Reference<ITLSPolicy> noverifypeers = Reference<ITLSPolicy>(plugin->create_policy());
-			noverifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size());
-			noverifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str());
-			noverifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size());
+			rc = noverifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size());
+			if (!rc) break;
+			rc = noverifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str());
+			if (!rc) break;
+			rc = noverifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size());
+			if (!rc) break;
+			break;
+		}
 
+		if (rc) {
+			TraceEvent(SevInfo, "TLSCertificateRefreshSucceeded");
 			realVerifyPeersPolicy->set(verifypeers);
 			realNoVerifyPeersPolicy->set(noverifypeers);
-		} catch (Error& e) {
+			mismatches = 0;
+		} else {
 			// Some files didn't match up, they should in the future, and we'll retry then.
-			TraceEvent(SevWarn, "TLSCertificateRefresh").error(e);
+			mismatches++;
+			TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").detail("mismatches", mismatches);
 		}
 	}
 }

From 31fb5479e1f8201e0d61b17932bd47df5c12c56d Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Thu, 23 Aug 2018 18:51:34 -0700
Subject: [PATCH 08/37] Pre-work for 6.0.9 patch release.

* Bump MSI GUID
* Bump 6.0.8 to 6.0.9 on downloads page
---
 documentation/sphinx/source/downloads.rst | 24 +++++++++++------------
 packaging/msi/FDBInstaller.wxs            |  2 +-
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst
index 4357cddabf..93a7711dec 100644
--- a/documentation/sphinx/source/downloads.rst
+++ b/documentation/sphinx/source/downloads.rst
@@ -10,38 +10,38 @@ macOS
 
 The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
 
-* `FoundationDB-6.0.8.pkg <https://www.foundationdb.org/downloads/6.0.8/macOS/installers/FoundationDB-6.0.8.pkg>`_
+* `FoundationDB-6.0.9.pkg <https://www.foundationdb.org/downloads/6.0.9/macOS/installers/FoundationDB-6.0.9.pkg>`_
 
 Ubuntu
 ------
 
 The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
 
-* `foundationdb-clients-6.0.8-1_amd64.deb <https://www.foundationdb.org/downloads/6.0.8/ubuntu/installers/foundationdb-clients_6.0.8-1_amd64.deb>`_
-* `foundationdb-server-6.0.8-1_amd64.deb <https://www.foundationdb.org/downloads/6.0.8/ubuntu/installers/foundationdb-server_6.0.8-1_amd64.deb>`_ (depends on the clients package)
+* `foundationdb-clients-6.0.9-1_amd64.deb <https://www.foundationdb.org/downloads/6.0.9/ubuntu/installers/foundationdb-clients_6.0.9-1_amd64.deb>`_
+* `foundationdb-server-6.0.9-1_amd64.deb <https://www.foundationdb.org/downloads/6.0.9/ubuntu/installers/foundationdb-server_6.0.9-1_amd64.deb>`_ (depends on the clients package)
 
 RHEL/CentOS EL6
 ---------------
 
 The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
 
-* `foundationdb-clients-6.0.8-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.8/rhel6/installers/foundationdb-clients-6.0.8-1.el6.x86_64.rpm>`_
-* `foundationdb-server-6.0.8-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.8/rhel6/installers/foundationdb-server-6.0.8-1.el6.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.0.9-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.9/rhel6/installers/foundationdb-clients-6.0.9-1.el6.x86_64.rpm>`_
+* `foundationdb-server-6.0.9-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.9/rhel6/installers/foundationdb-server-6.0.9-1.el6.x86_64.rpm>`_ (depends on the clients package)
 
 RHEL/CentOS EL7
 ---------------
 
 The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
 
-* `foundationdb-clients-6.0.8-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.8/rhel7/installers/foundationdb-clients-6.0.8-1.el7.x86_64.rpm>`_
-* `foundationdb-server-6.0.8-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.8/rhel7/installers/foundationdb-server-6.0.8-1.el7.x86_64.rpm>`_ (depends on the clients package)
+* `foundationdb-clients-6.0.9-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.9/rhel7/installers/foundationdb-clients-6.0.9-1.el7.x86_64.rpm>`_
+* `foundationdb-server-6.0.9-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/6.0.9/rhel7/installers/foundationdb-server-6.0.9-1.el7.x86_64.rpm>`_ (depends on the clients package)
 
 Windows
 -------
 
 The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
 
-* `foundationdb-6.0.8-x64.msi <https://www.foundationdb.org/downloads/6.0.8/windows/installers/foundationdb-6.0.8-x64.msi>`_
+* `foundationdb-6.0.9-x64.msi <https://www.foundationdb.org/downloads/6.0.9/windows/installers/foundationdb-6.0.9-x64.msi>`_
 
 API Language Bindings
 =====================
@@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
 
 If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package:
 
-* `foundationdb-6.0.8.tar.gz <https://www.foundationdb.org/downloads/6.0.8/bindings/python/foundationdb-6.0.8.tar.gz>`_
+* `foundationdb-6.0.9.tar.gz <https://www.foundationdb.org/downloads/6.0.9/bindings/python/foundationdb-6.0.9.tar.gz>`_
 
 Ruby 1.9.3/2.0.0+
 -----------------
 
-* `fdb-6.0.8.gem <https://www.foundationdb.org/downloads/6.0.8/bindings/ruby/fdb-6.0.8.gem>`_
+* `fdb-6.0.9.gem <https://www.foundationdb.org/downloads/6.0.9/bindings/ruby/fdb-6.0.9.gem>`_
 
 Java 8+
 -------
 
-* `fdb-java-6.0.8.jar <https://www.foundationdb.org/downloads/6.0.8/bindings/java/fdb-java-6.0.8.jar>`_
-* `fdb-java-6.0.8-javadoc.jar <https://www.foundationdb.org/downloads/6.0.8/bindings/java/fdb-java-6.0.8-javadoc.jar>`_
+* `fdb-java-6.0.9.jar <https://www.foundationdb.org/downloads/6.0.9/bindings/java/fdb-java-6.0.9.jar>`_
+* `fdb-java-6.0.9-javadoc.jar <https://www.foundationdb.org/downloads/6.0.9/bindings/java/fdb-java-6.0.9-javadoc.jar>`_
 
 Go 1.1+
 -------
diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
index 2adbcbc2a7..95696ca553 100644
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{87E8D968-B3DE-4C62-82E8-4429423D159C}'
+           Id='{F4C82ECC-493D-4363-A06C-139701B30473}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'

From b0d6ecd9d6a65e2f5ca255442f4c2cefa369b506 Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Thu, 23 Aug 2018 18:56:45 -0700
Subject: [PATCH 09/37] Add missing release notes for 6.0.9

---
 documentation/sphinx/source/release-notes.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 10358b62e6..623b74190b 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -48,6 +48,7 @@ Fixes
 * Sometimes a minority of coordinators would fail to converge after a new leader was elected. [6.0.6] `(PR #700) <https://github.com/apple/foundationdb/pull/700>`_
 * Calling status too many times in a 5 second interval caused the cluster controller to pause for a few seconds. [6.0.7] `(PR #711) <https://github.com/apple/foundationdb/pull/711>`_
 * Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.8] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_
+* TLS certificate reloading could cause TLS connections to drop until process restart [6.0.9] `(PR #717) <https://github.com/apple/foundationdb/pull/717>`_
 
 Status
 ------

From f60c927f16f5e5d22682236ff1059c2dfa2ffe5b Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Fri, 24 Aug 2018 00:52:34 -0700
Subject: [PATCH 10/37] Release 6.0.9 post-release MSI GUID+version bumps.

---
 packaging/msi/FDBInstaller.wxs | 2 +-
 versions.target                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs
index 95696ca553..507c5ffb10 100644
--- a/packaging/msi/FDBInstaller.wxs
+++ b/packaging/msi/FDBInstaller.wxs
@@ -32,7 +32,7 @@
 
 <Wix xmlns='http://schemas.microsoft.com/wix/2006/wi'>
   <Product Name='$(var.Title)'
-           Id='{F4C82ECC-493D-4363-A06C-139701B30473}'
+           Id='{90C29CFB-37C1-42F3-80B2-A1560145DA10}'
            UpgradeCode='{A95EA002-686E-4164-8356-C715B7F8B1C8}'
            Version='$(var.Version)'
            Manufacturer='$(var.Manufacturer)'
diff --git a/versions.target b/versions.target
index 96e95e324a..a5dba3c2ca 100644
--- a/versions.target
+++ b/versions.target
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
-    <Version>6.0.9</Version>
+    <Version>6.0.10</Version>
     <PackageName>6.0</PackageName>
   </PropertyGroup>
 </Project>

From c32102ad3e3c2d44cc78f412b47278fe648f40ef Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 24 Aug 2018 09:33:41 -0700
Subject: [PATCH 11/37] Add a release note

---
 documentation/sphinx/source/release-notes.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 623b74190b..38fb82d114 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -64,6 +64,7 @@ Bindings
 * API version updated to 600. There are no changes since API version 520.
 * Several cases where functions in go might previously cause a panic now return a non-``nil`` error. `(PR #532) <https://github.com/apple/foundationdb/pull/532>`_
 * C API calls made on the network thread could be reordered with calls made from other threads. [6.0.2] `(Issue #518) <https://github.com/apple/foundationdb/issues/518>`_
+* The TLS_PLUGIN option is now a no-op and has been deprecated. [6.0.10] `(PR #710) <https://github.com/apple/foundationdb/pull/710>`_
 
 Other Changes
 -------------

From adfaa618e607e47fe708b9d6df8a4bc1d8e10d84 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 24 Aug 2018 10:05:24 -0700
Subject: [PATCH 12/37] Filter version_too_old errors

---
 bindings/bindingtester/tests/directory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bindings/bindingtester/tests/directory.py b/bindings/bindingtester/tests/directory.py
index daa211b384..387381b5ea 100644
--- a/bindings/bindingtester/tests/directory.py
+++ b/bindings/bindingtester/tests/directory.py
@@ -378,7 +378,7 @@ class DirectoryTest(Test):
 
     def get_result_specifications(self):
         return [
-            ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1021]),
+            ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1007, 1021]),
             ResultSpecification(self.directory_log, ordering_index=0),
             ResultSpecification(self.subspace_log, ordering_index=0)
         ]

From b08792ba606f66a4f7a9be09c9f7829dae583a38 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 24 Aug 2018 14:16:57 -0700
Subject: [PATCH 13/37] Use the local flowKnobs and clientKnobs rather than
 const_casting the global ones

---
 fdbbackup/backup.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 0d18c5f207..9ec435c959 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -2630,8 +2630,8 @@ int main(int argc, char* argv[]) {
 
 		for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
 			try {
-				if (!const_cast<FlowKnobs*>(FLOW_KNOBS)->setKnob( k->first, k->second ) &&
-					!const_cast<ClientKnobs*>(CLIENT_KNOBS)->setKnob( k->first, k->second )) 
+				if (!flowKnobs->setKnob( k->first, k->second ) &&
+					!clientKnobs->setKnob( k->first, k->second )) 
 				{
 					fprintf(stderr, "Unrecognized knob option '%s'\n", k->first.c_str());
 					return FDB_EXIT_ERROR;

From affd7423b476fb455f8565149417fc79c212a740 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Fri, 31 Aug 2018 01:21:24 -0700
Subject: [PATCH 14/37] Added class to write json as objects are added
 Integrated JsonString class into status

---
 fdbclient/ClusterInterface.h |   4 +-
 fdbclient/Status.cpp         |   9 +
 fdbclient/Status.h           |   4 +-
 fdbclient/fdbclient.vcxproj  |   1 +
 fdbserver/Status.actor.cpp   | 461 ++++++++++++++++++-----------------
 flow/JsonString.cpp          | 396 ++++++++++++++++++++++++++++++
 flow/JsonString.h            | 138 +++++++++++
 flow/flow.vcxproj            |   2 +
 8 files changed, 794 insertions(+), 221 deletions(-)
 create mode 100644 fdbclient/Status.cpp
 create mode 100644 flow/JsonString.cpp
 create mode 100644 flow/JsonString.h

diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h
index 5fd8b6bb18..a35c897840 100644
--- a/fdbclient/ClusterInterface.h
+++ b/fdbclient/ClusterInterface.h
@@ -27,6 +27,7 @@
 #include "Status.h"
 #include "ClientDBInfo.h"
 #include "ClientWorkerInterface.h"
+#include "flow/JsonString.h"
 
 struct ClusterInterface {
 	RequestStream< struct OpenDatabaseRequest > openDatabase;
@@ -145,7 +146,7 @@ struct SystemFailureStatus {
 struct FailureMonitoringRequest {
 	// Sent by all participants to the cluster controller reply.clientRequestIntervalMS
 	//   ms after receiving the previous reply.
-	// Provides the controller the self-diagnosed status of the sender, and also 
+	// Provides the controller the self-diagnosed status of the sender, and also
 	//   requests the status of other systems.  Failure to timely send one of these implies
 	//   a failed status.
 	// If !senderStatus.present(), the sender wants to receive the latest failure information
@@ -192,6 +193,7 @@ struct StatusReply {
 
 	StatusReply() {}
 	explicit StatusReply(StatusObject obj) : statusObj(obj), statusStr(json_spirit::write_string(json_spirit::mValue(obj))) {}
+	explicit StatusReply(JsonString obj) : statusStr(obj.getJson()) {}
 
 	template <class Ar>
 	void serialize(Ar& ar) {
diff --git a/fdbclient/Status.cpp b/fdbclient/Status.cpp
new file mode 100644
index 0000000000..e666b601a5
--- /dev/null
+++ b/fdbclient/Status.cpp
@@ -0,0 +1,9 @@
+#include "flow/flow.h"
+#include "Status.h"
+
+JsonString	StatusObject::toJsonString() const {
+	JsonString	jsonStringObj;
+	std::string jsonText = json_spirit::write_string(json_spirit::mValue(*this));
+	jsonStringObj.setJson(jsonText);
+	return jsonStringObj;
+}
diff --git a/fdbclient/Status.h b/fdbclient/Status.h
index d061b71f56..d027113049 100644
--- a/fdbclient/Status.h
+++ b/fdbclient/Status.h
@@ -22,6 +22,7 @@
 #define FDBCLIENT_STATUS_H
 
 #include "../fdbrpc/JSONDoc.h"
+#include "../flow/JsonString.h"
 
 struct StatusObject : json_spirit::mObject {
 	typedef json_spirit::mObject Map;
@@ -29,6 +30,7 @@ struct StatusObject : json_spirit::mObject {
 
 	StatusObject() {}
 	StatusObject(json_spirit::mObject const& o) : json_spirit::mObject(o) {}
+	JsonString	toJsonString() const;
 };
 
 template <class Ar>
@@ -73,7 +75,7 @@ static StatusObject makeMessage(const char *name, const char *description) {
 // Typedef to cover older code that was written when this class was only a reader and called StatusObjectReader
 typedef JSONDoc StatusObjectReader;
 
-// Template specialization for get<JSONDoc> because is convenient to get() an 
+// Template specialization for get<JSONDoc> because is convenient to get() an
 // element from an object directly into a JSONDoc to have a handle to that sub-doc.
 template <> inline bool JSONDoc::get<JSONDoc>(const std::string path, StatusObjectReader &out, bool split) {
 	bool r = has(path, split);
diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj
index ceae4f38c3..00808c18bd 100755
--- a/fdbclient/fdbclient.vcxproj
+++ b/fdbclient/fdbclient.vcxproj
@@ -97,6 +97,7 @@
     <ActorCompiler Include="MultiVersionTransaction.actor.cpp" />
     <ClCompile Include="RYWIterator.cpp" />
     <ActorCompiler Include="StatusClient.actor.cpp" />
+    <ClCompile Include="Status.cpp" />
     <ClCompile Include="SystemData.cpp" />
     <ActorCompiler Include="ThreadSafeTransaction.actor.cpp" />
     <ActorCompiler Include="TaskBucket.actor.cpp" />
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index f851353b4b..8e05648075 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -158,38 +158,41 @@ static Optional<std::pair<WorkerInterface, ProcessClass>> getWorker(std::map<Net
 	return itr->second;
 }
 
-static StatusObject makeCounter(double hz=0.0, double r=0.0, int64_t c=0) {
-	StatusObject out;
-	out["hz"] = hz;
-	out["roughness"] = r;
-	out["counter"] = c;
-	return out;
-}
+class StatusCounter {
+	public:
+		StatusCounter(double hz=0.0, double roughness=0.0, int64_t counter=0) : _hz(hz), _roughness(roughness), _counter(counter) {}
+		StatusCounter(const std::string& parsableText) {
+			parseText(parsableText);
+		}
 
-static StatusObject parseCounter(std::string const& s) {
-	// Parse what traceCounters() in Stats.actor.cpp formats
-	double hz = 0.0, roughness = 0.0;
-	long long counter = 0;
-	sscanf(s.c_str(), "%lf %lf %lld", &hz, &roughness, &counter);
-	return makeCounter(hz, roughness, counter);
-}
+		StatusCounter& parseText(const std::string& parsableText) {
+			sscanf(parsableText.c_str(), "%lf %lf %lld", &_hz, &_roughness, &_counter);
+			return *this;
+		}
 
-static StatusObject addCounters(StatusObject c1, StatusObject c2) {
-	// "add" the given counter objects.  Roughness is averaged weighted by rate.
+		StatusCounter& updateValues(const StatusCounter& statusCounter) {
+			double hzNew = _hz + statusCounter._hz;
+			double roughnessNew = (_hz + statusCounter._hz) ? (_roughness*_hz + statusCounter._roughness*statusCounter._hz) / (_hz + statusCounter._hz) : 0.0;
+			int64_t counterNew = _counter + statusCounter._counter;
+			_hz = hzNew;
+			_roughness = roughnessNew;
+			_counter = counterNew;
+			return *this;
+		}
 
-	double c1hz = c1["hz"].get_real();
-	double c2hz = c2["hz"].get_real();
-	double c1r = c1["roughness"].get_real();
-	double c2r = c2["roughness"].get_real();
-	double c1c = c1["counter"].get_real();
-	double c2c = c2["counter"].get_real();
+		JsonString	getStatus() const {
+			JsonString statusObject;
+			statusObject["hz"] = _hz;
+			statusObject["roughness"] = _roughness;
+			statusObject["counter"] = _counter;
+			return statusObject;
+		}
 
-	return makeCounter(
-		c1hz+c2hz,
-		(c1hz + c2hz) ? (c1r*c1hz + c2r*c2hz) / (c1hz + c2hz) : 0.0,
-		c1c+c2c
-		);
-}
+	protected:
+		double	_hz;
+		double	_roughness;
+		int64_t	_counter;
+};
 
 static double parseDouble(std::string const& s, bool permissive = false) {
 	double d = 0;
@@ -222,23 +225,23 @@ static int64_t parseInt64(std::string const& s, bool permissive = false) {
 	throw attribute_not_found();
 }
 
-static StatusObject getLocalityInfo(const LocalityData& locality) {
-	StatusObject localityObj;
+static JsonString getLocalityInfo(const LocalityData& locality) {
+	JsonString localityObj;
 
 	for(auto it = locality._data.begin(); it != locality._data.end(); it++) {
 		if(it->second.present()) {
 			localityObj[it->first.toString()] = it->second.get().toString();
 		}
 		else {
-			localityObj[it->first.toString()] = json_spirit::mValue();
+			localityObj[it->first.toString()] = JsonString();
 		}
 	}
 
 	return localityObj;
 }
 
-static StatusObject getError(const TraceEventFields& errorFields) {
-	StatusObject statusObj;
+static JsonString getError(const TraceEventFields& errorFields) {
+	JsonString statusObj;
 	try {
 		if (errorFields.size()) {
 			double time = atof(errorFields.getValue("Time").c_str());
@@ -274,14 +277,17 @@ static StatusObject getError(const TraceEventFields& errorFields) {
 	return statusObj;
 }
 
-static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair<WorkerInterface, ProcessClass>> workers, Optional<DatabaseConfiguration> configuration, std::set<std::string> *incomplete_reasons) {
-	StatusObject machineMap;
+static JsonString machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair<WorkerInterface, ProcessClass>> workers, Optional<DatabaseConfiguration> configuration, std::set<std::string> *incomplete_reasons) {
+	JsonString machineMap;
 	double metric;
 	int failed = 0;
 
 	// map from machine networkAddress to datacenter ID
 	std::map<NetworkAddress, std::string> dcIds;
 	std::map<NetworkAddress, LocalityData> locality;
+	std::map<std::string, bool> notExcludedMap;
+	std::map<std::string, int32_t> workerContribMap;
+	std::map<std::string, JsonString> machineJsonMap;
 
 	for (auto worker : workers){
 		locality[worker.first.address()] = worker.first.locality;
@@ -295,7 +301,7 @@ static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair
 			continue;
 		}
 
-		StatusObject statusObj;  // Represents the status for a machine
+		JsonString statusObj;  // Represents the status for a machine
 		const TraceEventFields& event = it->second;
 
 		try {
@@ -304,7 +310,7 @@ static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair
 			std::string machineId = event.getValue("MachineID");
 
 			// If this machine ID does not already exist in the machineMap, add it
-			if (!machineMap.count(machineId)) {
+			if (!machineMap.isPresent(machineId)) {
 				statusObj["machine_id"] = machineId;
 
 				if (dcIds.count(it->first)){
@@ -317,7 +323,7 @@ static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair
 
 				statusObj["address"] = address;
 
-				StatusObject memoryObj;
+				JsonString memoryObj;
 
 				metric = parseDouble(event.getValue("TotalMemory"));
 				memoryObj["total_bytes"] = metric;
@@ -330,7 +336,7 @@ static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair
 
 				statusObj["memory"] = memoryObj;
 
-				StatusObject cpuObj;
+				JsonString cpuObj;
 
 				metric = parseDouble(event.getValue("CPUSeconds"));
 				double cpu_seconds = metric;
@@ -344,20 +350,20 @@ static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair
 
 				statusObj["cpu"] = cpuObj;
 
-				StatusObject networkObj;
+				JsonString networkObj;
 
 				metric = parseDouble(event.getValue("MbpsSent"));
-				StatusObject megabits_sent;
+				JsonString megabits_sent;
 				megabits_sent["hz"] = metric;
 				networkObj["megabits_sent"] = megabits_sent;
 
 				metric = parseDouble(event.getValue("MbpsReceived"));
-				StatusObject megabits_received;
+				JsonString megabits_received;
 				megabits_received["hz"] = metric;
 				networkObj["megabits_received"] = megabits_received;
 
 				metric = parseDouble(event.getValue("RetransSegs"));
-				StatusObject retransSegsObj;
+				JsonString retransSegsObj;
 				if (elapsed > 0){
 					retransSegsObj["hz"] = metric / elapsed;
 				}
@@ -366,23 +372,30 @@ static StatusObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair
 				statusObj["network"] = networkObj;
 
 				if (configuration.present()){
-					statusObj["excluded"] = true; // Will be set to false below if this or any later process is not excluded
+					notExcludedMap[machineId] = true; // Will be set to false below if this or any later process is not excluded
 				}
 
-				statusObj["contributing_workers"] = 0;
-
-				machineMap[machineId] = statusObj;
+				workerContribMap[machineId] = 0;
+				machineJsonMap[machineId] = statusObj;
 			}
-			if (configuration.present() && !configuration.get().isExcludedServer(it->first))
-				machineMap[machineId].get_obj()["excluded"] = false;
 
-			machineMap[machineId].get_obj()["contributing_workers"] = machineMap[machineId].get_obj()["contributing_workers"].get_int() + 1;
+			if (configuration.present() && !configuration.get().isExcludedServer(it->first))
+				notExcludedMap[machineId] = false;
+			workerContribMap[machineId] ++;
 		}
 		catch (Error& e) {
 			++failed;
 		}
 	}
 
+	// Add the status json for each machine with tracked values
+	for (auto mapPair : machineJsonMap) {
+		auto& machineId = mapPair.first;
+		machineJsonMap[machineId]["excluded"] = notExcludedMap[machineId];
+		machineJsonMap[machineId]["contributing_workers"] = workerContribMap[machineId];
+		machineMap[machineId] = machineJsonMap[machineId];
+	}
+
 	if(failed > 0)
 		incomplete_reasons->insert("Cannot retrieve all machine status information.");
 
@@ -400,15 +413,16 @@ struct MachineMemoryInfo {
 };
 
 struct RolesInfo {
-	std::multimap<NetworkAddress, StatusObject> roles;
-	StatusObject& addRole( NetworkAddress address, std::string const& role, UID id) {
-		StatusObject obj;
+	std::multimap<NetworkAddress, JsonString> roles;
+	JsonString& addRole( NetworkAddress address, std::string const& role, UID id) {
+		JsonString obj;
 		obj["id"] = id.shortString();
 		obj["role"] = role;
-		return roles.insert( make_pair(address, obj ))->second;
+		return roles.insert( std::make_pair(address, obj ))->second;
 	}
-	StatusObject& addRole(std::string const& role, StorageServerInterface& iface, TraceEventFields const& metrics, Version maxTLogVersion) {
-		StatusObject obj;
+	JsonString& addRole(std::string const& role, StorageServerInterface& iface, TraceEventFields const& metrics, Version maxTLogVersion, double* pDataLagSeconds) {
+		JsonString obj;
+		double dataLagSeconds = -1.0;
 		obj["id"] = iface.id().shortString();
 		obj["role"] = role;
 		try {
@@ -417,10 +431,10 @@ struct RolesInfo {
 			obj["kvstore_free_bytes"] = parseInt64(metrics.getValue("KvstoreBytesFree"));
 			obj["kvstore_available_bytes"] = parseInt64(metrics.getValue("KvstoreBytesAvailable"));
 			obj["kvstore_total_bytes"] = parseInt64(metrics.getValue("KvstoreBytesTotal"));
-			obj["input_bytes"] = parseCounter(metrics.getValue("BytesInput"));
-			obj["durable_bytes"] = parseCounter(metrics.getValue("BytesDurable"));
+			obj["input_bytes"] = StatusCounter(metrics.getValue("BytesInput")).getStatus();
+			obj["durable_bytes"] = StatusCounter(metrics.getValue("BytesDurable")).getStatus();
 			obj["query_queue_max"] = parseInt(metrics.getValue("QueryQueueMax"));
-			obj["finished_queries"] = parseCounter(metrics.getValue("FinishedQueries"));
+			obj["finished_queries"] = StatusCounter(metrics.getValue("FinishedQueries")).getStatus();
 
 			Version version = parseInt64(metrics.getValue("Version"));
 			obj["data_version"] = version;
@@ -435,9 +449,10 @@ struct RolesInfo {
 				versionLag = std::max<int64_t>(versionLag, maxTLogVersion - version - SERVER_KNOBS->STORAGE_LOGGING_DELAY * SERVER_KNOBS->VERSIONS_PER_SECOND);
 			}
 
-			StatusObject dataLag;
+			JsonString dataLag;
 			dataLag["versions"] = versionLag;
-			dataLag["seconds"] = versionLag / (double)SERVER_KNOBS->VERSIONS_PER_SECOND;
+			dataLagSeconds = versionLag / (double)SERVER_KNOBS->VERSIONS_PER_SECOND;
+			dataLag["seconds"] = dataLagSeconds;
 
 			obj["data_lag"] = dataLag;
 
@@ -445,10 +460,13 @@ struct RolesInfo {
 			if(e.code() != error_code_attribute_not_found)
 				throw e;
 		}
-		return roles.insert( make_pair(iface.address(), obj ))->second;
+		if (pDataLagSeconds)
+			*pDataLagSeconds = dataLagSeconds;
+		return roles.insert( std::make_pair(iface.address(), obj ))->second;
 	}
-	StatusObject& addRole(std::string const& role, TLogInterface& iface, TraceEventFields const& metrics) {
-		StatusObject obj;
+	JsonString& addRole(std::string const& role, TLogInterface& iface, TraceEventFields const& metrics, Version* pMetricVersion) {
+		JsonString obj;
+		Version	metricVersion = 0;
 		obj["id"] = iface.id().shortString();
 		obj["role"] = role;
 		try {
@@ -460,21 +478,24 @@ struct RolesInfo {
 			obj["queue_disk_free_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesFree"));
 			obj["queue_disk_available_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesAvailable"));
 			obj["queue_disk_total_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesTotal"));
-			obj["input_bytes"] = parseCounter(metrics.getValue("BytesInput"));
-			obj["durable_bytes"] = parseCounter(metrics.getValue("BytesDurable"));
-			obj["data_version"] = parseInt64(metrics.getValue("Version"));
+			obj["input_bytes"] = StatusCounter(metrics.getValue("BytesInput")).getStatus();
+			obj["durable_bytes"] = StatusCounter(metrics.getValue("BytesDurable")).getStatus();
+			metricVersion = parseInt64(metrics.getValue("Version"));
+			obj["data_version"] = metricVersion;
 		} catch (Error& e) {
 			if(e.code() != error_code_attribute_not_found)
 				throw e;
 		}
-		return roles.insert( make_pair(iface.address(), obj ))->second;
+		if (pMetricVersion)
+			*pMetricVersion = metricVersion;
+		return roles.insert( std::make_pair(iface.address(), obj ))->second;
 	}
 	template <class InterfaceType>
-	StatusObject& addRole(std::string const& role, InterfaceType& iface) {
+	JsonString& addRole(std::string const& role, InterfaceType& iface) {
 		return addRole(iface.address(), role, iface.id());
 	}
-	StatusArray getStatusForAddress( NetworkAddress a ) {
-		StatusArray v;
+	JsonStringArray getStatusForAddress( NetworkAddress a ) {
+		JsonStringArray v;
 		auto it = roles.lower_bound(a);
 		while (it != roles.end() && it->first == a) {
 			v.push_back(it->second);
@@ -484,7 +505,7 @@ struct RolesInfo {
 	}
 };
 
-ACTOR static Future<StatusObject> processStatusFetcher(
+ACTOR static Future<JsonString> processStatusFetcher(
 		Reference<AsyncVar<struct ServerDBInfo>> db,
 		std::vector<std::pair<WorkerInterface, ProcessClass>> workers,
 		WorkerEvents pMetrics,
@@ -492,7 +513,7 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 		WorkerEvents errors,
 		WorkerEvents traceFileOpenErrors,
 		WorkerEvents programStarts,
-		std::map<std::string, StatusObject> processIssues,
+		std::map<std::string, JsonString> processIssues,
 		vector<std::pair<StorageServerInterface, TraceEventFields>> storageServers,
 		vector<std::pair<TLogInterface, TraceEventFields>> tLogs,
 		Database cx,
@@ -500,12 +521,12 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 		std::set<std::string> *incomplete_reasons) {
 
 	// Array to hold one entry for each process
-	state StatusObject processMap;
+	state JsonString processMap;
 	state double metric;
 
 	// construct a map from a process address to a status object containing a trace file open error
 	// this is later added to the messages subsection
-	state std::map<std::string, StatusObject> tracefileOpenErrorMap;
+	state std::map<std::string, JsonString> tracefileOpenErrorMap;
 	state WorkerEvents::iterator traceFileErrorsItr;
 	for(traceFileErrorsItr = traceFileOpenErrors.begin(); traceFileErrorsItr != traceFileOpenErrors.end(); ++traceFileErrorsItr) {
 		Void _ = wait(yield());
@@ -514,7 +535,7 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 				// Have event fields, parse it and turn it into a message object describing the trace file opening error
 				const TraceEventFields& event = traceFileErrorsItr->second;
 				std::string fileName = event.getValue("Filename");
-				StatusObject msgObj = makeMessage("file_open_error", format("Could not open file '%s' (%s).", fileName.c_str(), event.getValue("Error").c_str()).c_str());
+				JsonString msgObj = JsonString::makeMessage("file_open_error", format("Could not open file '%s' (%s).", fileName.c_str(), event.getValue("Error").c_str()).c_str());
 				msgObj["file_name"] = fileName;
 
 				// Map the address of the worker to the error message object
@@ -565,21 +586,23 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 
 	state std::vector<std::pair<TLogInterface, TraceEventFields>>::iterator log;
 	state Version maxTLogVersion = 0;
+	state Version tLogVersion = 0;
+
+	// Get largest TLog version
 	for(log = tLogs.begin(); log != tLogs.end(); ++log) {
-		StatusObject const& roleStatus = roles.addRole( "log", log->first, log->second );
-		if(roleStatus.count("data_version") > 0) {
-			maxTLogVersion = std::max(maxTLogVersion, roleStatus.at("data_version").get_int64());
+		JsonString const& roleStatus = roles.addRole( "log", log->first, log->second, &tLogVersion );
+		if (roleStatus.isPresent("data_version")) {
+			maxTLogVersion = std::max(maxTLogVersion, tLogVersion);
 		}
 		Void _ = wait(yield());
 	}
 
 	state std::vector<std::pair<StorageServerInterface, TraceEventFields>>::iterator ss;
 	state std::map<NetworkAddress, double> ssLag;
+	state double lagSeconds;
 	for(ss = storageServers.begin(); ss != storageServers.end(); ++ss) {
-		StatusObject const& roleStatus = roles.addRole( "storage", ss->first, ss->second, maxTLogVersion );
-		JSONDoc doc(roleStatus);
-		double lagSeconds;
-		if(doc.tryGet("data_lag.seconds", lagSeconds)) {
+		JsonString const& roleStatus = roles.addRole( "storage", ss->first, ss->second, maxTLogVersion, &lagSeconds );
+		if (lagSeconds != -1.0) {
 			ssLag[ss->first.address()] = lagSeconds;
 		}
 		Void _ = wait(yield());
@@ -594,16 +617,16 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 
 	for(workerItr = workers.begin(); workerItr != workers.end(); ++workerItr) {
 		Void _ = wait(yield());
-		state StatusObject statusObj;
+		state JsonString statusObj;
 		try {
 			ASSERT(pMetrics.count(workerItr->first.address()));
 
-			processMap[printable(workerItr->first.locality.processId())] = StatusObject();
+			processMap[printable(workerItr->first.locality.processId())] = JsonString();
 
 			NetworkAddress address = workerItr->first.address();
 			const TraceEventFields& event = pMetrics[workerItr->first.address()];
 			statusObj["address"] = address.toString();
-			StatusObject memoryObj;
+			JsonString memoryObj;
 
 			if (event.size() > 0) {
 				std::string zoneID = event.getValue("ZoneID");
@@ -642,21 +665,21 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 				metric = parseDouble(event.getValue("DiskReadSectors"));
 				double diskReadSectors = metric;
 
-				StatusObject diskObj;
+				JsonString diskObj;
 				if (elapsed > 0){
-					StatusObject cpuObj;
+					JsonString cpuObj;
 					cpuObj["usage_cores"] = std::max(0.0, cpu_seconds / elapsed);
 					statusObj["cpu"] = cpuObj;
 
 					diskObj["busy"] = std::max(0.0, std::min((elapsed - diskIdleSeconds) / elapsed, 1.0));
 
-					StatusObject readsObj;
+					JsonString readsObj;
 					readsObj["counter"] = diskReadsCount;
 					if (elapsed > 0)
 						readsObj["hz"] = diskReads / elapsed;
 					readsObj["sectors"] = diskReadSectors;
 
-					StatusObject writesObj;
+					JsonString writesObj;
 					writesObj["counter"] = diskWritesCount;
 					if (elapsed > 0)
 						writesObj["hz"] = diskWrites / elapsed;
@@ -670,26 +693,26 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 				diskObj["free_bytes"] = parseInt64(event.getValue("DiskFreeBytes"));
 				statusObj["disk"] = diskObj;
 
-				StatusObject networkObj;
+				JsonString networkObj;
 
 				networkObj["current_connections"] = parseInt64(event.getValue("CurrentConnections"));
-				StatusObject connections_established;
+				JsonString connections_established;
 				connections_established["hz"] = parseDouble(event.getValue("ConnectionsEstablished"));
 				networkObj["connections_established"] = connections_established;
-				StatusObject connections_closed;
+				JsonString connections_closed;
 				connections_closed["hz"] = parseDouble(event.getValue("ConnectionsClosed"));
 				networkObj["connections_closed"] = connections_closed;
-				StatusObject connection_errors;
+				JsonString connection_errors;
 				connection_errors["hz"] = parseDouble(event.getValue("ConnectionErrors"));
 				networkObj["connection_errors"] = connection_errors;
 
 				metric = parseDouble(event.getValue("MbpsSent"));
-				StatusObject megabits_sent;
+				JsonString megabits_sent;
 				megabits_sent["hz"] = metric;
 				networkObj["megabits_sent"] = megabits_sent;
 
 				metric = parseDouble(event.getValue("MbpsReceived"));
-				StatusObject megabits_received;
+				JsonString megabits_received;
 				megabits_received["hz"] = metric;
 				networkObj["megabits_received"] = megabits_received;
 
@@ -736,7 +759,7 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 
 			statusObj["memory"] = memoryObj;
 
-			StatusArray messages;
+			JsonStringArray messages;
 
 			if (errors.count(address) && errors[address].size()) {
 				// returns status object with type and time of error
@@ -757,7 +780,7 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 			}
 
 			if(ssLag[address] >= 60) {
-				messages.push_back(makeMessage("storage_server_lagging", format("Storage server lagging by %ld seconds.", (int64_t)ssLag[address]).c_str()));
+				messages.push_back(JsonString::makeMessage("storage_server_lagging", format("Storage server lagging by %ld seconds.", (int64_t)ssLag[address]).c_str()));
 			}
 
 			// Store the message array into the status object that represents the worker process
@@ -784,8 +807,8 @@ ACTOR static Future<StatusObject> processStatusFetcher(
 	return processMap;
 }
 
-static StatusObject clientStatusFetcher(ClientVersionMap clientVersionMap, std::map<NetworkAddress, std::string> traceLogGroupMap) {
-	StatusObject clientStatus;
+static JsonString clientStatusFetcher(ClientVersionMap clientVersionMap, std::map<NetworkAddress, std::string> traceLogGroupMap) {
+	JsonString clientStatus;
 
 	clientStatus["count"] = (int64_t)clientVersionMap.size();
 
@@ -796,17 +819,17 @@ static StatusObject clientStatusFetcher(ClientVersionMap clientVersionMap, std::
 		}
 	}
 
-	StatusArray versionsArray = StatusArray();
+	JsonStringArray versionsArray = JsonStringArray();
 	for(auto cv : clientVersions) {
-		StatusObject ver;
+		JsonString ver;
 		ver["count"] = (int64_t)cv.second.size();
 		ver["client_version"] = cv.first.clientVersion.toString();
 		ver["protocol_version"] = cv.first.protocolVersion.toString();
 		ver["source_version"] = cv.first.sourceVersion.toString();
 
-		StatusArray clients = StatusArray();
+		JsonStringArray clients = JsonStringArray();
 		for(auto client : cv.second) {
-			StatusObject cli;
+			JsonString cli;
 			cli["address"] = client.toString();
 			cli["log_group"] = traceLogGroupMap[client];
 			clients.push_back(cli);
@@ -823,8 +846,8 @@ static StatusObject clientStatusFetcher(ClientVersionMap clientVersionMap, std::
 	return clientStatus;
 }
 
-ACTOR static Future<StatusObject> recoveryStateStatusFetcher(std::pair<WorkerInterface, ProcessClass> mWorker, int workerCount, std::set<std::string> *incomplete_reasons) {
-	state StatusObject message;
+ACTOR static Future<JsonString> recoveryStateStatusFetcher(std::pair<WorkerInterface, ProcessClass> mWorker, int workerCount, std::set<std::string> *incomplete_reasons, int* statusCode) {
+	state JsonString message;
 
 	try {
 		TraceEventFields md = wait( timeoutError(mWorker.first.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0) );
@@ -832,7 +855,8 @@ ACTOR static Future<StatusObject> recoveryStateStatusFetcher(std::pair<WorkerInt
 		if (mStatusCode < 0 || mStatusCode >= RecoveryStatus::END)
 			throw attribute_not_found();
 
-		message = makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]);
+		message = JsonString::makeMessage(RecoveryStatus::names[mStatusCode], RecoveryStatus::descriptions[mStatusCode]);
+		*statusCode = mStatusCode;
 
 		// Add additional metadata for certain statuses
 		if (mStatusCode == RecoveryStatus::recruiting_transaction_servers) {
@@ -857,7 +881,7 @@ ACTOR static Future<StatusObject> recoveryStateStatusFetcher(std::pair<WorkerInt
 	}
 
 	// If recovery status name is not know, status is incomplete
-	if (!message.count("name"))
+	if (!message.isPresent("name"))
 		incomplete_reasons->insert("Recovery Status unavailable.");
 
 	return message;
@@ -928,7 +952,7 @@ ACTOR static Future<double> doCommitProbe(Future<double> grvProbe, Transaction *
 	}
 }
 
-ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, const char* prefix, const char* description, StatusObject *probeObj, StatusArray *messages, std::set<std::string> *incomplete_reasons) {
+ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, const char* prefix, const char* description, JsonString *probeObj, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
 	choose {
 		when(ErrorOr<double> result = wait(errorOr(probe))) {
 			if(result.isError()) {
@@ -939,20 +963,20 @@ ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, cons
 			}
 		}
 		when(Void _ = wait(delay(timeoutSeconds))) {
-			messages->push_back(makeMessage(format("%s_probe_timeout", prefix).c_str(), format("Unable to %s after %d seconds.", description, timeoutSeconds).c_str()));
+			messages->push_back(JsonString::makeMessage(format("%s_probe_timeout", prefix).c_str(), format("Unable to %s after %d seconds.", description, timeoutSeconds).c_str()));
 		}
 	}
 
 	return Void();
 }
 
-ACTOR static Future<StatusObject> latencyProbeFetcher(Database cx, StatusArray *messages, std::set<std::string> *incomplete_reasons) {
+ACTOR static Future<JsonString> latencyProbeFetcher(Database cx, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
 	state Transaction trImmediate(cx);
 	state Transaction trDefault(cx);
 	state Transaction trBatch(cx);
 	state Transaction trWrite(cx);
 
-	state StatusObject statusObj;
+	state JsonString statusObj;
 
 	try {
 		Future<double> immediateGrvProbe = doGrvProbe(&trImmediate, FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@@ -980,7 +1004,7 @@ ACTOR static Future<StatusObject> latencyProbeFetcher(Database cx, StatusArray *
 	return statusObj;
 }
 
-ACTOR static Future<std::pair<Optional<DatabaseConfiguration>,Optional<bool>>> loadConfiguration(Database cx, StatusArray *messages, std::set<std::string> *status_incomplete_reasons){
+ACTOR static Future<std::pair<Optional<DatabaseConfiguration>,Optional<bool>>> loadConfiguration(Database cx, JsonStringArray *messages, std::set<std::string> *status_incomplete_reasons){
 	state Optional<DatabaseConfiguration> result;
 	state Optional<bool> fullReplication;
 	state Transaction tr(cx);
@@ -1004,9 +1028,9 @@ ACTOR static Future<std::pair<Optional<DatabaseConfiguration>,Optional<bool>>> l
 				}
 				when(Void _ = wait(getConfTimeout)) {
 					if(!result.present()) {
-						messages->push_back(makeMessage("unreadable_configuration", "Unable to read database configuration."));
+						messages->push_back(JsonString::makeMessage("unreadable_configuration", "Unable to read database configuration."));
 					} else {
-						messages->push_back(makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
+						messages->push_back(JsonString::makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
 					}
 					break;
 				}
@@ -1030,7 +1054,7 @@ ACTOR static Future<std::pair<Optional<DatabaseConfiguration>,Optional<bool>>> l
 					fullReplication = (!unreplicated || (result.get().usableRegions == 1 && unreplicated < result.get().regions.size()));
 				}
 				when(Void _ = wait(getConfTimeout)) {
-					messages->push_back(makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
+					messages->push_back(JsonString::makeMessage("full_replication_timeout", "Unable to read datacenter replicas."));
 				}
 			}
 			break;
@@ -1042,23 +1066,23 @@ ACTOR static Future<std::pair<Optional<DatabaseConfiguration>,Optional<bool>>> l
 	return std::make_pair(result, fullReplication);
 }
 
-static StatusObject configurationFetcher(Optional<DatabaseConfiguration> conf, ServerCoordinators coordinators, std::set<std::string> *incomplete_reasons) {
-	StatusObject statusObj;
+static JsonString configurationFetcher(Optional<DatabaseConfiguration> conf, ServerCoordinators coordinators, std::set<std::string> *incomplete_reasons) {
+	JsonString statusObj;
 	try {
 		if(conf.present()) {
 			DatabaseConfiguration configuration = conf.get();
-			statusObj = configuration.toJSON();
+			statusObj = configuration.toJSON().toJsonString();
+//ahm			statusObj.setJson(configuration.toJSON());
 
-			StatusArray excludedServersArr;
+			JsonStringArray excludedServersArr;
 			std::set<AddressExclusion> excludedServers = configuration.getExcludedServers();
 			for (std::set<AddressExclusion>::iterator it = excludedServers.begin(); it != excludedServers.end(); it++) {
-				StatusObject statusObj;
+				JsonString statusObj;
 				statusObj["address"] = it->toString();
 				excludedServersArr.push_back(statusObj);
 			}
 			statusObj["excluded_servers"] = excludedServersArr;
 		}
-		StatusArray coordinatorLeaderServersArr;
 		vector< ClientLeaderRegInterface > coordinatorLeaderServers = coordinators.clientLeaderServers;
 		int count = coordinatorLeaderServers.size();
 		statusObj["coordinators_count"] = count;
@@ -1069,8 +1093,8 @@ static StatusObject configurationFetcher(Optional<DatabaseConfiguration> conf, S
 	return statusObj;
 }
 
-ACTOR static Future<StatusObject> dataStatusFetcher(std::pair<WorkerInterface, ProcessClass> mWorker, std::string dbName, int *minReplicasRemaining) {
-	state StatusObject statusObjData;
+ACTOR static Future<JsonString> dataStatusFetcher(std::pair<WorkerInterface, ProcessClass> mWorker, std::string dbName, int *minReplicasRemaining) {
+	state JsonString statusObjData;
 
 	try {
 		std::vector<Future<TraceEventFields>> futures;
@@ -1088,7 +1112,7 @@ ACTOR static Future<StatusObject> dataStatusFetcher(std::pair<WorkerInterface, P
 		TraceEventFields dataStats = dataInfo[1];
 
 		if (startingStats.size() && startingStats.getValue("State") != "Active") {
-			StatusObject stateSectionObj;
+			JsonString stateSectionObj;
 			stateSectionObj["name"] = "initializing";
 			stateSectionObj["description"] = "(Re)initializing automatic data distribution";
 			statusObjData["state"] = stateSectionObj;
@@ -1107,7 +1131,7 @@ ACTOR static Future<StatusObject> dataStatusFetcher(std::pair<WorkerInterface, P
 			int highestPriority = parseInt(md.getValue("HighestPriority"));
 
 			if( averagePartitionSize >= 0 ) {
-				StatusObject moving_data;
+				JsonString moving_data;
 				moving_data["in_queue_bytes"] = partitionsInQueue * averagePartitionSize;
 				moving_data["in_flight_bytes"] = partitionsInFlight * averagePartitionSize;
 				moving_data["total_written_bytes"] = totalBytesWritten;
@@ -1128,7 +1152,7 @@ ACTOR static Future<StatusObject> dataStatusFetcher(std::pair<WorkerInterface, P
 			statusObjData["partitions_count"] = shards;
 		}
 
-		StatusArray teamTrackers;
+		JsonStringArray teamTrackers;
 		for(int i = 0; i < 2; i++) {
 			TraceEventFields inFlight = dataInfo[3 + i];
 			if (!inFlight.size()) {
@@ -1140,12 +1164,12 @@ ACTOR static Future<StatusObject> dataStatusFetcher(std::pair<WorkerInterface, P
 			int unhealthyServers = parseInt(inFlight.getValue("UnhealthyServers"));
 			int highestPriority = parseInt(inFlight.getValue("HighestPriority"));
 
-			StatusObject team_tracker;
+			JsonString team_tracker;
 			team_tracker["primary"] = primary;
 			team_tracker["in_flight_bytes"] = totalDataInFlight;
 			team_tracker["unhealthy_servers"] = unhealthyServers;
 
-			StatusObject stateSectionObj;
+			JsonString stateSectionObj;
 			if (highestPriority >= PRIORITY_TEAM_0_LEFT) {
 				stateSectionObj["healthy"] = false;
 				stateSectionObj["name"] = "missing_data";
@@ -1274,7 +1298,7 @@ static int getExtraTLogEligibleMachines(vector<std::pair<WorkerInterface, Proces
 
 	if(configuration.regions.size() == 0) {
 		return allMachines.size() - std::max(configuration.tLogReplicationFactor, configuration.storageTeamSize);
-	} 
+	}
 	int extraTlogEligibleMachines = std::numeric_limits<int>::max();
 	for(auto& region : configuration.regions) {
 		extraTlogEligibleMachines = std::min<int>( extraTlogEligibleMachines, dcId_machine[region.dcId].size() - std::max(configuration.remoteTLogReplicationFactor, std::max(configuration.tLogReplicationFactor, configuration.storageTeamSize) ) );
@@ -1290,13 +1314,13 @@ static int getExtraTLogEligibleMachines(vector<std::pair<WorkerInterface, Proces
 	return extraTlogEligibleMachines;
 }
 
-ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, vector<std::pair<WorkerInterface, ProcessClass>> workers, std::pair<WorkerInterface, ProcessClass> mWorker, 
-	std::string dbName, StatusObject *qos, StatusObject *data_overlay, std::set<std::string> *incomplete_reasons, Future<ErrorOr<vector<std::pair<StorageServerInterface, TraceEventFields>>>> storageServerFuture) 
+ACTOR static Future<JsonString> workloadStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, vector<std::pair<WorkerInterface, ProcessClass>> workers, std::pair<WorkerInterface, ProcessClass> mWorker,
+	std::string dbName, JsonString *qos, JsonString *data_overlay, std::set<std::string> *incomplete_reasons, Future<ErrorOr<vector<std::pair<StorageServerInterface, TraceEventFields>>>> storageServerFuture)
 {
-	state StatusObject statusObj;
-	state StatusObject operationsObj;
-	state StatusObject bytesObj;
-	state StatusObject keysObj;
+	state JsonString statusObj;
+	state JsonString operationsObj;
+	state JsonString bytesObj;
+	state JsonString keysObj;
 
 	// Writes and conflicts
 	try {
@@ -1314,23 +1338,23 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
 		}
 		vector<TraceEventFields> proxyStats = wait(getAll(proxyStatFutures));
 
-		StatusObject mutations=makeCounter(), mutationBytes=makeCounter(), txnConflicts=makeCounter(), txnStartOut=makeCounter(), txnCommitOutSuccess=makeCounter();
+		StatusCounter mutations, mutationBytes, txnConflicts, txnStartOut, txnCommitOutSuccess;
 
 		for (auto &ps : proxyStats) {
-			mutations = addCounters( mutations, parseCounter(ps.getValue("Mutations")) );
-			mutationBytes = addCounters( mutationBytes, parseCounter(ps.getValue("MutationBytes")) );
-			txnConflicts = addCounters( txnConflicts, parseCounter(ps.getValue("TxnConflicts")) );
-			txnStartOut = addCounters( txnStartOut, parseCounter(ps.getValue("TxnStartOut")) );
-			txnCommitOutSuccess = addCounters( txnCommitOutSuccess, parseCounter(ps.getValue("TxnCommitOutSuccess")) );
+			mutations.updateValues( StatusCounter(ps.getValue("Mutations")) );
+			mutationBytes.updateValues( StatusCounter(ps.getValue("MutationBytes")) );
+			txnConflicts.updateValues( StatusCounter(ps.getValue("TxnConflicts")) );
+			txnStartOut.updateValues( StatusCounter(ps.getValue("TxnStartOut")) );
+			txnCommitOutSuccess.updateValues( StatusCounter(ps.getValue("TxnCommitOutSuccess")) );
 		}
 
-		operationsObj["writes"] = mutations;
-		bytesObj["written"] = mutationBytes;
+		operationsObj["writes"] = mutations.getStatus();
+		bytesObj["written"] = mutationBytes.getStatus();
 
-		StatusObject transactions;
-		transactions["conflicted"] = txnConflicts;
-		transactions["started"] = txnStartOut;
-		transactions["committed"] = txnCommitOutSuccess;
+		JsonString transactions;
+		transactions["conflicted"] = txnConflicts.getStatus();
+		transactions["started"] = txnStartOut.getStatus();
+		transactions["committed"] = txnCommitOutSuccess.getStatus();
 
 		statusObj["transactions"] = transactions;
 	}
@@ -1374,18 +1398,18 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
 		(*qos)["released_transactions_per_second"] = transPerSec;
 
 		int reason = parseInt(md.getValue("Reason"));
-		StatusObject perfLimit;
+		JsonString perfLimit;
 		if (transPerSec > tpsLimit * 0.8) {
 			// If reason is known, set qos.performance_limited_by, otherwise omit
 			if (reason >= 0 && reason < limitReasonEnd) {
-				perfLimit = makeMessage(limitReasonName[reason], limitReasonDesc[reason]);
+				perfLimit = JsonString::makeMessage(limitReasonName[reason], limitReasonDesc[reason]);
 				std::string reason_server_id = md.getValue("ReasonServerID");
 				if (!reason_server_id.empty())
 					perfLimit["reason_server_id"] = reason_server_id;
 			}
 		}
 		else {
-			perfLimit = makeMessage("workload", "The database is not being saturated by the workload.");
+			perfLimit = JsonString::makeMessage("workload", "The database is not being saturated by the workload.");
 		}
 
 		if(!perfLimit.empty()) {
@@ -1405,19 +1429,19 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
 			throw storageServers.getError();
 		}
 
-		StatusObject reads = makeCounter();
-		StatusObject readKeys = makeCounter();
-		StatusObject readBytes = makeCounter();
+		StatusCounter reads;
+		StatusCounter readKeys;
+		StatusCounter readBytes;
 
 		for(auto &ss : storageServers.get()) {
-			reads = addCounters(reads, parseCounter(ss.second.getValue("FinishedQueries")));
-			readKeys = addCounters(readKeys, parseCounter(ss.second.getValue("RowsQueried")));
-			readBytes = addCounters(readBytes, parseCounter(ss.second.getValue("BytesQueried")));
+			reads.updateValues( StatusCounter(ss.second.getValue("FinishedQueries")));
+			readKeys.updateValues( StatusCounter(ss.second.getValue("RowsQueried")));
+			readBytes.updateValues( StatusCounter(ss.second.getValue("BytesQueried")));
 		}
 
-		operationsObj["reads"] = reads;
-		keysObj["read"] = readKeys;
-		bytesObj["read"] = readBytes;
+		operationsObj["reads"] = reads.getStatus();
+		keysObj["read"] = readKeys.getStatus();
+		bytesObj["read"] = readBytes.getStatus();
 
 	}
 	catch (Error& e) {
@@ -1426,7 +1450,6 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
 		incomplete_reasons->insert("Unknown read state.");
 	}
 
-
 	statusObj["operations"] = operationsObj;
 	statusObj["keys"] = keysObj;
 	statusObj["bytes"] = bytesObj;
@@ -1434,19 +1457,19 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
 	return statusObj;
 }
 
-static StatusArray oldTlogFetcher(int* oldLogFaultTolerance, Reference<AsyncVar<struct ServerDBInfo>> db, std::unordered_map<NetworkAddress, WorkerInterface> const& address_workers) {
-	StatusArray oldTlogsArray;
+static JsonStringArray oldTlogFetcher(int* oldLogFaultTolerance, Reference<AsyncVar<struct ServerDBInfo>> db, std::unordered_map<NetworkAddress, WorkerInterface> const& address_workers) {
+	JsonStringArray oldTlogsArray;
 
 	if(db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS) {
 		for(auto it : db->get().logSystemConfig.oldTLogs) {
-			StatusObject statusObj;
-			StatusArray logsObj;
+			JsonString statusObj;
+			JsonStringArray logsObj;
 			int maxFaultTolerance = 0;
-			
+
 			for(int i = 0; i < it.tLogs.size(); i++) {
 				int failedLogs = 0;
 				for(auto& log : it.tLogs[i].tLogs) {
-					StatusObject logObj;
+					JsonString logObj;
 					bool failed = !log.present() || !address_workers.count(log.interf().address());
 					logObj["id"] = log.id().shortString();
 					logObj["healthy"] = !failed;
@@ -1483,14 +1506,8 @@ static StatusArray oldTlogFetcher(int* oldLogFaultTolerance, Reference<AsyncVar<
 	return oldTlogsArray;
 }
 
-/*
-static StatusObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, int numTLogEligibleMachines, int minReplicasRemaining, int oldLogFaultTolerance) {
-=======
-static StatusObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, std::vector<std::pair<WorkerInterface, ProcessClass>>& workers, int numTLogEligibleMachines, int minReplicasRemaining) {
-*/
-
-static StatusObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, std::vector<std::pair<WorkerInterface, ProcessClass>>& workers, int extraTlogEligibleMachines, int minReplicasRemaining) {
-	StatusObject statusObj;
+static JsonString faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, std::vector<std::pair<WorkerInterface, ProcessClass>>& workers, int extraTlogEligibleMachines, int minReplicasRemaining) {
+	JsonString statusObj;
 
 	// without losing data
 	int32_t maxMachineFailures = configuration.maxMachineFailuresTolerated();
@@ -1524,8 +1541,6 @@ static StatusObject faultToleranceStatusFetcher(DatabaseConfiguration configurat
 	if (minReplicasRemaining >= 0){
 		machineFailuresWithoutLosingData = std::min(machineFailuresWithoutLosingData, minReplicasRemaining - 1);
 	}
-	// ahm
-	//	machineFailuresWithoutLosingData = std::min(machineFailuresWithoutLosingData, oldLogFaultTolerance);
 
 	statusObj["max_machine_failures_without_losing_data"] = std::max(machineFailuresWithoutLosingData, 0);
 
@@ -1543,13 +1558,13 @@ static std::string getIssueDescription(std::string name) {
 	return name;
 }
 
-static std::map<std::string, StatusObject> getProcessIssuesAsMessages( ProcessIssuesMap const& _issues ) {
-	std::map<std::string, StatusObject> issuesMap;
+static std::map<std::string, JsonString> getProcessIssuesAsMessages( ProcessIssuesMap const& _issues ) {
+	std::map<std::string, JsonString> issuesMap;
 
 	try {
 		ProcessIssuesMap issues = _issues;
 		for (auto i : issues) {
-			StatusObject message = makeMessage(i.second.first.c_str(), getIssueDescription(i.second.first).c_str());
+			JsonString message = JsonString::makeMessage(i.second.first.c_str(), getIssueDescription(i.second.first).c_str());
 			issuesMap[i.first.toString()] = message;
 		}
 	}
@@ -1561,8 +1576,8 @@ static std::map<std::string, StatusObject> getProcessIssuesAsMessages( ProcessIs
 	return issuesMap;
 }
 
-static StatusArray getClientIssuesAsMessages( ProcessIssuesMap const& _issues) {
-	StatusArray issuesList;
+static JsonStringArray getClientIssuesAsMessages( ProcessIssuesMap const& _issues) {
+	JsonStringArray issuesList;
 
 	try {
 		ProcessIssuesMap issues = _issues;
@@ -1573,8 +1588,8 @@ static StatusArray getClientIssuesAsMessages( ProcessIssuesMap const& _issues) {
 		}
 
 		for (auto i : deduplicatedIssues) {
-			StatusObject message = makeMessage(i.first.c_str(), getIssueDescription(i.first).c_str());
-			StatusArray addresses;
+			JsonString message = JsonString::makeMessage(i.first.c_str(), getIssueDescription(i.first).c_str());
+			JsonStringArray addresses;
 			for(auto addr : i.second) {
 				addresses.push_back(addr);
 			}
@@ -1591,9 +1606,11 @@ static StatusArray getClientIssuesAsMessages( ProcessIssuesMap const& _issues) {
 	return issuesList;
 }
 
-ACTOR Future<StatusObject> layerStatusFetcher(Database cx, StatusArray *messages, std::set<std::string> *incomplete_reasons) {
-	state StatusObject result;
-	state JSONDoc json(result);
+// ahm What is this doing?
+ACTOR Future<JsonString> layerStatusFetcher(Database cx, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
+	state JsonString result;
+//ahm	state JSONDoc json(result);
+	state JSONDoc json;
 
 	try {
 		state ReadYourWritesTransaction tr(cx);
@@ -1647,8 +1664,8 @@ ACTOR Future<StatusObject> layerStatusFetcher(Database cx, StatusArray *messages
 	return result;
 }
 
-ACTOR Future<StatusObject> lockedStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, StatusArray *messages, std::set<std::string> *incomplete_reasons) {
-	state StatusObject statusObj;
+ACTOR Future<JsonString> lockedStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
+	state JsonString statusObj;
 
 	state Database cx = openDBOnServer(db, TaskDefaultEndpoint, true, false); // Open a new database connection that isn't lock-aware
 	state Transaction tr(cx);
@@ -1705,7 +1722,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 	state std::string dbName = "DB";
 
 	// Check if master worker is present
-	state StatusArray messages;
+	state JsonStringArray messages;
 	state std::set<std::string> status_incomplete_reasons;
 	state std::pair<WorkerInterface, ProcessClass> mWorker;
 
@@ -1715,7 +1732,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		if (_mWorker.present()) {
 			mWorker = _mWorker.get();
 		} else {
-			messages.push_back(makeMessage("unreachable_master_worker", "Unable to locate the master worker."));
+			messages.push_back(JsonString::makeMessage("unreachable_master_worker", "Unable to locate the master worker."));
 		}
 
 		// Get latest events for various event types from ALL workers
@@ -1746,17 +1763,18 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		// We now have a unique set of workers who were in some way unreachable.  If there is anything in that set, create a message
 		// for it and include the list of unreachable processes.
 		if (mergeUnreachable.size()){
-			StatusObject message = makeMessage("unreachable_processes", "The cluster has some unreachable processes.");
-			StatusArray unreachableProcs;
+			JsonString message = JsonString::makeMessage("unreachable_processes", "The cluster has some unreachable processes.");
+			JsonStringArray unreachableProcs;
 			for (auto m : mergeUnreachable){
-				unreachableProcs.push_back(StatusObject({ {"address", m} }));
+				unreachableProcs.push_back(JsonString("address", m));	//ahm I'm pretty sure this is right
 			}
 			message["unreachable_processes"] = unreachableProcs;
 			messages.push_back(message);
 		}
 
 		// construct status information for cluster subsections
-		state StatusObject recoveryStateStatus = wait(recoveryStateStatusFetcher(mWorker, workers.size(), &status_incomplete_reasons));
+		state int statusCode = (int) RecoveryStatus::END;
+		state JsonString recoveryStateStatus = wait(recoveryStateStatusFetcher(mWorker, workers.size(), &status_incomplete_reasons, &statusCode));
 
 		// machine metrics
 		state WorkerEvents mMetrics = workerEventsVec[0].present() ? workerEventsVec[0].get().first : WorkerEvents();
@@ -1766,16 +1784,16 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		state WorkerEvents traceFileOpenErrors = workerEventsVec[3].present() ? workerEventsVec[3].get().first : WorkerEvents();
 		state WorkerEvents programStarts = workerEventsVec[4].present() ? workerEventsVec[4].get().first : WorkerEvents();
 
-		state StatusObject statusObj;
+		state JsonString statusObj;
 		if(db->get().recoveryCount > 0) {
 			statusObj["generation"] = db->get().recoveryCount;
 		}
 
-		state std::map<std::string, StatusObject> processIssues = getProcessIssuesAsMessages(workerIssues);
+		state std::map<std::string, JsonString> processIssues = getProcessIssuesAsMessages(workerIssues);
 		state vector<std::pair<StorageServerInterface, TraceEventFields>> storageServers;
 		state vector<std::pair<TLogInterface, TraceEventFields>> tLogs;
-		state StatusObject qos;
-		state StatusObject data_overlay;
+		state JsonString qos;
+		state JsonString data_overlay;
 
 		statusObj["protocol_version"] = format("%llx", currentProtocolVersion);
 		statusObj["connection_string"] = coordinators.ccf->getConnectionString().toString();
@@ -1783,7 +1801,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		state Optional<DatabaseConfiguration> configuration;
 		state Optional<bool> fullReplication;
 
-		if(!(recoveryStateStatus.count("name") && recoveryStateStatus["name"] == RecoveryStatus::names[RecoveryStatus::configuration_missing])) {
+		if(!(recoveryStateStatus.isPresent("name") && statusCode == RecoveryStatus::configuration_missing)) {
 			std::pair<Optional<DatabaseConfiguration>,Optional<bool>> loadResults = wait(loadConfiguration(cx, &messages, &status_incomplete_reasons));
 			configuration = loadResults.first;
 			fullReplication = loadResults.second;
@@ -1797,9 +1815,9 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		if (configuration.present()){
 			// Do the latency probe by itself to avoid interference from other status activities
-			StatusObject latencyProbeResults = wait(latencyProbeFetcher(cx, &messages, &status_incomplete_reasons));
+			JsonString latencyProbeResults = wait(latencyProbeFetcher(cx, &messages, &status_incomplete_reasons));
 
-			statusObj["database_available"] = latencyProbeResults.count("immediate_priority_transaction_start_seconds") && latencyProbeResults.count("read_seconds") && latencyProbeResults.count("commit_seconds");
+			statusObj["database_available"] = latencyProbeResults.isPresent("immediate_priority_transaction_start_seconds") && latencyProbeResults.isPresent("read_seconds") && latencyProbeResults.isPresent("commit_seconds");
 			if (!latencyProbeResults.empty()) {
 				statusObj["latency_probe"] = latencyProbeResults;
 			}
@@ -1813,13 +1831,13 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			state Future<ErrorOr<vector<std::pair<TLogInterface, TraceEventFields>>>> tLogFuture = errorOr(getTLogsAndMetrics(db, address_workers));
 
 			state int minReplicasRemaining = -1;
-			std::vector<Future<StatusObject>> futures2;
+			std::vector<Future<JsonString>> futures2;
 			futures2.push_back(dataStatusFetcher(mWorker, dbName, &minReplicasRemaining));
 			futures2.push_back(workloadStatusFetcher(db, workers, mWorker, dbName, &qos, &data_overlay, &status_incomplete_reasons, storageServerFuture));
 			futures2.push_back(layerStatusFetcher(cx, &messages, &status_incomplete_reasons));
 			futures2.push_back(lockedStatusFetcher(db, &messages, &status_incomplete_reasons));
 
-			state std::vector<StatusObject> workerStatuses = wait(getAll(futures2));
+			state std::vector<JsonString> workerStatuses = wait(getAll(futures2));
 
 			int oldLogFaultTolerance = 100;
 			if(db->get().recoveryState >= RecoveryState::ACCEPTING_COMMITS && db->get().logSystemConfig.oldTLogs.size() > 0) {
@@ -1831,7 +1849,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				statusObj["fault_tolerance"] = faultToleranceStatusFetcher(configuration.get(), coordinators, workers, extraTlogEligibleMachines, minReplicasRemaining);
 			}
 
-			StatusObject configObj = configurationFetcher(configuration, coordinators, &status_incomplete_reasons);
+			JsonString configObj = configurationFetcher(configuration, coordinators, &status_incomplete_reasons);
 
 			// configArr could be empty
 			if (!configObj.empty())
@@ -1848,8 +1866,9 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				statusObj["qos"] = qos;
 
 			// Merge data_overlay into data
-			StatusObject &clusterDataSection = workerStatuses[0];
-			clusterDataSection.insert(data_overlay.begin(), data_overlay.end());
+			JsonString &clusterDataSection = workerStatuses[0];
+			clusterDataSection.append(data_overlay); //ahm This is a problem, if dupe keys
+//ahm			clusterDataSection.insert(data_overlay.begin(), data_overlay.end());
 
 			// If data section not empty, add it to statusObj
 			if (!clusterDataSection.empty())
@@ -1857,7 +1876,8 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 			// Insert database_locked section
 			if(!workerStatuses[3].empty()) {
-				statusObj.insert(workerStatuses[3].begin(), workerStatuses[3].end());
+				statusObj.append(workerStatuses[3]); //ahm This is a problem, if dupe keys
+//ahm				statusObj.insert(workerStatuses[3].begin(), workerStatuses[3].end());
 			}
 
 			// Need storage servers now for processStatusFetcher() below.
@@ -1866,7 +1886,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				storageServers = _storageServers.get();
 			}
 			else
-				messages.push_back(makeMessage("storage_servers_error", "Timed out trying to retrieve storage servers."));
+				messages.push_back(JsonString::makeMessage("storage_servers_error", "Timed out trying to retrieve storage servers."));
 
 			// ...also tlogs
 			ErrorOr<vector<std::pair<TLogInterface, TraceEventFields>>> _tLogs = wait(tLogFuture);
@@ -1874,18 +1894,20 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				tLogs = _tLogs.get();
 			}
 			else
-				messages.push_back(makeMessage("log_servers_error", "Timed out trying to retrieve log servers."));
+				messages.push_back(JsonString::makeMessage("log_servers_error", "Timed out trying to retrieve log servers."));
 		}
 		else {
 			// Set layers status to { _valid: false, error: "configurationMissing"}
-			statusObj["layers"] = json_spirit::mObject({{"_valid", false}, {"_error", "configurationMissing"}});
+			JsonString	jsonString("_valid", false);
+			jsonString.append("_error", "configurationMissing");
+			statusObj["layers"] = jsonString;
 		}
 
-		StatusObject processStatus = wait(processStatusFetcher(db, workers, pMetrics, mMetrics, latestError, traceFileOpenErrors, programStarts, processIssues, storageServers, tLogs, cx, configuration, &status_incomplete_reasons));
+		JsonString processStatus = wait(processStatusFetcher(db, workers, pMetrics, mMetrics, latestError, traceFileOpenErrors, programStarts, processIssues, storageServers, tLogs, cx, configuration, &status_incomplete_reasons));
 		statusObj["processes"] = processStatus;
 		statusObj["clients"] = clientStatusFetcher(clientVersionMap, traceLogGroupMap);
 
-		StatusArray incompatibleConnectionsArray;
+		JsonStringArray incompatibleConnectionsArray;
 		for(auto it : incompatibleConnections) {
 			incompatibleConnectionsArray.push_back(it.toString());
 		}
@@ -1896,9 +1918,9 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			statusObj["recovery_state"] = recoveryStateStatus;
 
 		// cluster messages subsection;
-		StatusArray clientIssuesArr = getClientIssuesAsMessages(clientIssues);
+		JsonStringArray clientIssuesArr = getClientIssuesAsMessages(clientIssues);
 		if (clientIssuesArr.size() > 0) {
-			StatusObject clientIssueMessage = makeMessage("client_issues", "Some clients of this cluster have issues.");
+			JsonString clientIssueMessage = JsonString::makeMessage("client_issues", "Some clients of this cluster have issues.");
 			clientIssueMessage["issues"] = clientIssuesArr;
 			messages.push_back(clientIssueMessage);
 		}
@@ -1906,11 +1928,12 @@ ACTOR Future<StatusReply> clusterGetStatus(
 		// Create the status_incomplete message if there were any reasons that the status is incomplete.
 		if (!status_incomplete_reasons.empty())
 		{
-			StatusObject incomplete_message = makeMessage("status_incomplete", "Unable to retrieve all status information.");
+			JsonString incomplete_message = JsonString::makeMessage("status_incomplete", "Unable to retrieve all status information.");
 			// Make a JSON array of all of the reasons in the status_incomplete_reasons set.
-			StatusArray reasons;
-			for (auto i : status_incomplete_reasons)
-				reasons.push_back(StatusObject({ { "description", i } }));
+			JsonStringArray reasons;
+			for (auto i : status_incomplete_reasons) {
+				reasons.push_back(JsonString("description", i));	//ahm pretty sure this is right
+			}
 			incomplete_message["reasons"] = reasons;
 			messages.push_back(incomplete_message);
 		}
diff --git a/flow/JsonString.cpp b/flow/JsonString.cpp
new file mode 100644
index 0000000000..57716976b3
--- /dev/null
+++ b/flow/JsonString.cpp
@@ -0,0 +1,396 @@
+#include "JsonString.h"
+#include "Hash3.h"
+#include <iostream>
+
+std::string format( const char* form, ... );
+
+JsonString::JsonString() : _jsonText(), _keyNames() {
+}
+JsonString::JsonString( const JsonString& jsonString) : _jsonText(jsonString._jsonText), _keyNames(jsonString._keyNames) {
+}
+JsonString::JsonString( const JsonStringArray& jsonArray) : _jsonText(), _keyNames() {
+	append(jsonArray);
+}
+JsonString::JsonString( const std::string& value ) : _jsonText(), _keyNames() {
+	append(value);
+}
+JsonString::JsonString( const char* value ) : _jsonText(), _keyNames() {
+	append(value);
+}
+
+JsonString::JsonString( const std::string& name, const std::string& value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, const char* value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, double value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, long int value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, long unsigned int value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, long long int value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, long long unsigned int value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, int value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, unsigned value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, bool value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, const JsonString& value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+JsonString::JsonString( const std::string& name, const JsonStringArray& value ) : _jsonText(), _keyNames() {
+	append(name, value);
+}
+
+uint32_t JsonString::hash32( const std::string& name ) {
+	uint32_t a=0, b=0;
+	hashlittle2( (const void*) name.c_str(), name.length(), &a, &b );
+	return a;
+}
+
+bool	JsonString::isPresent(uint32_t nameHash) const {
+	return (_keyNames.find(nameHash) != _keyNames.end());
+}
+bool	JsonString::isPresent(const std::string& name) const {
+	return isPresent(hash32(name));
+}
+
+JsonString JsonString::makeMessage(const char *name, const char *description) {
+	JsonString out;
+	out["name"] = name;
+	out["description"] = description;
+	return out;
+}
+
+void JsonString::hashName( const std::string& name) {
+	_keyNames.insert(hash32(name));
+}
+
+JsonString& JsonString::append( const std::string& name, const std::string& value ) {
+	hashName(name);
+	_jsonText += (_jsonText.empty() ? "\"" : ",\n  \"") + name + "\": \"" + value + "\"";
+	return *this;
+}
+JsonString& JsonString::append( const std::string& name, const JsonString& value ) {
+	hashName(name);
+	_jsonText += (_jsonText.empty() ? "\"" : ",\n  \"") + name + "\": { " + value._jsonText + " }";
+	return *this;
+}
+JsonString& JsonString::append( const std::string& name, const JsonStringArray& values ) {
+	hashName(name);
+	_jsonText += (_jsonText.empty() ? "\"" : ",\n  \"") + name + "\": [ ";
+	size_t counter = 0;
+	for (auto const& value : values) {
+		if (counter)
+			_jsonText += ",\n  ";
+		_jsonText += value.getJson();
+		counter ++;
+	}
+	_jsonText += " ]";
+	return *this;
+}
+JsonString& JsonString::append( const std::string& name, const char* value ) {
+	std::string	textValue(value);
+	return append(name, textValue);
+}
+JsonString& JsonString::append( const std::string& name, double value ) {
+	return append(name, format("%g", value));
+}
+JsonString& JsonString::append( const std::string& name, long int value ) {
+	return append(name, format("%ld", value));
+}
+JsonString& JsonString::append( const std::string& name, long unsigned int value ) {
+	return append(name, format("%lu", value));
+}
+JsonString& JsonString::append( const std::string& name, long long int value ) {
+	return append(name, format("%lld", value));
+}
+JsonString& JsonString::append( const std::string& name, long long unsigned int value ) {
+	return append(name, format("%llu", value));
+}
+JsonString& JsonString::append( const std::string& name, int value ) {
+	return append(name, format("%d", value));
+}
+JsonString& JsonString::append( const std::string& name, unsigned value ) {
+	return append(name, format("%u", value));
+}
+JsonString& JsonString::append( const std::string& name, bool value ) {
+	hashName(name);
+	_jsonText += (_jsonText.empty() ? "\"" : ",\n  ") + name + (value ? "\": true" : "\": false");
+	return *this;
+}
+
+JsonString& JsonString::append( const std::string& value ) {
+	_jsonText += (_jsonText.empty() ? "\"" : ", \"") + value + "\"";
+	return *this;
+}
+JsonString& JsonString::append( const char* value ) {
+	std::string	textValue(value);
+	return append(textValue);
+}
+JsonString& JsonString::append( double value ) {
+	return append(format("%g", value));
+}
+JsonString& JsonString::append( long int value ) {
+	return append(format("%ld", value));
+}
+JsonString& JsonString::append( long unsigned int value ) {
+	return append(format("%lu", value));
+}
+JsonString& JsonString::append( long long int value ) {
+	return append(format("%lld", value));
+}
+JsonString& JsonString::append( long long unsigned int value ) {
+	return append(format("%llu", value));
+}
+JsonString& JsonString::append( int value ) {
+	return append(format("%d", value));
+}
+JsonString& JsonString::append( unsigned value ) {
+	return append(format("%u", value));
+}
+JsonString& JsonString::append( bool value ) {
+	if (!_jsonText.empty())
+		_jsonText += ", ";
+	_jsonText += value ? "true" : "false";
+	return *this;
+}
+JsonString& JsonString::append( const JsonString& value ) {
+	// Only do something, if not empty
+	if (!value.empty()) {
+		if (!_jsonText.empty())
+			_jsonText += ",\n  ";
+		_jsonText += value._jsonText;
+		_keyNames.insert(value._keyNames.begin(), value._keyNames.end());
+	}
+	return *this;
+}
+JsonString& JsonString::append( const JsonStringArray& values ) {
+	_jsonText += _jsonText.empty() ? "[ " : ",\n  [ ";
+	size_t counter = 0;
+	for (auto const& value : values) {
+		if (counter)
+			_jsonText += ",\n  ";
+		_jsonText += value.getJson();
+		counter ++;
+	}
+	_jsonText += " ]";
+	return *this;
+}
+
+JsonString& JsonString::clear() {
+	_keyNames.clear();
+	_jsonText.clear();
+	return *this;
+}
+
+bool	JsonString::empty() const {
+	return _jsonText.empty();
+}
+
+const std::string&	JsonString::getJsonText() const {
+	return _jsonText;
+}
+
+std::string	JsonString::getJson() const {
+	// If not empty with no names (only values), don't add brackets because prob in an array
+	return (!empty() && _keyNames.empty()) ? _jsonText : ("{ " + _jsonText + " }");
+}
+
+JsonString& JsonString::copy( const JsonString& jsonString ) {
+	_jsonText = jsonString._jsonText;
+	_keyNames = jsonString._keyNames;
+	return *this;
+}
+
+JsonString& JsonString::operator=( const JsonString& jsonString ) {
+	return copy(jsonString);
+}
+
+//ahm Remove the leading and ending {}
+void	JsonString::setJson(const std::string& jsonText) {
+	_jsonText = jsonText;
+}
+
+JsonStringSetter& JsonString::operator[]( const std::string& name ) {
+	JsonStringSetter* stringSetter = new JsonStringSetter(*this, name);
+	return *stringSetter;
+}
+
+JsonStringSetter& JsonString::operator[]( const char* name ) {
+	std::string	textName(name);
+	JsonStringSetter* stringSetter = new JsonStringSetter(*this, textName);
+	return *stringSetter;
+}
+
+int JsonString::compare( const JsonString& jsonString ) const {
+	return jsonString._jsonText.compare(_jsonText);
+}
+
+bool JsonString::equals( const JsonString& jsonString ) const {
+	return (compare(jsonString) == 0);
+}
+
+
+JsonStringArray::JsonStringArray() {
+}
+JsonStringArray::JsonStringArray( const JsonStringArray& arr) : std::vector<JsonString>(arr.begin(), arr.end()) {
+}
+JsonStringArray::~JsonStringArray() {
+}
+
+JsonStringSetter::JsonStringSetter( JsonString& jsonString, const std::string& name ) : _jsonString(jsonString), _name(name) {
+}
+
+JsonStringSetter& JsonStringSetter::operator=( const std::string& value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( const char* value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( double value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( long int value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( long unsigned int value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( long long int value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( long long unsigned int value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( int value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( unsigned value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( bool value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( const JsonString& value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+JsonStringSetter& JsonStringSetter::operator=( const JsonStringArray& value ) {
+	_jsonString.append(_name, value);
+	return *this;
+}
+
+inline void _fdbassert(const char* expression, const char* file, int line) {
+	fprintf(stderr, "Assertion '%s' failed, file '%s' line '%d'.", expression, file, line);
+	abort();
+}
+
+#define fdbassert(EXPRESSION) ((EXPRESSION) ? (void)0 : _fdbassert(#EXPRESSION, __FILE__, __LINE__))
+
+int jsonUnitTests() {
+	int	status = 0;
+
+	JsonString	jsonString1;
+	jsonString1["_valid"] = false;
+	jsonString1["error"] = "configurationMissing";
+
+	JsonString	jsonString2("_valid", false);
+	jsonString2["error"] = "configurationMissing";
+
+	JsonString	jsonString3("error", "configurationMissing");
+	jsonString3["_valid"] = false;
+
+	JsonString	jsonString4("_valid", false);
+	jsonString4.append("error", "configurationMissing");
+
+	std::cout << "Json1: " << jsonString1.getJson() << std::endl;
+	std::cout << "Json2: " << jsonString2.getJson() << std::endl;
+	std::cout << "Json3: " << jsonString3.getJson() << std::endl;
+	std::cout << "Json4: " << jsonString4.getJson() << std::endl;
+
+	fdbassert(jsonString1 == jsonString2);
+	fdbassert(jsonString1 != jsonString3); // wrong order
+	fdbassert(jsonString1 == jsonString4);
+
+	JsonString	jsonObj, testObj;
+	JsonStringArray jsonArray;
+
+	jsonObj["alvin"] = "moore";
+	jsonObj["count"] = 3;
+	testObj["pi"] = 3.14159;
+	jsonObj["piobj"] = testObj;
+
+	testObj.clear();
+	testObj["val1"] = 7.9;
+	testObj["val2"] = 34;
+	jsonArray.push_back(testObj);
+
+	testObj.clear();
+	testObj["name1"] = "alvin";
+	testObj["name2"] = "evan";
+	testObj["name3"] = "ben";
+	jsonArray.push_back(testObj);
+	jsonObj.append("name_vals", jsonArray);
+
+	std::cout << "Json (complex): " << jsonObj.getJson() << std::endl;
+
+	jsonObj.clear();
+	jsonObj.append(jsonArray);
+	std::cout << "Json (complex array): " << jsonObj.getJson() << std::endl;
+
+	jsonArray.clear();
+	jsonArray.push_back("nissan");
+	jsonArray.push_back("honda");
+	jsonArray.push_back("bmw");
+	jsonObj.clear();
+	jsonObj.append(jsonArray);
+	std::cout << "Array (simple array #1): " << jsonObj.getJson() << std::endl;
+
+	testObj.clear();
+	testObj.append("nissan");
+	testObj.append("honda");
+	testObj.append("bmw");
+	jsonArray.clear();
+	jsonArray.push_back(testObj);
+	jsonObj.clear();
+	jsonObj.append(jsonArray);
+	std::cout << "Array (simple array #2): " << jsonObj.getJson() << std::endl;
+
+	jsonObj.clear();
+	jsonObj["name1"] = "alvin";
+	jsonObj["name2"] = "evan";
+	jsonObj["name3"] = "ben";
+	testObj.clear();
+	testObj["val1"] = 7.9;
+	testObj["val2"] = 34;
+	std::cout << "Merge (obj #1): " << jsonObj.getJson() << std::endl;
+	std::cout << "Merge (obj #2): " << testObj.getJson() << std::endl;
+	jsonObj.append(testObj);
+	std::cout << "Merged: " << jsonObj.getJson() << std::endl;
+
+	return status;
+}
diff --git a/flow/JsonString.h b/flow/JsonString.h
new file mode 100644
index 0000000000..2a6b1e9999
--- /dev/null
+++ b/flow/JsonString.h
@@ -0,0 +1,138 @@
+
+#ifndef JSONSTRING_H
+#define JSONSTRING_H
+
+#include <string>
+#include <set>
+#include <vector>
+
+class JsonString;
+class JsonStringArray;
+class JsonStringSetter;
+
+class JsonString {
+	public:
+		JsonString();
+		JsonString( const JsonString& jsonString);
+		JsonString( const JsonStringArray& jsonArray);
+		JsonString( const char* value );
+		JsonString( const std::string& value );
+
+		JsonString( const std::string& name, const char* value );
+		JsonString( const std::string& name, const std::string& value );
+		JsonString( const std::string& name, double value );
+		JsonString( const std::string& name, long int value );
+		JsonString( const std::string& name, long unsigned int value );
+		JsonString( const std::string& name, long long int value );
+		JsonString( const std::string& name, long long unsigned int value );
+		JsonString( const std::string& name, int value );
+		JsonString( const std::string& name, unsigned value );
+		JsonString( const std::string& name, bool value );
+		JsonString( const std::string& name, const JsonString& value );
+		JsonString( const std::string& name, const JsonStringArray& value );
+
+		JsonString& append( const std::string& name, const char* value );
+		JsonString& append( const std::string& name, const std::string& value );
+		JsonString& append( const std::string& name, double value );
+		JsonString& append( const std::string& name, long int value );
+		JsonString& append( const std::string& name, long unsigned int value );
+		JsonString& append( const std::string& name, long long int value );
+		JsonString& append( const std::string& name, long long unsigned int value );
+		JsonString& append( const std::string& name, int value );
+		JsonString& append( const std::string& name, unsigned value );
+		JsonString& append( const std::string& name, bool value );
+		JsonString& append( const std::string& name, const JsonString& value );
+		JsonString& append( const std::string& name, const JsonStringArray& value );
+
+		JsonString& append( const std::string& value );
+		JsonString& append( const char* value );
+		JsonString& append( const JsonStringArray& value );
+
+		JsonString& append( double value );
+		JsonString& append( long int value );
+		JsonString& append( long unsigned int value );
+		JsonString& append( long long int value );
+		JsonString& append( long long unsigned int value );
+		JsonString& append( int value );
+		JsonString& append( unsigned value );
+		JsonString& append( bool value );
+		JsonString& append( const JsonString& value );
+
+
+		JsonStringSetter& operator[]( const std::string& name );
+		JsonStringSetter& operator[]( const char* name );
+
+		int compare( const JsonString& jsonString ) const;
+		bool equals( const JsonString& jsonString ) const;
+		bool operator==( const JsonString& jsonString ) const { return equals(jsonString); }
+		bool operator!=( const JsonString& jsonString ) const { return !equals(jsonString); }
+
+		JsonString& copy( const JsonString& jsonString );
+		JsonString& operator=( const JsonString& jsonString );
+
+		std::string	getJson() const;
+		void	setJson(const std::string& jsonText);
+
+		JsonString& clear();
+		bool	empty() const;
+
+		bool	isPresent(uint32_t nameHash) const;
+		bool	isPresent(const std::string& name) const;
+
+		static uint32_t hash32( const std::string& name );
+		static JsonString makeMessage(const char *name, const char *description);
+
+	protected:
+		void hashName( const std::string& name);
+
+	protected:
+		std::string		_jsonText;
+		std::set<uint32_t>	_keyNames;
+
+		// Uneditted text
+		const std::string&	getJsonText() const;
+};
+
+// Make protected because no virtual destructor
+class JsonStringArray : protected std::vector<JsonString>
+{
+	typedef JsonString T;
+	typedef std::vector<JsonString> vector;
+	public:
+		using vector::push_back;
+		using vector::operator[];
+		using vector::begin;
+		using vector::end;
+		using vector::size;
+		using vector::empty;
+		using vector::clear;
+	public:
+		JsonStringArray();
+		JsonStringArray( const JsonStringArray& jsonStringArray);
+		virtual ~JsonStringArray();
+};
+
+class JsonStringSetter {
+	public:
+
+	JsonStringSetter( JsonString& jsonString, const std::string& name );
+
+	JsonStringSetter& operator=( const std::string& value );
+	JsonStringSetter& operator=( const char* value );
+	JsonStringSetter& operator=( double value );
+	JsonStringSetter& operator=( long int value );
+	JsonStringSetter& operator=( long unsigned int value );
+	JsonStringSetter& operator=( long long int value );
+	JsonStringSetter& operator=( long long unsigned int value );
+	JsonStringSetter& operator=( int value );
+	JsonStringSetter& operator=( unsigned value );
+	JsonStringSetter& operator=( bool value );
+	JsonStringSetter& operator=( const JsonString& value );
+	JsonStringSetter& operator=( const JsonStringArray& value );
+
+	protected:
+		JsonString&		_jsonString;
+		std::string		_name;
+};
+
+#endif
diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj
index da4545142b..84e4858175 100644
--- a/flow/flow.vcxproj
+++ b/flow/flow.vcxproj
@@ -45,6 +45,7 @@
     <ClCompile Include="UnitTest.cpp" />
     <ClCompile Include="version.cpp" />
     <ClCompile Include="SignalSafeUnwind.cpp" />
+    <ClCompile Include="JsonString.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="CompressedInt.h" />
@@ -88,6 +89,7 @@
     <ClInclude Include="ThreadSafeQueue.h" />
     <ClInclude Include="Trace.h" />
     <ClInclude Include="SignalSafeUnwind.h" />
+    <ClInclude Include="JsonString.h" />
     <ClInclude Include="UnitTest.h" />
     <ActorCompiler Include="ThreadHelper.actor.h">
       <EnableCompile>false</EnableCompile>

From e60c668853cbd3ac1ae606315611aa74dbbbb839 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 10:51:55 -0700
Subject: [PATCH 15/37] The cluster controller will increase its failure
 monitoring delay after there have been many unfinishedRecoveries

---
 fdbclient/Knobs.cpp                   |  2 ++
 fdbclient/Knobs.h                     |  2 ++
 fdbserver/ClusterController.actor.cpp | 26 ++++++++++++++++++++------
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp
index b208302dae..9844b93228 100644
--- a/fdbclient/Knobs.cpp
+++ b/fdbclient/Knobs.cpp
@@ -39,6 +39,8 @@ ClientKnobs::ClientKnobs(bool randomize) {
 	init( FAILURE_MIN_DELAY,                       4.0 ); if( randomize && BUGGIFY ) FAILURE_MIN_DELAY = 1.0;
 	init( FAILURE_TIMEOUT_DELAY,     FAILURE_MIN_DELAY );
 	init( CLIENT_FAILURE_TIMEOUT_DELAY, FAILURE_MIN_DELAY );
+	init( FAILURE_EMERGENCY_DELAY,                60.0 );
+	init( FAILURE_MAX_GENERATIONS,                   4 );
 
 	// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
 
diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h
index 5ee8d3baa4..0bba81ba0e 100644
--- a/fdbclient/Knobs.h
+++ b/fdbclient/Knobs.h
@@ -38,6 +38,8 @@ public:
 	double FAILURE_MIN_DELAY;
 	double FAILURE_TIMEOUT_DELAY;
 	double CLIENT_FAILURE_TIMEOUT_DELAY;
+	double FAILURE_EMERGENCY_DELAY;
+	double FAILURE_MAX_GENERATIONS;
 
 	// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
 	double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test)
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index ca34413b96..3cb5e410d7 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -94,8 +94,10 @@ public:
 		DatabaseConfiguration config;   // Asynchronously updated via master registration
 		DatabaseConfiguration fullyRecoveredConfig;
 		Database db;
+		int unfinishedRecoveries;
+		int logGenerations;
 
-		DBInfo() : masterRegistrationCount(0), recoveryStalled(false), forceRecovery(false),
+		DBInfo() : masterRegistrationCount(0), recoveryStalled(false), forceRecovery(false), unfinishedRecoveries(0), logGenerations(0),
 			clientInfo( new AsyncVar<ClientDBInfo>( ClientDBInfo() ) ),
 			serverInfo( new AsyncVar<ServerDBInfo>( ServerDBInfo( LiteralStringRef("DB") ) ) ),
 			db( DatabaseContext::create( clientInfo, Future<Void>(), LocalityData(), true, TaskDefaultEndpoint, true ) )  // SOMEDAY: Locality!
@@ -1059,6 +1061,7 @@ ACTOR Future<Void> clusterWatchDatabase( ClusterControllerData* cluster, Cluster
 			rmq.forceRecovery = db->forceRecovery;
 
 			cluster->masterProcessId = masterWorker.worker.first.locality.processId();
+			cluster->db.unfinishedRecoveries++;
 			ErrorOr<MasterInterface> newMaster = wait( masterWorker.worker.first.master.tryGetReply( rmq ) );
 			if (newMaster.present()) {
 				TraceEvent("CCWDB", cluster->id).detail("Recruited", newMaster.get().id());
@@ -1345,7 +1348,7 @@ struct FailureStatusInfo {
 };
 
 //The failure monitor client relies on the fact that the failure detection server will not declare itself failed
-ACTOR Future<Void> failureDetectionServer( UID uniqueID, FutureStream< FailureMonitoringRequest > requests ) {
+ACTOR Future<Void> failureDetectionServer( UID uniqueID, ClusterControllerData::DBInfo* db, FutureStream< FailureMonitoringRequest > requests ) {
 	state Version currentVersion = 0;
 	state std::map<NetworkAddress, FailureStatusInfo> currentStatus;	// The status at currentVersion
 	state std::deque<SystemFailureStatus> statusHistory;	// The last change in statusHistory is from currentVersion-1 to currentVersion
@@ -1443,13 +1446,16 @@ ACTOR Future<Void> failureDetectionServer( UID uniqueID, FutureStream< FailureMo
 			//TraceEvent("FailureDetectionPoll", uniqueID).detail("PivotDelay", pivotDelay).detail("Clients", currentStatus.size());
 			//TraceEvent("FailureDetectionAcceptableDelay").detail("Delay", acceptableDelay1000);
 
+			bool tooManyLogGenerations = std::max(db->unfinishedRecoveries, db->logGenerations) > CLIENT_KNOBS->FAILURE_MAX_GENERATIONS;
+
 			for(auto it = currentStatus.begin(); it != currentStatus.end(); ) {
 				double delay = t - it->second.lastRequestTime;
-
-				if ( it->first != g_network->getLocalAddress() && ( delay > pivotDelay * 2 + FLOW_KNOBS->SERVER_REQUEST_INTERVAL + CLIENT_KNOBS->FAILURE_MIN_DELAY || delay > CLIENT_KNOBS->FAILURE_MAX_DELAY ) ) {
+				if ( it->first != g_network->getLocalAddress() && ( tooManyLogGenerations ?
+					( delay > CLIENT_KNOBS->FAILURE_EMERGENCY_DELAY ) :
+					( delay > pivotDelay * 2 + FLOW_KNOBS->SERVER_REQUEST_INTERVAL + CLIENT_KNOBS->FAILURE_MIN_DELAY || delay > CLIENT_KNOBS->FAILURE_MAX_DELAY ) ) ) {
 					//printf("Failure Detection Server: Status of '%s' is now '%s' after %f sec\n", it->first.toString().c_str(), "Failed", now() - it->second.lastRequestTime);
 					TraceEvent("FailureDetectionStatus", uniqueID).detail("System", it->first).detail("Status","Failed").detail("Why", "Timeout").detail("LastRequestAge", delay)
-						.detail("PivotDelay", pivotDelay);
+						.detail("PivotDelay", pivotDelay).detail("UnfinishedRecoveries", db->unfinishedRecoveries).detail("LogGenerations", db->logGenerations);
 					statusHistory.push_back( SystemFailureStatus( it->first, FailureStatus(true) ) );
 					++currentVersion;
 					it = currentStatus.erase(it);
@@ -1555,6 +1561,14 @@ void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest c
 		return;
 	}
 
+	if ( req.recoveryState == RecoveryState::FULLY_RECOVERED ) {
+		self->db.unfinishedRecoveries = 0;
+		self->db.logGenerations = 0;
+		ASSERT( !req.logSystemConfig.oldTLogs.size() );
+	} else {
+		self->db.logGenerations = std::max<int>(self->db.logGenerations, req.logSystemConfig.oldTLogs.size());
+	}
+
 	db->masterRegistrationCount = req.registrationCount;
 	db->recoveryStalled = req.recoveryStalled;
 	if ( req.configuration.present() ) {
@@ -2135,7 +2149,7 @@ ACTOR Future<Void> clusterControllerCore( ClusterControllerFullInterface interf,
 	state Future<ErrorOr<Void>> error = errorOr( actorCollection( addActor.getFuture() ) );
 
 	auto pSelf = &self;
-	addActor.send( failureDetectionServer( self.id, interf.clientInterface.failureMonitoring.getFuture() ) );
+	addActor.send( failureDetectionServer( self.id, &self.db, interf.clientInterface.failureMonitoring.getFuture() ) );
 	addActor.send( clusterWatchDatabase( &self, &self.db ) );  // Start the master database
 	addActor.send( self.updateWorkerList.init( self.db.db ) );
 	addActor.send( statusServer( interf.clientInterface.databaseStatus.getFuture(), &self, coordinators));

From 6496a6d9c8a32f2ae4709e6694ef8318209ba709 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 12:43:14 -0700
Subject: [PATCH 16/37] fix: start move keys will only move destination servers
 to become source servers if less than destination servers are healthy and the
 total number of sources is less than 2x the number of destinations

---
 fdbserver/Knobs.cpp          |  1 +
 fdbserver/Knobs.h            |  1 +
 fdbserver/MoveKeys.actor.cpp | 84 ++++++++++++++++++++++++++++--------
 3 files changed, 67 insertions(+), 19 deletions(-)

diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp
index d0b5f0f489..cb9c8e3019 100644
--- a/fdbserver/Knobs.cpp
+++ b/fdbserver/Knobs.cpp
@@ -305,6 +305,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
 	init( MOVE_KEYS_KRM_LIMIT,                                  2000 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT = 2;
 	init( MOVE_KEYS_KRM_LIMIT_BYTES,                             1e5 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT_BYTES = 5e4; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map
 	init( MAX_SKIP_TAGS,                                           1 ); //The TLogs require tags to be densely packed to be memory efficient, so be careful increasing this knob
+	init( MAX_ADDED_SOURCES_MULTIPLIER,                          2.0 );
 
 	//FdbServer
 	bool longReboots = randomize && BUGGIFY;
diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h
index 0a7e93724a..ae189fc372 100644
--- a/fdbserver/Knobs.h
+++ b/fdbserver/Knobs.h
@@ -247,6 +247,7 @@ public:
 	int MOVE_KEYS_KRM_LIMIT;
 	int MOVE_KEYS_KRM_LIMIT_BYTES; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map
 	int MAX_SKIP_TAGS;
+	double MAX_ADDED_SOURCES_MULTIPLIER;
 
 	//FdbServer
 	double MIN_REBOOT_TIME;
diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp
index 83f6b1334b..bfa451517a 100644
--- a/fdbserver/MoveKeys.actor.cpp
+++ b/fdbserver/MoveKeys.actor.cpp
@@ -118,16 +118,62 @@ Future<Void> removeOldDestinations(Transaction *tr, UID oldDest, VectorRef<KeyRa
 	return waitForAll(actors);
 }
 
-ACTOR Future<vector<vector<Optional<UID>>>> findReadWriteDestinations(Standalone<RangeResultRef> shards, UID relocationIntervalId, Transaction* tr) {
+ACTOR Future<vector<UID>> addReadWriteDestinations(KeyRangeRef shard, vector<StorageServerInterface> srcInterfs, vector<StorageServerInterface> destInterfs, Version version, int desiredHealthy, int maxServers) {
+	if(srcInterfs.size() >= maxServers) {
+		return vector<UID>();
+	}
+
+	state vector< Future<Optional<UID>> > srcChecks;
+	for(int s=0; s<srcInterfs.size(); s++) {
+		srcChecks.push_back( checkReadWrite( srcInterfs[s].getShardState.getReplyUnlessFailedFor( GetShardStateRequest( shard, GetShardStateRequest::NO_WAIT), SERVER_KNOBS->SERVER_READY_QUORUM_INTERVAL, 0, TaskMoveKeys ), srcInterfs[s].id(), 0 ) );
+	}
+
+	state vector< Future<Optional<UID>> > destChecks;
+	for(int s=0; s<destInterfs.size(); s++) {
+		destChecks.push_back( checkReadWrite( destInterfs[s].getShardState.getReplyUnlessFailedFor( GetShardStateRequest( shard, GetShardStateRequest::NO_WAIT), SERVER_KNOBS->SERVER_READY_QUORUM_INTERVAL, 0, TaskMoveKeys ), destInterfs[s].id(), version ) );
+	}
+
+	Void _ = wait( waitForAll(srcChecks) && waitForAll(destChecks) );
+
+	int healthySrcs = 0;
+	for(auto it : srcChecks) {
+		if( it.get().present() ) {
+			healthySrcs++;
+		}
+	}
+
+	vector<UID> result;
+	int totalDesired = std::min<int>(desiredHealthy - healthySrcs, maxServers - srcInterfs.size());
+	for(int s = 0; s < destInterfs.size() && result.size() < totalDesired; s++) {
+		if(destChecks[s].get().present()) {
+			result.push_back(destChecks[s].get().get());
+		}
+	}
+
+	return result;
+}
+
+ACTOR Future<vector<vector<UID>>> additionalSources(Standalone<RangeResultRef> shards, Transaction* tr, int desiredHealthy, int maxServers) {
 	vector<Future<Optional<Value>>> serverListEntries;
+	std::set<UID> fetching;
 	for(int i = 0; i < shards.size() - 1; ++i) {
 		vector<UID> src;
 		vector<UID> dest;
 
 		decodeKeyServersValue( shards[i].value, src, dest );
 
+		for(int s=0; s<src.size(); s++) {
+			if(!fetching.count(src[s])) {
+				fetching.insert(src[s]);
+				serverListEntries.push_back( tr->get( serverListKeyFor(src[s]) ) );
+			}
+		}
+
 		for(int s=0; s<dest.size(); s++) {
-			serverListEntries.push_back( tr->get( serverListKeyFor(dest[s]) ) );
+			if(!fetching.count(dest[s])) {
+				fetching.insert(dest[s]);
+				serverListEntries.push_back( tr->get( serverListKeyFor(dest[s]) ) );
+			}
 		}
 	}
 
@@ -140,29 +186,31 @@ ACTOR Future<vector<vector<Optional<UID>>>> findReadWriteDestinations(Standalone
 		ssiMap[ssi.id()] = ssi;
 	}
 
-	vector<Future<vector<Optional<UID>>>> allChecks;
+	vector<Future<vector<UID>>> allChecks;
 	for(int i = 0; i < shards.size() - 1; ++i) {
 		KeyRangeRef rangeIntersectKeys( shards[i].key, shards[i+1].key );
 		vector<UID> src;
 		vector<UID> dest;
-		vector<StorageServerInterface> storageServerInterfaces;
+		vector<StorageServerInterface> srcInterfs;
+		vector<StorageServerInterface> destInterfs;
 
 		decodeKeyServersValue( shards[i].value, src, dest );
 
-		for(int s=0; s<dest.size(); s++)
-			storageServerInterfaces.push_back( ssiMap[dest[s]] );
-
-		vector< Future<Optional<UID>> > checks;
-		for(int s=0; s<storageServerInterfaces.size(); s++) {
-			checks.push_back( checkReadWrite( storageServerInterfaces[s].getShardState.getReplyUnlessFailedFor(
-				GetShardStateRequest( rangeIntersectKeys, GetShardStateRequest::NO_WAIT), SERVER_KNOBS->SERVER_READY_QUORUM_INTERVAL, 0, TaskMoveKeys ), dest[s], tr->getReadVersion().get() ) );
+		for(int s=0; s<src.size(); s++) {
+			srcInterfs.push_back( ssiMap[src[s]] );
 		}
 
-		allChecks.push_back(getAll(checks));
+		for(int s=0; s<dest.size(); s++) {
+			if( std::find(src.begin(), src.end(), dest[s]) == dest.end() ) {
+				destInterfs.push_back( ssiMap[dest[s]] );
+			}
+		}
+
+		allChecks.push_back(addReadWriteDestinations(rangeIntersectKeys, srcInterfs, destInterfs, tr->getReadVersion().get(), desiredHealthy, maxServers));
 	}
 
-	vector<vector<Optional<UID>>> readWriteDestinations = wait(getAll(allChecks));
-	return readWriteDestinations;
+	vector<vector<UID>> result = wait(getAll(allChecks));
+	return result;
 }
 
 // Set keyServers[keys].dest = servers
@@ -238,7 +286,7 @@ ACTOR Future<Void> startMoveKeys( Database occ, KeyRange keys, vector<UID> serve
 					//	printf("'%s': '%s'\n", old[i].key.toString().c_str(), old[i].value.toString().c_str());
 
 					//Check that enough servers for each shard are in the correct state
-					vector<vector<Optional<UID>>> readWriteDestinations = wait(findReadWriteDestinations(old, relocationIntervalId, &tr));
+					vector<vector<UID>> addAsSource = wait(additionalSources(old, &tr, servers.size(), SERVER_KNOBS->MAX_ADDED_SOURCES_MULTIPLIER*servers.size()));
 
 					// For each intersecting range, update keyServers[range] dest to be servers and clear existing dest servers from serverKeys
 					for(int i = 0; i < old.size() - 1; ++i) {
@@ -254,10 +302,8 @@ ACTOR Future<Void> startMoveKeys( Database occ, KeyRange keys, vector<UID> serve
 							.detail("OldDest", describe(dest))
 							.detail("ReadVersion", tr.getReadVersion().get());*/
 
-						for(auto& uid : readWriteDestinations[i]) {
-							if(uid.present()) {
-								src.push_back(uid.get());
-							}
+						for(auto& uid : addAsSource[i]) {
+							src.push_back(uid);
 						}
 						uniquify(src);
 

From d8659a5822d0f104629e3901d1059725168a49ce Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 12:46:57 -0700
Subject: [PATCH 17/37] fix: bytesWritten would overflow and go negative

---
 fdbserver/DataDistributionQueue.actor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp
index 3fe0a0b8ff..91a3c43ccd 100644
--- a/fdbserver/DataDistributionQueue.actor.cpp
+++ b/fdbserver/DataDistributionQueue.actor.cpp
@@ -343,7 +343,7 @@ struct DDQueueData {
 
 	int activeRelocations;
 	int queuedRelocations;
-	int bytesWritten;
+	int64_t bytesWritten;
 	int teamSize;
 
 	std::map<UID, Busyness> busymap;

From b0d94597d46677b7b4bcc06f3e0432b3f0dc5e8c Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 13:01:36 -0700
Subject: [PATCH 18/37] Add additional metrics to track fetch key duration on
 the storage servers

---
 fdbserver/storageserver.actor.cpp | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp
index b536f75e16..b2ca09d914 100644
--- a/fdbserver/storageserver.actor.cpp
+++ b/fdbserver/storageserver.actor.cpp
@@ -408,6 +408,7 @@ public:
 		Counter mutations, setMutations, clearRangeMutations, atomicMutations;
 		Counter updateBatches, updateVersions;
 		Counter loops;
+		Counter fetchWaitingMS, fetchWaitingCount, fetchExecutingMS, fetchExecutingCount;
 
 		Counters(StorageServer* self)
 			: cc("StorageServer", self->thisServerID.toString()),
@@ -429,7 +430,11 @@ public:
 			atomicMutations("AtomicMutations", cc),
 			updateBatches("UpdateBatches", cc),
 			updateVersions("UpdateVersions", cc),
-			loops("Loops", cc)
+			loops("Loops", cc),
+			fetchWaitingMS("FetchWaitingMS", cc),
+			fetchWaitingCount("FetchWaitingCount", cc),
+			fetchExecutingMS("FetchExecutingMS", cc),
+			fetchExecutingCount("FetchExecutingCount", cc)
 		{
 			specialCounter(cc, "LastTLogVersion", [self](){ return self->lastTLogVersion; });
 			specialCounter(cc, "Version", [self](){ return self->version.get(); });
@@ -1814,6 +1819,10 @@ ACTOR Future<Void> fetchKeys( StorageServer *data, AddingShard* shard ) {
 		Void _ = wait( data->fetchKeysParallelismLock.take( TaskDefaultYield, fetchBlockBytes ) );
 		state FlowLock::Releaser holdingFKPL( data->fetchKeysParallelismLock, fetchBlockBytes );
 
+		state double executeStart = now();
+		++data->counters.fetchWaitingCount;
+		data->counters.fetchWaitingMS += 1000*(executeStart - startt);
+
 		Void _ = wait(delay(0));
 
 		shard->phase = AddingShard::Fetching;
@@ -2005,6 +2014,9 @@ ACTOR Future<Void> fetchKeys( StorageServer *data, AddingShard* shard ) {
 
 		validate(data);
 
+		++data->counters.fetchExecutingCount;
+		data->counters.fetchExecutingMS += 1000*(now() - executeStart);
+
 		TraceEvent(SevDebug, interval.end(), data->thisServerID);
 	} catch (Error &e){
 		TraceEvent(SevDebug, interval.end(), data->thisServerID).error(e, true).detail("Version", data->version.get());

From 1e2ce75ce4b89a6059a6febd1f3006cc0d63f1ce Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 13:04:00 -0700
Subject: [PATCH 19/37] fix: if usable_regions=1 extraTlogEligibleMachines was
 calculated incorrectly

---
 fdbserver/Status.actor.cpp | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index f851353b4b..6c395b7187 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -1274,17 +1274,24 @@ static int getExtraTLogEligibleMachines(vector<std::pair<WorkerInterface, Proces
 
 	if(configuration.regions.size() == 0) {
 		return allMachines.size() - std::max(configuration.tLogReplicationFactor, configuration.storageTeamSize);
-	} 
-	int extraTlogEligibleMachines = std::numeric_limits<int>::max();
+	}
+	int extraTlogEligibleMachines = configuration.usableRegions == 1 ? 0 : std::numeric_limits<int>::max();
 	for(auto& region : configuration.regions) {
-		extraTlogEligibleMachines = std::min<int>( extraTlogEligibleMachines, dcId_machine[region.dcId].size() - std::max(configuration.remoteTLogReplicationFactor, std::max(configuration.tLogReplicationFactor, configuration.storageTeamSize) ) );
+		int eligible = dcId_machine[region.dcId].size() - std::max(configuration.remoteTLogReplicationFactor, std::max(configuration.tLogReplicationFactor, configuration.storageTeamSize) );
 		//FIXME: does not take into account fallback satellite policies
 		if(region.satelliteTLogReplicationFactor > 0) {
 			int totalSatelliteEligible = 0;
 			for(auto& sat : region.satellites) {
 				totalSatelliteEligible += dcId_machine[sat.dcId].size();
 			}
-			extraTlogEligibleMachines = std::min<int>( extraTlogEligibleMachines, totalSatelliteEligible - region.satelliteTLogReplicationFactor );
+			eligible = std::min<int>( eligible, totalSatelliteEligible - region.satelliteTLogReplicationFactor );
+		}
+		if( configuration.usableRegions == 1 ) {
+			if( region.priority >= 0 ) {
+				extraTlogEligibleMachines = std::max( extraTlogEligibleMachines, eligible );
+			}
+		} else {
+			extraTlogEligibleMachines = std::min( extraTlogEligibleMachines, eligible );
 		}
 	}
 	return extraTlogEligibleMachines;

From 90bf277206616411ad877db7220b5c31e8115953 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 13:07:48 -0700
Subject: [PATCH 20/37] =?UTF-8?q?require=20key=20value=20store=20memory=20?=
 =?UTF-8?q?to=20recover=20cleanly=20when=20recovering=20the=20txnStateStor?=
 =?UTF-8?q?e,=20since=20all=20of=20the=20data=20it=20is=20recovering=20has?=
 =?UTF-8?q?=20been=20fsync=E2=80=99ed?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fdbserver/IKeyValueStore.h              |  4 ++--
 fdbserver/KeyValueStoreMemory.actor.cpp | 19 ++++++++++++-------
 fdbserver/MasterProxyServer.actor.cpp   |  2 +-
 fdbserver/masterserver.actor.cpp        |  2 +-
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/fdbserver/IKeyValueStore.h b/fdbserver/IKeyValueStore.h
index c548d891e0..5bef3f9fb3 100644
--- a/fdbserver/IKeyValueStore.h
+++ b/fdbserver/IKeyValueStore.h
@@ -78,7 +78,7 @@ protected:
 
 extern IKeyValueStore* keyValueStoreSQLite( std::string const& filename, UID logID, KeyValueStoreType storeType, bool checkChecksums=false, bool checkIntegrity=false );
 extern IKeyValueStore* keyValueStoreMemory( std::string const& basename, UID logID, int64_t memoryLimit );
-extern IKeyValueStore* keyValueStoreLogSystem( class IDiskQueue* queue, UID logID, int64_t memoryLimit, bool disableSnapshot, bool replaceContent );
+extern IKeyValueStore* keyValueStoreLogSystem( class IDiskQueue* queue, UID logID, int64_t memoryLimit, bool disableSnapshot, bool replaceContent, bool exactRecovery );
 
 inline IKeyValueStore* openKVStore( KeyValueStoreType storeType, std::string const& filename, UID logID, int64_t memoryLimit, bool checkChecksums=false, bool checkIntegrity=false ) {
 	switch( storeType ) {
@@ -97,4 +97,4 @@ inline IKeyValueStore* openKVStore( KeyValueStoreType storeType, std::string con
 Future<Void> GenerateIOLogChecksumFile(std::string const & filename);
 Future<Void> KVFileCheck(std::string const & filename, bool const &integrity);
 
-#endif
\ No newline at end of file
+#endif
diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp
index 37bd01b702..5856d967ec 100644
--- a/fdbserver/KeyValueStoreMemory.actor.cpp
+++ b/fdbserver/KeyValueStoreMemory.actor.cpp
@@ -56,7 +56,7 @@ extern bool noUnseed;
 
 class KeyValueStoreMemory : public IKeyValueStore, NonCopyable {
 public:
-	KeyValueStoreMemory( IDiskQueue* log, UID id, int64_t memoryLimit, bool disableSnapshot, bool replaceContent );
+	KeyValueStoreMemory( IDiskQueue* log, UID id, int64_t memoryLimit, bool disableSnapshot, bool replaceContent, bool exactRecovery );
 
 	// IClosable
 	virtual Future<Void> getError() { return log->getError(); }
@@ -427,7 +427,7 @@ private:
 		return log->push( LiteralStringRef("\x01") ); // Changes here should be reflected in OP_DISK_OVERHEAD
 	}
 
-	ACTOR static Future<Void> recover( KeyValueStoreMemory* self ) {
+	ACTOR static Future<Void> recover( KeyValueStoreMemory* self, bool exactRecovery ) {
 		// 'uncommitted' variables track something that might be rolled back by an OpRollback, and are copied into permanent variables
 		// (in self) in OpCommit.  OpRollback does the reverse (copying the permanent versions over the uncommitted versions)
 		// the uncommitted and committed variables should be equal initially (to whatever makes sense if there are no committed transactions recovered)
@@ -559,6 +559,11 @@ private:
 			}
 
 			if (zeroFillSize) {
+				if( exactRecovery ) {
+					TraceEvent(SevError, "KVSMemExpectedExact", self->id);
+					ASSERT(false);
+				}
+
 				TEST( true );  // Fixing a partial commit at the end of the KeyValueStoreMemory log
 				for(int i=0; i<zeroFillSize; i++)
 					self->log->push( StringRef((const uint8_t*)"",1) );
@@ -704,11 +709,11 @@ private:
 	}
 };
 
-KeyValueStoreMemory::KeyValueStoreMemory( IDiskQueue* log, UID id, int64_t memoryLimit, bool disableSnapshot, bool replaceContent )
+KeyValueStoreMemory::KeyValueStoreMemory( IDiskQueue* log, UID id, int64_t memoryLimit, bool disableSnapshot, bool replaceContent, bool exactRecovery )
 	: log(log), id(id), previousSnapshotEnd(-1), currentSnapshotEnd(-1), resetSnapshot(false), memoryLimit(memoryLimit), committedWriteBytes(0),
 	  committedDataSize(0), transactionSize(0), transactionIsLarge(false), disableSnapshot(disableSnapshot), replaceContent(replaceContent), snapshotCount(0), firstCommitWithSnapshot(true)
 {
-	recovering = recover( this );
+	recovering = recover( this, exactRecovery );
 	snapshotting = snapshot( this );
 	commitActors = actorCollection( addActor.getFuture() );
 }
@@ -716,9 +721,9 @@ KeyValueStoreMemory::KeyValueStoreMemory( IDiskQueue* log, UID id, int64_t memor
 IKeyValueStore* keyValueStoreMemory( std::string const& basename, UID logID, int64_t memoryLimit ) {
 	TraceEvent("KVSMemOpening", logID).detail("Basename", basename).detail("MemoryLimit", memoryLimit);
 	IDiskQueue *log = openDiskQueue( basename, logID );
-	return new KeyValueStoreMemory( log, logID, memoryLimit, false, false );
+	return new KeyValueStoreMemory( log, logID, memoryLimit, false, false, false );
 }
 
-IKeyValueStore* keyValueStoreLogSystem( class IDiskQueue* queue, UID logID, int64_t memoryLimit, bool disableSnapshot, bool replaceContent ) {
-	return new KeyValueStoreMemory( queue, logID, memoryLimit, disableSnapshot, replaceContent );
+IKeyValueStore* keyValueStoreLogSystem( class IDiskQueue* queue, UID logID, int64_t memoryLimit, bool disableSnapshot, bool replaceContent, bool exactRecovery ) {
+	return new KeyValueStoreMemory( queue, logID, memoryLimit, disableSnapshot, replaceContent, exactRecovery );
 }
diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp
index 7161ca0d36..94cd78434d 100644
--- a/fdbserver/MasterProxyServer.actor.cpp
+++ b/fdbserver/MasterProxyServer.actor.cpp
@@ -1232,7 +1232,7 @@ ACTOR Future<Void> masterProxyServerCore(
 
 	commitData.logSystem = ILogSystem::fromServerDBInfo(proxy.id(), db->get(), false, addActor);
 	commitData.logAdapter = new LogSystemDiskQueueAdapter(commitData.logSystem, txsTag, false);
-	commitData.txnStateStore = keyValueStoreLogSystem(commitData.logAdapter, proxy.id(), 2e9, true, true);
+	commitData.txnStateStore = keyValueStoreLogSystem(commitData.logAdapter, proxy.id(), 2e9, true, true, true);
 
 	// ((SERVER_MEM_LIMIT * COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) / COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR) is only a approximate formula for limiting the memory used.
 	// COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR is an estimate based on experiments and not an accurate one.
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index 10b71358b9..cf5b2f354f 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -596,7 +596,7 @@ ACTOR Future<Void> readTransactionSystemState( Reference<MasterData> self, Refer
 	// Recover transaction state store
 	if(self->txnStateStore) self->txnStateStore->close();
 	self->txnStateLogAdapter = openDiskQueueAdapter( oldLogSystem, txsTag );
-	self->txnStateStore = keyValueStoreLogSystem( self->txnStateLogAdapter, self->dbgid, self->memoryLimit, false, false );
+	self->txnStateStore = keyValueStoreLogSystem( self->txnStateLogAdapter, self->dbgid, self->memoryLimit, false, false, true );
 
 	// Versionstamped operations (particularly those applied from DR) define a minimum commit version
 	// that we may recover to, as they embed the version in user-readable data and require that no

From 72c86e909e57b0046fd287bd3e2ce186f63f4037 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 17:40:27 -0700
Subject: [PATCH 21/37] fix: tracking of the number of unhealthy servers was
 incorrect fix: locality equality was only checking zoneId

---
 fdbserver/DataDistribution.actor.cpp | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index bd4efdf13c..c3011c9c53 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -260,7 +260,7 @@ struct ServerStatus {
 	bool isUnhealthy() const { return isFailed || isUndesired; }
 	const char* toString() const { return isFailed ? "Failed" : isUndesired ? "Undesired" : "Healthy"; }
 
-	bool operator == (ServerStatus const& r) const { return isFailed == r.isFailed && isUndesired == r.isUndesired && isWrongConfiguration == r.isWrongConfiguration && locality.zoneId() == r.locality.zoneId(); }
+	bool operator == (ServerStatus const& r) const { return isFailed == r.isFailed && isUndesired == r.isUndesired && isWrongConfiguration == r.isWrongConfiguration && locality == r.locality; }
 
 	//If a process has reappeared without the storage server that was on it (isFailed == true), we don't need to exclude it
 	//We also don't need to exclude processes who are in the wrong configuration (since those servers will be removed)
@@ -305,11 +305,15 @@ ACTOR Future<Void> storageServerFailureTracker(
 	Version addedVersion )
 {
 	loop {
-		bool unhealthy = statusMap->count(server.id()) && statusMap->get(server.id()).isUnhealthy();
-		if(unhealthy && !status->isUnhealthy()) {
-			(*unhealthyServers)--;
-		}
-		if(!unhealthy && status->isUnhealthy()) {
+		if( statusMap->count(server.id()) ) {
+			bool unhealthy = statusMap->get(server.id()).isUnhealthy();
+			if(unhealthy && !status->isUnhealthy()) {
+				(*unhealthyServers)--;
+			}
+			if(!unhealthy && status->isUnhealthy()) {
+				(*unhealthyServers)++;
+			}
+		} else if(status->isUnhealthy()) {
 			(*unhealthyServers)++;
 		}
 
@@ -593,7 +597,7 @@ struct DDTeamCollection {
 		teamBuilder.cancel();
 	}
 
-	ACTOR Future<Void> logOnCompletion( Future<Void> signal, DDTeamCollection *self ) {
+	ACTOR static Future<Void> logOnCompletion( Future<Void> signal, DDTeamCollection *self ) {
 		Void _ = wait(signal);
 		Void _ = wait(delay(SERVER_KNOBS->LOG_ON_COMPLETION_DELAY, TaskDataDistribution));
 
@@ -606,7 +610,7 @@ struct DDTeamCollection {
 		return Void();
 	}
 
-	ACTOR Future<Void> checkBuildTeams( DDTeamCollection* self ) {
+	ACTOR static Future<Void> checkBuildTeams( DDTeamCollection* self ) {
 		state Promise<Void> restart;
 
 		Void _ = wait( self->checkTeamDelay );
@@ -640,7 +644,7 @@ struct DDTeamCollection {
 
 	// SOMEDAY: Make bestTeam better about deciding to leave a shard where it is (e.g. in PRIORITY_TEAM_HEALTHY case)
 	//		    use keys, src, dest, metrics, priority, system load, etc.. to decide...
-	ACTOR Future<Void> getTeam( DDTeamCollection* self, GetTeamRequest req ) {
+	ACTOR static Future<Void> getTeam( DDTeamCollection* self, GetTeamRequest req ) {
 		try {
 			Void _ = wait( self->checkBuildTeams( self ) );
 
@@ -1242,6 +1246,10 @@ struct DDTeamCollection {
 		}
 		server_info.erase( removedServer );
 
+		if(server_status.count(removedServer) && server_status.get(removedServer).isUnhealthy()) {
+			unhealthyServers--;
+		}
+		server_status.clear( removedServer );
 		// remove all teams that contain removedServer
 		// SOMEDAY: can we avoid walking through all teams, since we have an index of teams in which removedServer participated
 		for(int t=0; t<teams.size(); t++) {

From 65eabedb6c24af7f6a388127d73bd8f1380985f8 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 31 Aug 2018 17:54:55 -0700
Subject: [PATCH 22/37] fix: addSubsetOfEmergencyTeams could add unhealthy
 teams optimized teamTracker to check if it satisfies the policy more
 efficiently added yields to initialization to avoid slow tasks when adding
 lots of teams

---
 fdbserver/DataDistribution.actor.cpp | 223 +++++++++++++++++++--------
 1 file changed, 157 insertions(+), 66 deletions(-)

diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index c3011c9c53..f6ec7d6bd9 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -48,8 +48,12 @@ struct TCServerInfo : public ReferenceCounted<TCServerInfo> {
 	Promise<Void> removed;
 	Future<Void> onRemoved;
 	Promise<Void> wakeUpTracker;
+	bool inDesiredDC;
+	LocalityEntry localityEntry;
 
-	TCServerInfo(StorageServerInterface ssi, ProcessClass processClass) : id(ssi.id()), lastKnownInterface(ssi), lastKnownClass(processClass), dataInFlightToServer(0), onInterfaceChanged(interfaceChanged.getFuture()), onRemoved(removed.getFuture()) {}
+	TCServerInfo(StorageServerInterface ssi, ProcessClass processClass, bool inDesiredDC, Reference<LocalitySet> storageServerSet) : id(ssi.id()), lastKnownInterface(ssi), lastKnownClass(processClass), dataInFlightToServer(0), onInterfaceChanged(interfaceChanged.getFuture()), onRemoved(removed.getFuture()), inDesiredDC(inDesiredDC) {
+		localityEntry = ((LocalityMap<UID>*) storageServerSet.getPtr())->add(ssi.locality, &id);
+	}
 };
 
 ACTOR Future<Void> updateServerMetrics( TCServerInfo *server ) {
@@ -505,7 +509,7 @@ Future<Void> storageServerTracker(
 	Promise<Void> const& errorOut,
 	Version const& addedVersion);
 
-Future<Void> teamTracker( struct DDTeamCollection* const& self, Reference<IDataDistributionTeam> const& team );
+Future<Void> teamTracker( struct DDTeamCollection* const& self, Reference<TCTeamInfo> const& team );
 
 struct DDTeamCollection {
 	enum { REQUESTING_WORKER = 0, GETTING_WORKER = 1, GETTING_STORAGE = 2 };
@@ -554,6 +558,33 @@ struct DDTeamCollection {
 	Future<Void> readyToStart;
 	Future<Void> checkTeamDelay;
 
+	Reference<LocalitySet> storageServerSet;
+	std::vector<LocalityEntry> forcedEntries, resultEntries;
+
+	void resetLocalitySet() {
+		storageServerSet = Reference<LocalitySet>(new LocalityMap<UID>());
+		LocalityMap<UID>* storageServerMap = (LocalityMap<UID>*) storageServerSet.getPtr();
+
+		for( auto& it : server_info ) {
+			it.second->localityEntry = storageServerMap->add(it.second->lastKnownInterface.locality, &it.second->id);
+		}
+	}
+
+	bool satisfiesPolicy(const std::vector<Reference<TCServerInfo>>& team, int amount = -1) {
+		forcedEntries.clear();
+		resultEntries.clear();
+		if(amount == -1) {
+			amount = team.size();
+		}
+
+		for(int i = 0; i < amount; i++) {
+			forcedEntries.push_back(team[i]->localityEntry);
+		}
+
+		bool result = storageServerSet->selectReplicas(configuration.storagePolicy, forcedEntries, resultEntries);
+		return result && resultEntries.size() == 0;
+	}
+
 	DDTeamCollection(
 		Database const& cx,
 		UID masterId,
@@ -568,7 +599,7 @@ struct DDTeamCollection {
 		Reference<AsyncVar<bool>> processingUnhealthy)
 		:cx(cx), masterId(masterId), lock(lock), output(output), shardsAffectedByTeamFailure(shardsAffectedByTeamFailure), doBuildTeams( true ), teamBuilder( Void() ),
 		 configuration(configuration), serverChanges(serverChanges), readyToStart(readyToStart), checkTeamDelay( delay( SERVER_KNOBS->CHECK_TEAM_DELAY, TaskDataDistribution) ),
-		 initialFailureReactionDelay( delayed( readyToStart, SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskDataDistribution ) ), healthyTeamCount( 0 ),
+		 initialFailureReactionDelay( delayed( readyToStart, SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskDataDistribution ) ), healthyTeamCount( 0 ), storageServerSet(new LocalityMap<UID>()),
 		 initializationDoneActor(logOnCompletion(readyToStart && initialFailureReactionDelay, this)), optimalTeamCount( 0 ), recruitingStream(0), restartRecruiting( SERVER_KNOBS->DEBOUNCE_RECRUITING_DELAY ),
 		 unhealthyServers(0), includedDCs(includedDCs), otherTrackedDCs(otherTrackedDCs), zeroHealthyTeams(zeroHealthyTeams), zeroOptimalTeams(true), primary(primary), processingUnhealthy(processingUnhealthy)
 	{
@@ -803,53 +834,101 @@ struct DDTeamCollection {
 		return total;
 	}
 
-	void addSubsetOfEmergencyTeams() {
-		for( int i = 0; i < badTeams.size(); i++ ) {
-			auto& serverIds = badTeams[i]->getServerIDs();
-			bool foundTeam = false;
-			for( int j = 0; j < std::max( 1, (int)(serverIds.size() - configuration.storageTeamSize + 1) ) && !foundTeam; j++ ) {
-				auto& serverTeams = server_info[serverIds[j]]->teams;
-				for( int k = 0; k < serverTeams.size(); k++ ) {
-					auto &testTeam = serverTeams[k]->getServerIDs();
-					bool allInTeam = true;
-					for( int l = 0; l < testTeam.size(); l++ ) {
-						if( std::find( serverIds.begin(), serverIds.end(), testTeam[l] ) == serverIds.end() ) {
-							allInTeam = false;
+	ACTOR static Future<Void> addSubsetOfEmergencyTeams( DDTeamCollection *self ) {
+		state int idx = 0;
+		state std::vector<Reference<TCServerInfo>> servers;
+		state std::vector<UID> serverIds;
+		state Reference<LocalitySet> tempSet = Reference<LocalitySet>(new LocalityMap<UID>());
+		state LocalityMap<UID>* tempMap = (LocalityMap<UID>*) tempSet.getPtr();
+
+		for(; idx < self->badTeams.size(); idx++ ) {
+			servers.clear();
+			for(auto server : self->badTeams[idx]->servers) {
+				if(server->inDesiredDC) {
+					servers.push_back(server);
+				}
+			}
+
+			if(servers.size() >= self->configuration.storageTeamSize) {
+				bool foundTeam = false;
+				for( int j = 0; j < std::max( 1, (int)(servers.size() - self->configuration.storageTeamSize + 1) ) && !foundTeam; j++ ) {
+					auto& serverTeams = self->server_info[servers[j]->id]->teams;
+					for( int k = 0; k < serverTeams.size(); k++ ) {
+						auto &testTeam = serverTeams[k]->getServerIDs();
+						bool allInTeam = true;
+						for( int l = 0; l < testTeam.size(); l++ ) {
+							bool foundServer = false;
+							for( auto it : servers ) {
+								if( it->id == testTeam[l] ) {
+									foundServer = true;
+									break;
+								}
+							}
+							if(!foundServer) {
+								allInTeam = false;
+								break;
+							}
+						}
+						if( allInTeam ) {
+							foundTeam = true;
 							break;
 						}
 					}
-					if( allInTeam ) {
-						foundTeam = true;
-						break;
+				}
+				if( !foundTeam ) {
+					if( self->satisfiesPolicy(servers) ) {
+						if(servers.size() == self->configuration.storageTeamSize || self->satisfiesPolicy(servers, self->configuration.storageTeamSize)) {
+							servers.resize(self->configuration.storageTeamSize);
+							self->addTeam(servers);
+						} else {
+							tempSet->clear();
+							for( auto it : servers ) {
+								tempMap->add(it->lastKnownInterface.locality, &it->id);
+							}
+
+							self->resultEntries.clear();
+							self->forcedEntries.clear();
+							bool result = tempSet->selectReplicas(self->configuration.storagePolicy, self->forcedEntries, self->resultEntries);
+							ASSERT(result && self->resultEntries.size() == self->configuration.storageTeamSize);
+
+							serverIds.clear();
+							for(auto& it : self->resultEntries) {
+								serverIds.push_back(*tempMap->getObject(it));
+							}
+							self->addTeam(serverIds.begin(), serverIds.end());
+						}
+					} else {
+						serverIds.clear();
+						for(auto it : servers) {
+							serverIds.push_back(it->id);
+						}
+						TraceEvent(SevWarnAlways, "CannotAddSubset", self->masterId).detail("Servers", describe(serverIds));
 					}
 				}
 			}
-			if( !foundTeam ) {
-				addTeam(serverIds.begin(), serverIds.begin() + configuration.storageTeamSize );
-			}
+			Void _ = wait( yield() );
 		}
+		return Void();
 	}
 
-	void init( InitialDataDistribution const& initTeams ) {
+	ACTOR static Future<Void> init( DDTeamCollection *self, Reference<InitialDataDistribution> initTeams ) {
 		// SOMEDAY: If some servers have teams and not others (or some servers have more data than others) and there is an address/locality collision, should
 		// we preferentially mark the least used server as undesirable?
-		for (auto i = initTeams.allServers.begin(); i != initTeams.allServers.end(); ++i) {
-			if (shouldHandleServer(i->first)) {
-				addServer(i->first, i->second, serverTrackerErrorOut, 0);
+		for (auto i = initTeams->allServers.begin(); i != initTeams->allServers.end(); ++i) {
+			if (self->shouldHandleServer(i->first)) {
+				self->addServer(i->first, i->second, self->serverTrackerErrorOut, 0);
 			}
 		}
 
-		if(primary) {
-			for(auto t = initTeams.primaryTeams.begin(); t != initTeams.primaryTeams.end(); ++t) {
-				addTeam(t->begin(), t->end() );
-			}
-		} else {
-			for(auto t = initTeams.remoteTeams.begin(); t != initTeams.remoteTeams.end(); ++t) {
-				addTeam(t->begin(), t->end() );
-			}
+		state std::set<std::vector<UID>>::iterator teamIter = self->primary ? initTeams->primaryTeams.begin() : initTeams->remoteTeams.begin();
+		state std::set<std::vector<UID>>::iterator teamIterEnd = self->primary ? initTeams->primaryTeams.end() : initTeams->remoteTeams.end();
+		for(; teamIter != teamIterEnd; ++teamIter) {
+			self->addTeam(teamIter->begin(), teamIter->end() );
+			Void _ = wait( yield() );
 		}
 
-		addSubsetOfEmergencyTeams();
+		Void _ = wait( addSubsetOfEmergencyTeams(self) );
+		return Void();
 	}
 
 	void evaluateTeamQuality() {
@@ -903,8 +982,18 @@ struct DDTeamCollection {
 		return exists;
 	}
 
-	void addTeam( std::set<UID> const& team ) {
-		addTeam(team.begin(), team.end());
+	void addTeam( const vector<Reference<TCServerInfo>>& newTeamServers ) {
+		Reference<TCTeamInfo> teamInfo( new TCTeamInfo( newTeamServers ) );
+
+		teamInfo->tracker = teamTracker( this, teamInfo );
+		if( teamInfo->servers.size() > configuration.storageTeamSize ) {
+			badTeams.push_back( teamInfo );
+		} else {
+			teams.push_back( teamInfo );
+			for (int i=0;i<newTeamServers.size();i++) {
+				server_info[ newTeamServers[i]->id ]->teams.push_back( teamInfo );
+			}
+		}
 	}
 
 	template<class InputIt>
@@ -916,17 +1005,11 @@ struct DDTeamCollection {
 			}
 		}
 
-		Reference<TCTeamInfo> teamInfo( new TCTeamInfo( newTeamServers ) );
-		TraceEvent("TeamCreation", masterId).detail("Team", teamInfo->getDesc());
-		teamInfo->tracker = teamTracker( this, teamInfo );
-		if( teamInfo->servers.size() > configuration.storageTeamSize ) {
-			badTeams.push_back( teamInfo );
-		} else {
-			teams.push_back( teamInfo );
-			for (int i=0;i<newTeamServers.size();i++) {
-				server_info[ newTeamServers[i]->id ]->teams.push_back( teamInfo );
-			}
-		}
+		addTeam( newTeamServers );
+	}
+
+	void addTeam( std::set<UID> const& team ) {
+		addTeam(team.begin(), team.end());
 	}
 
 	ACTOR Future<Void> addAllTeams( DDTeamCollection *self, int location, vector<LocalityEntry>* history, Reference<LocalityMap<UID>> processes, vector<std::vector<UID>>* output, int teamLimit, int* addedTeams ) {
@@ -1207,7 +1290,7 @@ struct DDTeamCollection {
 		allServers.push_back( newServer.id() );
 
 		TraceEvent("AddedStorageServer", masterId).detail("ServerID", newServer.id()).detail("ProcessClass", processClass.toString()).detail("WaitFailureToken", newServer.waitFailure.getEndpoint().token).detail("Address", newServer.waitFailure.getEndpoint().address);
-		auto &r = server_info[newServer.id()] = Reference<TCServerInfo>( new TCServerInfo( newServer, processClass ) );
+		auto &r = server_info[newServer.id()] = Reference<TCServerInfo>( new TCServerInfo( newServer, processClass, includedDCs.empty() || std::find(includedDCs.begin(), includedDCs.end(), newServer.locality.dcId()) != includedDCs.end(), storageServerSet ) );
 		r->tracker = storageServerTracker( this, cx, r.getPtr(), &server_status, lock, masterId, &server_info, serverChanges, errorOut, addedVersion );
 		restartTeamBuilder.trigger();
 	}
@@ -1250,6 +1333,10 @@ struct DDTeamCollection {
 			unhealthyServers--;
 		}
 		server_status.clear( removedServer );
+
+		//FIXME: add remove support to localitySet so we do not have to recreate it
+		resetLocalitySet();
+
 		// remove all teams that contain removedServer
 		// SOMEDAY: can we avoid walking through all teams, since we have an index of teams in which removedServer participated
 		for(int t=0; t<teams.size(); t++) {
@@ -1278,7 +1365,7 @@ struct DDTeamCollection {
 };
 
 // Track a team and issue RelocateShards when the level of degradation changes
-ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<IDataDistributionTeam> team) {
+ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<TCTeamInfo> team) {
 	state int lastServersLeft = team->getServerIDs().size();
 	state bool lastAnyUndesired = false;
 	state bool wrongSize = team->getServerIDs().size() != self->configuration.storageTeamSize;
@@ -1311,28 +1398,29 @@ ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<IDataDistribut
 			auto servers = team->getServerIDs();
 			bool anyUndesired = false;
 			bool anyWrongConfiguration = false;
-			Reference<LocalityGroup> teamLocality(new LocalityGroup());
+			int serversLeft = 0;
 
 			for(auto s = servers.begin(); s != servers.end(); ++s) {
 				change.push_back( self->server_status.onChange( *s ) );
 				auto& status = self->server_status.get(*s);
-				if (!status.isFailed)
-					teamLocality->add( status.locality );
-				if (status.isUndesired)
+				if (!status.isFailed) {
+					serversLeft++;
+				}
+				if (status.isUndesired) {
 					anyUndesired = true;
-				if (status.isWrongConfiguration)
+				}
+				if (status.isWrongConfiguration) {
 					anyWrongConfiguration = true;
+				}
 			}
 
-			int serversLeft = teamLocality->size();
-			bool matchesPolicy = self->configuration.storagePolicy->validate(teamLocality->getEntries(), teamLocality);
-
 			if( !self->initialFailureReactionDelay.isReady() ) {
 				change.push_back( self->initialFailureReactionDelay );
 			}
 			change.push_back( self->zeroHealthyTeams->onChange() );
 
-			bool recheck = (lastReady != self->initialFailureReactionDelay.isReady() || (lastZeroHealthy && !self->zeroHealthyTeams->get())) && (!matchesPolicy || anyUndesired || team->getServerIDs().size() != self->configuration.storageTeamSize);
+			bool healthy = self->satisfiesPolicy(team->servers) && !anyUndesired && team->getServerIDs().size() == self->configuration.storageTeamSize && serversLeft == self->configuration.storageTeamSize;
+			bool recheck = !healthy && (lastReady != self->initialFailureReactionDelay.isReady() || (lastZeroHealthy && !self->zeroHealthyTeams->get()));
 			lastReady = self->initialFailureReactionDelay.isReady();
 			lastZeroHealthy = self->zeroHealthyTeams->get();
 
@@ -1342,9 +1430,7 @@ ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<IDataDistribut
 					.detail("LastServersLeft", lastServersLeft).detail("ContainsUndesiredServer", anyUndesired)
 					.detail("HealthyTeamsCount", self->healthyTeamCount).detail("IsWrongConfiguration", anyWrongConfiguration);
 
-				bool healthy = matchesPolicy && !anyUndesired && team->getServerIDs().size() == self->configuration.storageTeamSize && team->getServerIDs().size() == serversLeft;
 				team->setHealthy( healthy );	// Unhealthy teams won't be chosen by bestTeam
-
 				team->setWrongConfiguration( anyWrongConfiguration );
 
 				bool optimal = team->isOptimal() && healthy;
@@ -1752,11 +1838,19 @@ ACTOR Future<Void> storageServerTracker(
 				}
 				when( std::pair<StorageServerInterface, ProcessClass> newInterface = wait( interfaceChanged ) ) {
 					bool restartRecruiting =  newInterface.first.waitFailure.getEndpoint().address != server->lastKnownInterface.waitFailure.getEndpoint().address;
+					bool localityChanged = server->lastKnownInterface.locality != newInterface.first.locality;
 					TraceEvent("StorageServerInterfaceChanged", masterId).detail("ServerID", server->id)
 						.detail("NewWaitFailureToken", newInterface.first.waitFailure.getEndpoint().token)
-						.detail("OldWaitFailureToken", server->lastKnownInterface.waitFailure.getEndpoint().token);
+						.detail("OldWaitFailureToken", server->lastKnownInterface.waitFailure.getEndpoint().token)
+						.detail("LocalityChanged", localityChanged);
+
 					server->lastKnownInterface = newInterface.first;
 					server->lastKnownClass = newInterface.second;
+					if(localityChanged) {
+						server->inDesiredDC = (self->includedDCs.empty() || std::find(self->includedDCs.begin(), self->includedDCs.end(), server->lastKnownInterface.locality.dcId()) != self->includedDCs.end());
+						self->resetLocalitySet();
+					}
+
 					interfaceChanged = server->onInterfaceChanged;
 					if(changes.present()) {
 						changes.get().send( std::make_pair(server->id, server->lastKnownInterface) );
@@ -1996,7 +2090,7 @@ ACTOR Future<Void> dataDistributionTeamCollection(
 	state Future<Void> error = actorCollection( self.addActor.getFuture() );
 
 	try {
-		self.init( *initData );
+		Void _ = wait( self.init( &self, initData ) );
 		initData = Reference<InitialDataDistribution>();
 		self.addActor.send(serverGetTeamRequests(tci, &self));
 
@@ -2336,10 +2430,7 @@ DDTeamCollection* testTeamCollection(int teamSize, IRepPolicyRef policy, int pro
 	 	interface.locality.set(LiteralStringRef("machineid"), Standalone<StringRef>(std::to_string(id)));
 		interface.locality.set(LiteralStringRef("zoneid"), Standalone<StringRef>(std::to_string(id % 5)));
 		interface.locality.set(LiteralStringRef("data_hall"), Standalone<StringRef>(std::to_string(id % 3)));
-		collection->server_info[uid] = Reference<TCServerInfo>(new TCServerInfo(
-			interface,
-			ProcessClass()
-		));
+		collection->server_info[uid] = Reference<TCServerInfo>(new TCServerInfo(interface, ProcessClass(), true, collection->storageServerSet));
 		collection->server_status.set(uid, ServerStatus(false, false, interface.locality));
 	}
 

From 21f5cf9ce9e972a2f38b682f99a87bf9fc00a676 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Tue, 4 Sep 2018 17:12:26 -0700
Subject: [PATCH 23/37] suppress spammy trace events

---
 fdbclient/MonitorLeader.actor.cpp              | 2 +-
 fdbrpc/FlowTransport.actor.cpp                 | 4 +++-
 fdbrpc/TLSConnection.actor.cpp                 | 3 +--
 fdbserver/workloads/ConsistencyCheck.actor.cpp | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp
index 6d14742137..44f25ac0ac 100644
--- a/fdbclient/MonitorLeader.actor.cpp
+++ b/fdbclient/MonitorLeader.actor.cpp
@@ -311,7 +311,7 @@ ACTOR Future<Void> monitorNominee( Key key, ClientLeaderRegInterface coord, Asyn
 		state Optional<LeaderInfo> li = wait( retryBrokenPromise( coord.getLeader, GetLeaderRequest( key, info->present() ? info->get().changeID : UID() ), TaskCoordinationReply ) );
 		Void _ = wait( Future<Void>(Void()) ); // Make sure we weren't cancelled
 
-		TraceEvent("GetLeaderReply").detail("Coordinator", coord.getLeader.getEndpoint().address).detail("Nominee", li.present() ? li.get().changeID : UID()).detail("Generation", generation);
+		TraceEvent("GetLeaderReply").suppressFor(1.0).detail("Coordinator", coord.getLeader.getEndpoint().address).detail("Nominee", li.present() ? li.get().changeID : UID()).detail("Generation", generation);
 
 		if (li != *info) {
 			*info = li;
diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp
index b991fdedb0..c593b89ea7 100644
--- a/fdbrpc/FlowTransport.actor.cpp
+++ b/fdbrpc/FlowTransport.actor.cpp
@@ -283,6 +283,7 @@ struct Peer : NonCopyable {
 		if ( !destination.isPublic() || outgoingConnectionIdle || destination > transport->localAddress ) {
 			// Keep the new connection
 			TraceEvent("IncomingConnection", conn->getDebugID())
+				.suppressFor(1.0)
 				.detail("FromAddr", conn->getPeerAddress())
 				.detail("CanonicalAddr", destination)
 				.detail("IsPublic", destination.isPublic());
@@ -292,6 +293,7 @@ struct Peer : NonCopyable {
 			connect = connectionKeeper( this, conn, reader );
 		} else {
 			TraceEvent("RedundantConnection", conn->getDebugID())
+				.suppressFor(1.0)
 				.detail("FromAddr", conn->getPeerAddress().toString())
 				.detail("CanonicalAddr", destination);
 
@@ -321,7 +323,7 @@ struct Peer : NonCopyable {
 			FlowTransport::transport().sendUnreliable( SerializeSource<ReplyPromise<Void>>(reply), remotePing.getEndpoint() );
 
 			choose {
-				when (Void _ = wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) { TraceEvent("ConnectionTimeout").detail("WithAddr", peer->destination); throw connection_failed(); }
+				when (Void _ = wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) { TraceEvent("ConnectionTimeout").suppressFor(1.0).detail("WithAddr", peer->destination); throw connection_failed(); }
 				when (Void _ = wait( reply.getFuture() )) {}
 				when (Void _ = wait( peer->incompatibleDataRead.onTrigger())) {}
 			}
diff --git a/fdbrpc/TLSConnection.actor.cpp b/fdbrpc/TLSConnection.actor.cpp
index 95468dee23..bc8142432c 100644
--- a/fdbrpc/TLSConnection.actor.cpp
+++ b/fdbrpc/TLSConnection.actor.cpp
@@ -83,8 +83,7 @@ ACTOR static Future<Void> handshake( TLSConnection* self ) {
 		Void _ = wait( r == ITLSSession::WANT_WRITE ? self->conn->onWritable() : self->conn->onReadable() );
 	}
 
-	TraceEvent("TLSConnectionHandshakeSuccessful", self->getDebugID())
-		.detail("Peer", self->getPeerAddress());
+	TraceEvent("TLSConnectionHandshakeSuccessful", self->getDebugID()).suppressFor(1.0).detail("Peer", self->getPeerAddress());
 
 	return Void();
 }
diff --git a/fdbserver/workloads/ConsistencyCheck.actor.cpp b/fdbserver/workloads/ConsistencyCheck.actor.cpp
index 57679857a7..76946df6a5 100644
--- a/fdbserver/workloads/ConsistencyCheck.actor.cpp
+++ b/fdbserver/workloads/ConsistencyCheck.actor.cpp
@@ -985,7 +985,7 @@ struct ConsistencyCheckWorkload : TestWorkload
 			}
 
 			if(bytesReadInRange > 0) {
-				TraceEvent("ConsistencyCheck_ReadRange").detail("Range", printable(range)).detail("BytesRead", bytesReadInRange);
+				TraceEvent("ConsistencyCheck_ReadRange").suppressFor(1.0).detail("Range", printable(range)).detail("BytesRead", bytesReadInRange);
 			}
 		}
 

From 1022e0a5c63e745a20cd4fc3e42b460024558da6 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Tue, 4 Sep 2018 17:16:44 -0700
Subject: [PATCH 24/37] added yields to the log router and tlogs after
 processing a version

---
 fdbserver/LogRouter.actor.cpp  |  2 ++
 fdbserver/TLogServer.actor.cpp | 16 +++++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/fdbserver/LogRouter.actor.cpp b/fdbserver/LogRouter.actor.cpp
index e45adc57b4..e9420ce19d 100644
--- a/fdbserver/LogRouter.actor.cpp
+++ b/fdbserver/LogRouter.actor.cpp
@@ -206,6 +206,7 @@ ACTOR Future<Void> pullAsyncData( LogRouterData *self ) {
 					Void _ = wait(self->minPopped.whenAtLeast(std::min(self->version.get(), ver - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)));
 					commitMessages(self, ver, messages);
 					self->version.set( ver );
+					Void _ = wait(yield(TaskTLogCommit));
 					//TraceEvent("LogRouterVersion").detail("Ver",ver);
 				}
 				lastVer = ver;
@@ -217,6 +218,7 @@ ACTOR Future<Void> pullAsyncData( LogRouterData *self ) {
 					if(ver > self->version.get()) {
 						Void _ = wait(self->minPopped.whenAtLeast(std::min(self->version.get(), ver - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)));
 						self->version.set( ver );
+						Void _ = wait(yield(TaskTLogCommit));
 					}
 					break;
 				}
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 36c0175f2d..984ec603df 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -1477,14 +1477,14 @@ ACTOR Future<Void> pullAsyncData( TLogData* self, Reference<LogData> logData, st
 			Void _ = wait( delayJittered(.005, TaskTLogCommit) );
 		}
 
-		if(logData->stopped) {
-			return Void();
-		}
+		state Version ver = 0;
+		state std::vector<TagsAndMessage> messages;
+		loop {
+			if(logData->stopped) {
+				return Void();
+			}
 
-		Version ver = 0;
-		std::vector<TagsAndMessage> messages;
-		while (true) {
-			bool foundMessage = r->hasMessage();
+			state bool foundMessage = r->hasMessage();
 			if (!foundMessage || r->version().version != ver) {
 				ASSERT(r->version().version > lastVer);
 				if (ver) {
@@ -1519,6 +1519,7 @@ ACTOR Future<Void> pullAsyncData( TLogData* self, Reference<LogData> logData, st
 					//FIXME: could we just use the ver and lastVer variables, or replace them with this?
 					self->prevVersion = logData->version.get();
 					logData->version.set( ver );
+					Void _ = wait( yield(TaskTLogCommit) );
 				}
 				lastVer = ver;
 				ver = r->version().version;
@@ -1556,6 +1557,7 @@ ACTOR Future<Void> pullAsyncData( TLogData* self, Reference<LogData> logData, st
 						//FIXME: could we just use the ver and lastVer variables, or replace them with this?
 						self->prevVersion = logData->version.get();
 						logData->version.set( ver );
+						Void _ = wait( yield(TaskTLogCommit) );
 					}
 					break;
 				}

From 40f5dbe4232679c64ee2b0b7d695ce9b081bbd7a Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Tue, 4 Sep 2018 22:16:35 -0700
Subject: [PATCH 25/37] fixed issues from review, added a safeguard to prevent
 configuring a cluster to an invalid configuration

---
 fdbcli/fdbcli.actor.cpp                      | 60 ++++++++++++-----
 fdbclient/ManagementAPI.actor.cpp            | 69 ++++++++++++--------
 fdbclient/ManagementAPI.h                    |  3 +-
 fdbclient/Schemas.cpp                        | 42 ++++++------
 fdbclient/Schemas.h                          |  6 +-
 fdbserver/workloads/StatusWorkload.actor.cpp |  9 +--
 6 files changed, 115 insertions(+), 74 deletions(-)

diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index f389cc878d..50342767c0 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -447,9 +447,9 @@ void initHelp() {
 		"change the database configuration",
 		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n  single - one copy of the data.  Not fault tolerant.\n  double - two copies of data (survive one failure).\n  triple - three copies of data (survive two failures).\n  three_data_hall - See the Admin Guide.\n  three_datacenter - See the Admin Guide.\n\nStorage engine:\n  ssd - B-Tree storage engine optimized for solid state disks.\n  memory - Durable in-memory storage engine for small datasets.\n\nproxies=<PROXIES>: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=<LOGS>: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=<RESOLVERS>: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information.");
 	helpMap["fileconfigure"] = CommandHelp(
-		"fileconfigure <FILENAME>",
+		"fileconfigure [new] <FILENAME>",
 		"change the database configuration from a file",
-		"Load a JSON document from the provided file, and change the database configuration to match the contents of the JSON document. The format should be the same as the value of the \"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\". Add \"create\":\"new\" to initialize a new database.");
+		"The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. Load a JSON document from the provided file, and change the database configuration to match the contents of the JSON document. The format should be the same as the value of the \"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\".");
 	helpMap["coordinators"] = CommandHelp(
 		"coordinators auto|<ADDRESS>+ [description=new_cluster_description]",
 		"change cluster coordinators or description",
@@ -1578,9 +1578,12 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
 	case ConfigurationResult::UNKNOWN_OPTION:
 	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
 		printUsage(tokens[0]);
-		ret = true;
+		ret=true;
+		break;
+	case ConfigurationResult::INVALID_CONFIGURATION:
+		printf("ERROR: These changes would make the configuration invalid\n");
+		ret=true;
 		break;
-
 	case ConfigurationResult::DATABASE_ALREADY_CREATED:
 		printf("ERROR: Database already exists! To change configuration, don't say `new'\n");
 		ret=true;
@@ -1600,7 +1603,7 @@ ACTOR Future<bool> configure( Database db, std::vector<StringRef> tokens, Refere
 	return ret;
 }
 
-ACTOR Future<bool> fileConfigure(Database db, std::string filePath) {
+ACTOR Future<bool> fileConfigure(Database db, std::string filePath, bool isNewDatabase) {
 	std::string contents(readFileBytes(filePath, 100000));
 	json_spirit::mValue config;
 	if(!json_spirit::read_string( contents, config )) {
@@ -1610,12 +1613,21 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath) {
 	StatusObject configJSON = config.get_obj();
 
 	json_spirit::mValue schema;
-	json_spirit::read_string( configurationSchema.toString(), schema );
-	if( !schemaMatch(schema.get_obj(), configJSON) ) {
+	if(!json_spirit::read_string( JSONSchemas::configurationSchema.toString(), schema )) {
+		ASSERT(false);
+	}
+
+	std::string errorStr;
+	if( !schemaMatch(schema.get_obj(), configJSON, errorStr) ) {
+		printf("%s", errorStr.c_str());
 		return true;
 	}
 
 	std::string configString;
+	if(isNewDatabase) {
+		configString = "new";
+	}
+
 	for(auto kv : configJSON) {
 		if(!configString.empty()) {
 			configString += " ";
@@ -1637,15 +1649,27 @@ ACTOR Future<bool> fileConfigure(Database db, std::string filePath) {
 	bool ret;
 	switch(result) {
 	case ConfigurationResult::NO_OPTIONS_PROVIDED:
-	case ConfigurationResult::CONFLICTING_OPTIONS:
-	case ConfigurationResult::UNKNOWN_OPTION:
-	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
-		printUsage(LiteralStringRef("fileconfigure"));
-		ret = true;
+		printf("ERROR: No options provided\n");
+		ret=true;
+		break;
+	case ConfigurationResult::CONFLICTING_OPTIONS:
+		printf("ERROR: Conflicting options\n");
+		ret=true;
+		break;
+	case ConfigurationResult::UNKNOWN_OPTION:
+		printf("ERROR: Unknown option\n"); //This should not be possible because of schema match
+		ret=true;
+		break;
+	case ConfigurationResult::INCOMPLETE_CONFIGURATION:
+		printf("ERROR: Must specify both a replication level and a storage engine when creating a new database\n");
+		ret=true;
+		break;
+	case ConfigurationResult::INVALID_CONFIGURATION:
+		printf("ERROR: These changes would make the configuration invalid\n");
+		ret=true;
 		break;
-
 	case ConfigurationResult::DATABASE_ALREADY_CREATED:
-		printf("ERROR: Database already exists! To change the configuration, remove \"create\":\"new\"\n");
+		printf("ERROR: Database already exists! To change configuration, don't say `new'\n");
 		ret=true;
 		break;
 	case ConfigurationResult::DATABASE_CREATED:
@@ -2526,12 +2550,12 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
 				}
 
 				if (tokencmp(tokens[0], "fileconfigure")) {
-					if (tokens.size() != 2) {
+					if (tokens.size() == 2 || (tokens.size() == 3 && tokens[1] == LiteralStringRef("new"))) {
+						bool err = wait( fileConfigure( db, tokens.back().toString(), tokens.size() == 3 ) );
+						if (err) is_error = true;
+					} else {
 						printUsage(tokens[0]);
 						is_error = true;
-					} else {
-						bool err = wait( fileConfigure( db, tokens[1].toString() ) );
-						if (err) is_error = true;
 					}
 					continue;
 				}
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index c7641ce6f7..979a6c444a 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -250,6 +250,23 @@ bool isCompleteConfiguration( std::map<std::string, std::string> const& options
 			options.count( p+"storage_engine" ) == 1;
 }
 
+ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration( Database cx ) {
+	state Transaction tr(cx);
+	loop {
+		try {
+			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
+			Standalone<RangeResultRef> res = wait( tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY) );
+			ASSERT( res.size() < CLIENT_KNOBS->TOO_MANY );
+			DatabaseConfiguration config;
+			config.fromKeyValues((VectorRef<KeyValueRef>) res);
+			return config;
+		} catch( Error &e ) {
+			Void _ = wait( tr.onError(e) );
+		}
+	}
+}
+
 ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std::string, std::string> m ) {
 	state StringRef initIdKey = LiteralStringRef( "\xff/init_id" );
 	state Transaction tr(cx);
@@ -264,6 +281,19 @@ ACTOR Future<ConfigurationResult::Type> changeConfig( Database cx, std::map<std:
 		m[initIdKey.toString()] = g_random->randomUniqueID().toString();
 		if (!isCompleteConfiguration(m))
 			return ConfigurationResult::INCOMPLETE_CONFIGURATION;
+	} else {
+		state Future<DatabaseConfiguration> fConfig = getDatabaseConfiguration(cx);
+		Void _ = wait( success(fConfig) || delay(1.0) );
+
+		if(fConfig.isReady()) {
+			DatabaseConfiguration config = fConfig.get();
+			for(auto kv : m) {
+				config.set(kv.first, kv.second);
+			}
+			if(!config.isValid()) {
+				return ConfigurationResult::INVALID_CONFIGURATION;
+			}
+		}
 	}
 
 	loop {
@@ -1172,23 +1202,6 @@ ACTOR Future<Void> waitForExcludedServers( Database cx, vector<AddressExclusion>
 	}
 }
 
-ACTOR Future<DatabaseConfiguration> getDatabaseConfiguration( Database cx ) {
-	state Transaction tr(cx);
-	loop {
-		try {
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
-			Standalone<RangeResultRef> res = wait( tr.getRange(configKeys, CLIENT_KNOBS->TOO_MANY) );
-			ASSERT( res.size() < CLIENT_KNOBS->TOO_MANY );
-			DatabaseConfiguration config;
-			config.fromKeyValues((VectorRef<KeyValueRef>) res);
-			return config;
-		} catch( Error &e ) {
-			Void _ = wait( tr.onError(e) );
-		}
-	}
-
-}
-
 ACTOR Future<Void> waitForFullReplication( Database cx ) {
 	state ReadYourWritesTransaction tr(cx);
 	loop {
@@ -1401,7 +1414,7 @@ void schemaCoverage( std::string const& spath, bool covered ) {
 	}
 }
 
-bool schemaMatch( StatusObject const schema, StatusObject const result, Severity sev, bool printErrors, bool checkCoverage, std::string path, std::string schema_path ) {
+bool schemaMatch( StatusObject const schema, StatusObject const result, std::string& errorStr, Severity sev, bool checkCoverage, std::string path, std::string schema_path ) {
 	// Returns true if everything in `result` is permitted by `schema`
 
 	// Really this should recurse on "values" rather than "objects"?
@@ -1418,7 +1431,7 @@ bool schemaMatch( StatusObject const schema, StatusObject const result, Severity
 			if(checkCoverage) schemaCoverage(spath);
 
 			if (!schema.count(key)) {
-				if(printErrors) printf("ERROR: Unknown key `%s'\n", kpath.c_str());
+				errorStr += format("ERROR: Unknown key `%s'\n", kpath.c_str());
 				TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaPath", spath);
 				ok = false;
 				continue;
@@ -1436,14 +1449,14 @@ bool schemaMatch( StatusObject const schema, StatusObject const result, Severity
 						break;
 					}
 				if (!any_match) {
-					if(printErrors) printf("ERROR: Unknown value `%s' for key `%s'\n", json_spirit::write_string(rv).c_str(), kpath.c_str());
+					errorStr += format("ERROR: Unknown value `%s' for key `%s'\n", json_spirit::write_string(rv).c_str(), kpath.c_str());
 					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaEnumItems", enum_values.size()).detail("Value", json_spirit::write_string(rv));
 					if(checkCoverage) schemaCoverage(spath + ".$enum." + json_spirit::write_string(rv));
 					ok = false;
 				}
 			} else if (sv.type() == json_spirit::obj_type && sv.get_obj().count("$map")) {
 				if (rv.type() != json_spirit::obj_type) {
-					if(printErrors) printf("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str());
+					errorStr += format("ERROR: Expected an object as the value for key `%s'\n", kpath.c_str());
 					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
 					ok = false;
 					continue;
@@ -1460,18 +1473,18 @@ bool schemaMatch( StatusObject const schema, StatusObject const result, Severity
 					auto vpath = kpath + "[" + value_pair.first + "]";
 					auto upath = spath + ".$map";
 					if (value_pair.second.type() != json_spirit::obj_type) {
-						if(printErrors) printf("ERROR: Expected an object for `%s'\n", vpath.c_str());
+						errorStr += format("ERROR: Expected an object for `%s'\n", vpath.c_str());
 						TraceEvent(sev, "SchemaMismatch").detail("Path", vpath).detail("ValueType", value_pair.second.type());
 						ok = false;
 						continue;
 					}
-					if (!schemaMatch(schema_obj, value_pair.second.get_obj(), sev, printErrors, checkCoverage, vpath, upath))
+					if (!schemaMatch(schema_obj, value_pair.second.get_obj(), errorStr, sev, checkCoverage, vpath, upath))
 						ok = false;
 				}
 			} else {
 				// The schema entry isn't an operator, so it asserts a type and (depending on the type) recursive schema definition
 				if (normJSONType(sv.type()) != normJSONType(rv.type())) {
-					if(printErrors) printf("ERROR: Incorrect value type for key `%s'\n", kpath.c_str());
+					errorStr += format("ERROR: Incorrect value type for key `%s'\n", kpath.c_str());
 					TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaType", sv.type()).detail("ValueType", rv.type());
 					ok = false;
 					continue;
@@ -1482,7 +1495,7 @@ bool schemaMatch( StatusObject const schema, StatusObject const result, Severity
 					if (!schema_array.size()) {
 						// An empty schema array means that the value array is required to be empty
 						if (value_array.size()) {
-							if(printErrors) printf("ERROR: Expected an empty array for key `%s'\n", kpath.c_str());
+							errorStr += format("ERROR: Expected an empty array for key `%s'\n", kpath.c_str());
 							TraceEvent(sev, "SchemaMismatch").detail("Path", kpath).detail("SchemaSize", schema_array.size()).detail("ValueSize", value_array.size());
 							ok = false;
 							continue;
@@ -1493,12 +1506,12 @@ bool schemaMatch( StatusObject const schema, StatusObject const result, Severity
 						int index = 0;
 						for(auto &value_item : value_array) {
 							if (value_item.type() != json_spirit::obj_type) {
-								if(printErrors) printf("ERROR: Expected all array elements to be objects for key `%s'\n", kpath.c_str());
+								errorStr += format("ERROR: Expected all array elements to be objects for key `%s'\n", kpath.c_str());
 								TraceEvent(sev, "SchemaMismatch").detail("Path", kpath + format("[%d]",index)).detail("ValueType", value_item.type());
 								ok = false;
 								continue;
 							}
-							if (!schemaMatch(schema_obj, value_item.get_obj(), sev, printErrors, checkCoverage, kpath + format("[%d]", index), spath + "[0]"))
+							if (!schemaMatch(schema_obj, value_item.get_obj(), errorStr, sev, checkCoverage, kpath + format("[%d]", index), spath + "[0]"))
 								ok = false;
 							index++;
 						}
@@ -1507,7 +1520,7 @@ bool schemaMatch( StatusObject const schema, StatusObject const result, Severity
 				} else if (rv.type() == json_spirit::obj_type) {
 					auto& schema_obj = sv.get_obj();
 					auto& value_obj = rv.get_obj();
-					if (!schemaMatch(schema_obj, value_obj, sev, printErrors, checkCoverage, kpath, spath))
+					if (!schemaMatch(schema_obj, value_obj, errorStr, sev, checkCoverage, kpath, spath))
 						ok = false;
 				}
 			}
diff --git a/fdbclient/ManagementAPI.h b/fdbclient/ManagementAPI.h
index b588f1d008..18fce9d87a 100644
--- a/fdbclient/ManagementAPI.h
+++ b/fdbclient/ManagementAPI.h
@@ -46,6 +46,7 @@ public:
 		CONFLICTING_OPTIONS,
 		UNKNOWN_OPTION,
 		INCOMPLETE_CONFIGURATION,
+		INVALID_CONFIGURATION,
 		DATABASE_ALREADY_CREATED,
 		DATABASE_CREATED,
 		SUCCESS
@@ -168,6 +169,6 @@ Future<Void> forceRecovery (Reference<ClusterConnectionFile> const& clusterFile)
 Future<std::vector<NetworkAddress>> getCoordinators( Database const& cx );
 
 void schemaCoverage( std::string const& spath, bool covered=true );
-bool schemaMatch( StatusObject const schema, StatusObject const result, Severity sev=SevError, bool printErrors=true, bool checkCoverage=false, std::string path = std::string(), std::string schema_path = std::string() );
+bool schemaMatch( StatusObject const schema, StatusObject const result, std::string& errorStr, Severity sev=SevError, bool checkCoverage=false, std::string path = std::string(), std::string schema_path = std::string() );
 
 #endif
diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index f33415f7d8..64ce8f428f 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -20,7 +20,7 @@
 
 #include "Schemas.h"
 
-const KeyRef statusSchema = LiteralStringRef(R"statusSchema(
+const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
 {
    "cluster":{
       "layers":{
@@ -567,13 +567,13 @@ const KeyRef statusSchema = LiteralStringRef(R"statusSchema(
    }
 })statusSchema");
 
-const KeyRef configurationSchema = LiteralStringRef(R"configSchema(
+const KeyRef JSONSchemas::configurationSchema = LiteralStringRef(R"configSchema(
 {
     "create":{
     "$enum":[
         "new"
     ]},
-	"log_anti_quorum":0,
+    "log_anti_quorum":0,
     "log_replicas":2,
     "log_replication_policy":"(zoneid^3x1)",
     "redundancy_mode":{
@@ -586,24 +586,24 @@ const KeyRef configurationSchema = LiteralStringRef(R"configSchema(
         "three_data_hall"
     ]},
     "regions":[{
-    "datacenters":[{
-        "id":"mr",
-        "priority":1,
-        "satellite":1
-    }],
-    "satellite_redundancy_mode":{
-    "$enum":[
-        "one_satellite_single",
-        "one_satellite_double",
-        "one_satellite_triple",
-        "two_satellite_safe",
-        "two_satellite_fast"
-    ]},
-    "satellite_log_replicas":1,
-    "satellite_usable_dcs":1,
-    "satellite_anti_quorum":0,
-    "satellite_log_policy":"(zoneid^3x1)",
-    "satellite_logs":2
+        "datacenters":[{
+            "id":"mr",
+            "priority":1,
+            "satellite":1
+        }],
+        "satellite_redundancy_mode":{
+        "$enum":[
+            "one_satellite_single",
+            "one_satellite_double",
+            "one_satellite_triple",
+            "two_satellite_safe",
+            "two_satellite_fast"
+        ]},
+        "satellite_log_replicas":1,
+        "satellite_usable_dcs":1,
+        "satellite_anti_quorum":0,
+        "satellite_log_policy":"(zoneid^3x1)",
+        "satellite_logs":2
     }],
     "remote_redundancy_mode":{
     "$enum":[
diff --git a/fdbclient/Schemas.h b/fdbclient/Schemas.h
index 5cdce5390b..8070fceac3 100644
--- a/fdbclient/Schemas.h
+++ b/fdbclient/Schemas.h
@@ -26,7 +26,9 @@
 #include "flow/flow.h"
 #include "FDBTypes.h"
 
-extern const KeyRef statusSchema;
-extern const KeyRef configurationSchema;
+struct JSONSchemas {
+	static const KeyRef statusSchema;
+	static const KeyRef configurationSchema;
+};
 
 #endif /* FDBCLIENT_SCHEMAS_H */
diff --git a/fdbserver/workloads/StatusWorkload.actor.cpp b/fdbserver/workloads/StatusWorkload.actor.cpp
index e2bd6566c9..d5c88e5508 100644
--- a/fdbserver/workloads/StatusWorkload.actor.cpp
+++ b/fdbserver/workloads/StatusWorkload.actor.cpp
@@ -41,7 +41,7 @@ struct StatusWorkload : TestWorkload {
 	{
 		testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
 		requestsPerSecond = getOption(options, LiteralStringRef("requestsPerSecond"), 0.5);
-		auto statusSchemaStr = getOption(options, LiteralStringRef("schema"), statusSchema);
+		auto statusSchemaStr = getOption(options, LiteralStringRef("schema"), JSONSchemas::statusSchema);
 		if (statusSchemaStr.size()) {
 			json_spirit::mValue schema;
 			json_spirit::read_string( statusSchemaStr.toString(), schema );
@@ -123,8 +123,8 @@ struct StatusWorkload : TestWorkload {
 				save(br, result);
 				self->totalSize += br.getLength();
 				TraceEvent("StatusWorkloadReply").detail("ReplySize", br.getLength()).detail("Latency", now() - issued);//.detail("Reply", json_spirit::write_string(json_spirit::mValue(result)));
-
-				if (self->parsedSchema.present() && !schemaMatch(self->parsedSchema.get(), result, SevError, false, true) )
+				std::string errorStr;
+				if (self->parsedSchema.present() && !schemaMatch(self->parsedSchema.get(), result, errorStr, SevError, true) )
 					TraceEvent(SevError, "StatusWorkloadValidationFailed").detail("JSON", json_spirit::write_string(json_spirit::mValue(result)));
 			}
 			catch (Error& e) {
@@ -149,7 +149,8 @@ TEST_CASE("fdbserver/status/schema/basic") {
 		json_spirit::mValue test;
 		json_spirit::read_string( t, test );
 		TraceEvent("SchemaMatch").detail("Schema", json_spirit::write_string(schema)).detail("Value", json_spirit::write_string(test)).detail("Expect", expect_ok);
-		ASSERT( expect_ok == schemaMatch(schema.get_obj(), test.get_obj(), expect_ok ? SevError : SevInfo, false, true) );
+		std::string errorStr;
+		ASSERT( expect_ok == schemaMatch(schema.get_obj(), test.get_obj(), errorStr, expect_ok ? SevError : SevInfo, true) );
 	};
 	check(true, "{}");
 	check(true, "{\"apple\":4}");

From 04a768042aa9007cd3225890d3e1e4c534163db8 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 5 Sep 2018 03:50:53 -0700
Subject: [PATCH 26/37] Added TraceEvent to measure time to create Status Json
 Simplified JsonString class to use implementation method for reuse of methods
 Removed quotes from non-string values within json Added Tests for jsonstring
 Removed hashing of names for JsonString Switched name tracker to unordered
 set

---
 fdbserver/Status.actor.cpp | 162 ++++++++++++++++++++++++----
 flow/JsonString.cpp        | 216 +++++++++++++------------------------
 flow/JsonString.h          |  10 +-
 3 files changed, 224 insertions(+), 164 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 8e05648075..018d3b5453 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -621,8 +621,6 @@ ACTOR static Future<JsonString> processStatusFetcher(
 		try {
 			ASSERT(pMetrics.count(workerItr->first.address()));
 
-			processMap[printable(workerItr->first.locality.processId())] = JsonString();
-
 			NetworkAddress address = workerItr->first.address();
 			const TraceEventFields& event = pMetrics[workerItr->first.address()];
 			statusObj["address"] = address.toString();
@@ -1072,7 +1070,6 @@ static JsonString configurationFetcher(Optional<DatabaseConfiguration> conf, Ser
 		if(conf.present()) {
 			DatabaseConfiguration configuration = conf.get();
 			statusObj = configuration.toJSON().toJsonString();
-//ahm			statusObj.setJson(configuration.toJSON());
 
 			JsonStringArray excludedServersArr;
 			std::set<AddressExclusion> excludedServers = configuration.getExcludedServers();
@@ -1464,6 +1461,8 @@ static JsonStringArray oldTlogFetcher(int* oldLogFaultTolerance, Reference<Async
 		for(auto it : db->get().logSystemConfig.oldTLogs) {
 			JsonString statusObj;
 			JsonStringArray logsObj;
+			Optional<int32_t> sat_log_replication_factor, sat_log_write_anti_quorum, sat_log_fault_tolerance, log_replication_factor, log_write_anti_quorum, log_fault_tolerance, remote_log_replication_factor, remote_log_fault_tolerance;
+
 			int maxFaultTolerance = 0;
 
 			for(int i = 0; i < it.tLogs.size(); i++) {
@@ -1483,22 +1482,42 @@ static JsonStringArray oldTlogFetcher(int* oldLogFaultTolerance, Reference<Async
 				}
 				maxFaultTolerance = std::max(maxFaultTolerance, it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs);
 				if(it.tLogs[i].isLocal && it.tLogs[i].locality == tagLocalitySatellite) {
-					statusObj["satellite_log_replication_factor"] = it.tLogs[i].tLogReplicationFactor;
-					statusObj["satellite_log_write_anti_quorum"] = it.tLogs[i].tLogWriteAntiQuorum;
-					statusObj["satellite_log_fault_tolerance"] = it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs;
+					sat_log_replication_factor = it.tLogs[i].tLogReplicationFactor;
+					sat_log_write_anti_quorum = it.tLogs[i].tLogWriteAntiQuorum;
+					sat_log_fault_tolerance = it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs;
 				}
 				else if(it.tLogs[i].isLocal) {
-					statusObj["log_replication_factor"] = it.tLogs[i].tLogReplicationFactor;
-					statusObj["log_write_anti_quorum"] = it.tLogs[i].tLogWriteAntiQuorum;
-					statusObj["log_fault_tolerance"] = it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs;
+					log_replication_factor = it.tLogs[i].tLogReplicationFactor;
+					log_write_anti_quorum = it.tLogs[i].tLogWriteAntiQuorum;
+					log_fault_tolerance = it.tLogs[i].tLogReplicationFactor - 1 - it.tLogs[i].tLogWriteAntiQuorum - failedLogs;
 				}
 				else {
-					statusObj["remote_log_replication_factor"] = it.tLogs[i].tLogReplicationFactor;
-					statusObj["remote_log_fault_tolerance"] = it.tLogs[i].tLogReplicationFactor - 1 - failedLogs;
+					remote_log_replication_factor = it.tLogs[i].tLogReplicationFactor;
+					remote_log_fault_tolerance = it.tLogs[i].tLogReplicationFactor - 1 - failedLogs;
 				}
 			}
 			*oldLogFaultTolerance = std::min(*oldLogFaultTolerance, maxFaultTolerance);
 			statusObj["logs"] = logsObj;
+
+			if (sat_log_replication_factor.present())
+				statusObj["satellite_log_replication_factor"] = sat_log_replication_factor.get();
+			if (sat_log_write_anti_quorum.present())
+				statusObj["satellite_log_write_anti_quorum"] = sat_log_write_anti_quorum.get();
+			if (sat_log_fault_tolerance.present())
+				statusObj["satellite_log_fault_tolerance"] = sat_log_fault_tolerance.get();
+
+			if (log_replication_factor.present())
+				statusObj["log_replication_factor"] = log_replication_factor.get();
+			if (log_write_anti_quorum.present())
+				statusObj["log_write_anti_quorum"] = log_write_anti_quorum.get();
+			if (log_fault_tolerance.present())
+				statusObj["log_fault_tolerance"] = log_fault_tolerance.get();
+
+			if (remote_log_replication_factor.present())
+				statusObj["remote_log_replication_factor"] = remote_log_replication_factor.get();
+			if (remote_log_fault_tolerance.present())
+				statusObj["remote_log_fault_tolerance"] = remote_log_fault_tolerance.get();
+
 			oldTlogsArray.push_back(statusObj);
 		}
 	}
@@ -1606,11 +1625,10 @@ static JsonStringArray getClientIssuesAsMessages( ProcessIssuesMap const& _issue
 	return issuesList;
 }
 
-// ahm What is this doing?
 ACTOR Future<JsonString> layerStatusFetcher(Database cx, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
-	state JsonString result;
-//ahm	state JSONDoc json(result);
-	state JSONDoc json;
+	state StatusObject result;
+	state JSONDoc json(result);
+	state double tStart = now();
 
 	try {
 		state ReadYourWritesTransaction tr(cx);
@@ -1661,7 +1679,9 @@ ACTOR Future<JsonString> layerStatusFetcher(Database cx, JsonStringArray *messag
 	}
 
 	json.cleanOps();
-	return result;
+	JsonString statusObj = result.toJsonString();
+	TraceEvent("LayerStatusFetcher").detail("Duration", now()-tStart).detail("StatusSize",statusObj.getLength()).detail("StatusNameTotal",statusObj.getNameTotal());
+	return statusObj;
 }
 
 ACTOR Future<JsonString> lockedStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
@@ -1720,6 +1740,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 {
 	// since we no longer offer multi-database support, all databases must be named DB
 	state std::string dbName = "DB";
+	state double tStart = now();
 
 	// Check if master worker is present
 	state JsonStringArray messages;
@@ -1868,7 +1889,6 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			// Merge data_overlay into data
 			JsonString &clusterDataSection = workerStatuses[0];
 			clusterDataSection.append(data_overlay); //ahm This is a problem, if dupe keys
-//ahm			clusterDataSection.insert(data_overlay.begin(), data_overlay.end());
 
 			// If data section not empty, add it to statusObj
 			if (!clusterDataSection.empty())
@@ -1877,7 +1897,6 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			// Insert database_locked section
 			if(!workerStatuses[3].empty()) {
 				statusObj.append(workerStatuses[3]); //ahm This is a problem, if dupe keys
-//ahm				statusObj.insert(workerStatuses[3].begin(), workerStatuses[3].end());
 			}
 
 			// Need storage servers now for processStatusFetcher() below.
@@ -1945,6 +1964,9 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			statusObj["cluster_controller_timestamp"] = clusterTime;
 		}
 
+		TraceEvent("ClusterGetStatus").detail("Duration", now()-tStart)
+.detail("StatusSize",statusObj.getLength()) .detail("StatusNameTotal",statusObj.getNameTotal());
+
 		return StatusReply(statusObj);
 	} catch( Error&e ) {
 		TraceEvent(SevError, "StatusError").error(e);
@@ -1952,6 +1974,110 @@ ACTOR Future<StatusReply> clusterGetStatus(
 	}
 }
 
+TEST_CASE("status/json/jsonstring") {
+	JsonString	jsonObj, testObj;
+	JsonStringArray jsonArray;
+
+	JsonString	jsonString1;
+	jsonString1["_valid"] = false;
+	jsonString1["error"] = "configurationMissing";
+
+	JsonString	jsonString2("_valid", false);
+	jsonString2["error"] = "configurationMissing";
+
+	JsonString	jsonString3("error", "configurationMissing");
+	jsonString3["_valid"] = false;
+
+	JsonString	jsonString4("_valid", false);
+	jsonString4.append("error", "configurationMissing");
+
+	std::cout << "Json: " << jsonString1.getJson() << std::endl;
+
+	// Validate equality
+	ASSERT(jsonString1 == jsonString2);
+	ASSERT(jsonString1 != jsonString3); // wrong order
+	ASSERT(jsonString1.equals(jsonString4));
+	ASSERT(jsonString4.equals(jsonString1));
+
+	// Check presence of top level keys
+	ASSERT(jsonString1.isPresent("_valid"));
+	ASSERT(jsonString1.isPresent("error"));
+	ASSERT(!jsonString1.isPresent("configurationMissing"));
+	ASSERT(jsonString1.getNameTotal() == 2);
+
+	// Check empty
+	ASSERT(!jsonString1.empty());
+	ASSERT(JsonString().empty());
+
+	// Check clear
+	jsonString1.clear();
+	ASSERT(jsonString1.empty());
+
+	// Set object
+	jsonObj.clear();
+	testObj.clear();
+	jsonObj["first"] = "last";
+	jsonObj["count"] = 3;
+	testObj["pi"] = 3.14159;
+	jsonObj["piobj"] = testObj;
+
+	// Check presence of top level keys
+	ASSERT(jsonObj.isPresent("count"));
+	ASSERT(testObj.isPresent("pi"));
+	ASSERT(!jsonObj.isPresent("pi"));
+	ASSERT(jsonObj.getNameTotal() == 3);
+
+	testObj.clear();
+	testObj["val1"] = 7.9;
+	testObj["val2"] = 34;
+	jsonArray.push_back(testObj);
+
+	testObj.clear();
+	testObj["name1"] = "robert";
+	testObj["name2"] = "john";
+	testObj["name3"] = "eric";
+	jsonArray.push_back(testObj);
+	jsonObj.append("name_vals", jsonArray);
+
+	std::cout << "Json (complex): " << jsonObj.getJson() << std::endl;
+
+	jsonObj.clear();
+	jsonObj.append(jsonArray);
+	std::cout << "Json (complex array): " << jsonObj.getJson() << std::endl;
+
+	jsonArray.clear();
+	jsonArray.push_back("nissan");
+	jsonArray.push_back("honda");
+	jsonArray.push_back("bmw");
+	jsonObj.clear();
+	jsonObj.append(jsonArray);
+	std::cout << "Array (simple array #1): " << jsonObj.getJson() << std::endl;
+
+	jsonObj.clear();
+	testObj.clear();
+	jsonArray.clear();
+	testObj.append("nissan");
+	testObj.append("honda");
+	testObj.append("bmw");
+	jsonArray.push_back(testObj);
+	jsonObj.append(jsonArray);
+	std::cout << "Array (simple array #2): " << jsonObj.getJson() << std::endl;
+
+	jsonObj.clear();
+	testObj.clear();
+	jsonObj["name1"] = "robert";
+	jsonObj["name2"] = "john";
+	jsonObj["name3"] = "eric";
+	testObj["val1"] = 7.9;
+	testObj["val2"] = 34;
+	std::cout << "Merge (obj #1): " << jsonObj.getJson() << std::endl;
+	std::cout << "Merge (obj #2): " << testObj.getJson() << std::endl;
+	jsonObj.append(testObj);
+	std::cout << "Merged: " << jsonObj.getJson() << std::endl;
+
+	return Void();
+}
+
 TEST_CASE("status/json/merging") {
 	StatusObject objA, objB, objC;
 	JSONDoc a(objA), b(objB), c(objC);
diff --git a/flow/JsonString.cpp b/flow/JsonString.cpp
index 57716976b3..c7470cd083 100644
--- a/flow/JsonString.cpp
+++ b/flow/JsonString.cpp
@@ -1,6 +1,8 @@
 #include "JsonString.h"
 #include "Hash3.h"
 #include <iostream>
+#include "Trace.h"
+#include "flow.h"
 
 std::string format( const char* form, ... );
 
@@ -61,11 +63,8 @@ uint32_t JsonString::hash32( const std::string& name ) {
 	return a;
 }
 
-bool	JsonString::isPresent(uint32_t nameHash) const {
-	return (_keyNames.find(nameHash) != _keyNames.end());
-}
 bool	JsonString::isPresent(const std::string& name) const {
-	return isPresent(hash32(name));
+	return (_keyNames.find(name) != _keyNames.end());
 }
 
 JsonString JsonString::makeMessage(const char *name, const char *description) {
@@ -75,14 +74,63 @@ JsonString JsonString::makeMessage(const char *name, const char *description) {
 	return out;
 }
 
+// ahm Todo: Added assert for correctness when name hash already present to identify collisions
 void JsonString::hashName( const std::string& name) {
-	_keyNames.insert(hash32(name));
+	if (isPresent(name)) {
+		TraceEvent(g_network && g_network->isSimulated() ? SevError : SevWarnAlways, "JsonError").detail("KeyPresent", name).backtrace();
+	}
+	_keyNames.insert(name);
+}
+
+JsonString& JsonString::appendImpl( const std::string& name, const std::string& value, bool quote ) {
+	hashName(name);
+	_jsonText += (_jsonText.empty() ? "\"" : ",\n  \"") + name + (quote ? "\": \"" : "\": ") + value;
+	if (quote)
+		_jsonText += "\"";
+	return *this;
+}
+JsonString& JsonString::appendImpl( const std::string& value, bool quote ) {
+	if (quote) {
+		_jsonText += (_jsonText.empty() ? "\"" : ", \"") + value + "\"";
+	}
+	else {
+		if (_jsonText.empty())
+			_jsonText += ", ";
+		_jsonText += value;
+	}
+	return *this;
 }
 
 JsonString& JsonString::append( const std::string& name, const std::string& value ) {
-	hashName(name);
-	_jsonText += (_jsonText.empty() ? "\"" : ",\n  \"") + name + "\": \"" + value + "\"";
-	return *this;
+	return appendImpl(name, value, true);
+}
+JsonString& JsonString::append( const std::string& name, const char* value ) {
+	std::string	textValue(value);
+	return appendImpl(name, textValue, true);
+}
+JsonString& JsonString::append( const std::string& name, double value ) {
+	return appendImpl(name, format("%g", value), false);
+}
+JsonString& JsonString::append( const std::string& name, long int value ) {
+	return appendImpl(name, format("%ld", value), false);
+}
+JsonString& JsonString::append( const std::string& name, long unsigned int value ) {
+	return appendImpl(name, format("%lu", value), false);
+}
+JsonString& JsonString::append( const std::string& name, long long int value ) {
+	return appendImpl(name, format("%lld", value), false);
+}
+JsonString& JsonString::append( const std::string& name, long long unsigned int value ) {
+	return appendImpl(name, format("%llu", value), false);
+}
+JsonString& JsonString::append( const std::string& name, int value ) {
+	return appendImpl(name, format("%d", value), false);
+}
+JsonString& JsonString::append( const std::string& name, unsigned value ) {
+	return appendImpl(name, format("%u", value), false);
+}
+JsonString& JsonString::append( const std::string& name, bool value ) {
+	return appendImpl(name, value ? "true" : "false", false);
 }
 JsonString& JsonString::append( const std::string& name, const JsonString& value ) {
 	hashName(name);
@@ -102,71 +150,37 @@ JsonString& JsonString::append( const std::string& name, const JsonStringArray&
 	_jsonText += " ]";
 	return *this;
 }
-JsonString& JsonString::append( const std::string& name, const char* value ) {
-	std::string	textValue(value);
-	return append(name, textValue);
-}
-JsonString& JsonString::append( const std::string& name, double value ) {
-	return append(name, format("%g", value));
-}
-JsonString& JsonString::append( const std::string& name, long int value ) {
-	return append(name, format("%ld", value));
-}
-JsonString& JsonString::append( const std::string& name, long unsigned int value ) {
-	return append(name, format("%lu", value));
-}
-JsonString& JsonString::append( const std::string& name, long long int value ) {
-	return append(name, format("%lld", value));
-}
-JsonString& JsonString::append( const std::string& name, long long unsigned int value ) {
-	return append(name, format("%llu", value));
-}
-JsonString& JsonString::append( const std::string& name, int value ) {
-	return append(name, format("%d", value));
-}
-JsonString& JsonString::append( const std::string& name, unsigned value ) {
-	return append(name, format("%u", value));
-}
-JsonString& JsonString::append( const std::string& name, bool value ) {
-	hashName(name);
-	_jsonText += (_jsonText.empty() ? "\"" : ",\n  ") + name + (value ? "\": true" : "\": false");
-	return *this;
-}
 
 JsonString& JsonString::append( const std::string& value ) {
-	_jsonText += (_jsonText.empty() ? "\"" : ", \"") + value + "\"";
-	return *this;
+	return appendImpl(value, true);
 }
 JsonString& JsonString::append( const char* value ) {
 	std::string	textValue(value);
-	return append(textValue);
+	return appendImpl(textValue, true);
 }
 JsonString& JsonString::append( double value ) {
-	return append(format("%g", value));
+	return appendImpl(format("%g", value), false);
 }
 JsonString& JsonString::append( long int value ) {
-	return append(format("%ld", value));
+	return appendImpl(format("%ld", value), false);
 }
 JsonString& JsonString::append( long unsigned int value ) {
-	return append(format("%lu", value));
+	return appendImpl(format("%lu", value), false);
 }
 JsonString& JsonString::append( long long int value ) {
-	return append(format("%lld", value));
+	return appendImpl(format("%lld", value), false);
 }
 JsonString& JsonString::append( long long unsigned int value ) {
-	return append(format("%llu", value));
+	return appendImpl(format("%llu", value), false);
 }
 JsonString& JsonString::append( int value ) {
-	return append(format("%d", value));
+	return appendImpl(format("%d", value), false);
 }
 JsonString& JsonString::append( unsigned value ) {
-	return append(format("%u", value));
+	return appendImpl(format("%u", value), false);
 }
 JsonString& JsonString::append( bool value ) {
-	if (!_jsonText.empty())
-		_jsonText += ", ";
-	_jsonText += value ? "true" : "false";
-	return *this;
+	return appendImpl(value ? "true" : "false", false);
 }
 JsonString& JsonString::append( const JsonString& value ) {
 	// Only do something, if not empty
@@ -205,6 +219,13 @@ const std::string&	JsonString::getJsonText() const {
 	return _jsonText;
 }
 
+size_t	JsonString::getLength() const {
+	return _jsonText.length() + ((!empty() && _keyNames.empty()) ? 0 : 2);
+}
+size_t	JsonString::getNameTotal() const {
+	return _keyNames.size();
+}
+
 std::string	JsonString::getJson() const {
 	// If not empty with no names (only values), don't add brackets because prob in an array
 	return (!empty() && _keyNames.empty()) ? _jsonText : ("{ " + _jsonText + " }");
@@ -303,94 +324,3 @@ JsonStringSetter& JsonStringSetter::operator=( const JsonStringArray& value ) {
 	_jsonString.append(_name, value);
 	return *this;
 }
-
-inline void _fdbassert(const char* expression, const char* file, int line) {
-	fprintf(stderr, "Assertion '%s' failed, file '%s' line '%d'.", expression, file, line);
-	abort();
-}
-
-#define fdbassert(EXPRESSION) ((EXPRESSION) ? (void)0 : _fdbassert(#EXPRESSION, __FILE__, __LINE__))
-
-int jsonUnitTests() {
-	int	status = 0;
-
-	JsonString	jsonString1;
-	jsonString1["_valid"] = false;
-	jsonString1["error"] = "configurationMissing";
-
-	JsonString	jsonString2("_valid", false);
-	jsonString2["error"] = "configurationMissing";
-
-	JsonString	jsonString3("error", "configurationMissing");
-	jsonString3["_valid"] = false;
-
-	JsonString	jsonString4("_valid", false);
-	jsonString4.append("error", "configurationMissing");
-
-	std::cout << "Json1: " << jsonString1.getJson() << std::endl;
-	std::cout << "Json2: " << jsonString2.getJson() << std::endl;
-	std::cout << "Json3: " << jsonString3.getJson() << std::endl;
-	std::cout << "Json4: " << jsonString4.getJson() << std::endl;
-
-	fdbassert(jsonString1 == jsonString2);
-	fdbassert(jsonString1 != jsonString3); // wrong order
-	fdbassert(jsonString1 == jsonString4);
-
-	JsonString	jsonObj, testObj;
-	JsonStringArray jsonArray;
-
-	jsonObj["alvin"] = "moore";
-	jsonObj["count"] = 3;
-	testObj["pi"] = 3.14159;
-	jsonObj["piobj"] = testObj;
-
-	testObj.clear();
-	testObj["val1"] = 7.9;
-	testObj["val2"] = 34;
-	jsonArray.push_back(testObj);
-
-	testObj.clear();
-	testObj["name1"] = "alvin";
-	testObj["name2"] = "evan";
-	testObj["name3"] = "ben";
-	jsonArray.push_back(testObj);
-	jsonObj.append("name_vals", jsonArray);
-
-	std::cout << "Json (complex): " << jsonObj.getJson() << std::endl;
-
-	jsonObj.clear();
-	jsonObj.append(jsonArray);
-	std::cout << "Json (complex array): " << jsonObj.getJson() << std::endl;
-
-	jsonArray.clear();
-	jsonArray.push_back("nissan");
-	jsonArray.push_back("honda");
-	jsonArray.push_back("bmw");
-	jsonObj.clear();
-	jsonObj.append(jsonArray);
-	std::cout << "Array (simple array #1): " << jsonObj.getJson() << std::endl;
-
-	testObj.clear();
-	testObj.append("nissan");
-	testObj.append("honda");
-	testObj.append("bmw");
-	jsonArray.clear();
-	jsonArray.push_back(testObj);
-	jsonObj.clear();
-	jsonObj.append(jsonArray);
-	std::cout << "Array (simple array #2): " << jsonObj.getJson() << std::endl;
-
-	jsonObj.clear();
-	jsonObj["name1"] = "alvin";
-	jsonObj["name2"] = "evan";
-	jsonObj["name3"] = "ben";
-	testObj.clear();
-	testObj["val1"] = 7.9;
-	testObj["val2"] = 34;
-	std::cout << "Merge (obj #1): " << jsonObj.getJson() << std::endl;
-	std::cout << "Merge (obj #2): " << testObj.getJson() << std::endl;
-	jsonObj.append(testObj);
-	std::cout << "Merged: " << jsonObj.getJson() << std::endl;
-
-	return status;
-}
diff --git a/flow/JsonString.h b/flow/JsonString.h
index 2a6b1e9999..e89b31ca3e 100644
--- a/flow/JsonString.h
+++ b/flow/JsonString.h
@@ -3,7 +3,7 @@
 #define JSONSTRING_H
 
 #include <string>
-#include <set>
+#include <unordered_set>
 #include <vector>
 
 class JsonString;
@@ -73,10 +73,12 @@ class JsonString {
 		std::string	getJson() const;
 		void	setJson(const std::string& jsonText);
 
+		size_t	getLength() const;
+		size_t	getNameTotal() const;
+
 		JsonString& clear();
 		bool	empty() const;
 
-		bool	isPresent(uint32_t nameHash) const;
 		bool	isPresent(const std::string& name) const;
 
 		static uint32_t hash32( const std::string& name );
@@ -84,10 +86,12 @@ class JsonString {
 
 	protected:
 		void hashName( const std::string& name);
+		JsonString& appendImpl( const std::string& name, const std::string& value, bool quote);
+		JsonString& appendImpl( const std::string& value, bool quote);
 
 	protected:
 		std::string		_jsonText;
-		std::set<uint32_t>	_keyNames;
+		std::unordered_set<std::string>	_keyNames;
 
 		// Uneditted text
 		const std::string&	getJsonText() const;

From 43a2afc3b6f448a07f51c6910f405e6702eba676 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 5 Sep 2018 08:06:19 -0700
Subject: [PATCH 27/37] Added TODO comment Removed debug comments

---
 fdbserver/Status.actor.cpp | 8 ++++----
 flow/JsonString.cpp        | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 018d3b5453..203312626c 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -1787,7 +1787,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			JsonString message = JsonString::makeMessage("unreachable_processes", "The cluster has some unreachable processes.");
 			JsonStringArray unreachableProcs;
 			for (auto m : mergeUnreachable){
-				unreachableProcs.push_back(JsonString("address", m));	//ahm I'm pretty sure this is right
+				unreachableProcs.push_back(JsonString("address", m));
 			}
 			message["unreachable_processes"] = unreachableProcs;
 			messages.push_back(message);
@@ -1888,7 +1888,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 			// Merge data_overlay into data
 			JsonString &clusterDataSection = workerStatuses[0];
-			clusterDataSection.append(data_overlay); //ahm This is a problem, if dupe keys
+			clusterDataSection.append(data_overlay);
 
 			// If data section not empty, add it to statusObj
 			if (!clusterDataSection.empty())
@@ -1896,7 +1896,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 			// Insert database_locked section
 			if(!workerStatuses[3].empty()) {
-				statusObj.append(workerStatuses[3]); //ahm This is a problem, if dupe keys
+				statusObj.append(workerStatuses[3]);
 			}
 
 			// Need storage servers now for processStatusFetcher() below.
@@ -1951,7 +1951,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 			// Make a JSON array of all of the reasons in the status_incomplete_reasons set.
 			JsonStringArray reasons;
 			for (auto i : status_incomplete_reasons) {
-				reasons.push_back(JsonString("description", i));	//ahm pretty sure this is right
+				reasons.push_back(JsonString("description", i));
 			}
 			incomplete_message["reasons"] = reasons;
 			messages.push_back(incomplete_message);
diff --git a/flow/JsonString.cpp b/flow/JsonString.cpp
index c7470cd083..fb69f0500e 100644
--- a/flow/JsonString.cpp
+++ b/flow/JsonString.cpp
@@ -74,7 +74,6 @@ JsonString JsonString::makeMessage(const char *name, const char *description) {
 	return out;
 }
 
-// ahm Todo: Added assert for correctness when name hash already present to identify collisions
 void JsonString::hashName( const std::string& name) {
 	if (isPresent(name)) {
 		TraceEvent(g_network && g_network->isSimulated() ? SevError : SevWarnAlways, "JsonError").detail("KeyPresent", name).backtrace();
@@ -241,8 +240,9 @@ JsonString& JsonString::operator=( const JsonString& jsonString ) {
 	return copy(jsonString);
 }
 
-//ahm Remove the leading and ending {}
+//TODO: Populate key names member
 void	JsonString::setJson(const std::string& jsonText) {
+	_keyNames.clear();
 	_jsonText = jsonText;
 }
 

From 282e9e41c2573d7b9670ca4e1bb7fa901e65f200 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 5 Sep 2018 11:40:04 -0700
Subject: [PATCH 28/37] fix: windows build was broken because statusSchema
 string was too long

---
 fdbclient/Schemas.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp
index 64ce8f428f..b3a598d7a0 100644
--- a/fdbclient/Schemas.cpp
+++ b/fdbclient/Schemas.cpp
@@ -291,6 +291,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
             "description":"abc"
          }
       ],
+)statusSchema" R"statusSchema(
       "recovery_state":{
          "required_resolvers":1,
          "required_proxies":1,

From 6aa22af83be1164af3d49f77a4d3d96630c7d4ca Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 5 Sep 2018 12:22:04 -0700
Subject: [PATCH 29/37] Added explicit keyword for appropriate constructor
 Templatized some methods Removed unused hash function

---
 fdbserver/Status.actor.cpp |  19 +++---
 flow/JsonString.cpp        | 122 ++++++++++++++-----------------------
 flow/JsonString.h          |  34 ++++++-----
 3 files changed, 75 insertions(+), 100 deletions(-)

diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index 203312626c..ca91151178 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -389,11 +389,12 @@ static JsonString machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair<W
 	}
 
 	// Add the status json for each machine with tracked values
-	for (auto mapPair : machineJsonMap) {
+	for (auto& mapPair : machineJsonMap) {
 		auto& machineId = mapPair.first;
-		machineJsonMap[machineId]["excluded"] = notExcludedMap[machineId];
-		machineJsonMap[machineId]["contributing_workers"] = workerContribMap[machineId];
-		machineMap[machineId] = machineJsonMap[machineId];
+		auto& jsonItem = machineJsonMap[machineId];
+		jsonItem["excluded"] = notExcludedMap[machineId];
+		jsonItem["contributing_workers"] = workerContribMap[machineId];
+		machineMap[machineId] = jsonItem;
 	}
 
 	if(failed > 0)
@@ -1610,7 +1611,7 @@ static JsonStringArray getClientIssuesAsMessages( ProcessIssuesMap const& _issue
 			JsonString message = JsonString::makeMessage(i.first.c_str(), getIssueDescription(i.first).c_str());
 			JsonStringArray addresses;
 			for(auto addr : i.second) {
-				addresses.push_back(addr);
+				addresses.push_back(JsonString(addr));
 			}
 
 			message["addresses"] = addresses;
@@ -1928,7 +1929,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
 
 		JsonStringArray incompatibleConnectionsArray;
 		for(auto it : incompatibleConnections) {
-			incompatibleConnectionsArray.push_back(it.toString());
+			incompatibleConnectionsArray.push_back(JsonString(it.toString()));
 		}
 		statusObj["incompatible_connections"] = incompatibleConnectionsArray;
 		statusObj["datacenter_version_difference"] = datacenterVersionDifference;
@@ -2046,9 +2047,9 @@ TEST_CASE("status/json/jsonstring") {
 	std::cout << "Json (complex array): " << jsonObj.getJson() << std::endl;
 
 	jsonArray.clear();
-	jsonArray.push_back("nissan");
-	jsonArray.push_back("honda");
-	jsonArray.push_back("bmw");
+	jsonArray.push_back(JsonString("nissan"));
+	jsonArray.push_back(JsonString("honda"));
+	jsonArray.push_back(JsonString("bmw"));
 	jsonObj.clear();
 	jsonObj.append(jsonArray);
 	std::cout << "Array (simple array #1): " << jsonObj.getJson() << std::endl;
diff --git a/flow/JsonString.cpp b/flow/JsonString.cpp
index fb69f0500e..731e1e0e3d 100644
--- a/flow/JsonString.cpp
+++ b/flow/JsonString.cpp
@@ -57,12 +57,6 @@ JsonString::JsonString( const std::string& name, const JsonStringArray& value )
 	append(name, value);
 }
 
-uint32_t JsonString::hash32( const std::string& name ) {
-	uint32_t a=0, b=0;
-	hashlittle2( (const void*) name.c_str(), name.length(), &a, &b );
-	return a;
-}
-
 bool	JsonString::isPresent(const std::string& name) const {
 	return (_keyNames.find(name) != _keyNames.end());
 }
@@ -100,36 +94,64 @@ JsonString& JsonString::appendImpl( const std::string& value, bool quote ) {
 	return *this;
 }
 
+std::string	JsonString::stringify(const char* value) {
+	std::string	textValue(value);
+	return appendImpl(name, textValue, true);
+}
+std::string	JsonString::stringify(double value) {
+	return format("%g", value);
+}
+std::string	JsonString::stringify(long int value) {
+	return format("%ld", value);
+}
+std::string	JsonString::stringify(long unsigned int value) {
+	return format("%lu", value);
+}
+std::string	JsonString::stringify(long long int value) {
+	return format("%lld", value);
+}
+std::string	JsonString::stringify(long long unsigned int value) {
+	return format("%llu", value);
+}
+std::string	JsonString::stringify(int value) {
+	return format("%d", value);
+}
+std::string	JsonString::stringify(unsigned value) {
+	return format("%u", value);
+}
+std::string	JsonString::stringify(bool value) {
+	return value ? "true" : "false";
+}
+
 JsonString& JsonString::append( const std::string& name, const std::string& value ) {
 	return appendImpl(name, value, true);
 }
 JsonString& JsonString::append( const std::string& name, const char* value ) {
-	std::string	textValue(value);
-	return appendImpl(name, textValue, true);
+	return appendImpl(name, stringify(value), true);
 }
 JsonString& JsonString::append( const std::string& name, double value ) {
-	return appendImpl(name, format("%g", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, long int value ) {
-	return appendImpl(name, format("%ld", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, long unsigned int value ) {
-	return appendImpl(name, format("%lu", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, long long int value ) {
-	return appendImpl(name, format("%lld", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, long long unsigned int value ) {
-	return appendImpl(name, format("%llu", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, int value ) {
-	return appendImpl(name, format("%d", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, unsigned value ) {
-	return appendImpl(name, format("%u", value), false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, bool value ) {
-	return appendImpl(name, value ? "true" : "false", false);
+	return appendImpl(name, stringify(value), false);
 }
 JsonString& JsonString::append( const std::string& name, const JsonString& value ) {
 	hashName(name);
@@ -154,32 +176,31 @@ JsonString& JsonString::append( const std::string& value ) {
 	return appendImpl(value, true);
 }
 JsonString& JsonString::append( const char* value ) {
-	std::string	textValue(value);
-	return appendImpl(textValue, true);
+	return appendImpl(stringify(value), true);
 }
 JsonString& JsonString::append( double value ) {
-	return appendImpl(format("%g", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( long int value ) {
-	return appendImpl(format("%ld", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( long unsigned int value ) {
-	return appendImpl(format("%lu", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( long long int value ) {
-	return appendImpl(format("%lld", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( long long unsigned int value ) {
-	return appendImpl(format("%llu", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( int value ) {
-	return appendImpl(format("%d", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( unsigned value ) {
-	return appendImpl(format("%u", value), false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( bool value ) {
-	return appendImpl(value ? "true" : "false", false);
+	return appendImpl(stringify(value), false);
 }
 JsonString& JsonString::append( const JsonString& value ) {
 	// Only do something, if not empty
@@ -275,52 +296,3 @@ JsonStringArray::~JsonStringArray() {
 
 JsonStringSetter::JsonStringSetter( JsonString& jsonString, const std::string& name ) : _jsonString(jsonString), _name(name) {
 }
-
-JsonStringSetter& JsonStringSetter::operator=( const std::string& value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( const char* value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( double value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( long int value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( long unsigned int value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( long long int value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( long long unsigned int value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( int value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( unsigned value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( bool value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( const JsonString& value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
-JsonStringSetter& JsonStringSetter::operator=( const JsonStringArray& value ) {
-	_jsonString.append(_name, value);
-	return *this;
-}
diff --git a/flow/JsonString.h b/flow/JsonString.h
index e89b31ca3e..2a68f27376 100644
--- a/flow/JsonString.h
+++ b/flow/JsonString.h
@@ -14,9 +14,9 @@ class JsonString {
 	public:
 		JsonString();
 		JsonString( const JsonString& jsonString);
-		JsonString( const JsonStringArray& jsonArray);
-		JsonString( const char* value );
-		JsonString( const std::string& value );
+		explicit JsonString( const JsonStringArray& jsonArray);
+		explicit JsonString( const char* value );
+		explicit JsonString( const std::string& value );
 
 		JsonString( const std::string& name, const char* value );
 		JsonString( const std::string& name, const std::string& value );
@@ -81,7 +81,6 @@ class JsonString {
 
 		bool	isPresent(const std::string& name) const;
 
-		static uint32_t hash32( const std::string& name );
 		static JsonString makeMessage(const char *name, const char *description);
 
 	protected:
@@ -89,6 +88,16 @@ class JsonString {
 		JsonString& appendImpl( const std::string& name, const std::string& value, bool quote);
 		JsonString& appendImpl( const std::string& value, bool quote);
 
+		static std::string	stringify(const char* value);
+		static std::string	stringify(double value);
+		static std::string	stringify(long int value);
+		static std::string	stringify(long unsigned int value);
+		static std::string	stringify(long long int value);
+		static std::string	stringify(long long unsigned int value);
+		static std::string	stringify(int value);
+		static std::string	stringify(unsigned value);
+		static std::string	stringify(bool value);
+
 	protected:
 		std::string		_jsonText;
 		std::unordered_set<std::string>	_keyNames;
@@ -121,18 +130,11 @@ class JsonStringSetter {
 
 	JsonStringSetter( JsonString& jsonString, const std::string& name );
 
-	JsonStringSetter& operator=( const std::string& value );
-	JsonStringSetter& operator=( const char* value );
-	JsonStringSetter& operator=( double value );
-	JsonStringSetter& operator=( long int value );
-	JsonStringSetter& operator=( long unsigned int value );
-	JsonStringSetter& operator=( long long int value );
-	JsonStringSetter& operator=( long long unsigned int value );
-	JsonStringSetter& operator=( int value );
-	JsonStringSetter& operator=( unsigned value );
-	JsonStringSetter& operator=( bool value );
-	JsonStringSetter& operator=( const JsonString& value );
-	JsonStringSetter& operator=( const JsonStringArray& value );
+	template <class valClass>
+	JsonStringSetter& operator=( const valClass& value ) {
+		_jsonString.append(_name, value);
+		return *this;
+	}
 
 	protected:
 		JsonString&		_jsonString;

From e7cc9ca41ac2d16838d6b126740e200d97e5002a Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 5 Sep 2018 13:00:52 -0700
Subject: [PATCH 30/37] it is not valid to configure regions when using
 three_datacenter replication

---
 fdbclient/DatabaseConfiguration.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index 4a2487718d..e2ae0b5735 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -177,7 +177,8 @@ bool DatabaseConfiguration::isValid() const {
 		usableRegions <= 2 &&
 		regions.size() <= 2 &&
 		( usableRegions == 1 || regions.size() == 2 ) &&
-		( regions.size() == 0 || regions[0].priority >= 0 ) ) ) {
+		( regions.size() == 0 || regions[0].priority >= 0 ) &&
+		( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) {
 		return false;
 	}
 

From d9906d7d6a66bbca3e51a9c2d5d74820a327ff9d Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 5 Sep 2018 13:42:10 -0700
Subject: [PATCH 31/37] code cleanup

---
 fdbclient/DatabaseConfiguration.cpp  | 2 +-
 fdbserver/DataDistribution.actor.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index e2ae0b5735..b47822630e 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -178,7 +178,7 @@ bool DatabaseConfiguration::isValid() const {
 		regions.size() <= 2 &&
 		( usableRegions == 1 || regions.size() == 2 ) &&
 		( regions.size() == 0 || regions[0].priority >= 0 ) &&
-		( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) {
+		( regions.size() == 0 || tLogPolicy->info() != "dcid^2 x zoneid^2 x 1") ) ) { //We cannot specify regions with three_datacenter replication
 		return false;
 	}
 
diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp
index f6ec7d6bd9..b477472a52 100644
--- a/fdbserver/DataDistribution.actor.cpp
+++ b/fdbserver/DataDistribution.actor.cpp
@@ -851,7 +851,7 @@ struct DDTeamCollection {
 
 			if(servers.size() >= self->configuration.storageTeamSize) {
 				bool foundTeam = false;
-				for( int j = 0; j < std::max( 1, (int)(servers.size() - self->configuration.storageTeamSize + 1) ) && !foundTeam; j++ ) {
+				for( int j = 0; j < servers.size() - self->configuration.storageTeamSize + 1 && !foundTeam; j++ ) {
 					auto& serverTeams = self->server_info[servers[j]->id]->teams;
 					for( int k = 0; k < serverTeams.size(); k++ ) {
 						auto &testTeam = serverTeams[k]->getServerIDs();

From 821adeae4f29096efb722d33875ef03a80be4416 Mon Sep 17 00:00:00 2001
From: Alvin Moore <alvinm@apple.com>
Date: Wed, 5 Sep 2018 14:17:33 -0700
Subject: [PATCH 32/37] Added method to swap in json text

---
 fdbclient/Status.cpp | 4 +---
 flow/JsonString.cpp  | 9 +++++++--
 flow/JsonString.h    | 7 +++++--
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/fdbclient/Status.cpp b/fdbclient/Status.cpp
index e666b601a5..8851477282 100644
--- a/fdbclient/Status.cpp
+++ b/fdbclient/Status.cpp
@@ -2,8 +2,6 @@
 #include "Status.h"
 
 JsonString	StatusObject::toJsonString() const {
-	JsonString	jsonStringObj;
 	std::string jsonText = json_spirit::write_string(json_spirit::mValue(*this));
-	jsonStringObj.setJson(jsonText);
-	return jsonStringObj;
+	return JsonString().swapJsonText(jsonText);
 }
diff --git a/flow/JsonString.cpp b/flow/JsonString.cpp
index 731e1e0e3d..2da675c30e 100644
--- a/flow/JsonString.cpp
+++ b/flow/JsonString.cpp
@@ -13,6 +13,7 @@ JsonString::JsonString( const JsonString& jsonString) : _jsonText(jsonString._js
 JsonString::JsonString( const JsonStringArray& jsonArray) : _jsonText(), _keyNames() {
 	append(jsonArray);
 }
+
 JsonString::JsonString( const std::string& value ) : _jsonText(), _keyNames() {
 	append(value);
 }
@@ -95,8 +96,7 @@ JsonString& JsonString::appendImpl( const std::string& value, bool quote ) {
 }
 
 std::string	JsonString::stringify(const char* value) {
-	std::string	textValue(value);
-	return appendImpl(name, textValue, true);
+	return std::string(value);
 }
 std::string	JsonString::stringify(double value) {
 	return format("%g", value);
@@ -266,6 +266,11 @@ void	JsonString::setJson(const std::string& jsonText) {
 	_keyNames.clear();
 	_jsonText = jsonText;
 }
+JsonString&	JsonString::swapJsonText(std::string& jsonText) {
+	_keyNames.clear();
+	_jsonText.swap(jsonText);
+	return *this;
+}
 
 JsonStringSetter& JsonString::operator[]( const std::string& name ) {
 	JsonStringSetter* stringSetter = new JsonStringSetter(*this, name);
diff --git a/flow/JsonString.h b/flow/JsonString.h
index 2a68f27376..4ee57a8d4e 100644
--- a/flow/JsonString.h
+++ b/flow/JsonString.h
@@ -14,9 +14,10 @@ class JsonString {
 	public:
 		JsonString();
 		JsonString( const JsonString& jsonString);
+		JsonString( JsonString&& jsonString) = default;
 		explicit JsonString( const JsonStringArray& jsonArray);
-		explicit JsonString( const char* value );
-		explicit JsonString( const std::string& value );
+		explicit JsonString( const char* value ); // Used to define values (used in an Array)
+		explicit JsonString( const std::string& value ); // Used to define values (used in an Array)
 
 		JsonString( const std::string& name, const char* value );
 		JsonString( const std::string& name, const std::string& value );
@@ -72,6 +73,7 @@ class JsonString {
 
 		std::string	getJson() const;
 		void	setJson(const std::string& jsonText);
+		JsonString&	swapJsonText(std::string& jsonText);
 
 		size_t	getLength() const;
 		size_t	getNameTotal() const;
@@ -122,6 +124,7 @@ class JsonStringArray : protected std::vector<JsonString>
 	public:
 		JsonStringArray();
 		JsonStringArray( const JsonStringArray& jsonStringArray);
+		JsonStringArray( JsonStringArray&& jsonStringArray) = default;
 		virtual ~JsonStringArray();
 };
 

From 21d9695ca4b6f2bdabe34b8c579a5b9ab360efc6 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 5 Sep 2018 14:49:23 -0700
Subject: [PATCH 33/37] updated release notes for 6.0.10

---
 documentation/sphinx/source/release-notes.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index 0259743e23..c2b8a4a9d3 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -2,7 +2,7 @@
 Release Notes
 #############
 
-6.0.9
+6.0.10
 =====
 
 Features
@@ -14,6 +14,7 @@ Features
 * TLS peer verification now supports verifiying on Subject Alternative Name. `(Issue #514) <https://github.com/apple/foundationdb/issues/514>`_
 * TLS peer verification now supports suffix matching by field. `(Issue #515) <https://github.com/apple/foundationdb/issues/515>`_
 * TLS certificates are automatically reloaded after being updated. [6.0.5] `(Issue #505) <https://github.com/apple/foundationdb/issues/505>`_
+* Added the ``fileconfigure`` command to fdbcli, which configures a database from a JSON document. [6.0.10] `(PR #713) <https://github.com/apple/foundationdb/pull/713>`_
 
 Performance
 -----------
@@ -25,6 +26,7 @@ Performance
 * Network connections are now closed after no interface needs the connection. [6.0.1] `(Issue #375) <https://github.com/apple/foundationdb/issues/375>`_
 * Significantly improved the CPU efficiency of copy mutations to transaction logs during recovery. [6.0.2] `(PR #595) <https://github.com/apple/foundationdb/pull/595>`_
 * A cluster configured with usable_regions=2 did not limit the rate at which it could copy data from the primary DC to the remote DC. This caused poor performance when recovering from a DC outage. [6.0.5] `(PR #673) <https://github.com/apple/foundationdb/pull/673>`_
+* Significantly improved the CPU efficiency of generating status on the cluster controller. [6.0.10] `(PR #738) <https://github.com/apple/foundationdb/pull/738>`_
 
 Fixes
 -----
@@ -47,10 +49,12 @@ Fixes
 * Databases with more than 10TB of data would pause for a few seconds after recovery. [6.0.6] `(PR #705) <https://github.com/apple/foundationdb/pull/705>`_
 * Sometimes a minority of coordinators would fail to converge after a new leader was elected. [6.0.6] `(PR #700) <https://github.com/apple/foundationdb/pull/700>`_
 * Calling status too many times in a 5 second interval caused the cluster controller to pause for a few seconds. [6.0.7] `(PR #711) <https://github.com/apple/foundationdb/pull/711>`_
-* Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.8] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_
+* Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.8] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_ [6.0.10] `(PR #739) <https://github.com/apple/foundationdb/pull/739>`_
 * TLS certificate reloading could cause TLS connections to drop until process restart. [6.0.9] `(PR #717) <https://github.com/apple/foundationdb/pull/717>`_
 * Watches polled the server much more frequently than intended. [6.0.10] `(PR #728) <https://github.com/apple/foundationdb/pull/728>`_
 * Backup and DR didn't allow setting certain knobs. [6.0.10] `(Issue #715) <https://github.com/apple/foundationdb/issues/715>`_
+* The failure monitor will become much less reactive after multiple successive failed recoveries. [6.0.10] `(PR #739) <https://github.com/apple/foundationdb/pull/739>`_
+* Data distribution did not limit the number of source servers for a shard. [6.0.10] `(PR #739) <https://github.com/apple/foundationdb/pull/739>`_
 
 Status
 ------

From 2de0b5d6d7a85ff22e54b3fe09036efd80d6ada0 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 5 Sep 2018 15:06:14 -0700
Subject: [PATCH 34/37] Add the roles running on a process as a field on trace
 events in the form of a comma delimited string of role abbreviations.

---
 fdbrpc/simulator.h                    |  8 +--
 fdbserver/ClusterController.actor.cpp | 10 +--
 fdbserver/OldTLogServer.actor.cpp     |  4 +-
 fdbserver/TLogServer.actor.cpp        |  8 +--
 fdbserver/WorkerInterface.h           | 33 +++++++++-
 fdbserver/tester.actor.cpp            |  6 +-
 fdbserver/worker.actor.cpp            | 90 ++++++++++++++++-----------
 flow/Trace.cpp                        | 66 ++++++++++++++++++++
 flow/Trace.h                          |  3 +
 9 files changed, 171 insertions(+), 57 deletions(-)

diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h
index 8335da18bb..8bc66d9eb2 100644
--- a/fdbrpc/simulator.h
+++ b/fdbrpc/simulator.h
@@ -159,7 +159,7 @@ public:
 
 	virtual void addRole(NetworkAddress const& address, std::string const& role) {
 		roleAddresses[address][role] ++;
-		TraceEvent("RoleAdd").detail("Address", address).detail("Role", role).detail("Roles", roleAddresses[address].size()).detail("Value", roleAddresses[address][role]);
+		TraceEvent("RoleAdd").detail("Address", address).detail("Role", role).detail("NumRoles", roleAddresses[address].size()).detail("Value", roleAddresses[address][role]);
 	}
 
 	virtual void removeRole(NetworkAddress const& address, std::string const& role) {
@@ -169,16 +169,16 @@ public:
 			if (rolesIt != addressIt->second.end()) {
 				if (rolesIt->second > 1) {
 					rolesIt->second --;
-					TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", addressIt->second.size()).detail("Value", rolesIt->second).detail("Result", "Decremented Role");
+					TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("NumRoles", addressIt->second.size()).detail("Value", rolesIt->second).detail("Result", "Decremented Role");
 				}
 				else {
 					addressIt->second.erase(rolesIt);
 					if (addressIt->second.size()) {
-						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", addressIt->second.size()).detail("Value", 0).detail("Result", "Removed Role");
+						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("NumRoles", addressIt->second.size()).detail("Value", 0).detail("Result", "Removed Role");
 					}
 					else {
 						roleAddresses.erase(addressIt);
-						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("Roles", 0).detail("Value", 0).detail("Result", "Removed Address");
+						TraceEvent("RoleRemove").detail("Address", address).detail("Role", role).detail("NumRoles", 0).detail("Value", 0).detail("Result", "Removed Address");
 					}
 				}
 			}
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index ca34413b96..969e51e101 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -2150,10 +2150,10 @@ ACTOR Future<Void> clusterControllerCore( ClusterControllerFullInterface interf,
 	loop choose {
 		when( ErrorOr<Void> err = wait( error ) ) {
 			if (err.isError()) {
-				endRole(interf.id(), "ClusterController", "Stop Received Error", false, err.getError());
+				endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Stop Received Error", false, err.getError());
 			}
 			else {
-				endRole(interf.id(), "ClusterController", "Stop Received Signal", true);
+				endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Stop Received Signal", true);
 			}
 
 			// We shut down normally even if there was a serious error (so this fdbserver may be re-elected cluster controller)
@@ -2225,7 +2225,7 @@ ACTOR Future<Void> clusterControllerCore( ClusterControllerFullInterface interf,
 		}
 		when( Void _ = wait( leaderFail ) ) {
 			// We are no longer the leader if this has changed.
-			endRole(interf.id(), "ClusterController", "Leader Replaced", true);
+			endRole(Role::CLUSTER_CONTROLLER, interf.id(), "Leader Replaced", true);
 			TEST(true); // Lost Cluster Controller Role
 			return Void();
 		}
@@ -2252,13 +2252,13 @@ ACTOR Future<Void> clusterController( ServerCoordinators coordinators, Reference
 			}
 
 			hasConnected = true;
-			startRole(cci.id(), UID(), "ClusterController");
+			startRole(Role::CLUSTER_CONTROLLER, cci.id(), UID());
 			inRole = true;
 
 			Void _ = wait( clusterControllerCore( cci, leaderFail, coordinators ) );
 		} catch(Error& e) {
 			if (inRole)
-				endRole(cci.id(), "ClusterController", "Error", e.code() == error_code_actor_cancelled || e.code() == error_code_coordinators_changed, e);
+				endRole(Role::CLUSTER_CONTROLLER, cci.id(), "Error", e.code() == error_code_actor_cancelled || e.code() == error_code_coordinators_changed, e);
 			else
 				TraceEvent( e.code() == error_code_coordinators_changed ? SevInfo : SevError, "ClusterControllerCandidateError", cci.id()).error(e);
 			throw;
diff --git a/fdbserver/OldTLogServer.actor.cpp b/fdbserver/OldTLogServer.actor.cpp
index d2a5d6d1fc..7e31e98d7c 100644
--- a/fdbserver/OldTLogServer.actor.cpp
+++ b/fdbserver/OldTLogServer.actor.cpp
@@ -412,7 +412,7 @@ namespace oldTLog {
 				// These are initialized differently on init() or recovery
 				recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), recovery(Void())
 		{
-			startRole(interf.id(), UID(), "TLog");
+			startRole(Role::TRANSACTION_LOG,interf.id(), UID());
 
 			persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
 			persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
@@ -437,7 +437,7 @@ namespace oldTLog {
 			TraceEvent("TLogBytesWhenRemoved", tli.id()).detail("SharedBytesInput", tLogData->bytesInput).detail("SharedBytesDurable", tLogData->bytesDurable).detail("LocalBytesInput", bytesInput.getValue()).detail("LocalBytesDurable", bytesDurable.getValue());
 
 			ASSERT_ABORT(tLogData->bytesDurable <= tLogData->bytesInput);
-			endRole(tli.id(), "TLog", "Error", true);
+			endRole(Role::TRANSACTION_LOG, tli.id(), "Error", true);
 
 			if(!tLogData->terminated) {
 				Key logIdKey = BinaryWriter::toValue(logId,Unversioned());
diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp
index 36c0175f2d..4b3173fc75 100644
--- a/fdbserver/TLogServer.actor.cpp
+++ b/fdbserver/TLogServer.actor.cpp
@@ -423,7 +423,7 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 			recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), unrecoveredBefore(1), recoveredAt(1), unpoppedRecoveredTags(0),
 			logRouterPopToVersion(0), locality(tagLocalityInvalid)
 	{
-		startRole(interf.id(), UID(), "TLog");
+		startRole(Role::TRANSACTION_LOG, interf.id(), UID());
 
 		persistentDataVersion.init(LiteralStringRef("TLog.PersistentDataVersion"), cc.id);
 		persistentDataDurableVersion.init(LiteralStringRef("TLog.PersistentDataDurableVersion"), cc.id);
@@ -446,7 +446,7 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
 	}
 
 	~LogData() {
-		endRole(logId, "TLog", "Error", true);
+		endRole(Role::TRANSACTION_LOG, logId, "Error", true);
 
 		if(!terminated.isReady()) {
 			tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue();
@@ -2027,7 +2027,7 @@ ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQ
 
 	TraceEvent("SharedTlog", tlogId);
 	// FIXME: Pass the worker id instead of stubbing it
-	startRole(tlogId, UID(), "SharedTLog");
+	startRole(Role::SHARED_TRANSACTION_LOG, tlogId, UID());
 	try {
 		if(restoreFromDisk) {
 			Void _ = wait( restorePersistentState( &self, locality, oldLog, recovered, tlogRequests ) );
@@ -2060,7 +2060,7 @@ ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQ
 	} catch (Error& e) {
 		self.terminated.send(Void());
 		TraceEvent("TLogError", tlogId).error(e, true);
-		endRole(tlogId, "SharedTLog", "Error", true);
+		endRole(Role::SHARED_TRANSACTION_LOG, tlogId, "Error", true);
 		if(recovered.canBeSet()) recovered.send(Void());
 
 		while(!tlogRequests.isEmpty()) {
diff --git a/fdbserver/WorkerInterface.h b/fdbserver/WorkerInterface.h
index fa124ff440..bbea56b638 100644
--- a/fdbserver/WorkerInterface.h
+++ b/fdbserver/WorkerInterface.h
@@ -281,8 +281,37 @@ struct DiskStoreRequest {
 	}
 };
 
-void startRole(UID roleId, UID workerId, std::string as, std::map<std::string, std::string> details = std::map<std::string, std::string>(), std::string origination = "Recruited");
-void endRole(UID id, std::string as, std::string reason, bool ok = true, Error e = Error());
+struct Role {
+	static const Role WORKER;
+	static const Role STORAGE_SERVER;
+	static const Role TRANSACTION_LOG;
+	static const Role SHARED_TRANSACTION_LOG;
+	static const Role MASTER_PROXY;
+	static const Role MASTER;
+	static const Role RESOLVER;
+	static const Role CLUSTER_CONTROLLER;
+	static const Role TESTER;
+	static const Role LOG_ROUTER;
+	static const Role NONE; 
+
+	std::string roleName;
+	std::string abbreviation;
+
+	bool operator==(const Role &r) const {
+		return roleName == r.roleName;
+	}
+	bool operator!=(const Role &r) const {
+		return !(*this == r);
+	}
+
+private:
+	Role(std::string roleName, std::string abbreviation) : roleName(roleName), abbreviation(abbreviation) {
+		ASSERT(abbreviation.size() == 2); // Having a fixed size makes log queries more straightforward
+	}
+};
+
+void startRole(const Role &role, UID roleId, UID workerId, std::map<std::string, std::string> details = std::map<std::string, std::string>(), std::string origination = "Recruited");
+void endRole(const Role &role, UID id, std::string reason, bool ok = true, Error e = Error());
 
 struct ServerDBInfo;
 
diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp
index 0add045023..be6cf740b2 100644
--- a/fdbserver/tester.actor.cpp
+++ b/fdbserver/tester.actor.cpp
@@ -496,7 +496,7 @@ ACTOR Future<Void> testerServerWorkload( WorkloadRequest work, Reference<Cluster
 		details["ClientId"] = format("%d", work.clientId);
 		details["ClientCount"] = format("%d", work.clientCount);
 		details["WorkloadTimeout"] = format("%d", work.timeout);
-		startRole(workIface.id(), UID(), "Tester", details);
+		startRole(Role::TESTER, workIface.id(), UID(), details);
 
 		Standalone<StringRef> database = work.database;
 
@@ -526,7 +526,7 @@ ACTOR Future<Void> testerServerWorkload( WorkloadRequest work, Reference<Cluster
 
 		Void _ = wait(test);
 		
-		endRole(workIface.id(), "Tester", "Complete");
+		endRole(Role::TESTER, workIface.id(), "Complete");
 	} catch (Error& e) {
 		if (!replied) {
 			if (e.code() == error_code_test_specification_invalid)
@@ -536,7 +536,7 @@ ACTOR Future<Void> testerServerWorkload( WorkloadRequest work, Reference<Cluster
 		}
 
 		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_please_reboot_delete || e.code() == error_code_actor_cancelled;
-		endRole(workIface.id(), "Tester", "Error", ok, e);
+		endRole(Role::TESTER, workIface.id(), "Error", ok, e);
 
 		if (e.code() != error_code_test_specification_invalid && e.code() != error_code_timed_out) {
 			throw;  // fatal errors will kill the testerServer as well
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index b02b1f3bd3..cc63fc283b 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -71,10 +71,9 @@ Database openDBOnServer( Reference<AsyncVar<ServerDBInfo>> const& db, int taskID
 
 struct ErrorInfo {
 	Error error;
-	const char* context;
+	const Role &role;
 	UID id;
-	ErrorInfo() {}
-	ErrorInfo( Error e, const char* context, UID id ) : error(e), context(context), id(id) {}
+	ErrorInfo( Error e, const Role &role, UID id ) : error(e), role(role), id(id) {}
 	template <class Ar> void serialize(Ar&) { ASSERT(false); }
 };
 
@@ -97,15 +96,15 @@ Error checkIOTimeout(Error const &e) {
 }
 
 ACTOR Future<Void> forwardError( PromiseStream<ErrorInfo> errors,
-	const char* context, UID id,
+	Role role, UID id,
 	Future<Void> process )
 {
 	try {
 		Void _ = wait(process);
-		errors.send( ErrorInfo(success(), context, id) );
+		errors.send( ErrorInfo(success(), role, id) );
 		return Void();
 	} catch (Error& e) {
-		errors.send( ErrorInfo(e, context, id) );
+		errors.send( ErrorInfo(e, role, id) );
 		return Void();
 	}
 }
@@ -144,7 +143,7 @@ ACTOR Future<Void> workerHandleErrors(FutureStream<ErrorInfo> errors) {
 			if(!ok)
 				err.error = checkIOTimeout(err.error);  // Possibly convert error to io_timeout
 
-			endRole(err.id, err.context, "Error", ok, err.error);
+			endRole(err.role, err.id, "Error", ok, err.error);
 
 			if (err.error.code() == error_code_please_reboot || err.error.code() == error_code_io_timeout) throw err.error;
 		}
@@ -385,9 +384,13 @@ Standalone<StringRef> roleString(std::set<std::pair<std::string, std::string>> r
 	return StringRef(result);
 }
 
-void startRole(UID roleId, UID workerId, std::string as, std::map<std::string, std::string> details, std::string origination) {
+void startRole(const Role &role, UID roleId, UID workerId, std::map<std::string, std::string> details, std::string origination) {
+	if(role != Role::WORKER) {
+		addTraceRole(role.abbreviation);
+	}
+
 	TraceEvent ev("Role", roleId);
-	ev.detail("As", as)
+	ev.detail("As", role.roleName)
 		.detail("Transition", "Begin")
 		.detail("Origination", origination)
 		.detail("OnWorker", workerId);
@@ -397,26 +400,26 @@ void startRole(UID roleId, UID workerId, std::string as, std::map<std::string, s
 	ev.trackLatest( (roleId.shortString() + ".Role" ).c_str() );
 
 	// Update roles map, log Roles metrics
-	g_roles.insert({as, roleId.shortString()});
+	g_roles.insert({role.roleName, roleId.shortString()});
 	StringMetricHandle(LiteralStringRef("Roles")) = roleString(g_roles, false);
 	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
-	if (g_network->isSimulated()) g_simulator.addRole(g_network->getLocalAddress(), as);
+	if (g_network->isSimulated()) g_simulator.addRole(g_network->getLocalAddress(), role.roleName);
 }
 
-void endRole(UID id, std::string as, std::string reason, bool ok, Error e) {
+void endRole(const Role &role, UID id, std::string reason, bool ok, Error e) {
 	{
 		TraceEvent ev("Role", id);
 		if(e.code() != invalid_error_code)
 			ev.error(e, true);
 		ev.detail("Transition", "End")
-			.detail("As", as)
+			.detail("As", role.roleName)
 			.detail("Reason", reason);
 
 		ev.trackLatest( (id.shortString() + ".Role").c_str() );
 	}
 
 	if(!ok) {
-		std::string type = as + "Failed";
+		std::string type = role.roleName + "Failed";
 
 		TraceEvent err(SevError, type.c_str(), id);
 		if(e.code() != invalid_error_code) {
@@ -428,10 +431,14 @@ void endRole(UID id, std::string as, std::string reason, bool ok, Error e) {
 	latestEventCache.clear( id.shortString() );
 
 	// Update roles map, log Roles metrics
-	g_roles.erase({as, id.shortString()});
+	g_roles.erase({role.roleName, id.shortString()});
 	StringMetricHandle(LiteralStringRef("Roles")) = roleString(g_roles, false);
 	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
-	if (g_network->isSimulated()) g_simulator.removeRole(g_network->getLocalAddress(), as);
+	if (g_network->isSimulated()) g_simulator.removeRole(g_network->getLocalAddress(), role.roleName);
+
+	if(role != Role::WORKER) {
+		removeTraceRole(role.abbreviation);
+	}
 }
 
 ACTOR Future<Void> monitorServerDBInfo( Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> ccInterface, Reference<ClusterConnectionFile> connFile, LocalityData locality, Reference<AsyncVar<ServerDBInfo>> dbInfo ) {
@@ -560,7 +567,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 				std::map<std::string, std::string> details;
 				details["StorageEngine"] = s.storeType.toString();
-				startRole( recruited.id(), interf.id(), "StorageServer", details, "Restored" );
+				startRole( Role::STORAGE_SERVER, recruited.id(), interf.id(), details, "Restored" );
 
 				DUMPTOKEN(recruited.getVersion);
 				DUMPTOKEN(recruited.getValue);
@@ -580,7 +587,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				recoveries.push_back(recovery.getFuture());
 				f =  handleIOErrors( f, kv, s.storeID, kvClosed );
 				f = storageServerRollbackRebooter( f, s.storeType, s.filename, recruited.id(), recruited.locality, dbInfo, folder, &filesClosed, memoryLimit );
-				errorForwarders.add( forwardError( errors, "StorageServer", recruited.id(), f ) );
+				errorForwarders.add( forwardError( errors, Role::STORAGE_SERVER, recruited.id(), f ) );
 			} else if( s.storedComponent == DiskStore::TLogData ) {
 				IKeyValueStore* kv = openKVStore( s.storeType, s.filename, s.storeID, memoryLimit, validateDataFiles );
 				IDiskQueue* queue = openDiskQueue(
@@ -590,7 +597,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 				std::map<std::string, std::string> details;
 				details["StorageEngine"] = s.storeType.toString();
-				startRole( s.storeID, interf.id(), "SharedTLog", details, "Restored" );
+				startRole( Role::SHARED_TRANSACTION_LOG, s.storeID, interf.id(), details, "Restored" );
 
 				Promise<Void> oldLog;
 				Promise<Void> recovery;
@@ -602,7 +609,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				if(tlog.isReady()) {
 					tlog = oldLog.getFuture() || tl;
 				}
-				errorForwarders.add( forwardError( errors, "SharedTLog", s.storeID, tl ) );
+				errorForwarders.add( forwardError( errors, Role::SHARED_TRANSACTION_LOG, s.storeID, tl ) );
 			}
 		}
 
@@ -610,7 +617,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 		details["Locality"] = locality.toString();
 		details["DataFolder"] = folder;
 		details["StoresPresent"] = format("%d", stores.size());
-		startRole( interf.id(), interf.id(), "Worker", details );
+		startRole( Role::WORKER, interf.id(), interf.id(), details );
 
 		Void _ = wait(waitForAll(recoveries));
 		recoveredDiskFiles.send(Void());
@@ -667,7 +674,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				recruited.locality = locality;
 				recruited.initEndpoints();
 
-				startRole( recruited.id(), interf.id(), "MasterServer" );
+				startRole( Role::MASTER, recruited.id(), interf.id() );
 
 				DUMPTOKEN( recruited.waitFailure );
 				DUMPTOKEN( recruited.getRateInfo );
@@ -677,7 +684,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 				//printf("Recruited as masterServer\n");
 				Future<Void> masterProcess = masterServer( recruited, dbInfo, ServerCoordinators( connFile ), req.lifetime, req.forceRecovery );
-				errorForwarders.add( zombie(recruited, forwardError( errors, "MasterServer", recruited.id(), masterProcess )) );
+				errorForwarders.add( zombie(recruited, forwardError( errors, Role::MASTER, recruited.id(), masterProcess )) );
 				req.reply.send(recruited);
 			}
 			when( InitializeTLogRequest req = waitNext(interf.tLog.getFuture()) ) {
@@ -689,7 +696,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 					details["StorageEngine"] = req.storeType.toString();
 
 					//FIXME: start role for every tlog instance, rather that just for the shared actor, also use a different role type for the shared actor
-					startRole( logId, interf.id(), "SharedTLog", details );
+					startRole( Role::SHARED_TRANSACTION_LOG, logId, interf.id(), details );
 
 					std::string filename = filenameFromId( req.storeType, folder, fileLogDataPrefix.toString(), logId );
 					IKeyValueStore* data = openKVStore( req.storeType, filename, logId, memoryLimit );
@@ -700,7 +707,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 					tlog = tLog( data, queue, dbInfo, locality, tlogRequests, logId, false, Promise<Void>(), Promise<Void>() );
 					tlog = handleIOErrors( tlog, data, logId );
 					tlog = handleIOErrors( tlog, queue, logId );
-					errorForwarders.add( forwardError( errors, "SharedTLog", logId, tlog ) );
+					errorForwarders.add( forwardError( errors, Role::SHARED_TRANSACTION_LOG, logId, tlog ) );
 				}
 			}
 			when( InitializeStorageRequest req = waitNext(interf.storage.getFuture()) ) {
@@ -711,7 +718,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 					std::map<std::string, std::string> details;
 					details["StorageEngine"] = req.storeType.toString();
-					startRole( recruited.id(), interf.id(), "StorageServer", details );
+					startRole( Role::STORAGE_SERVER, recruited.id(), interf.id(), details );
 
 					DUMPTOKEN(recruited.getVersion);
 					DUMPTOKEN(recruited.getValue);
@@ -737,7 +744,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 					s = handleIOErrors(s, data, recruited.id(), kvClosed);
 					s = storageCache.removeOnReady( req.reqId, s );
 					s = storageServerRollbackRebooter( s, req.storeType, filename, recruited.id(), recruited.locality, dbInfo, folder, &filesClosed, memoryLimit );
-					errorForwarders.add( forwardError( errors, "StorageServer", recruited.id(), s ) );
+					errorForwarders.add( forwardError( errors, Role::STORAGE_SERVER, recruited.id(), s ) );
 				} else
 					forwardPromise( req.reply, storageCache.get( req.reqId ) );
 			}
@@ -748,7 +755,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 
 				std::map<std::string, std::string> details;
 				details["ForMaster"] = req.master.id().shortString();
-				startRole( recruited.id(), interf.id(), "MasterProxyServer", details );
+				startRole( Role::MASTER_PROXY, recruited.id(), interf.id(), details );
 
 				DUMPTOKEN(recruited.commit);
 				DUMPTOKEN(recruited.getConsistentReadVersion);
@@ -759,7 +766,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				DUMPTOKEN(recruited.txnState);
 
 				//printf("Recruited as masterProxyServer\n");
-				errorForwarders.add( zombie(recruited, forwardError( errors, "MasterProxyServer", recruited.id(),
+				errorForwarders.add( zombie(recruited, forwardError( errors, Role::MASTER_PROXY, recruited.id(),
 						masterProxyServer( recruited, req, dbInfo ) ) ) );
 				req.reply.send(recruited);
 			}
@@ -769,14 +776,14 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				recruited.initEndpoints();
 
 				std::map<std::string, std::string> details;
-				startRole( recruited.id(), interf.id(), "Resolver", details );
+				startRole( Role::RESOLVER, recruited.id(), interf.id(), details );
 
 				DUMPTOKEN(recruited.resolve);
 				DUMPTOKEN(recruited.metrics);
 				DUMPTOKEN(recruited.split);
 				DUMPTOKEN(recruited.waitFailure);
 
-				errorForwarders.add( zombie(recruited, forwardError( errors, "Resolver", recruited.id(),
+				errorForwarders.add( zombie(recruited, forwardError( errors, Role::RESOLVER, recruited.id(),
 						resolver( recruited, req, dbInfo ) ) ) );
 				req.reply.send(recruited);
 			}
@@ -785,7 +792,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				recruited.initEndpoints();
 
 				std::map<std::string, std::string> details;
-				startRole( recruited.id(), interf.id(), "LogRouter", details );
+				startRole( Role::LOG_ROUTER, recruited.id(), interf.id(), details );
 
 				DUMPTOKEN( recruited.peekMessages );
 				DUMPTOKEN( recruited.popMessages );
@@ -794,13 +801,11 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 				DUMPTOKEN( recruited.getQueuingMetrics );
 				DUMPTOKEN( recruited.confirmRunning );
 
-				errorForwarders.add( zombie(recruited, forwardError( errors, "logRouter", recruited.id(), 
+				errorForwarders.add( zombie(recruited, forwardError( errors, Role::LOG_ROUTER, recruited.id(), 
 						logRouter( recruited, req, dbInfo ) ) ) );
 				req.reply.send(recruited);
 			}
-			when( DebugQueryRequest req = waitNext(interf.debugQuery.getFuture()) ) {
-				errorForwarders.add( forwardError( errors, "DebugQuery", UID(), debugQueryServer(req) ) );
-			}
+			when( DebugQueryRequest req = waitNext(interf.debugQuery.getFuture()) ) { }
 			when( CoordinationPingMessage m = waitNext( interf.coordinationPing.getFuture() ) ) {
 				TraceEvent("CoordinationPing", interf.id()).detail("CCID", m.clusterControllerId).detail("TimeStep", m.timeStep);
 			}
@@ -860,7 +865,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
 		state Error e = err;
 		bool ok = e.code() == error_code_please_reboot || e.code() == error_code_actor_cancelled || e.code() == error_code_please_reboot_delete;
 
-		endRole(interf.id(), "Worker", "WorkerError", ok, e);
+		endRole(Role::WORKER, interf.id(), "WorkerError", ok, e);
 		errorForwarders.clear(false);
 		tlog = Void();
 
@@ -1040,3 +1045,14 @@ ACTOR Future<Void> fdbd(
 		throw err;
 	}
 }
+
+const Role Role::WORKER("Worker", "WK");
+const Role Role::STORAGE_SERVER("StorageServer", "SS");
+const Role Role::TRANSACTION_LOG("TLog", "TL");
+const Role Role::SHARED_TRANSACTION_LOG("SharedTLog", "SL");
+const Role Role::MASTER_PROXY("MasterProxyServer", "MP");
+const Role Role::MASTER("MasterServer", "MS");
+const Role Role::RESOLVER("Resolver", "RV");
+const Role Role::CLUSTER_CONTROLLER("ClusterController", "CC");
+const Role Role::TESTER("Tester", "TS");
+const Role Role::LOG_ROUTER("LogRouter", "LR");
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index f676aa3be8..68e9f5b05c 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -148,6 +148,34 @@ private:
 	EventMetricHandle<TraceEventNameID> SevInfoNames;
 	EventMetricHandle<TraceEventNameID> SevDebugNames;
 
+	struct RoleInfo {
+		std::map<std::string, int> roles;
+		std::string rolesString;
+
+		void refreshRolesString() {
+			rolesString = "";
+			for(auto itr : roles) {
+				if(!rolesString.empty()) {
+					rolesString += ",";
+				}
+				rolesString += itr.first;
+			}
+		}
+	};
+
+	RoleInfo roleInfo;
+	std::map<NetworkAddress, RoleInfo> roleInfoMap;
+
+	RoleInfo& mutateRoleInfo() {
+		ASSERT(g_network);
+
+		if(g_network->isSimulated()) {
+			return roleInfoMap[g_network->getLocalAddress()];
+		}
+		
+		return roleInfo;
+	}
+
 public:
 	bool logTraceEventMetrics;
 
@@ -307,6 +335,11 @@ public:
 		}
 
 		fields.addField("LogGroup", logGroup);
+
+		RoleInfo const& r = mutateRoleInfo();
+		if(r.rolesString.size() > 0) {
+			fields.addField("Roles", r.rolesString);
+		}
 	}
 
 	void writeEvent( TraceEventFields fields, std::string trackLatestKey, bool trackError ) {
@@ -436,6 +469,31 @@ public:
 		}
 	}
 
+	void addRole(std::string role) {
+		MutexHolder holder(mutex);
+
+		RoleInfo &r = mutateRoleInfo();
+		++r.roles[role];
+		r.refreshRolesString();
+	}
+
+	void removeRole(std::string role) {
+		MutexHolder holder(mutex);
+
+		RoleInfo &r = mutateRoleInfo();
+
+		auto itr = r.roles.find(role);
+		ASSERT(itr != r.roles.end() || g_network->isSimulated() && g_network->getLocalAddress() == NetworkAddress());
+
+		if(itr != r.roles.end()) {
+			if(--(*itr).second == 0) {
+				r.roles.erase(itr);
+			}
+
+			r.refreshRolesString();
+		}
+	}
+
 	~TraceLog() {
 		close();
 		if (writer) writer->addref(); // FIXME: We are not shutting down the writer thread at all, because the ThreadPool shutdown mechanism is blocking (necessarily waits for current work items to finish) and we might not be able to finish everything.
@@ -550,6 +608,14 @@ bool traceFileIsOpen() {
 	return g_traceLog.isOpen();
 }
 
+void addTraceRole(std::string role) {
+	g_traceLog.addRole(role);
+}
+
+void removeTraceRole(std::string role) {
+	g_traceLog.removeRole(role);
+}
+
 TraceEvent::TraceEvent( const char* type, UID id ) : id(id), type(type), severity(SevInfo), initialized(false), enabled(true) {}
 TraceEvent::TraceEvent( Severity severity, const char* type, UID id ) : id(id), type(type), severity(severity), initialized(false), enabled(true) {}
 TraceEvent::TraceEvent( TraceInterval& interval, UID id ) : id(id), type(interval.type), severity(interval.severity), initialized(false), enabled(true) {
diff --git a/flow/Trace.h b/flow/Trace.h
index 6b0e988c27..638e61e34b 100644
--- a/flow/Trace.h
+++ b/flow/Trace.h
@@ -262,6 +262,9 @@ void initTraceEventMetrics();
 void closeTraceFile();
 bool traceFileIsOpen();
 
+void addTraceRole(std::string role);
+void removeTraceRole(std::string role);
+
 enum trace_clock_t { TRACE_CLOCK_NOW, TRACE_CLOCK_REALTIME };
 extern trace_clock_t g_trace_clock;
 extern TraceBatch g_traceBatch;

From f87443a6c0f21d027237acb322c1b8bb9f799b52 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 5 Sep 2018 15:07:10 -0700
Subject: [PATCH 35/37] minor release note change

---
 documentation/sphinx/source/release-notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst
index c2b8a4a9d3..d70497e24d 100644
--- a/documentation/sphinx/source/release-notes.rst
+++ b/documentation/sphinx/source/release-notes.rst
@@ -49,8 +49,8 @@ Fixes
 * Databases with more than 10TB of data would pause for a few seconds after recovery. [6.0.6] `(PR #705) <https://github.com/apple/foundationdb/pull/705>`_
 * Sometimes a minority of coordinators would fail to converge after a new leader was elected. [6.0.6] `(PR #700) <https://github.com/apple/foundationdb/pull/700>`_
 * Calling status too many times in a 5 second interval caused the cluster controller to pause for a few seconds. [6.0.7] `(PR #711) <https://github.com/apple/foundationdb/pull/711>`_
-* Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.8] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_ [6.0.10] `(PR #739) <https://github.com/apple/foundationdb/pull/739>`_
 * TLS certificate reloading could cause TLS connections to drop until process restart. [6.0.9] `(PR #717) <https://github.com/apple/foundationdb/pull/717>`_
+* Configuring from usable_regions=2 to usable_regions=1 on a cluster with a large number of processes would prevent data distribution from completing. [6.0.10] `(PR #721) <https://github.com/apple/foundationdb/pull/721>`_ `(PR #739) <https://github.com/apple/foundationdb/pull/739>`_
 * Watches polled the server much more frequently than intended. [6.0.10] `(PR #728) <https://github.com/apple/foundationdb/pull/728>`_
 * Backup and DR didn't allow setting certain knobs. [6.0.10] `(Issue #715) <https://github.com/apple/foundationdb/issues/715>`_
 * The failure monitor will become much less reactive after multiple successive failed recoveries. [6.0.10] `(PR #739) <https://github.com/apple/foundationdb/pull/739>`_

From 9e79f9ec59f9c2fe3479c68c3fce3685fffb3853 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 5 Sep 2018 15:53:12 -0700
Subject: [PATCH 36/37] Cleanup from review

---
 fdbserver/WorkerInterface.h | 4 ++--
 fdbserver/worker.actor.cpp  | 8 ++++----
 flow/Trace.cpp              | 9 +++------
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/fdbserver/WorkerInterface.h b/fdbserver/WorkerInterface.h
index bbea56b638..26fedb0e3b 100644
--- a/fdbserver/WorkerInterface.h
+++ b/fdbserver/WorkerInterface.h
@@ -292,10 +292,10 @@ struct Role {
 	static const Role CLUSTER_CONTROLLER;
 	static const Role TESTER;
 	static const Role LOG_ROUTER;
-	static const Role NONE; 
 
 	std::string roleName;
 	std::string abbreviation;
+	bool includeInTraceRoles;
 
 	bool operator==(const Role &r) const {
 		return roleName == r.roleName;
@@ -305,7 +305,7 @@ struct Role {
 	}
 
 private:
-	Role(std::string roleName, std::string abbreviation) : roleName(roleName), abbreviation(abbreviation) {
+	Role(std::string roleName, std::string abbreviation, bool includeInTraceRoles=true) : roleName(roleName), abbreviation(abbreviation), includeInTraceRoles(includeInTraceRoles) {
 		ASSERT(abbreviation.size() == 2); // Having a fixed size makes log queries more straightforward
 	}
 };
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index cc63fc283b..cac8138129 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -385,7 +385,7 @@ Standalone<StringRef> roleString(std::set<std::pair<std::string, std::string>> r
 }
 
 void startRole(const Role &role, UID roleId, UID workerId, std::map<std::string, std::string> details, std::string origination) {
-	if(role != Role::WORKER) {
+	if(role.includeInTraceRoles) {
 		addTraceRole(role.abbreviation);
 	}
 
@@ -436,7 +436,7 @@ void endRole(const Role &role, UID id, std::string reason, bool ok, Error e) {
 	StringMetricHandle(LiteralStringRef("RolesWithIDs")) = roleString(g_roles, true);
 	if (g_network->isSimulated()) g_simulator.removeRole(g_network->getLocalAddress(), role.roleName);
 
-	if(role != Role::WORKER) {
+	if(role.includeInTraceRoles) {
 		removeTraceRole(role.abbreviation);
 	}
 }
@@ -1046,10 +1046,10 @@ ACTOR Future<Void> fdbd(
 	}
 }
 
-const Role Role::WORKER("Worker", "WK");
+const Role Role::WORKER("Worker", "WK", false);
 const Role Role::STORAGE_SERVER("StorageServer", "SS");
 const Role Role::TRANSACTION_LOG("TLog", "TL");
-const Role Role::SHARED_TRANSACTION_LOG("SharedTLog", "SL");
+const Role Role::SHARED_TRANSACTION_LOG("SharedTLog", "SL", false);
 const Role Role::MASTER_PROXY("MasterProxyServer", "MP");
 const Role Role::MASTER("MasterServer", "MS");
 const Role Role::RESOLVER("Resolver", "RV");
diff --git a/flow/Trace.cpp b/flow/Trace.cpp
index 68e9f5b05c..e409ff63ea 100644
--- a/flow/Trace.cpp
+++ b/flow/Trace.cpp
@@ -483,13 +483,10 @@ public:
 		RoleInfo &r = mutateRoleInfo();
 
 		auto itr = r.roles.find(role);
-		ASSERT(itr != r.roles.end() || g_network->isSimulated() && g_network->getLocalAddress() == NetworkAddress());
-
-		if(itr != r.roles.end()) {
-			if(--(*itr).second == 0) {
-				r.roles.erase(itr);
-			}
+		ASSERT(itr != r.roles.end() || (g_network->isSimulated() && g_network->getLocalAddress() == NetworkAddress()));
 
+		if(itr != r.roles.end() && --(*itr).second == 0) {
+			r.roles.erase(itr);
 			r.refreshRolesString();
 		}
 	}

From 29c2783d21a57a76f4bc9338c5b0fd8edb1bd2e3 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Wed, 5 Sep 2018 16:11:24 -0700
Subject: [PATCH 37/37] fixed merge issue

---
 fdbserver/worker.actor.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 1aaad1d41f..ad977cbf5c 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -100,13 +100,8 @@ ACTOR Future<Void> forwardError( PromiseStream<ErrorInfo> errors,
 	Future<Void> process )
 {
 	try {
-<<<<<<< HEAD
 		wait(process);
-		errors.send( ErrorInfo(success(), context, id) );
-=======
-		Void _ = wait(process);
 		errors.send( ErrorInfo(success(), role, id) );
->>>>>>> release-6.0
 		return Void();
 	} catch (Error& e) {
 		errors.send( ErrorInfo(e, role, id) );