Merge branch 'master' of https://github.com/apple/foundationdb into feature-streaming-reply-master

# Conflicts:
#	fdbclient/CMakeLists.txt
#	fdbclient/Knobs.h
#	fdbserver/Knobs.h

documentation/sphinx/source/release-notes/release-notes-700.rst

@@ -24,13 +24,15 @@ Performance
 
 Reliability
 -----------
-
-
+* Improved worker recruitment logic to avoid unnecessary recoveries when processes are added or removed from a cluster. `(PR #4695) <https://github.com/apple/foundationdb/pull/4695>`_ `(PR #4631) <https://github.com/apple/foundationdb/pull/4631>`_ `(PR #4509) <https://github.com/apple/foundationdb/pull/4509>`_
+* Log class processes are prioritized above transaction class proceses for becoming tlogs. `(PR #4509) <https://github.com/apple/foundationdb/pull/4509>`_ 
 
 Fixes
 -----
-
-
+* Fixed a rare crash on the cluster controller when using multi-region configurations. `(PR #4547) <https://github.com/apple/foundationdb/pull/4547>`_ 
+* Using the ``exclude failed`` command could leave the data distributor in a state where it cannot complete relocations. `(PR #4495) <https://github.com/apple/foundationdb/pull/4495>`_ 
+* Fixed a rare crash that could happen on the sequencer during recovery. `(PR #4548) <https://github.com/apple/foundationdb/pull/4548>`_ 
+* When configured with ``usable_regions=2``, a cluster would not fail over to a region which contained only storage class processes. `(PR #4599) <https://github.com/apple/foundationdb/pull/4599>`_ 
 
 Status
 ------

fdbbackup/backup.actor.cpp

@@ -38,6 +38,7 @@
 #include "fdbclient/Status.h"
 #include "fdbclient/BackupContainer.h"
 #include "fdbclient/KeyBackedTypes.h"
+#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/RunTransaction.actor.h"
 #include "fdbclient/S3BlobStore.h"
 #include "fdbclient/json_spirit/json_spirit_writer_template.h"
@@ -3703,27 +3704,26 @@ int main(int argc, char* argv[]) {
 			}
 		}
 
-		for (auto k = knobs.begin(); k != knobs.end(); ++k) {
+		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO);
+		auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
+		for (const auto& [knobName, knobValueString] : knobs) {
 			try {
-				if (!globalFlowKnobs->setKnob(k->first, k->second) &&
-				    !globalClientKnobs->setKnob(k->first, k->second)) {
-					fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
-					TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
-				}
+				auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
+				g_knobs.setKnob(knobName, knobValue);
 			} catch (Error& e) {
 				if (e.code() == error_code_invalid_option_value) {
 					fprintf(stderr,
 					        "WARNING: Invalid value '%s' for knob option '%s'\n",
-					        k->second.c_str(),
-					        k->first.c_str());
+					        knobValueString.c_str(),
+					        knobName.c_str());
 					TraceEvent(SevWarnAlways, "InvalidKnobValue")
-					    .detail("Knob", printable(k->first))
-					    .detail("Value", printable(k->second));
+					    .detail("Knob", printable(knobName))
+					    .detail("Value", printable(knobValueString));
 				} else {
-					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what());
+					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
 					TraceEvent(SevError, "FailedToSetKnob")
-					    .detail("Knob", printable(k->first))
-					    .detail("Value", printable(k->second))
+					    .detail("Knob", printable(knobName))
+					    .detail("Value", printable(knobValueString))
 					    .error(e);
 					throw;
 				}
@@ -3731,8 +3731,7 @@ int main(int argc, char* argv[]) {
 		}
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		globalFlowKnobs->initialize(true);
-		globalClientKnobs->initialize(true);
+		g_knobs.initialize(Randomize::NO, IsSimulated::NO);
 
 		if (trace) {
 			if (!traceLogGroup.empty())

fdbcli/fdbcli.actor.cpp

@@ -28,6 +28,7 @@
 #include "fdbclient/StatusClient.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/GlobalConfig.actor.h"
+#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/ClusterInterface.h"
@@ -3060,23 +3061,25 @@ struct CLIOptions {
 			return;
 		}
 
-		for (const auto& [knob, value] : knobs) {
+		auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
+		for (const auto& [knobName, knobValueString] : knobs) {
 			try {
-				if (!globalFlowKnobs->setKnob(knob, value) && !globalClientKnobs->setKnob(knob, value)) {
-					fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", knob.c_str());
-					TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(knob));
-				}
+				auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
+				g_knobs.setKnob(knobName, knobValue);
 			} catch (Error& e) {
 				if (e.code() == error_code_invalid_option_value) {
-					fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", value.c_str(), knob.c_str());
+					fprintf(stderr,
+					        "WARNING: Invalid value '%s' for knob option '%s'\n",
+					        knobValueString.c_str(),
+					        knobName.c_str());
 					TraceEvent(SevWarnAlways, "InvalidKnobValue")
-					    .detail("Knob", printable(knob))
-					    .detail("Value", printable(value));
+					    .detail("Knob", printable(knobName))
+					    .detail("Value", printable(knobValueString));
 				} else {
-					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knob.c_str(), e.what());
+					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
 					TraceEvent(SevError, "FailedToSetKnob")
-					    .detail("Knob", printable(knob))
-					    .detail("Value", printable(value))
+					    .detail("Knob", printable(knobName))
+					    .detail("Value", printable(knobValueString))
 					    .error(e);
 					exit_code = FDB_EXIT_ERROR;
 				}
@@ -3084,8 +3087,7 @@ struct CLIOptions {
 		}
 
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		globalFlowKnobs->initialize(true);
-		globalClientKnobs->initialize(true);
+		g_knobs.initialize(Randomize::NO, IsSimulated::NO);
 	}
 
 	int processArg(CSimpleOpt& args) {
@@ -4857,6 +4859,8 @@ int main(int argc, char** argv) {
 
 	registerCrashHandler();
 
+	IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO);
+
 #ifdef __unixish__
 	struct sigaction act;
 

fdbclient/CMakeLists.txt

@@ -15,11 +15,19 @@ set(FDBCLIENT_SRCS
   BackupContainerLocalDirectory.h
   BackupContainerS3BlobStore.actor.cpp
   BackupContainerS3BlobStore.h
+  ClientKnobCollection.cpp
+  ClientKnobCollection.h
+  ClientKnobs.cpp
+  ClientKnobs.h
   ClientLogEvents.h
   ClientWorkerInterface.h
   ClusterInterface.h
   CommitProxyInterface.h
   CommitTransaction.h
+  ConfigKnobs.cpp
+  ConfigKnobs.h
+  ConfigTransactionInterface.cpp
+  ConfigTransactionInterface.h
   CoordinationInterface.h
   DatabaseBackupAgent.actor.cpp
   DatabaseConfiguration.cpp
@@ -27,6 +35,7 @@ set(FDBCLIENT_SRCS
   DatabaseContext.h
   EventTypes.actor.h
   FDBOptions.h
+  FDBTypes.cpp
   FDBTypes.h
   FileBackupAgent.actor.cpp
   GlobalConfig.h
@@ -35,13 +44,18 @@ set(FDBCLIENT_SRCS
   GrvProxyInterface.h
   HTTP.actor.cpp
   IClientApi.h
+  IConfigTransaction.cpp
+  IConfigTransaction.h
+  ISingleThreadTransaction.cpp
+  ISingleThreadTransaction.h
   JsonBuilder.cpp
   JsonBuilder.h
   KeyBackedTypes.h
   KeyRangeMap.actor.cpp
   KeyRangeMap.h
-  Knobs.cpp
   Knobs.h
+  IKnobCollection.cpp
+  IKnobCollection.h
   ManagementAPI.actor.cpp
   ManagementAPI.actor.h
   MonitorLeader.actor.cpp
@@ -55,6 +69,11 @@ set(FDBCLIENT_SRCS
   Notified.h
   ParallelStream.actor.cpp
   ParallelStream.actor.h
+  PaxosConfigTransaction.actor.cpp
+  PaxosConfigTransaction.h
+  SimpleConfigTransaction.actor.cpp
+  SpecialKeySpace.actor.cpp
+  SpecialKeySpace.actor.h
   ReadYourWrites.actor.cpp
   ReadYourWrites.h
   RestoreInterface.cpp
@@ -65,6 +84,11 @@ set(FDBCLIENT_SRCS
   S3BlobStore.actor.cpp
   Schemas.cpp
   Schemas.h
+  ServerKnobCollection.cpp
+  ServerKnobCollection.h
+  ServerKnobs.cpp
+  ServerKnobs.h
+  SimpleConfigTransaction.h
   SnapshotCache.h
   SpecialKeySpace.actor.cpp
   SpecialKeySpace.actor.h
@@ -81,6 +105,8 @@ set(FDBCLIENT_SRCS
   TagThrottle.h
   TaskBucket.actor.cpp
   TaskBucket.h
+  TestKnobCollection.cpp
+  TestKnobCollection.h
   ThreadSafeTransaction.cpp
   ThreadSafeTransaction.h
   Tuple.cpp

fdbclient/ClientKnobCollection.cpp

@@ -0,0 +1,51 @@
+/*
+ * ClientKnobCollection.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ClientKnobCollection.h"
+
+ClientKnobCollection::ClientKnobCollection(Randomize randomize, IsSimulated isSimulated)
+  : flowKnobs(randomize, isSimulated), clientKnobs(randomize) {}
+
+void ClientKnobCollection::initialize(Randomize randomize, IsSimulated isSimulated) {
+	flowKnobs.initialize(randomize, isSimulated);
+	clientKnobs.initialize(randomize);
+}
+
+void ClientKnobCollection::reset(Randomize randomize, IsSimulated isSimulated) {
+	flowKnobs.reset(randomize, isSimulated);
+	clientKnobs.reset(randomize);
+}
+
+Optional<KnobValue> ClientKnobCollection::tryParseKnobValue(std::string const& knobName,
+                                                            std::string const& knobValue) const {
+	auto parsedKnobValue = flowKnobs.parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+	parsedKnobValue = clientKnobs.parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+	return {};
+}
+
+bool ClientKnobCollection::trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	return knobValue.visitSetKnob(knobName, flowKnobs) || knobValue.visitSetKnob(knobName, clientKnobs);
+}

fdbclient/ClientKnobCollection.h

@@ -0,0 +1,46 @@
+/*
+ * ClientKnobCollection.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/ClientKnobs.h"
+#include "fdbclient/IKnobCollection.h"
+#include "flow/Knobs.h"
+
+/*
+ * Stores both flow knobs and client knobs, attempting to access server knobs or test knobs
+ * results in a run-time error
+ */
+class ClientKnobCollection : public IKnobCollection {
+	FlowKnobs flowKnobs;
+	ClientKnobs clientKnobs;
+
+public:
+	ClientKnobCollection(Randomize randomize, IsSimulated isSimulated);
+	void initialize(Randomize randomize, IsSimulated isSimulated) override;
+	void reset(Randomize randomize, IsSimulated isSimulated) override;
+	FlowKnobs const& getFlowKnobs() const override { return flowKnobs; }
+	ClientKnobs const& getClientKnobs() const override { return clientKnobs; }
+	ClientKnobs& getMutableClientKnobs() { return clientKnobs; }
+	ServerKnobs const& getServerKnobs() const override { throw internal_error(); }
+	TestKnobs const& getTestKnobs() const override { throw internal_error(); }
+	Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const override;
+	bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
+};

fdbclient/ClientKnobs.cpp

@@ -1,5 +1,5 @@
 /*
- * Knobs.cpp
+ * ClientKnobs.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -23,16 +23,14 @@
 #include "fdbclient/SystemData.h"
 #include "flow/UnitTest.h"
 
-std::unique_ptr<ClientKnobs> globalClientKnobs = std::make_unique<ClientKnobs>();
-ClientKnobs const* CLIENT_KNOBS = globalClientKnobs.get();
-
 #define init(knob, value) initKnob(knob, value, #knob)
 
-ClientKnobs::ClientKnobs() {
-	initialize();
+ClientKnobs::ClientKnobs(Randomize randomize) {
+	initialize(randomize);
 }
 
-void ClientKnobs::initialize(bool randomize) {
+void ClientKnobs::initialize(Randomize _randomize) {
+	bool const randomize = (_randomize == Randomize::YES);
 	// clang-format off
 
 	init( TOO_MANY,                            1000000 );
@@ -114,7 +112,7 @@ void ClientKnobs::initialize(bool randomize) {
 
 	// Core
 	init( CORE_VERSIONSPERSECOND,		           1e6 );
-	init( LOG_RANGE_BLOCK_SIZE,                    1e6 ); //Dependent on CORE_VERSIONSPERSECOND
+	init( LOG_RANGE_BLOCK_SIZE, CORE_VERSIONSPERSECOND );
 	init( MUTATION_BLOCK_SIZE,	            	  10000);
 
 	// TaskBucket
@@ -255,14 +253,14 @@ void ClientKnobs::initialize(bool randomize) {
 
 TEST_CASE("/fdbclient/knobs/initialize") {
 	// This test depends on TASKBUCKET_TIMEOUT_VERSIONS being defined as a constant multiple of CORE_VERSIONSPERSECOND
-	ClientKnobs clientKnobs;
-	int initialCoreVersionsPerSecond = clientKnobs.CORE_VERSIONSPERSECOND;
+	ClientKnobs clientKnobs(Randomize::NO);
+	int64_t initialCoreVersionsPerSecond = clientKnobs.CORE_VERSIONSPERSECOND;
 	int initialTaskBucketTimeoutVersions = clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS;
-	clientKnobs.setKnob("core_versionspersecond", format("%ld", initialCoreVersionsPerSecond * 2));
-	ASSERT(clientKnobs.CORE_VERSIONSPERSECOND == initialCoreVersionsPerSecond * 2);
-	ASSERT(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS == initialTaskBucketTimeoutVersions);
-	clientKnobs.initialize();
-	ASSERT(clientKnobs.CORE_VERSIONSPERSECOND == initialCoreVersionsPerSecond * 2);
-	ASSERT(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS == initialTaskBucketTimeoutVersions * 2);
+	clientKnobs.setKnob("core_versionspersecond", initialCoreVersionsPerSecond * 2);
+	ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2);
+	ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions);
+	clientKnobs.initialize(Randomize::NO);
+	ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2);
+	ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions * 2);
 	return Void();
 }

fdbclient/ClientKnobs.h

@@ -0,0 +1,241 @@
+/*
+ * ClientKnobs.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBCLIENT_KNOBS_H
+#define FDBCLIENT_KNOBS_H
+#pragma once
+
+#include "flow/Knobs.h"
+#include "flow/flow.h"
+
+class ClientKnobs : public KnobsImpl<ClientKnobs> {
+public:
+	int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
+
+	double SYSTEM_MONITOR_INTERVAL;
+	double NETWORK_BUSYNESS_MONITOR_INTERVAL; // The interval in which we should update the network busyness metric
+
+	double FAILURE_MAX_DELAY;
+	double FAILURE_MIN_DELAY;
+	double FAILURE_TIMEOUT_DELAY;
+	double CLIENT_FAILURE_TIMEOUT_DELAY;
+	double FAILURE_EMERGENCY_DELAY;
+	double FAILURE_MAX_GENERATIONS;
+	double RECOVERY_DELAY_START_GENERATION;
+	double RECOVERY_DELAY_SECONDS_PER_GENERATION;
+	double MAX_GENERATIONS;
+	double MAX_GENERATIONS_OVERRIDE;
+	double MAX_GENERATIONS_SIM;
+
+	double COORDINATOR_RECONNECTION_DELAY;
+	int CLIENT_EXAMPLE_AMOUNT;
+	double MAX_CLIENT_STATUS_AGE;
+	int MAX_COMMIT_PROXY_CONNECTIONS;
+	int MAX_GRV_PROXY_CONNECTIONS;
+	double STATUS_IDLE_TIMEOUT;
+
+	// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
+	double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
+	                                 // mostly wrong (e.g. dumping the database after a test)
+	double FUTURE_VERSION_RETRY_DELAY;
+	int REPLY_BYTE_LIMIT;
+	double DEFAULT_BACKOFF;
+	double DEFAULT_MAX_BACKOFF;
+	double BACKOFF_GROWTH_RATE;
+	double RESOURCE_CONSTRAINED_MAX_BACKOFF;
+	int PROXY_COMMIT_OVERHEAD_BYTES;
+	double SHARD_STAT_SMOOTH_AMOUNT;
+	int INIT_MID_SHARD_BYTES;
+
+	int TRANSACTION_SIZE_LIMIT;
+	int64_t KEY_SIZE_LIMIT;
+	int64_t SYSTEM_KEY_SIZE_LIMIT;
+	int64_t VALUE_SIZE_LIMIT;
+	int64_t SPLIT_KEY_SIZE_LIMIT;
+	int METADATA_VERSION_CACHE_SIZE;
+
+	int MAX_BATCH_SIZE;
+	double GRV_BATCH_TIMEOUT;
+	int BROADCAST_BATCH_SIZE;
+	double TRANSACTION_TIMEOUT_DELAY_INTERVAL;
+
+	// When locationCache in DatabaseContext gets to be this size, items will be evicted
+	int LOCATION_CACHE_EVICTION_SIZE;
+	int LOCATION_CACHE_EVICTION_SIZE_SIM;
+
+	int GET_RANGE_SHARD_LIMIT;
+	int WARM_RANGE_SHARD_LIMIT;
+	int STORAGE_METRICS_SHARD_LIMIT;
+	int SHARD_COUNT_LIMIT;
+	double STORAGE_METRICS_UNFAIR_SPLIT_LIMIT;
+	double STORAGE_METRICS_TOO_MANY_SHARDS_DELAY;
+	double AGGREGATE_HEALTH_METRICS_MAX_STALENESS;
+	double DETAILED_HEALTH_METRICS_MAX_STALENESS;
+	double MID_SHARD_SIZE_MAX_STALENESS;
+	bool TAG_ENCODE_KEY_SERVERS;
+	int64_t RANGESTREAM_FRAGMENT_SIZE;
+	int RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT;
+	bool QUARANTINE_TSS_ON_MISMATCH;
+
+	// KeyRangeMap
+	int KRM_GET_RANGE_LIMIT;
+	int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
+	                               // entries will be returned from an attempt to read a key range map
+
+	int DEFAULT_MAX_OUTSTANDING_WATCHES;
+	int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
+	double WATCH_POLLING_TIME;
+	double NO_RECENT_UPDATES_DURATION;
+	double FAST_WATCH_TIMEOUT;
+	double WATCH_TIMEOUT;
+
+	double IS_ACCEPTABLE_DELAY;
+
+	// Core
+	int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
+	int LOG_RANGE_BLOCK_SIZE;
+	int MUTATION_BLOCK_SIZE;
+
+	// Taskbucket
+	double TASKBUCKET_LOGGING_DELAY;
+	int TASKBUCKET_MAX_PRIORITY;
+	double TASKBUCKET_CHECK_TIMEOUT_CHANCE;
+	double TASKBUCKET_TIMEOUT_JITTER_OFFSET;
+	double TASKBUCKET_TIMEOUT_JITTER_RANGE;
+	double TASKBUCKET_CHECK_ACTIVE_DELAY;
+	int TASKBUCKET_CHECK_ACTIVE_AMOUNT;
+	int TASKBUCKET_TIMEOUT_VERSIONS;
+	int TASKBUCKET_MAX_TASK_KEYS;
+
+	// Backup
+	int BACKUP_LOCAL_FILE_WRITE_BLOCK;
+	int BACKUP_CONCURRENT_DELETES;
+	int BACKUP_SIMULATED_LIMIT_BYTES;
+	int BACKUP_GET_RANGE_LIMIT_BYTES;
+	int BACKUP_LOCK_BYTES;
+	double BACKUP_RANGE_TIMEOUT;
+	double BACKUP_RANGE_MINWAIT;
+	int BACKUP_SNAPSHOT_DISPATCH_INTERVAL_SEC;
+	int BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
+	int BACKUP_SHARD_TASK_LIMIT;
+	double BACKUP_AGGREGATE_POLL_RATE;
+	double BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL;
+	int BACKUP_LOG_WRITE_BATCH_MAX_SIZE;
+	int BACKUP_LOG_ATOMIC_OPS_SIZE;
+	int BACKUP_MAX_LOG_RANGES;
+	int BACKUP_SIM_COPY_LOG_RANGES;
+	int BACKUP_OPERATION_COST_OVERHEAD;
+	int BACKUP_VERSION_DELAY;
+	int BACKUP_MAP_KEY_LOWER_LIMIT;
+	int BACKUP_MAP_KEY_UPPER_LIMIT;
+	int BACKUP_COPY_TASKS;
+	int BACKUP_BLOCK_SIZE;
+	int COPY_LOG_BLOCK_SIZE;
+	int COPY_LOG_BLOCKS_PER_TASK;
+	int COPY_LOG_PREFETCH_BLOCKS;
+	int COPY_LOG_READ_AHEAD_BYTES;
+	double COPY_LOG_TASK_DURATION_NANOS;
+	int BACKUP_TASKS_PER_AGENT;
+	int BACKUP_POLL_PROGRESS_SECONDS;
+	int64_t VERSIONS_PER_SECOND; // Copy of SERVER_KNOBS, as we can't link with it
+	int SIM_BACKUP_TASKS_PER_AGENT;
+	int BACKUP_RANGEFILE_BLOCK_SIZE;
+	int BACKUP_LOGFILE_BLOCK_SIZE;
+	int BACKUP_DISPATCH_ADDTASK_SIZE;
+	int RESTORE_DISPATCH_ADDTASK_SIZE;
+	int RESTORE_DISPATCH_BATCH_SIZE;
+	int RESTORE_WRITE_TX_SIZE;
+	int APPLY_MAX_LOCK_BYTES;
+	int APPLY_MIN_LOCK_BYTES;
+	int APPLY_BLOCK_SIZE;
+	double APPLY_MAX_DECAY_RATE;
+	double APPLY_MAX_INCREASE_FACTOR;
+	double BACKUP_ERROR_DELAY;
+	double BACKUP_STATUS_DELAY;
+	double BACKUP_STATUS_JITTER;
+	double MIN_CLEANUP_SECONDS;
+	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
+
+	// Configuration
+	int32_t DEFAULT_AUTO_COMMIT_PROXIES;
+	int32_t DEFAULT_AUTO_GRV_PROXIES;
+	int32_t DEFAULT_COMMIT_GRV_PROXIES_RATIO;
+	int32_t DEFAULT_MAX_GRV_PROXIES;
+	int32_t DEFAULT_AUTO_RESOLVERS;
+	int32_t DEFAULT_AUTO_LOGS;
+
+	// Client Status Info
+	double CSI_SAMPLING_PROBABILITY;
+	int64_t CSI_SIZE_LIMIT;
+	double CSI_STATUS_DELAY;
+
+	int HTTP_SEND_SIZE;
+	int HTTP_READ_SIZE;
+	int HTTP_VERBOSE_LEVEL;
+	std::string HTTP_REQUEST_ID_HEADER;
+	int BLOBSTORE_CONNECT_TRIES;
+	int BLOBSTORE_CONNECT_TIMEOUT;
+	int BLOBSTORE_MAX_CONNECTION_LIFE;
+	int BLOBSTORE_REQUEST_TRIES;
+	int BLOBSTORE_REQUEST_TIMEOUT_MIN;
+	int BLOBSTORE_REQUESTS_PER_SECOND;
+	int BLOBSTORE_LIST_REQUESTS_PER_SECOND;
+	int BLOBSTORE_WRITE_REQUESTS_PER_SECOND;
+	int BLOBSTORE_READ_REQUESTS_PER_SECOND;
+	int BLOBSTORE_DELETE_REQUESTS_PER_SECOND;
+	int BLOBSTORE_CONCURRENT_REQUESTS;
+	int BLOBSTORE_MULTIPART_MAX_PART_SIZE;
+	int BLOBSTORE_MULTIPART_MIN_PART_SIZE;
+	int BLOBSTORE_CONCURRENT_UPLOADS;
+	int BLOBSTORE_CONCURRENT_LISTS;
+	int BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
+	int BLOBSTORE_CONCURRENT_READS_PER_FILE;
+	int BLOBSTORE_READ_BLOCK_SIZE;
+	int BLOBSTORE_READ_AHEAD_BLOCKS;
+	int BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
+	int BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
+	int BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
+
+	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
+	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
+
+	// fdbcli
+	int CLI_CONNECT_PARALLELISM;
+	double CLI_CONNECT_TIMEOUT;
+
+	// trace
+	int TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH;
+
+	// transaction tags
+	int MAX_TRANSACTION_TAG_LENGTH;
+	int MAX_TAGS_PER_TRANSACTION;
+	int COMMIT_SAMPLE_COST; // The expectation of sampling is every COMMIT_SAMPLE_COST sample once
+	int WRITE_COST_BYTE_FACTOR;
+	int INCOMPLETE_SHARD_PLUS; // The size of (possible) incomplete shard when estimate clear range
+	double READ_TAG_SAMPLE_RATE; // Communicated to clients from cluster
+	double TAG_THROTTLE_SMOOTHING_WINDOW;
+	double TAG_THROTTLE_RECHECK_INTERVAL;
+	double TAG_THROTTLE_EXPIRATION_INTERVAL;
+
+	ClientKnobs(Randomize randomize);
+	void initialize(Randomize randomize);
+};
+
+#endif

fdbclient/ClusterInterface.h

@@ -42,7 +42,7 @@ struct ClusterInterface {
 	UID id() const { return openDatabase.getEndpoint().token; }
 	NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
 
-	bool hasMessage() {
+	bool hasMessage() const {
 		return openDatabase.getFuture().isReady() || failureMonitoring.getFuture().isReady() ||
 		       databaseStatus.getFuture().isReady() || ping.getFuture().isReady() ||
 		       getClientWorkers.getFuture().isReady() || forceRecovery.getFuture().isReady();

fdbclient/CommitProxyInterface.h

@@ -1,4 +1,3 @@
-
 /*
  * CommitProxyInterface.h
  *

fdbclient/ConfigKnobs.cpp

@@ -0,0 +1,159 @@
+/*
+ * ConfigKnobs.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ConfigKnobs.h"
+#include "fdbclient/Tuple.h"
+#include "flow/UnitTest.h"
+
+ConfigKey ConfigKeyRef::decodeKey(KeyRef const& key) {
+	Tuple tuple;
+	try {
+		tuple = Tuple::unpack(key);
+	} catch (Error& e) {
+		TraceEvent(SevWarnAlways, "FailedToUnpackConfigKey").detail("Key", printable(key)).error(e);
+		throw invalid_config_db_key();
+	}
+	if (tuple.size() != 2) {
+		throw invalid_config_db_key();
+	}
+	if (tuple.getType(0) == Tuple::NULL_TYPE) {
+		return ConfigKeyRef({}, tuple.getString(1));
+	} else {
+		if (tuple.getType(0) != Tuple::BYTES || tuple.getType(1) != Tuple::BYTES) {
+			throw invalid_config_db_key();
+		}
+		return ConfigKeyRef(tuple.getString(0), tuple.getString(1));
+	}
+}
+
+Value KnobValueRef::ToValueFunc::operator()(int v) const {
+	return BinaryWriter::toValue(v, Unversioned());
+}
+Value KnobValueRef::ToValueFunc::operator()(int64_t v) const {
+	return BinaryWriter::toValue(v, Unversioned());
+}
+Value KnobValueRef::ToValueFunc::operator()(bool v) const {
+	return BinaryWriter::toValue(v, Unversioned());
+}
+Value KnobValueRef::ToValueFunc::operator()(ValueRef v) const {
+	return v;
+}
+Value KnobValueRef::ToValueFunc::operator()(double v) const {
+	return BinaryWriter::toValue(v, Unversioned());
+}
+
+KnobValue KnobValueRef::CreatorFunc::operator()(NoKnobFound) const {
+	ASSERT(false);
+	return {};
+}
+KnobValue KnobValueRef::CreatorFunc::operator()(int v) const {
+	return KnobValueRef(v);
+}
+KnobValue KnobValueRef::CreatorFunc::operator()(double v) const {
+	return KnobValueRef(v);
+}
+KnobValue KnobValueRef::CreatorFunc::operator()(int64_t v) const {
+	return KnobValueRef(v);
+}
+KnobValue KnobValueRef::CreatorFunc::operator()(bool v) const {
+	return KnobValueRef(v);
+}
+KnobValue KnobValueRef::CreatorFunc::operator()(std::string const& v) const {
+	return KnobValueRef(ValueRef(reinterpret_cast<uint8_t const*>(v.c_str()), v.size()));
+}
+
+namespace {
+
+class SetKnobFunc {
+	Knobs* knobs;
+	std::string const* knobName;
+
+public:
+	SetKnobFunc(Knobs& knobs, std::string const& knobName) : knobs(&knobs), knobName(&knobName) {}
+	template <class T>
+	bool operator()(T const& v) const {
+		return knobs->setKnob(*knobName, v);
+	}
+	bool operator()(StringRef const& v) const { return knobs->setKnob(*knobName, v.toString()); }
+};
+
+struct ToStringFunc {
+	std::string operator()(int v) const { return format("int:%d", v); }
+	std::string operator()(int64_t v) const { return format("int64_t:%ld", v); }
+	std::string operator()(bool v) const { return format("bool:%d", v); }
+	std::string operator()(ValueRef v) const { return "string:" + v.toString(); }
+	std::string operator()(double v) const { return format("double:%lf", v); }
+};
+
+} // namespace
+
+KnobValue KnobValueRef::create(ParsedKnobValue const& v) {
+	return std::visit(CreatorFunc{}, v);
+}
+
+bool KnobValueRef::visitSetKnob(std::string const& knobName, Knobs& knobs) const {
+	return std::visit(SetKnobFunc{ knobs, knobName }, value);
+}
+
+std::string KnobValueRef::toString() const {
+	return std::visit(ToStringFunc{}, value);
+}
+
+TEST_CASE("/fdbclient/ConfigDB/ConfigKey/EncodeDecode") {
+	Tuple tuple;
+	tuple << "class-A"_sr
+	      << "test_long"_sr;
+	auto packed = tuple.pack();
+	auto unpacked = ConfigKeyRef::decodeKey(packed);
+	ASSERT(unpacked.configClass.get() == "class-A"_sr);
+	ASSERT(unpacked.knobName == "test_long"_sr);
+	return Void();
+}
+
+namespace {
+
+void decodeFailureTest(KeyRef key) {
+	try {
+		ConfigKeyRef::decodeKey(key);
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_invalid_config_db_key);
+		return;
+	}
+	ASSERT(false);
+}
+
+} // namespace
+
+TEST_CASE("/fdbclient/ConfigDB/ConfigKey/DecodeFailure") {
+	{
+		Tuple tuple;
+		tuple << "s1"_sr
+		      << "s2"_sr
+		      << "s3"_sr;
+		decodeFailureTest(tuple.pack());
+	}
+	{
+		Tuple tuple;
+		tuple << "s1"_sr << 5;
+		decodeFailureTest(tuple.pack());
+	}
+	decodeFailureTest("non-tuple-key"_sr);
+	return Void();
+}

fdbclient/ConfigKnobs.h

@@ -0,0 +1,206 @@
+/*
+ * ConfigKnobs.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <variant>
+
+#include "fdbclient/FDBTypes.h"
+
+/*
+ * KnobValueRefs are stored in the configuration database, and in local configuration files. They are created from
+ * ParsedKnobValue objects, so it is assumed that the value type is correct for the corresponding knob name
+ */
+class KnobValueRef {
+	std::variant<int, double, int64_t, bool, ValueRef> value;
+	template <class T>
+	explicit KnobValueRef(T const& v) : value(std::in_place_type<T>, v) {}
+
+	explicit KnobValueRef(Arena& arena, ValueRef const& v) : value(std::in_place_type<ValueRef>, arena, v) {}
+
+	struct CreatorFunc {
+		Standalone<KnobValueRef> operator()(NoKnobFound) const;
+		Standalone<KnobValueRef> operator()(int) const;
+		Standalone<KnobValueRef> operator()(double) const;
+		Standalone<KnobValueRef> operator()(int64_t) const;
+		Standalone<KnobValueRef> operator()(bool) const;
+		Standalone<KnobValueRef> operator()(std::string const& v) const;
+	};
+
+	struct ToValueFunc {
+		Value operator()(int) const;
+		Value operator()(int64_t) const;
+		Value operator()(bool) const;
+		Value operator()(ValueRef) const;
+		Value operator()(double) const;
+	};
+
+public:
+	static constexpr FileIdentifier file_identifier = 9297109;
+
+	template <class T>
+	static Value toValue(T const& v) {
+		return ToValueFunc{}(v);
+	}
+
+	KnobValueRef() = default;
+
+	explicit KnobValueRef(Arena& arena, KnobValueRef const& rhs) : value(rhs.value) {
+		if (std::holds_alternative<ValueRef>(value)) {
+			value = ValueRef(arena, std::get<ValueRef>(value));
+		}
+	}
+
+	static Standalone<KnobValueRef> create(ParsedKnobValue const& v);
+
+	size_t expectedSize() const {
+		return std::holds_alternative<KeyRef>(value) ? std::get<KeyRef>(value).expectedSize() : 0;
+	}
+
+	Value toValue() const { return std::visit(ToValueFunc{}, value); }
+
+	// Returns true if and only if the knob was successfully found and set
+	bool visitSetKnob(std::string const& knobName, Knobs& knobs) const;
+
+	std::string toString() const;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, value);
+	}
+};
+
+using KnobValue = Standalone<KnobValueRef>;
+
+/*
+ * In the configuration database, each key contains a configuration class (or no configuration class, in the case of
+ * global updates), and a knob name
+ */
+struct ConfigKeyRef {
+	static constexpr FileIdentifier file_identifier = 5918726;
+
+	// Empty config class means the update is global
+	Optional<KeyRef> configClass;
+	KeyRef knobName;
+
+	ConfigKeyRef() = default;
+	explicit ConfigKeyRef(Optional<KeyRef> configClass, KeyRef knobName)
+	  : configClass(configClass), knobName(knobName) {}
+	explicit ConfigKeyRef(Arena& arena, Optional<KeyRef> configClass, KeyRef knobName)
+	  : configClass(arena, configClass), knobName(arena, knobName) {}
+	explicit ConfigKeyRef(Arena& arena, ConfigKeyRef const& rhs) : ConfigKeyRef(arena, rhs.configClass, rhs.knobName) {}
+
+	static Standalone<ConfigKeyRef> decodeKey(KeyRef const&);
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, configClass, knobName);
+	}
+
+	bool operator==(ConfigKeyRef const& rhs) const {
+		return (configClass == rhs.configClass) && (knobName == rhs.knobName);
+	}
+	bool operator!=(ConfigKeyRef const& rhs) const { return !(*this == rhs); }
+	size_t expectedSize() const { return configClass.expectedSize() + knobName.expectedSize(); }
+};
+using ConfigKey = Standalone<ConfigKeyRef>;
+
+inline bool operator<(ConfigKeyRef const& lhs, ConfigKeyRef const& rhs) {
+	if (lhs.configClass != rhs.configClass) {
+		return lhs.configClass < rhs.configClass;
+	} else {
+		return lhs.knobName < rhs.knobName;
+	}
+}
+
+/*
+ * Only set and point clear configuration database mutations are currently permitted.
+ */
+class ConfigMutationRef {
+	ConfigKeyRef key;
+	// Empty value means this is a clear mutation
+	Optional<KnobValueRef> value;
+
+public:
+	static constexpr FileIdentifier file_identifier = 7219528;
+
+	ConfigMutationRef() = default;
+
+	explicit ConfigMutationRef(Arena& arena, ConfigKeyRef key, Optional<KnobValueRef> value)
+	  : key(arena, key), value(arena, value) {}
+
+	explicit ConfigMutationRef(ConfigKeyRef key, Optional<KnobValueRef> value) : key(key), value(value) {}
+
+	ConfigKeyRef getKey() const { return key; }
+
+	Optional<KeyRef> getConfigClass() const { return key.configClass; }
+
+	KeyRef getKnobName() const { return key.knobName; }
+
+	KnobValueRef getValue() const { return value.get(); }
+
+	ConfigMutationRef(Arena& arena, ConfigMutationRef const& rhs) : key(arena, rhs.key), value(arena, rhs.value) {}
+
+	bool isSet() const { return value.present(); }
+
+	static Standalone<ConfigMutationRef> createConfigMutation(KeyRef encodedKey, KnobValueRef value) {
+		auto key = ConfigKeyRef::decodeKey(encodedKey);
+		return ConfigMutationRef(key, value);
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, key, value);
+	}
+
+	size_t expectedSize() const { return key.expectedSize() + value.expectedSize(); }
+};
+using ConfigMutation = Standalone<ConfigMutationRef>;
+
+/*
+ * Each configuration database commit is annotated with:
+ *   - A description (set manually by the client)
+ *   - A commit timestamp (automatically generated at commit time)
+ */
+struct ConfigCommitAnnotationRef {
+	KeyRef description;
+	double timestamp{ 0.0 };
+
+	ConfigCommitAnnotationRef() = default;
+	explicit ConfigCommitAnnotationRef(KeyRef description, double timestamp)
+	  : description(description), timestamp(timestamp) {}
+	explicit ConfigCommitAnnotationRef(Arena& arena, ConfigCommitAnnotationRef& rhs)
+	  : description(arena, rhs.description), timestamp(rhs.timestamp) {}
+
+	size_t expectedSize() const { return description.expectedSize(); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, timestamp, description);
+	}
+};
+using ConfigCommitAnnotation = Standalone<ConfigCommitAnnotationRef>;
+
+enum class UseConfigDB {
+	DISABLED,
+	SIMPLE,
+	PAXOS,
+};

fdbclient/ConfigTransactionInterface.cpp

@@ -0,0 +1,47 @@
+/*
+ * ConfigTransactionInterface.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ConfigTransactionInterface.h"
+#include "fdbclient/CoordinationInterface.h"
+#include "flow/IRandom.h"
+
+ConfigTransactionInterface::ConfigTransactionInterface() : _id(deterministicRandom()->randomUniqueID()) {}
+
+void ConfigTransactionInterface::setupWellKnownEndpoints() {
+	getVersion.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETVERSION, TaskPriority::Coordination);
+	get.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GET, TaskPriority::Coordination);
+	getClasses.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETCLASSES, TaskPriority::Coordination);
+	getKnobs.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_GETKNOBS, TaskPriority::Coordination);
+	commit.makeWellKnownEndpoint(WLTOKEN_CONFIGTXN_COMMIT, TaskPriority::Coordination);
+}
+
+ConfigTransactionInterface::ConfigTransactionInterface(NetworkAddress const& remote)
+  : getVersion(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETVERSION)), get(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GET)),
+    getClasses(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETCLASSES)),
+    getKnobs(Endpoint({ remote }, WLTOKEN_CONFIGTXN_GETKNOBS)), commit(Endpoint({ remote }, WLTOKEN_CONFIGTXN_COMMIT)) {
+}
+
+bool ConfigTransactionInterface::operator==(ConfigTransactionInterface const& rhs) const {
+	return _id == rhs._id;
+}
+
+bool ConfigTransactionInterface::operator!=(ConfigTransactionInterface const& rhs) const {
+	return !(*this == rhs);
+}

fdbclient/ConfigTransactionInterface.h

@@ -0,0 +1,193 @@
+/*
+ * ConfigTransactionInterface.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/CommitTransaction.h"
+#include "fdbclient/ConfigKnobs.h"
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbrpc/fdbrpc.h"
+#include "flow/flow.h"
+
+struct ConfigTransactionGetVersionReply {
+	static constexpr FileIdentifier file_identifier = 2934851;
+	ConfigTransactionGetVersionReply() = default;
+	explicit ConfigTransactionGetVersionReply(Version version) : version(version) {}
+	Version version;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version);
+	}
+};
+
+struct ConfigTransactionGetVersionRequest {
+	static constexpr FileIdentifier file_identifier = 138941;
+	ReplyPromise<ConfigTransactionGetVersionReply> reply;
+	ConfigTransactionGetVersionRequest() = default;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
+struct ConfigTransactionGetReply {
+	static constexpr FileIdentifier file_identifier = 2034110;
+	Optional<KnobValue> value;
+	ConfigTransactionGetReply() = default;
+	explicit ConfigTransactionGetReply(Optional<KnobValue> const& value) : value(value) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, value);
+	}
+};
+
+struct ConfigTransactionGetRequest {
+	static constexpr FileIdentifier file_identifier = 923040;
+	Version version;
+	ConfigKey key;
+	ReplyPromise<ConfigTransactionGetReply> reply;
+
+	ConfigTransactionGetRequest() = default;
+	explicit ConfigTransactionGetRequest(Version version, ConfigKey key) : version(version), key(key) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, key, reply);
+	}
+};
+
+struct ConfigTransactionCommitRequest {
+	static constexpr FileIdentifier file_identifier = 103841;
+	Arena arena;
+	Version version{ ::invalidVersion };
+	VectorRef<ConfigMutationRef> mutations;
+	ConfigCommitAnnotationRef annotation;
+	ReplyPromise<Void> reply;
+
+	size_t expectedSize() const { return mutations.expectedSize() + annotation.expectedSize(); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, arena, version, mutations, annotation, reply);
+	}
+};
+
+struct ConfigTransactionGetRangeReply {
+	static constexpr FileIdentifier file_identifier = 430263;
+	Standalone<RangeResultRef> range;
+
+	ConfigTransactionGetRangeReply() = default;
+	explicit ConfigTransactionGetRangeReply(Standalone<RangeResultRef> range) : range(range) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, range);
+	}
+};
+
+struct ConfigTransactionGetConfigClassesReply {
+	static constexpr FileIdentifier file_identifier = 5309618;
+	Standalone<VectorRef<KeyRef>> configClasses;
+
+	ConfigTransactionGetConfigClassesReply() = default;
+	explicit ConfigTransactionGetConfigClassesReply(Standalone<VectorRef<KeyRef>> const& configClasses)
+	  : configClasses(configClasses) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, configClasses);
+	}
+};
+
+struct ConfigTransactionGetConfigClassesRequest {
+	static constexpr FileIdentifier file_identifier = 7163400;
+	Version version;
+	ReplyPromise<ConfigTransactionGetConfigClassesReply> reply;
+
+	ConfigTransactionGetConfigClassesRequest() = default;
+	explicit ConfigTransactionGetConfigClassesRequest(Version version) : version(version) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version);
+	}
+};
+
+struct ConfigTransactionGetKnobsReply {
+	static constexpr FileIdentifier file_identifier = 4109852;
+	Standalone<VectorRef<KeyRef>> knobNames;
+
+	ConfigTransactionGetKnobsReply() = default;
+	explicit ConfigTransactionGetKnobsReply(Standalone<VectorRef<KeyRef>> const& knobNames) : knobNames(knobNames) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, knobNames);
+	}
+};
+
+struct ConfigTransactionGetKnobsRequest {
+	static constexpr FileIdentifier file_identifier = 987410;
+	Version version;
+	Optional<Key> configClass;
+	ReplyPromise<ConfigTransactionGetKnobsReply> reply;
+
+	ConfigTransactionGetKnobsRequest() = default;
+	explicit ConfigTransactionGetKnobsRequest(Version version, Optional<Key> configClass)
+	  : version(version), configClass(configClass) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, configClass, reply);
+	}
+};
+
+/*
+ * Configuration database nodes serve a ConfigTransactionInterface which contains well known endpoints,
+ * used by clients to transactionally update the configuration database
+ */
+struct ConfigTransactionInterface {
+	UID _id;
+
+public:
+	static constexpr FileIdentifier file_identifier = 982485;
+	struct RequestStream<ConfigTransactionGetVersionRequest> getVersion;
+	struct RequestStream<ConfigTransactionGetRequest> get;
+	struct RequestStream<ConfigTransactionGetConfigClassesRequest> getClasses;
+	struct RequestStream<ConfigTransactionGetKnobsRequest> getKnobs;
+	struct RequestStream<ConfigTransactionCommitRequest> commit;
+
+	ConfigTransactionInterface();
+	void setupWellKnownEndpoints();
+	ConfigTransactionInterface(NetworkAddress const& remote);
+
+	bool operator==(ConfigTransactionInterface const& rhs) const;
+	bool operator!=(ConfigTransactionInterface const& rhs) const;
+	UID id() const { return _id; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, getVersion, get, getClasses, getKnobs, commit);
+	}
+};

fdbclient/CoordinationInterface.h

@@ -30,6 +30,7 @@
 
 const int MAX_CLUSTER_FILE_BYTES = 60000;
 
+// well known endpoints published to the client.
 constexpr UID WLTOKEN_CLIENTLEADERREG_GETLEADER(-1, 2);
 constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
 
@@ -37,7 +38,12 @@ constexpr UID WLTOKEN_CLIENTLEADERREG_OPENDATABASE(-1, 3);
 constexpr UID WLTOKEN_PROTOCOL_INFO(-1, 10);
 constexpr UID WLTOKEN_CLIENTLEADERREG_DESCRIPTOR_MUTABLE(-1, 11);
 
-// well known endpoints published to the client.
+constexpr UID WLTOKEN_CONFIGTXN_GETVERSION(-1, 12);
+constexpr UID WLTOKEN_CONFIGTXN_GET(-1, 13);
+constexpr UID WLTOKEN_CONFIGTXN_GETCLASSES(-1, 14);
+constexpr UID WLTOKEN_CONFIGTXN_GETKNOBS(-1, 15);
+constexpr UID WLTOKEN_CONFIGTXN_COMMIT(-1, 16);
+
 struct ClientLeaderRegInterface {
 	RequestStream<struct GetLeaderRequest> getLeader;
 	RequestStream<struct OpenDatabaseCoordRequest> openDatabase;

fdbclient/DatabaseContext.h

@@ -219,15 +219,15 @@ public:
 	Error deferredError;
 	bool lockAware;
 
-	bool isError() { return deferredError.code() != invalid_error_code; }
+	bool isError() const { return deferredError.code() != invalid_error_code; }
 
-	void checkDeferredError() {
+	void checkDeferredError() const {
 		if (isError()) {
 			throw deferredError;
 		}
 	}
 
-	int apiVersionAtLeast(int minVersion) { return apiVersion < 0 || apiVersion >= minVersion; }
+	int apiVersionAtLeast(int minVersion) const { return apiVersion < 0 || apiVersion >= minVersion; }
 
 	Future<Void> onConnected(); // Returns after a majority of coordination servers are available and have reported a
 	                            // leader. The cluster file therefore is valid, but the database might be unavailable.
@@ -414,6 +414,7 @@ public:
 	double healthMetricsLastUpdated;
 	double detailedHealthMetricsLastUpdated;
 	Smoother smoothMidShardSize;
+	bool useConfigDatabase{ false };
 
 	UniqueOrderedOptionList<FDBTransactionOptions> transactionDefaults;
 

fdbclient/FDBTypes.cpp

@@ -0,0 +1,67 @@
+/*
+ * FDBTypes.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/Knobs.h"
+
+KeyRef keyBetween(const KeyRangeRef& keys) {
+	int pos = 0; // will be the position of the first difference between keys.begin and keys.end
+	int minSize = std::min(keys.begin.size(), keys.end.size());
+	for (; pos < minSize && pos < CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT; pos++) {
+		if (keys.begin[pos] != keys.end[pos]) {
+			return keys.end.substr(0, pos + 1);
+		}
+	}
+
+	// If one more character keeps us in the limit, and the latter key is simply
+	// longer, then we only need one more byte of the end string.
+	if (pos < CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT && keys.begin.size() < keys.end.size()) {
+		return keys.end.substr(0, pos + 1);
+	}
+
+	return keys.end;
+}
+
+void KeySelectorRef::setKey(KeyRef const& key) {
+	// There are no keys in the database with size greater than KEY_SIZE_LIMIT, so if this key selector has a key
+	// which is large, then we can translate it to an equivalent key selector with a smaller key
+	if (key.size() >
+	    (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT))
+		this->key = key.substr(0,
+		                       (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
+		                                                                 : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
+		                           1);
+	else
+		this->key = key;
+}
+
+std::string KeySelectorRef::toString() const {
+	if (offset > 0) {
+		if (orEqual)
+			return format("%d+firstGreaterThan(%s)", offset - 1, printable(key).c_str());
+		else
+			return format("%d+firstGreaterOrEqual(%s)", offset - 1, printable(key).c_str());
+	} else {
+		if (orEqual)
+			return format("%d+lastLessOrEqual(%s)", offset, printable(key).c_str());
+		else
+			return format("%d+lastLessThan(%s)", offset, printable(key).c_str());
+	}
+}

fdbclient/FDBTypes.h

@@ -28,7 +28,6 @@
 
 #include "flow/Arena.h"
 #include "flow/flow.h"
-#include "fdbclient/Knobs.h"
 
 typedef int64_t Version;
 typedef uint64_t LogEpoch;
@@ -514,28 +513,12 @@ inline KeyRange prefixRange(KeyRef prefix) {
 	range.contents() = KeyRangeRef(start, end);
 	return range;
 }
-inline KeyRef keyBetween(const KeyRangeRef& keys) {
+
 // Returns (one of) the shortest key(s) either contained in keys or equal to keys.end,
 // assuming its length is no more than CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT. If the length of
 // the shortest key exceeds that limit, then the end key is returned.
 // The returned reference is valid as long as keys is valid.
-
-	int pos = 0; // will be the position of the first difference between keys.begin and keys.end
-	int minSize = std::min(keys.begin.size(), keys.end.size());
-	for (; pos < minSize && pos < CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT; pos++) {
-		if (keys.begin[pos] != keys.end[pos]) {
-			return keys.end.substr(0, pos + 1);
-		}
-	}
-
-	// If one more character keeps us in the limit, and the latter key is simply
-	// longer, then we only need one more byte of the end string.
-	if (pos < CLIENT_KNOBS->SPLIT_KEY_SIZE_LIMIT && keys.begin.size() < keys.end.size()) {
-		return keys.end.substr(0, pos + 1);
-	}
-
-	return keys.end;
-}
+KeyRef keyBetween(const KeyRangeRef& keys);
 
 struct KeySelectorRef {
 private:
@@ -560,32 +543,9 @@ public:
 
 	KeyRef getKey() const { return key; }
 
-	void setKey(KeyRef const& key) {
-		// There are no keys in the database with size greater than KEY_SIZE_LIMIT, so if this key selector has a key
-		// which is large, then we can translate it to an equivalent key selector with a smaller key
-		if (key.size() > (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-		                                                           : CLIENT_KNOBS->KEY_SIZE_LIMIT))
-			this->key = key.substr(0,
-			                       (key.startsWith(LiteralStringRef("\xff")) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT
-			                                                                 : CLIENT_KNOBS->KEY_SIZE_LIMIT) +
-			                           1);
-		else
-			this->key = key;
-	}
+	void setKey(KeyRef const& key);
 
-	std::string toString() const {
-		if (offset > 0) {
-			if (orEqual)
-				return format("%d+firstGreaterThan(%s)", offset - 1, printable(key).c_str());
-			else
-				return format("%d+firstGreaterOrEqual(%s)", offset - 1, printable(key).c_str());
-		} else {
-			if (orEqual)
-				return format("%d+lastLessOrEqual(%s)", offset, printable(key).c_str());
-			else
-				return format("%d+lastLessThan(%s)", offset, printable(key).c_str());
-		}
-	}
+	std::string toString() const;
 
 	bool isBackward() const {
 		return !orEqual && offset <= 0;

fdbclient/IConfigTransaction.cpp

@@ -0,0 +1,35 @@
+/*
+ * IConfigTransaction.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/IConfigTransaction.h"
+#include "fdbclient/SimpleConfigTransaction.h"
+#include "fdbclient/PaxosConfigTransaction.h"
+
+Reference<IConfigTransaction> IConfigTransaction::createTestSimple(ConfigTransactionInterface const& cti) {
+	return makeReference<SimpleConfigTransaction>(cti);
+}
+
+Reference<IConfigTransaction> IConfigTransaction::createSimple(Database const& cx) {
+	return makeReference<SimpleConfigTransaction>(cx);
+}
+
+Reference<IConfigTransaction> IConfigTransaction::createPaxos(Database const& cx) {
+	return makeReference<PaxosConfigTransaction>(cx);
+}

fdbclient/IConfigTransaction.h

@@ -0,0 +1,67 @@
+/*
+ * IConfigTransaction.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "fdbclient/ConfigKnobs.h"
+#include "fdbclient/ConfigTransactionInterface.h"
+#include "fdbclient/ISingleThreadTransaction.h"
+#include "fdbclient/NativeAPI.actor.h"
+
+/*
+ * Configuration transactions are used by clients to update the configuration database, in order
+ * to dynamically update knobs. The interface is similar to that of regular transactions, but simpler, and
+ * many virtual methods of ISingleThreadTransaction are disallowed here
+ */
+class IConfigTransaction : public ISingleThreadTransaction {
+protected:
+	IConfigTransaction() = default;
+
+public:
+	virtual ~IConfigTransaction() = default;
+
+	static Reference<IConfigTransaction> createTestSimple(ConfigTransactionInterface const&);
+	static Reference<IConfigTransaction> createSimple(Database const&);
+	static Reference<IConfigTransaction> createPaxos(Database const&);
+
+	// Not implemented:
+	void setVersion(Version) override { throw client_invalid_operation(); }
+	Future<Key> getKey(KeySelector const& key, bool snapshot = false) override { throw client_invalid_operation(); }
+	Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) override {
+		throw client_invalid_operation();
+	}
+	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) override {
+		throw client_invalid_operation();
+	}
+	Future<int64_t> getEstimatedRangeSizeBytes(KeyRange const& keys) override { throw client_invalid_operation(); }
+	void addReadConflictRange(KeyRangeRef const& keys) override { throw client_invalid_operation(); }
+	void makeSelfConflicting() override { throw client_invalid_operation(); }
+	void atomicOp(KeyRef const& key, ValueRef const& operand, uint32_t operationType) override {
+		throw client_invalid_operation();
+	}
+	Future<Void> watch(Key const& key) override { throw client_invalid_operation(); }
+	void addWriteConflictRange(KeyRangeRef const& keys) override { throw client_invalid_operation(); }
+	Future<Standalone<StringRef>> getVersionstamp() override { throw client_invalid_operation(); }
+
+	// Implemented:
+	void getWriteConflicts(KeyRangeMap<bool>* result) override{};
+};

fdbclient/IKnobCollection.cpp

@@ -0,0 +1,90 @@
+/*
+ * IKnobCollection.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/IKnobCollection.h"
+#include "fdbclient/ClientKnobCollection.h"
+#include "fdbclient/ServerKnobCollection.h"
+#include "fdbclient/TestKnobCollection.h"
+
+std::unique_ptr<IKnobCollection> IKnobCollection::create(Type type, Randomize randomize, IsSimulated isSimulated) {
+	if (type == Type::CLIENT) {
+		return std::make_unique<ClientKnobCollection>(randomize, isSimulated);
+	} else if (type == Type::SERVER) {
+		return std::make_unique<ServerKnobCollection>(randomize, isSimulated);
+	} else if (type == Type::TEST) {
+		return std::make_unique<TestKnobCollection>(randomize, isSimulated);
+	}
+	UNSTOPPABLE_ASSERT(false);
+}
+
+KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::string const& knobValue) const {
+	auto result = tryParseKnobValue(knobName, knobValue);
+	if (!result.present()) {
+		throw invalid_option();
+	}
+	return result.get();
+}
+
+void IKnobCollection::setKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	if (!trySetKnob(knobName, knobValue)) {
+		TraceEvent(SevWarnAlways, "FailedToSetKnob")
+		    .detail("KnobName", knobName)
+		    .detail("KnobValue", knobValue.toString());
+		throw invalid_option_value();
+	}
+}
+
+KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::string const& knobValue, Type type) {
+	// TODO: Ideally it should not be necessary to create a template object to parse knobs
+	static std::unique_ptr<IKnobCollection> clientKnobCollection, serverKnobCollection, testKnobCollection;
+	if (type == Type::CLIENT) {
+		if (!clientKnobCollection) {
+			clientKnobCollection = create(type, Randomize::NO, IsSimulated::NO);
+		}
+		return clientKnobCollection->parseKnobValue(knobName, knobValue);
+	} else if (type == Type::SERVER) {
+		if (!serverKnobCollection) {
+			serverKnobCollection = create(type, Randomize::NO, IsSimulated::NO);
+		}
+		return serverKnobCollection->parseKnobValue(knobName, knobValue);
+	} else if (type == Type::TEST) {
+		if (!testKnobCollection) {
+			testKnobCollection = create(type, Randomize::NO, IsSimulated::NO);
+		}
+		return testKnobCollection->parseKnobValue(knobName, knobValue);
+	}
+	UNSTOPPABLE_ASSERT(false);
+}
+
+std::unique_ptr<IKnobCollection> IKnobCollection::globalKnobCollection =
+    IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO);
+
+void IKnobCollection::setGlobalKnobCollection(Type type, Randomize randomize, IsSimulated isSimulated) {
+	globalKnobCollection = create(type, randomize, isSimulated);
+	FLOW_KNOBS = &globalKnobCollection->getFlowKnobs();
+}
+
+IKnobCollection const& IKnobCollection::getGlobalKnobCollection() {
+	return *globalKnobCollection;
+}
+
+IKnobCollection& IKnobCollection::getMutableGlobalKnobCollection() {
+	return *globalKnobCollection;
+}

fdbclient/IKnobCollection.h

@@ -0,0 +1,65 @@
+/*
+ * IKnobCollection.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "fdbclient/ClientKnobs.h"
+#include "fdbclient/ConfigKnobs.h"
+#include "fdbclient/ServerKnobs.h"
+#include "flow/Knobs.h"
+
+/*
+ * Each IKnobCollection instantiation stores several Knobs objects, and when parsing or setting a knob,
+ * these Knobs objects will be traversed in order, until the requested knob name is found. The order of traversal is:
+ *  - FlowKnobs
+ *  - ClientKnobs
+ *  - ServerKnobs
+ *  - TestKnobs
+ */
+class IKnobCollection {
+	static std::unique_ptr<IKnobCollection> globalKnobCollection;
+
+public:
+	enum class Type {
+		CLIENT,
+		SERVER,
+		TEST,
+	};
+
+	static std::unique_ptr<IKnobCollection> create(Type, Randomize, IsSimulated);
+	virtual void initialize(Randomize randomize, IsSimulated isSimulated) = 0;
+	virtual void reset(Randomize randomize, IsSimulated isSimulated) = 0;
+	virtual FlowKnobs const& getFlowKnobs() const = 0;
+	virtual ClientKnobs const& getClientKnobs() const = 0;
+	virtual ServerKnobs const& getServerKnobs() const = 0;
+	virtual class TestKnobs const& getTestKnobs() const = 0;
+	virtual Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const = 0;
+	KnobValue parseKnobValue(std::string const& knobName, std::string const& knobValue) const;
+	static KnobValue parseKnobValue(std::string const& knobName, std::string const& knobValue, Type);
+	// Result indicates whether or not knob was successfully set:
+	virtual bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) = 0;
+	void setKnob(std::string const& knobName, KnobValueRef const& knobValue);
+
+	static void setGlobalKnobCollection(Type, Randomize, IsSimulated);
+	static IKnobCollection const& getGlobalKnobCollection();
+	static IKnobCollection& getMutableGlobalKnobCollection();
+};

fdbclient/ISingleThreadTransaction.cpp

@@ -0,0 +1,58 @@
+/*
+ * ISingleThreadTransaction.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/ISingleThreadTransaction.h"
+#include "fdbclient/PaxosConfigTransaction.h"
+#include "fdbclient/ReadYourWrites.h"
+#include "fdbclient/SimpleConfigTransaction.h"
+
+ISingleThreadTransaction* ISingleThreadTransaction::allocateOnForeignThread(Type type) {
+	if (type == Type::RYW) {
+		auto tr =
+		    (ReadYourWritesTransaction*)(ReadYourWritesTransaction::operator new(sizeof(ReadYourWritesTransaction)));
+		tr->preinitializeOnForeignThread();
+		return tr;
+	} else if (type == Type::SIMPLE_CONFIG) {
+		auto tr = (SimpleConfigTransaction*)(SimpleConfigTransaction::operator new(sizeof(SimpleConfigTransaction)));
+		return tr;
+	} else if (type == Type::PAXOS_CONFIG) {
+		auto tr = (PaxosConfigTransaction*)(PaxosConfigTransaction::operator new(sizeof(PaxosConfigTransaction)));
+		return tr;
+	}
+	ASSERT(false);
+	return nullptr;
+}
+
+void ISingleThreadTransaction::create(ISingleThreadTransaction* tr, Type type, Database db) {
+	switch (type) {
+	case Type::RYW:
+		new (tr) ReadYourWritesTransaction(db);
+		break;
+	case Type::SIMPLE_CONFIG:
+		new (tr) SimpleConfigTransaction(db);
+		break;
+	case Type::PAXOS_CONFIG:
+		new (tr) PaxosConfigTransaction(db);
+		break;
+	default:
+		ASSERT(false);
+	}
+}

fdbclient/ISingleThreadTransaction.h

@@ -0,0 +1,89 @@
+/*
+ * ISingleThreadTransaction.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/FDBOptions.g.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/KeyRangeMap.h"
+#include "flow/Error.h"
+#include "flow/FastRef.h"
+
+/*
+ * Used by ThreadSafeTransaction to execute normal or configuration transaction operations on the network thread
+ */
+class ISingleThreadTransaction : public ReferenceCounted<ISingleThreadTransaction> {
+protected:
+	ISingleThreadTransaction() = default;
+	ISingleThreadTransaction(Error const& deferredError) : deferredError(deferredError) {}
+
+public:
+	virtual ~ISingleThreadTransaction() = default;
+
+	enum class Type {
+		RYW,
+		SIMPLE_CONFIG,
+		PAXOS_CONFIG,
+	};
+
+	static ISingleThreadTransaction* allocateOnForeignThread(Type type);
+	static void create(ISingleThreadTransaction* tr, Type type, Database db);
+
+	virtual void setVersion(Version v) = 0;
+	virtual Future<Version> getReadVersion() = 0;
+	virtual Optional<Version> getCachedReadVersion() const = 0;
+	virtual Future<Optional<Value>> get(const Key& key, bool snapshot = false) = 0;
+	virtual Future<Key> getKey(const KeySelector& key, bool snapshot = false) = 0;
+	virtual Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
+	                                                    const KeySelector& end,
+	                                                    int limit,
+	                                                    bool snapshot = false,
+	                                                    bool reverse = false) = 0;
+	virtual Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
+	                                                    KeySelector end,
+	                                                    GetRangeLimits limits,
+	                                                    bool snapshot = false,
+	                                                    bool reverse = false) = 0;
+	virtual Future<Standalone<VectorRef<const char*>>> getAddressesForKey(Key const& key) = 0;
+	virtual Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) = 0;
+	virtual Future<int64_t> getEstimatedRangeSizeBytes(KeyRange const& keys) = 0;
+	virtual void addReadConflictRange(KeyRangeRef const& keys) = 0;
+	virtual void makeSelfConflicting() = 0;
+	virtual void atomicOp(KeyRef const& key, ValueRef const& operand, uint32_t operationType) = 0;
+	virtual void set(KeyRef const& key, ValueRef const& value) = 0;
+	virtual void clear(const KeyRangeRef& range) = 0;
+	virtual void clear(KeyRef const& key) = 0;
+	virtual Future<Void> watch(Key const& key) = 0;
+	virtual void addWriteConflictRange(KeyRangeRef const& keys) = 0;
+	virtual Future<Void> commit() = 0;
+	virtual Version getCommittedVersion() const = 0;
+	virtual int64_t getApproximateSize() const = 0;
+	virtual Future<Standalone<StringRef>> getVersionstamp() = 0;
+	virtual void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) = 0;
+	virtual Future<Void> onError(Error const& e) = 0;
+	virtual void cancel() = 0;
+	virtual void reset() = 0;
+	virtual void debugTransaction(UID dID) = 0;
+	virtual void checkDeferredError() const = 0;
+	virtual void getWriteConflicts(KeyRangeMap<bool>* result) = 0;
+
+	// Used by ThreadSafeTransaction for exceptions thrown in void methods
+	Error deferredError;
+};

fdbclient/Knobs.h

@@ -18,227 +18,8 @@
  * limitations under the License.
  */
 
-#ifndef FDBCLIENT_KNOBS_H
-#define FDBCLIENT_KNOBS_H
 #pragma once
 
-#include "flow/Knobs.h"
-#include "flow/flow.h"
+#include "fdbclient/IKnobCollection.h"
 
-class ClientKnobs : public Knobs {
-public:
-	int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically
-
-	double SYSTEM_MONITOR_INTERVAL;
-	double NETWORK_BUSYNESS_MONITOR_INTERVAL; // The interval in which we should update the network busyness metric
-
-	double FAILURE_MAX_DELAY;
-	double FAILURE_MIN_DELAY;
-	double FAILURE_TIMEOUT_DELAY;
-	double CLIENT_FAILURE_TIMEOUT_DELAY;
-	double FAILURE_EMERGENCY_DELAY;
-	double FAILURE_MAX_GENERATIONS;
-	double RECOVERY_DELAY_START_GENERATION;
-	double RECOVERY_DELAY_SECONDS_PER_GENERATION;
-	double MAX_GENERATIONS;
-	double MAX_GENERATIONS_OVERRIDE;
-	double MAX_GENERATIONS_SIM;
-
-	double COORDINATOR_RECONNECTION_DELAY;
-	int CLIENT_EXAMPLE_AMOUNT;
-	double MAX_CLIENT_STATUS_AGE;
-	int MAX_COMMIT_PROXY_CONNECTIONS;
-	int MAX_GRV_PROXY_CONNECTIONS;
-	double STATUS_IDLE_TIMEOUT;
-
-	// wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin
-	double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is
-	                                 // mostly wrong (e.g. dumping the database after a test)
-	double FUTURE_VERSION_RETRY_DELAY;
-	int REPLY_BYTE_LIMIT;
-	double DEFAULT_BACKOFF;
-	double DEFAULT_MAX_BACKOFF;
-	double BACKOFF_GROWTH_RATE;
-	double RESOURCE_CONSTRAINED_MAX_BACKOFF;
-	int PROXY_COMMIT_OVERHEAD_BYTES;
-	double SHARD_STAT_SMOOTH_AMOUNT;
-	int INIT_MID_SHARD_BYTES;
-
-	int TRANSACTION_SIZE_LIMIT;
-	int64_t KEY_SIZE_LIMIT;
-	int64_t SYSTEM_KEY_SIZE_LIMIT;
-	int64_t VALUE_SIZE_LIMIT;
-	int64_t SPLIT_KEY_SIZE_LIMIT;
-	int METADATA_VERSION_CACHE_SIZE;
-
-	int MAX_BATCH_SIZE;
-	double GRV_BATCH_TIMEOUT;
-	int BROADCAST_BATCH_SIZE;
-	double TRANSACTION_TIMEOUT_DELAY_INTERVAL;
-
-	// When locationCache in DatabaseContext gets to be this size, items will be evicted
-	int LOCATION_CACHE_EVICTION_SIZE;
-	int LOCATION_CACHE_EVICTION_SIZE_SIM;
-
-	int GET_RANGE_SHARD_LIMIT;
-	int WARM_RANGE_SHARD_LIMIT;
-	int STORAGE_METRICS_SHARD_LIMIT;
-	int SHARD_COUNT_LIMIT;
-	double STORAGE_METRICS_UNFAIR_SPLIT_LIMIT;
-	double STORAGE_METRICS_TOO_MANY_SHARDS_DELAY;
-	double AGGREGATE_HEALTH_METRICS_MAX_STALENESS;
-	double DETAILED_HEALTH_METRICS_MAX_STALENESS;
-	double MID_SHARD_SIZE_MAX_STALENESS;
-	bool TAG_ENCODE_KEY_SERVERS;
-	int64_t RANGESTREAM_FRAGMENT_SIZE;
-	int RANGESTREAM_BUFFERED_FRAGMENTS_LIMIT;
-	bool QUARANTINE_TSS_ON_MISMATCH;
-
-	// KeyRangeMap
-	int KRM_GET_RANGE_LIMIT;
-	int KRM_GET_RANGE_LIMIT_BYTES; // This must be sufficiently larger than KEY_SIZE_LIMIT to ensure that at least two
-	                               // entries will be returned from an attempt to read a key range map
-
-	int DEFAULT_MAX_OUTSTANDING_WATCHES;
-	int ABSOLUTE_MAX_WATCHES; // The client cannot set the max outstanding watches higher than this
-	double WATCH_POLLING_TIME;
-	double NO_RECENT_UPDATES_DURATION;
-	double FAST_WATCH_TIMEOUT;
-	double WATCH_TIMEOUT;
-
-	double IS_ACCEPTABLE_DELAY;
-
-	// Core
-	int64_t CORE_VERSIONSPERSECOND; // This is defined within the server but used for knobs based on server value
-	int LOG_RANGE_BLOCK_SIZE;
-	int MUTATION_BLOCK_SIZE;
-
-	// Taskbucket
-	double TASKBUCKET_LOGGING_DELAY;
-	int TASKBUCKET_MAX_PRIORITY;
-	double TASKBUCKET_CHECK_TIMEOUT_CHANCE;
-	double TASKBUCKET_TIMEOUT_JITTER_OFFSET;
-	double TASKBUCKET_TIMEOUT_JITTER_RANGE;
-	double TASKBUCKET_CHECK_ACTIVE_DELAY;
-	int TASKBUCKET_CHECK_ACTIVE_AMOUNT;
-	int TASKBUCKET_TIMEOUT_VERSIONS;
-	int TASKBUCKET_MAX_TASK_KEYS;
-
-	// Backup
-	int BACKUP_LOCAL_FILE_WRITE_BLOCK;
-	int BACKUP_CONCURRENT_DELETES;
-	int BACKUP_SIMULATED_LIMIT_BYTES;
-	int BACKUP_GET_RANGE_LIMIT_BYTES;
-	int BACKUP_LOCK_BYTES;
-	double BACKUP_RANGE_TIMEOUT;
-	double BACKUP_RANGE_MINWAIT;
-	int BACKUP_SNAPSHOT_DISPATCH_INTERVAL_SEC;
-	int BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
-	int BACKUP_SHARD_TASK_LIMIT;
-	double BACKUP_AGGREGATE_POLL_RATE;
-	double BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL;
-	int BACKUP_LOG_WRITE_BATCH_MAX_SIZE;
-	int BACKUP_LOG_ATOMIC_OPS_SIZE;
-	int BACKUP_MAX_LOG_RANGES;
-	int BACKUP_SIM_COPY_LOG_RANGES;
-	int BACKUP_OPERATION_COST_OVERHEAD;
-	int BACKUP_VERSION_DELAY;
-	int BACKUP_MAP_KEY_LOWER_LIMIT;
-	int BACKUP_MAP_KEY_UPPER_LIMIT;
-	int BACKUP_COPY_TASKS;
-	int BACKUP_BLOCK_SIZE;
-	int COPY_LOG_BLOCK_SIZE;
-	int COPY_LOG_BLOCKS_PER_TASK;
-	int COPY_LOG_PREFETCH_BLOCKS;
-	int COPY_LOG_READ_AHEAD_BYTES;
-	double COPY_LOG_TASK_DURATION_NANOS;
-	int BACKUP_TASKS_PER_AGENT;
-	int BACKUP_POLL_PROGRESS_SECONDS;
-	int64_t VERSIONS_PER_SECOND; // Copy of SERVER_KNOBS, as we can't link with it
-	int SIM_BACKUP_TASKS_PER_AGENT;
-	int BACKUP_RANGEFILE_BLOCK_SIZE;
-	int BACKUP_LOGFILE_BLOCK_SIZE;
-	int BACKUP_DISPATCH_ADDTASK_SIZE;
-	int RESTORE_DISPATCH_ADDTASK_SIZE;
-	int RESTORE_DISPATCH_BATCH_SIZE;
-	int RESTORE_WRITE_TX_SIZE;
-	int APPLY_MAX_LOCK_BYTES;
-	int APPLY_MIN_LOCK_BYTES;
-	int APPLY_BLOCK_SIZE;
-	double APPLY_MAX_DECAY_RATE;
-	double APPLY_MAX_INCREASE_FACTOR;
-	double BACKUP_ERROR_DELAY;
-	double BACKUP_STATUS_DELAY;
-	double BACKUP_STATUS_JITTER;
-	double MIN_CLEANUP_SECONDS;
-	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
-
-	// Configuration
-	int32_t DEFAULT_AUTO_COMMIT_PROXIES;
-	int32_t DEFAULT_AUTO_GRV_PROXIES;
-	int32_t DEFAULT_COMMIT_GRV_PROXIES_RATIO;
-	int32_t DEFAULT_MAX_GRV_PROXIES;
-	int32_t DEFAULT_AUTO_RESOLVERS;
-	int32_t DEFAULT_AUTO_LOGS;
-
-	// Client Status Info
-	double CSI_SAMPLING_PROBABILITY;
-	int64_t CSI_SIZE_LIMIT;
-	double CSI_STATUS_DELAY;
-
-	int HTTP_SEND_SIZE;
-	int HTTP_READ_SIZE;
-	int HTTP_VERBOSE_LEVEL;
-	std::string HTTP_REQUEST_ID_HEADER;
-	int BLOBSTORE_CONNECT_TRIES;
-	int BLOBSTORE_CONNECT_TIMEOUT;
-	int BLOBSTORE_MAX_CONNECTION_LIFE;
-	int BLOBSTORE_REQUEST_TRIES;
-	int BLOBSTORE_REQUEST_TIMEOUT_MIN;
-	int BLOBSTORE_REQUESTS_PER_SECOND;
-	int BLOBSTORE_LIST_REQUESTS_PER_SECOND;
-	int BLOBSTORE_WRITE_REQUESTS_PER_SECOND;
-	int BLOBSTORE_READ_REQUESTS_PER_SECOND;
-	int BLOBSTORE_DELETE_REQUESTS_PER_SECOND;
-	int BLOBSTORE_CONCURRENT_REQUESTS;
-	int BLOBSTORE_MULTIPART_MAX_PART_SIZE;
-	int BLOBSTORE_MULTIPART_MIN_PART_SIZE;
-	int BLOBSTORE_CONCURRENT_UPLOADS;
-	int BLOBSTORE_CONCURRENT_LISTS;
-	int BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
-	int BLOBSTORE_CONCURRENT_READS_PER_FILE;
-	int BLOBSTORE_READ_BLOCK_SIZE;
-	int BLOBSTORE_READ_AHEAD_BLOCKS;
-	int BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
-	int BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
-	int BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
-
-	int CONSISTENCY_CHECK_RATE_LIMIT_MAX;
-	int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME;
-
-	// fdbcli
-	int CLI_CONNECT_PARALLELISM;
-	double CLI_CONNECT_TIMEOUT;
-
-	// trace
-	int TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH;
-
-	// transaction tags
-	int MAX_TRANSACTION_TAG_LENGTH;
-	int MAX_TAGS_PER_TRANSACTION;
-	int COMMIT_SAMPLE_COST; // The expectation of sampling is every COMMIT_SAMPLE_COST sample once
-	int WRITE_COST_BYTE_FACTOR;
-	int INCOMPLETE_SHARD_PLUS; // The size of (possible) incomplete shard when estimate clear range
-	double READ_TAG_SAMPLE_RATE; // Communicated to clients from cluster
-	double TAG_THROTTLE_SMOOTHING_WINDOW;
-	double TAG_THROTTLE_RECHECK_INTERVAL;
-	double TAG_THROTTLE_EXPIRATION_INTERVAL;
-
-	ClientKnobs();
-	void initialize(bool randomize = false);
-};
-
-extern std::unique_ptr<ClientKnobs> globalClientKnobs;
-extern ClientKnobs const* CLIENT_KNOBS;
-
-#endif
+#define CLIENT_KNOBS (&IKnobCollection::getGlobalKnobCollection().getClientKnobs())

fdbclient/NativeAPI.actor.cpp

@@ -37,10 +37,10 @@
 #include "fdbclient/CoordinationInterface.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/GlobalConfig.actor.h"
+#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/JsonBuilder.h"
 #include "fdbclient/KeyBackedTypes.h"
 #include "fdbclient/KeyRangeMap.h"
-#include "fdbclient/Knobs.h"
 #include "fdbclient/ManagementAPI.actor.h"
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/MonitorLeader.h"
@@ -1550,6 +1550,10 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional<Stri
 			validateOptionValue(value, false);
 			transactionTracingEnabled--;
 			break;
+		case FDBDatabaseOptions::USE_CONFIG_DATABASE:
+			validateOptionValue(value, false);
+			useConfigDatabase = true;
+			break;
 		default:
 			break;
 		}
@@ -1838,14 +1842,12 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> valu
 		}
 
 		std::string knobName = optionValue.substr(0, eq);
-		std::string knobValue = optionValue.substr(eq + 1);
-		if (globalFlowKnobs->setKnob(knobName, knobValue)) {
-			// update dependent knobs
-			globalFlowKnobs->initialize();
-		} else if (globalClientKnobs->setKnob(knobName, knobValue)) {
-			// update dependent knobs
-			globalClientKnobs->initialize();
-		} else {
+		std::string knobValueString = optionValue.substr(eq + 1);
+
+		try {
+			auto knobValue = IKnobCollection::parseKnobValue(knobName, knobValueString, IKnobCollection::Type::CLIENT);
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob(knobName, knobValue);
+		} catch (Error& e) {
 			TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
 			fprintf(stderr, "FoundationDB client ignoring unrecognized knob option '%s'\n", knobName.c_str());
 		}
@@ -5488,7 +5490,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 	return readVersion;
 }
 
-Optional<Version> Transaction::getCachedReadVersion() {
+Optional<Version> Transaction::getCachedReadVersion() const {
 	if (readVersion.isValid() && readVersion.isReady() && !readVersion.isError()) {
 		return readVersion.get();
 	} else {
@@ -6053,7 +6055,7 @@ Future<Standalone<VectorRef<KeyRef>>> Transaction::splitStorageMetrics(KeyRange
 	return ::splitStorageMetrics(cx, keys, limit, estimated);
 }
 
-void Transaction::checkDeferredError() {
+void Transaction::checkDeferredError() const {
 	cx->checkDeferredError();
 }
 

fdbclient/NativeAPI.actor.h

@@ -246,7 +246,7 @@ public:
 	void setVersion(Version v);
 	Future<Version> getReadVersion() { return getReadVersion(0); }
 	Future<Version> getRawReadVersion();
-	Optional<Version> getCachedReadVersion();
+	Optional<Version> getCachedReadVersion() const;
 
 	[[nodiscard]] Future<Optional<Value>> get(const Key& key, bool snapshot = false);
 	[[nodiscard]] Future<Void> watch(Reference<Watch> watch);
@@ -359,7 +359,9 @@ public:
 
 	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
 
-	Version getCommittedVersion() { return committedVersion; } // May be called only after commit() returns success
+	Version getCommittedVersion() const {
+		return committedVersion;
+	} // May be called only after commit() returns success
 	[[nodiscard]] Future<Standalone<StringRef>>
 	getVersionstamp(); // Will be fulfilled only after commit() returns success
 
@@ -391,7 +393,7 @@ public:
 
 	int apiVersionAtLeast(int minVersion) const;
 
-	void checkDeferredError();
+	void checkDeferredError() const;
 
 	Database getDatabase() const { return cx; }
 	static Reference<TransactionLogInfo> createTrLogInfoProbabilistically(const Database& cx);

fdbclient/PaxosConfigTransaction.actor.cpp

@@ -0,0 +1,130 @@
+/*
+ * PaxosConfigTransaction.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/PaxosConfigTransaction.h"
+#include "flow/actorcompiler.h" // must be last include
+
+class PaxosConfigTransactionImpl {};
+
+Future<Version> PaxosConfigTransaction::getReadVersion() {
+	// TODO: Implement
+	return ::invalidVersion;
+}
+
+Optional<Version> PaxosConfigTransaction::getCachedReadVersion() const {
+	// TODO: Implement
+	return ::invalidVersion;
+}
+
+Future<Optional<Value>> PaxosConfigTransaction::get(Key const& key, bool snapshot) {
+	// TODO: Implement
+	return Optional<Value>{};
+}
+
+Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector const& begin,
+                                                                    KeySelector const& end,
+                                                                    int limit,
+                                                                    bool snapshot,
+                                                                    bool reverse) {
+	// TODO: Implement
+	ASSERT(false);
+	return Standalone<RangeResultRef>{};
+}
+
+Future<Standalone<RangeResultRef>> PaxosConfigTransaction::getRange(KeySelector begin,
+                                                                    KeySelector end,
+                                                                    GetRangeLimits limits,
+                                                                    bool snapshot,
+                                                                    bool reverse) {
+	// TODO: Implememnt
+	ASSERT(false);
+	return Standalone<RangeResultRef>{};
+}
+
+void PaxosConfigTransaction::set(KeyRef const& key, ValueRef const& value) {
+	// TODO: Implememnt
+	ASSERT(false);
+}
+
+void PaxosConfigTransaction::clear(KeyRef const& key) {
+	// TODO: Implememnt
+	ASSERT(false);
+}
+
+Future<Void> PaxosConfigTransaction::commit() {
+	// TODO: Implememnt
+	ASSERT(false);
+	return Void();
+}
+
+Version PaxosConfigTransaction::getCommittedVersion() const {
+	// TODO: Implement
+	ASSERT(false);
+	return ::invalidVersion;
+}
+
+int64_t PaxosConfigTransaction::getApproximateSize() const {
+	// TODO: Implement
+	ASSERT(false);
+	return 0;
+}
+
+void PaxosConfigTransaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+Future<Void> PaxosConfigTransaction::onError(Error const& e) {
+	// TODO: Implement
+	ASSERT(false);
+	return Void();
+}
+
+void PaxosConfigTransaction::cancel() {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+void PaxosConfigTransaction::reset() {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+void PaxosConfigTransaction::fullReset() {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+void PaxosConfigTransaction::debugTransaction(UID dID) {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+void PaxosConfigTransaction::checkDeferredError() const {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+PaxosConfigTransaction::PaxosConfigTransaction(Database const& cx) {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+PaxosConfigTransaction::~PaxosConfigTransaction() = default;

fdbclient/PaxosConfigTransaction.h

@@ -0,0 +1,66 @@
+/*
+ * PaxosConfigTransaction.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "fdbclient/IConfigTransaction.h"
+
+/*
+ * Fault-tolerant configuration transaction implementation
+ */
+class PaxosConfigTransaction final : public IConfigTransaction, public FastAllocated<PaxosConfigTransaction> {
+	std::unique_ptr<class PaxosConfigTransactionImpl> _impl;
+	PaxosConfigTransactionImpl const& impl() const { return *_impl; }
+	PaxosConfigTransactionImpl& impl() { return *_impl; }
+
+public:
+	PaxosConfigTransaction(Database const&);
+	~PaxosConfigTransaction();
+	Future<Version> getReadVersion() override;
+	Optional<Version> getCachedReadVersion() const override;
+
+	Future<Optional<Value>> get(Key const& key, bool snapshot = false) override;
+	Future<Standalone<RangeResultRef>> getRange(KeySelector const& begin,
+	                                            KeySelector const& end,
+	                                            int limit,
+	                                            bool snapshot = false,
+	                                            bool reverse = false) override;
+	Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
+	                                            KeySelector end,
+	                                            GetRangeLimits limits,
+	                                            bool snapshot = false,
+	                                            bool reverse = false) override;
+	void set(KeyRef const& key, ValueRef const& value) override;
+	void clear(KeyRangeRef const&) override { throw client_invalid_operation(); }
+	void clear(KeyRef const&) override;
+	Future<Void> commit() override;
+	Version getCommittedVersion() const override;
+	int64_t getApproximateSize() const override;
+	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
+	Future<Void> onError(Error const& e) override;
+	void cancel() override;
+	void reset() override;
+	void debugTransaction(UID dID) override;
+	void checkDeferredError() const override;
+
+	void fullReset();
+};

fdbclient/ReadYourWrites.actor.cpp

@@ -1280,8 +1280,8 @@ public:
 };
 
 ReadYourWritesTransaction::ReadYourWritesTransaction(Database const& cx)
-  : cache(&arena), writes(&arena), tr(cx), retries(0), approximateSize(0), creationTime(now()), commitStarted(false),
-    options(tr), deferredError(cx->deferredError), versionStampFuture(tr.getVersionstamp()),
+  : ISingleThreadTransaction(cx->deferredError), cache(&arena), writes(&arena), tr(cx), retries(0), approximateSize(0),
+    creationTime(now()), commitStarted(false), options(tr), versionStampFuture(tr.getVersionstamp()),
     specialKeySpaceWriteMap(std::make_pair(false, Optional<Value>()), specialKeys.end) {
 	std::copy(
 	    cx.getTransactionDefaults().begin(), cx.getTransactionDefaults().end(), std::back_inserter(persistentOptions));
@@ -1729,6 +1729,10 @@ void ReadYourWritesTransaction::getWriteConflicts(KeyRangeMap<bool>* result) {
 	}
 }
 
+void ReadYourWritesTransaction::preinitializeOnForeignThread() {
+	tr.preinitializeOnForeignThread();
+}
+
 void ReadYourWritesTransaction::setTransactionID(uint64_t id) {
 	tr.setTransactionID(id);
 }
@@ -2274,11 +2278,10 @@ void ReadYourWritesTransaction::operator=(ReadYourWritesTransaction&& r) noexcep
 }
 
 ReadYourWritesTransaction::ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept
-  : cache(std::move(r.cache)), writes(std::move(r.writes)), arena(std::move(r.arena)), reading(std::move(r.reading)),
-    retries(r.retries), approximateSize(r.approximateSize), creationTime(r.creationTime),
-    deferredError(std::move(r.deferredError)), timeoutActor(std::move(r.timeoutActor)),
-    resetPromise(std::move(r.resetPromise)), commitStarted(r.commitStarted), options(r.options),
-    transactionDebugInfo(r.transactionDebugInfo) {
+  : ISingleThreadTransaction(std::move(r.deferredError)), cache(std::move(r.cache)), writes(std::move(r.writes)),
+    arena(std::move(r.arena)), reading(std::move(r.reading)), retries(r.retries), approximateSize(r.approximateSize),
+    creationTime(r.creationTime), timeoutActor(std::move(r.timeoutActor)), resetPromise(std::move(r.resetPromise)),
+    commitStarted(r.commitStarted), options(r.options), transactionDebugInfo(r.transactionDebugInfo) {
 	cache.arena = &arena;
 	writes.arena = &arena;
 	tr = std::move(r.tr);

fdbclient/ReadYourWrites.h

@@ -25,6 +25,7 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/KeyRangeMap.h"
 #include "fdbclient/RYWIterator.h"
+#include "fdbclient/ISingleThreadTransaction.h"
 #include <list>
 
 // SOMEDAY: Optimize getKey to avoid using getRange
@@ -61,35 +62,31 @@ struct TransactionDebugInfo : public ReferenceCounted<TransactionDebugInfo> {
 // keeping a reference to a value longer than its creating transaction would hold all of the memory generated by the
 // transaction
 class ReadYourWritesTransaction final : NonCopyable,
-                                        public ReferenceCounted<ReadYourWritesTransaction>,
+                                        public ISingleThreadTransaction,
                                         public FastAllocated<ReadYourWritesTransaction> {
 public:
-	static ReadYourWritesTransaction* allocateOnForeignThread() {
-		ReadYourWritesTransaction* tr =
-		    (ReadYourWritesTransaction*)ReadYourWritesTransaction::operator new(sizeof(ReadYourWritesTransaction));
-		tr->tr.preinitializeOnForeignThread();
-		return tr;
-	}
-
 	explicit ReadYourWritesTransaction(Database const& cx);
 	~ReadYourWritesTransaction();
 
-	void setVersion(Version v) { tr.setVersion(v); }
-	Future<Version> getReadVersion();
-	Optional<Version> getCachedReadVersion() { return tr.getCachedReadVersion(); }
-	Future<Optional<Value>> get(const Key& key, bool snapshot = false);
-	Future<Key> getKey(const KeySelector& key, bool snapshot = false);
-	Future<RangeResult> getRange(const KeySelector& begin,
+	void setVersion(Version v) override { tr.setVersion(v); }
+	Future<Version> getReadVersion() override;
+	Optional<Version> getCachedReadVersion() const override { return tr.getCachedReadVersion(); }
+	Future<Optional<Value>> get(const Key& key, bool snapshot = false) override;
+	Future<Key> getKey(const KeySelector& key, bool snapshot = false) override;
+	Future<Standalone<RangeResultRef>> getRange(const KeySelector& begin,
 	                                            const KeySelector& end,
 	                                            int limit,
 	                                            bool snapshot = false,
-	                             bool reverse = false);
-	Future<RangeResult> getRange(KeySelector begin,
+	                                            bool reverse = false) override;
+	Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
 	                                            KeySelector end,
 	                                            GetRangeLimits limits,
 	                                            bool snapshot = false,
-	                             bool reverse = false);
-	Future<RangeResult> getRange(const KeyRange& keys, int limit, bool snapshot = false, bool reverse = false) {
+	                                            bool reverse = false) override;
+	Future<Standalone<RangeResultRef>> getRange(const KeyRange& keys,
+	                                            int limit,
+	                                            bool snapshot = false,
+	                                            bool reverse = false) {
 		return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()),
 		                KeySelector(firstGreaterOrEqual(keys.end), keys.arena()),
 		                limit,
@@ -107,39 +104,39 @@ public:
 		                reverse);
 	}
 
-	[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key);
-	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize);
-	Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys);
+	[[nodiscard]] Future<Standalone<VectorRef<const char*>>> getAddressesForKey(const Key& key) override;
+	Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override;
+	Future<int64_t> getEstimatedRangeSizeBytes(const KeyRange& keys) override;
 
-	void addReadConflictRange(KeyRangeRef const& keys);
-	void makeSelfConflicting() { tr.makeSelfConflicting(); }
+	void addReadConflictRange(KeyRangeRef const& keys) override;
+	void makeSelfConflicting() override { tr.makeSelfConflicting(); }
 
-	void atomicOp(const KeyRef& key, const ValueRef& operand, uint32_t operationType);
-	void set(const KeyRef& key, const ValueRef& value);
-	void clear(const KeyRangeRef& range);
-	void clear(const KeyRef& key);
+	void atomicOp(const KeyRef& key, const ValueRef& operand, uint32_t operationType) override;
+	void set(const KeyRef& key, const ValueRef& value) override;
+	void clear(const KeyRangeRef& range) override;
+	void clear(const KeyRef& key) override;
 
-	[[nodiscard]] Future<Void> watch(const Key& key);
+	[[nodiscard]] Future<Void> watch(const Key& key) override;
 
-	void addWriteConflictRange(KeyRangeRef const& keys);
+	void addWriteConflictRange(KeyRangeRef const& keys) override;
 
-	[[nodiscard]] Future<Void> commit();
-	Version getCommittedVersion() { return tr.getCommittedVersion(); }
-	int64_t getApproximateSize() { return approximateSize; }
-	[[nodiscard]] Future<Standalone<StringRef>> getVersionstamp();
+	[[nodiscard]] Future<Void> commit() override;
+	Version getCommittedVersion() const override { return tr.getCommittedVersion(); }
+	int64_t getApproximateSize() const override { return approximateSize; }
+	[[nodiscard]] Future<Standalone<StringRef>> getVersionstamp() override;
 
-	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>());
+	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
 
-	[[nodiscard]] Future<Void> onError(Error const& e);
+	[[nodiscard]] Future<Void> onError(Error const& e) override;
 
 	// These are to permit use as state variables in actors:
 	ReadYourWritesTransaction() : cache(&arena), writes(&arena) {}
 	void operator=(ReadYourWritesTransaction&& r) noexcept;
 	ReadYourWritesTransaction(ReadYourWritesTransaction&& r) noexcept;
 
-	void cancel();
-	void reset();
-	void debugTransaction(UID dID) { tr.debugTransaction(dID); }
+	void cancel() override;
+	void reset() override;
+	void debugTransaction(UID dID) override { tr.debugTransaction(dID); }
 
 	Future<Void> debug_onIdle() { return reading; }
 
@@ -148,16 +145,15 @@ public:
 	// Throws before the lifetime of this transaction ends
 	Future<Void> resetFuture() { return resetPromise.getFuture(); }
 
-	// Used by ThreadSafeTransaction for exceptions thrown in void methods
-	Error deferredError;
-
-	void checkDeferredError() {
+	void checkDeferredError() const override {
 		tr.checkDeferredError();
 		if (deferredError.code() != invalid_error_code)
 			throw deferredError;
 	}
 
-	void getWriteConflicts(KeyRangeMap<bool>* result);
+	void getWriteConflicts(KeyRangeMap<bool>* result) override;
+
+	void preinitializeOnForeignThread();
 
 	Database getDatabase() const { return tr.getDatabase(); }
 

fdbclient/ServerKnobCollection.cpp

@@ -0,0 +1,52 @@
+/*
+ * ServerKnobCollection.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/ServerKnobCollection.h"
+
+ServerKnobCollection::ServerKnobCollection(Randomize randomize, IsSimulated isSimulated)
+  : clientKnobCollection(randomize, isSimulated),
+    serverKnobs(randomize, &clientKnobCollection.getMutableClientKnobs(), isSimulated) {}
+
+void ServerKnobCollection::initialize(Randomize randomize, IsSimulated isSimulated) {
+	clientKnobCollection.initialize(randomize, isSimulated);
+	serverKnobs.initialize(randomize, &clientKnobCollection.getMutableClientKnobs(), isSimulated);
+}
+
+void ServerKnobCollection::reset(Randomize randomize, IsSimulated isSimulated) {
+	clientKnobCollection.reset(randomize, isSimulated);
+	serverKnobs.reset(randomize, &clientKnobCollection.getMutableClientKnobs(), isSimulated);
+}
+
+Optional<KnobValue> ServerKnobCollection::tryParseKnobValue(std::string const& knobName,
+                                                            std::string const& knobValue) const {
+	auto result = clientKnobCollection.tryParseKnobValue(knobName, knobValue);
+	if (result.present()) {
+		return result;
+	}
+	auto parsedKnobValue = serverKnobs.parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+	return {};
+}
+
+bool ServerKnobCollection::trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	return clientKnobCollection.trySetKnob(knobName, knobValue) || knobValue.visitSetKnob(knobName, serverKnobs);
+}

fdbclient/ServerKnobCollection.h

@@ -0,0 +1,44 @@
+/*
+ * ServerKnobCollection.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/ClientKnobCollection.h"
+#include "fdbclient/IKnobCollection.h"
+
+/*
+ * Stores both flow knobs, client knobs, and server knobs. Attempting to access test knobs
+ * results in a run-time error
+ */
+class ServerKnobCollection : public IKnobCollection {
+	ClientKnobCollection clientKnobCollection;
+	ServerKnobs serverKnobs;
+
+public:
+	ServerKnobCollection(Randomize randomize, IsSimulated isSimulated);
+	void initialize(Randomize randomize, IsSimulated isSimulated) override;
+	void reset(Randomize randomize, IsSimulated isSimulated) override;
+	FlowKnobs const& getFlowKnobs() const override { return clientKnobCollection.getFlowKnobs(); }
+	ClientKnobs const& getClientKnobs() const override { return clientKnobCollection.getClientKnobs(); }
+	ServerKnobs const& getServerKnobs() const override { return serverKnobs; }
+	TestKnobs const& getTestKnobs() const override { throw internal_error(); }
+	Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const override;
+	bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
+};

fdbclient/ServerKnobs.cpp

@@ -1,5 +1,5 @@
 /*
- * Knobs.cpp
+ * ServerKnobs.cpp
  *
  * This source file is part of the FoundationDB open source project
  *
@@ -18,20 +18,17 @@
  * limitations under the License.
  */
 
-#include "fdbserver/Knobs.h"
-#include "fdbrpc/Locality.h"
-#include <cmath>
-
-std::unique_ptr<ServerKnobs> globalServerKnobs = std::make_unique<ServerKnobs>();
-ServerKnobs const* SERVER_KNOBS = globalServerKnobs.get();
+#include "fdbclient/ServerKnobs.h"
 
 #define init(knob, value) initKnob(knob, value, #knob)
 
-ServerKnobs::ServerKnobs() {
-	initialize();
+ServerKnobs::ServerKnobs(Randomize randomize, ClientKnobs* clientKnobs, IsSimulated isSimulated) {
+	initialize(randomize, clientKnobs, isSimulated);
 }
 
-void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSimulated) {
+void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsSimulated _isSimulated) {
+	bool const randomize = _randomize == Randomize::YES;
+	bool const isSimulated = _isSimulated == IsSimulated::YES;
 	// clang-format off
 	// Versions
 	init( VERSIONS_PER_SECOND,                                   1e6 );

fdbclient/ServerKnobs.h

@@ -0,0 +1,679 @@
+/*
+ * ServerKnobs.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "flow/Knobs.h"
+#include "fdbrpc/fdbrpc.h"
+#include "fdbrpc/Locality.h"
+#include "fdbclient/ClientKnobs.h"
+
+// Disk queue
+static constexpr int _PAGE_SIZE = 4096;
+
+class ServerKnobs : public KnobsImpl<ServerKnobs> {
+public:
+	// Versions
+	int64_t VERSIONS_PER_SECOND;
+	int64_t MAX_VERSIONS_IN_FLIGHT;
+	int64_t MAX_VERSIONS_IN_FLIGHT_FORCED;
+	int64_t MAX_READ_TRANSACTION_LIFE_VERSIONS;
+	int64_t MAX_WRITE_TRANSACTION_LIFE_VERSIONS;
+	double MAX_COMMIT_BATCH_INTERVAL; // Each commit proxy generates a CommitTransactionBatchRequest at least this
+	                                  // often, so that versions always advance smoothly
+
+	// TLogs
+	double TLOG_TIMEOUT; // tlog OR commit proxy failure - master's reaction time
+	double TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS; // Warns if a tlog takes too long to rejoin
+	double RECOVERY_TLOG_SMART_QUORUM_DELAY; // smaller might be better for bug amplification
+	double TLOG_STORAGE_MIN_UPDATE_INTERVAL;
+	double BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL;
+	int DESIRED_TOTAL_BYTES;
+	int DESIRED_UPDATE_BYTES;
+	double UPDATE_DELAY;
+	int MAXIMUM_PEEK_BYTES;
+	int APPLY_MUTATION_BYTES;
+	int RECOVERY_DATA_BYTE_LIMIT;
+	int BUGGIFY_RECOVERY_DATA_LIMIT;
+	double LONG_TLOG_COMMIT_TIME;
+	int64_t LARGE_TLOG_COMMIT_BYTES;
+	double BUGGIFY_RECOVER_MEMORY_LIMIT;
+	double BUGGIFY_WORKER_REMOVED_MAX_LAG;
+	int64_t UPDATE_STORAGE_BYTE_LIMIT;
+	int64_t REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT;
+	double TLOG_PEEK_DELAY;
+	int LEGACY_TLOG_UPGRADE_ENTRIES_PER_VERSION;
+	int VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS; // Multiplicative factor to bound total space used to store a version
+	                                              // message (measured in 1/1024ths, e.g. a value of 2048 yields a
+	                                              // factor of 2).
+	int64_t VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD;
+	double TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
+	int64_t TLOG_MESSAGE_BLOCK_BYTES;
+	int64_t MAX_MESSAGE_SIZE;
+	int LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE;
+	double PEEK_TRACKER_EXPIRATION_TIME;
+	int PARALLEL_GET_MORE_REQUESTS;
+	int MULTI_CURSOR_PRE_FETCH_LIMIT;
+	int64_t MAX_QUEUE_COMMIT_BYTES;
+	int DESIRED_OUTSTANDING_MESSAGES;
+	double DESIRED_GET_MORE_DELAY;
+	int CONCURRENT_LOG_ROUTER_READS;
+	int LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED; // 0==peek from primary, non-zero==peek from satellites
+	double DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME;
+	double DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME;
+	int64_t TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES;
+	int64_t TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK;
+	int64_t TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH;
+	int64_t DISK_QUEUE_FILE_EXTENSION_BYTES; // When we grow the disk queue, by how many bytes should it grow?
+	int64_t DISK_QUEUE_FILE_SHRINK_BYTES; // When we shrink the disk queue, by how many bytes should it shrink?
+	int64_t DISK_QUEUE_MAX_TRUNCATE_BYTES; // A truncate larger than this will cause the file to be replaced instead.
+	double TLOG_DEGRADED_DURATION;
+	int64_t MAX_CACHE_VERSIONS;
+	double TXS_POPPED_MAX_DELAY;
+	double TLOG_MAX_CREATE_DURATION;
+	int PEEK_LOGGING_AMOUNT;
+	double PEEK_LOGGING_DELAY;
+	double PEEK_RESET_INTERVAL;
+	double PEEK_MAX_LATENCY;
+	bool PEEK_COUNT_SMALL_MESSAGES;
+	double PEEK_STATS_INTERVAL;
+	double PEEK_STATS_SLOW_AMOUNT;
+	double PEEK_STATS_SLOW_RATIO;
+	double PUSH_RESET_INTERVAL;
+	double PUSH_MAX_LATENCY;
+	double PUSH_STATS_INTERVAL;
+	double PUSH_STATS_SLOW_AMOUNT;
+	double PUSH_STATS_SLOW_RATIO;
+	int TLOG_POP_BATCH_SIZE;
+
+	// Data distribution queue
+	double HEALTH_POLL_TIME;
+	double BEST_TEAM_STUCK_DELAY;
+	double BG_REBALANCE_POLLING_INTERVAL;
+	double BG_REBALANCE_SWITCH_CHECK_INTERVAL;
+	double DD_QUEUE_LOGGING_INTERVAL;
+	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
+	int DD_QUEUE_MAX_KEY_SERVERS;
+	int DD_REBALANCE_PARALLELISM;
+	int DD_REBALANCE_RESET_AMOUNT;
+	double BG_DD_MAX_WAIT;
+	double BG_DD_MIN_WAIT;
+	double BG_DD_INCREASE_RATE;
+	double BG_DD_DECREASE_RATE;
+	double BG_DD_SATURATION_DELAY;
+	double INFLIGHT_PENALTY_HEALTHY;
+	double INFLIGHT_PENALTY_REDUNDANT;
+	double INFLIGHT_PENALTY_UNHEALTHY;
+	double INFLIGHT_PENALTY_ONE_LEFT;
+	bool USE_OLD_NEEDED_SERVERS;
+
+	// Higher priorities are executed first
+	// Priority/100 is the "priority group"/"superpriority".  Priority inversion
+	//   is possible within but not between priority groups; fewer priority groups
+	//   mean better worst case time bounds
+	// Maximum allowable priority is 999.
+	int PRIORITY_RECOVER_MOVE;
+	int PRIORITY_REBALANCE_UNDERUTILIZED_TEAM;
+	int PRIORITY_REBALANCE_OVERUTILIZED_TEAM;
+	int PRIORITY_PERPETUAL_STORAGE_WIGGLE;
+	int PRIORITY_TEAM_HEALTHY;
+	int PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER;
+	int PRIORITY_TEAM_REDUNDANT;
+	int PRIORITY_MERGE_SHARD;
+	int PRIORITY_POPULATE_REGION;
+	int PRIORITY_TEAM_UNHEALTHY;
+	int PRIORITY_TEAM_2_LEFT;
+	int PRIORITY_TEAM_1_LEFT;
+	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
+	int PRIORITY_TEAM_0_LEFT;
+	int PRIORITY_SPLIT_SHARD;
+
+	// Data distribution
+	double RETRY_RELOCATESHARD_DELAY;
+	double DATA_DISTRIBUTION_FAILURE_REACTION_TIME;
+	int MIN_SHARD_BYTES, SHARD_BYTES_RATIO, SHARD_BYTES_PER_SQRT_BYTES, MAX_SHARD_BYTES, KEY_SERVER_SHARD_BYTES;
+	int64_t SHARD_MAX_BYTES_PER_KSEC, // Shards with more than this bandwidth will be split immediately
+	    SHARD_MIN_BYTES_PER_KSEC, // Shards with more than this bandwidth will not be merged
+	    SHARD_SPLIT_BYTES_PER_KSEC; // When splitting a shard, it is split into pieces with less than this bandwidth
+	double SHARD_MAX_READ_DENSITY_RATIO;
+	int64_t SHARD_READ_HOT_BANDWITH_MIN_PER_KSECONDS;
+	double SHARD_MAX_BYTES_READ_PER_KSEC_JITTER;
+	double STORAGE_METRIC_TIMEOUT;
+	double METRIC_DELAY;
+	double ALL_DATA_REMOVED_DELAY;
+	double INITIAL_FAILURE_REACTION_DELAY;
+	double CHECK_TEAM_DELAY;
+	double LOG_ON_COMPLETION_DELAY;
+	int BEST_TEAM_MAX_TEAM_TRIES;
+	int BEST_TEAM_OPTION_COUNT;
+	int BEST_OF_AMT;
+	double SERVER_LIST_DELAY;
+	double RECRUITMENT_IDLE_DELAY;
+	double STORAGE_RECRUITMENT_DELAY;
+	bool TSS_HACK_IDENTITY_MAPPING;
+	double TSS_RECRUITMENT_TIMEOUT;
+	double TSS_DD_CHECK_INTERVAL;
+	double DATA_DISTRIBUTION_LOGGING_INTERVAL;
+	double DD_ENABLED_CHECK_DELAY;
+	double DD_STALL_CHECK_DELAY;
+	double DD_LOW_BANDWIDTH_DELAY;
+	double DD_MERGE_COALESCE_DELAY;
+	double STORAGE_METRICS_POLLING_DELAY;
+	double STORAGE_METRICS_RANDOM_DELAY;
+	double AVAILABLE_SPACE_RATIO_CUTOFF;
+	int DESIRED_TEAMS_PER_SERVER;
+	int MAX_TEAMS_PER_SERVER;
+	int64_t DD_SHARD_SIZE_GRANULARITY;
+	int64_t DD_SHARD_SIZE_GRANULARITY_SIM;
+	int DD_MOVE_KEYS_PARALLELISM;
+	int DD_FETCH_SOURCE_PARALLELISM;
+	int DD_MERGE_LIMIT;
+	double DD_SHARD_METRICS_TIMEOUT;
+	int64_t DD_LOCATION_CACHE_SIZE;
+	double MOVEKEYS_LOCK_POLLING_DELAY;
+	double DEBOUNCE_RECRUITING_DELAY;
+	int REBALANCE_MAX_RETRIES;
+	int DD_OVERLAP_PENALTY;
+	int DD_EXCLUDE_MIN_REPLICAS;
+	bool DD_VALIDATE_LOCALITY;
+	int DD_CHECK_INVALID_LOCALITY_DELAY;
+	bool DD_ENABLE_VERBOSE_TRACING;
+	int64_t
+	    DD_SS_FAILURE_VERSIONLAG; // Allowed SS version lag from the current read version before marking it as failed.
+	int64_t DD_SS_ALLOWED_VERSIONLAG; // SS will be marked as healthy if it's version lag goes below this value.
+	double DD_SS_STUCK_TIME_LIMIT; // If a storage server is not getting new versions for this amount of time, then it
+	                               // becomes undesired.
+	int DD_TEAMS_INFO_PRINT_INTERVAL;
+	int DD_TEAMS_INFO_PRINT_YIELD_COUNT;
+	int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY;
+	int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle
+
+	// TeamRemover to remove redundant teams
+	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
+	double TR_REMOVE_MACHINE_TEAM_DELAY; // wait for the specified time before try to remove next machine team
+	bool TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS; // guard to select which machineTeamRemover logic to use
+
+	bool TR_FLAG_DISABLE_SERVER_TEAM_REMOVER; // disable the serverTeamRemover actor
+	double TR_REMOVE_SERVER_TEAM_DELAY; // wait for the specified time before try to remove next server team
+	double TR_REMOVE_SERVER_TEAM_EXTRA_DELAY; // serverTeamRemover waits for the delay and check DD healthyness again to
+	                                          // ensure it runs after machineTeamRemover
+
+	// Remove wrong storage engines
+	double DD_REMOVE_STORE_ENGINE_DELAY; // wait for the specified time before remove the next batch
+
+	double DD_FAILURE_TIME;
+	double DD_ZERO_HEALTHY_TEAM_DELAY;
+
+	// Redwood Storage Engine
+	int PREFIX_TREE_IMMEDIATE_KEY_SIZE_LIMIT;
+	int PREFIX_TREE_IMMEDIATE_KEY_SIZE_MIN;
+
+	// KeyValueStore SQLITE
+	int CLEAR_BUFFER_SIZE;
+	double READ_VALUE_TIME_ESTIMATE;
+	double READ_RANGE_TIME_ESTIMATE;
+	double SET_TIME_ESTIMATE;
+	double CLEAR_TIME_ESTIMATE;
+	double COMMIT_TIME_ESTIMATE;
+	int CHECK_FREE_PAGE_AMOUNT;
+	double DISK_METRIC_LOGGING_INTERVAL;
+	int64_t SOFT_HEAP_LIMIT;
+
+	int SQLITE_PAGE_SCAN_ERROR_LIMIT;
+	int SQLITE_BTREE_PAGE_USABLE;
+	int SQLITE_BTREE_CELL_MAX_LOCAL;
+	int SQLITE_BTREE_CELL_MIN_LOCAL;
+	int SQLITE_FRAGMENT_PRIMARY_PAGE_USABLE;
+	int SQLITE_FRAGMENT_OVERFLOW_PAGE_USABLE;
+	double SQLITE_FRAGMENT_MIN_SAVINGS;
+	int SQLITE_CHUNK_SIZE_PAGES;
+	int SQLITE_CHUNK_SIZE_PAGES_SIM;
+	int SQLITE_READER_THREADS;
+	int SQLITE_WRITE_WINDOW_LIMIT;
+	double SQLITE_WRITE_WINDOW_SECONDS;
+
+	// KeyValueStoreSqlite spring cleaning
+	double SPRING_CLEANING_NO_ACTION_INTERVAL;
+	double SPRING_CLEANING_LAZY_DELETE_INTERVAL;
+	double SPRING_CLEANING_VACUUM_INTERVAL;
+	double SPRING_CLEANING_LAZY_DELETE_TIME_ESTIMATE;
+	double SPRING_CLEANING_VACUUM_TIME_ESTIMATE;
+	double SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE;
+	int SPRING_CLEANING_MIN_LAZY_DELETE_PAGES;
+	int SPRING_CLEANING_MAX_LAZY_DELETE_PAGES;
+	int SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE;
+	int SPRING_CLEANING_MIN_VACUUM_PAGES;
+	int SPRING_CLEANING_MAX_VACUUM_PAGES;
+
+	// KeyValueStoreMemory
+	int64_t REPLACE_CONTENTS_BYTES;
+
+	// KeyValueStoreRocksDB
+	int ROCKSDB_BACKGROUND_PARALLELISM;
+	int ROCKSDB_READ_PARALLELISM;
+	int64_t ROCKSDB_MEMTABLE_BYTES;
+	bool ROCKSDB_UNSAFE_AUTO_FSYNC;
+	int64_t ROCKSDB_PERIODIC_COMPACTION_SECONDS;
+	int ROCKSDB_PREFIX_LEN;
+	int64_t ROCKSDB_BLOCK_CACHE_SIZE;
+
+	// Leader election
+	int MAX_NOTIFICATIONS;
+	int MIN_NOTIFICATIONS;
+	double NOTIFICATION_FULL_CLEAR_TIME;
+	double CANDIDATE_MIN_DELAY;
+	double CANDIDATE_MAX_DELAY;
+	double CANDIDATE_GROWTH_RATE;
+	double POLLING_FREQUENCY;
+	double HEARTBEAT_FREQUENCY;
+
+	// Commit CommitProxy
+	double START_TRANSACTION_BATCH_INTERVAL_MIN;
+	double START_TRANSACTION_BATCH_INTERVAL_MAX;
+	double START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION;
+	double START_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA;
+	double START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL;
+	double START_TRANSACTION_MAX_TRANSACTIONS_TO_START;
+	int START_TRANSACTION_MAX_REQUESTS_TO_START;
+	double START_TRANSACTION_RATE_WINDOW;
+	double START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET;
+	int START_TRANSACTION_MAX_QUEUE_SIZE;
+	int KEY_LOCATION_MAX_QUEUE_SIZE;
+
+	double COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE;
+	double COMMIT_TRANSACTION_BATCH_INTERVAL_MIN;
+	double COMMIT_TRANSACTION_BATCH_INTERVAL_MAX;
+	double COMMIT_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION;
+	double COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA;
+	int COMMIT_TRANSACTION_BATCH_COUNT_MAX;
+	int COMMIT_TRANSACTION_BATCH_BYTES_MIN;
+	int COMMIT_TRANSACTION_BATCH_BYTES_MAX;
+	double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE;
+	double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER;
+	int64_t COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT;
+	double COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL;
+	double COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR;
+
+	double RESOLVER_COALESCE_TIME;
+	int BUGGIFIED_ROW_LIMIT;
+	double PROXY_SPIN_DELAY;
+	double UPDATE_REMOTE_LOG_VERSION_INTERVAL;
+	int MAX_TXS_POP_VERSION_HISTORY;
+	double MIN_CONFIRM_INTERVAL;
+	double ENFORCED_MIN_RECOVERY_DURATION;
+	double REQUIRED_MIN_RECOVERY_DURATION;
+	bool ALWAYS_CAUSAL_READ_RISKY;
+	int MAX_COMMIT_UPDATES;
+	double MAX_PROXY_COMPUTE;
+	double MAX_COMPUTE_PER_OPERATION;
+	int PROXY_COMPUTE_BUCKETS;
+	double PROXY_COMPUTE_GROWTH_RATE;
+	int TXN_STATE_SEND_AMOUNT;
+	double REPORT_TRANSACTION_COST_ESTIMATION_DELAY;
+	bool PROXY_REJECT_BATCH_QUEUED_TOO_LONG;
+
+	int RESET_MASTER_BATCHES;
+	int RESET_RESOLVER_BATCHES;
+	double RESET_MASTER_DELAY;
+	double RESET_RESOLVER_DELAY;
+
+	// Master Server
+	double COMMIT_SLEEP_TIME;
+	double MIN_BALANCE_TIME;
+	int64_t MIN_BALANCE_DIFFERENCE;
+	double SECONDS_BEFORE_NO_FAILURE_DELAY;
+	int64_t MAX_TXS_SEND_MEMORY;
+	int64_t MAX_RECOVERY_VERSIONS;
+	double MAX_RECOVERY_TIME;
+	double PROVISIONAL_START_DELAY;
+	double PROVISIONAL_DELAY_GROWTH;
+	double PROVISIONAL_MAX_DELAY;
+	double SECONDS_BEFORE_RECRUIT_BACKUP_WORKER;
+	double CC_INTERFACE_TIMEOUT;
+
+	// Resolver
+	int64_t KEY_BYTES_PER_SAMPLE;
+	int64_t SAMPLE_OFFSET_PER_KEY;
+	double SAMPLE_EXPIRATION_TIME;
+	double SAMPLE_POLL_TIME;
+	int64_t RESOLVER_STATE_MEMORY_LIMIT;
+
+	// Backup Worker
+	double BACKUP_TIMEOUT; // master's reaction time for backup failure
+	double BACKUP_NOOP_POP_DELAY;
+	int BACKUP_FILE_BLOCK_BYTES;
+	int64_t BACKUP_LOCK_BYTES;
+	double BACKUP_UPLOAD_DELAY;
+
+	// Cluster Controller
+	double CLUSTER_CONTROLLER_LOGGING_DELAY;
+	double MASTER_FAILURE_REACTION_TIME;
+	double MASTER_FAILURE_SLOPE_DURING_RECOVERY;
+	int WORKER_COORDINATION_PING_DELAY;
+	double SIM_SHUTDOWN_TIMEOUT;
+	double SHUTDOWN_TIMEOUT;
+	double MASTER_SPIN_DELAY;
+	double CC_CHANGE_DELAY;
+	double CC_CLASS_DELAY;
+	double WAIT_FOR_GOOD_RECRUITMENT_DELAY;
+	double WAIT_FOR_GOOD_REMOTE_RECRUITMENT_DELAY;
+	double ATTEMPT_RECRUITMENT_DELAY;
+	double WAIT_FOR_DISTRIBUTOR_JOIN_DELAY;
+	double WAIT_FOR_RATEKEEPER_JOIN_DELAY;
+	double WORKER_FAILURE_TIME;
+	double CHECK_OUTSTANDING_INTERVAL;
+	double INCOMPATIBLE_PEERS_LOGGING_INTERVAL;
+	double VERSION_LAG_METRIC_INTERVAL;
+	int64_t MAX_VERSION_DIFFERENCE;
+	double FORCE_RECOVERY_CHECK_DELAY;
+	double RATEKEEPER_FAILURE_TIME;
+	double REPLACE_INTERFACE_DELAY;
+	double REPLACE_INTERFACE_CHECK_DELAY;
+	double COORDINATOR_REGISTER_INTERVAL;
+	double CLIENT_REGISTER_INTERVAL;
+
+	// Knobs used to select the best policy (via monte carlo)
+	int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)
+	int POLICY_GENERATIONS; // number of policies to generate
+
+	int EXPECTED_MASTER_FITNESS;
+	int EXPECTED_TLOG_FITNESS;
+	int EXPECTED_LOG_ROUTER_FITNESS;
+	int EXPECTED_COMMIT_PROXY_FITNESS;
+	int EXPECTED_GRV_PROXY_FITNESS;
+	int EXPECTED_RESOLVER_FITNESS;
+	double RECRUITMENT_TIMEOUT;
+	int DBINFO_SEND_AMOUNT;
+	double DBINFO_BATCH_DELAY;
+
+	// Move Keys
+	double SHARD_READY_DELAY;
+	double SERVER_READY_QUORUM_INTERVAL;
+	double SERVER_READY_QUORUM_TIMEOUT;
+	double REMOVE_RETRY_DELAY;
+	int MOVE_KEYS_KRM_LIMIT;
+	int MOVE_KEYS_KRM_LIMIT_BYTES; // This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT
+	                               // (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an
+	                               // attempt to read a key range map
+	int MAX_SKIP_TAGS;
+	double MAX_ADDED_SOURCES_MULTIPLIER;
+
+	// FdbServer
+	double MIN_REBOOT_TIME;
+	double MAX_REBOOT_TIME;
+	std::string LOG_DIRECTORY;
+	int64_t SERVER_MEM_LIMIT;
+	double SYSTEM_MONITOR_FREQUENCY;
+
+	// Ratekeeper
+	double SMOOTHING_AMOUNT;
+	double SLOW_SMOOTHING_AMOUNT;
+	double METRIC_UPDATE_RATE;
+	double DETAILED_METRIC_UPDATE_RATE;
+	double LAST_LIMITED_RATIO;
+	double RATEKEEPER_DEFAULT_LIMIT;
+
+	int64_t TARGET_BYTES_PER_STORAGE_SERVER;
+	int64_t SPRING_BYTES_STORAGE_SERVER;
+	int64_t AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES;
+	int64_t TARGET_BYTES_PER_STORAGE_SERVER_BATCH;
+	int64_t SPRING_BYTES_STORAGE_SERVER_BATCH;
+	int64_t STORAGE_HARD_LIMIT_BYTES;
+	int64_t STORAGE_DURABILITY_LAG_HARD_MAX;
+	int64_t STORAGE_DURABILITY_LAG_SOFT_MAX;
+
+	int64_t LOW_PRIORITY_STORAGE_QUEUE_BYTES;
+	int64_t LOW_PRIORITY_DURABILITY_LAG;
+
+	int64_t TARGET_BYTES_PER_TLOG;
+	int64_t SPRING_BYTES_TLOG;
+	int64_t TARGET_BYTES_PER_TLOG_BATCH;
+	int64_t SPRING_BYTES_TLOG_BATCH;
+	int64_t TLOG_SPILL_THRESHOLD;
+	int64_t TLOG_HARD_LIMIT_BYTES;
+	int64_t TLOG_RECOVER_MEMORY_LIMIT;
+	double TLOG_IGNORE_POP_AUTO_ENABLE_DELAY;
+
+	int64_t MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
+	int64_t MAX_AUTO_THROTTLED_TRANSACTION_TAGS;
+	double MIN_TAG_COST;
+	double AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
+	double AUTO_THROTTLE_RAMP_TAG_BUSYNESS;
+	double AUTO_TAG_THROTTLE_RAMP_UP_TIME;
+	double AUTO_TAG_THROTTLE_DURATION;
+	double TAG_THROTTLE_PUSH_INTERVAL;
+	double AUTO_TAG_THROTTLE_START_AGGREGATION_TIME;
+	double AUTO_TAG_THROTTLE_UPDATE_FREQUENCY;
+	double TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL;
+	bool AUTO_TAG_THROTTLING_ENABLED;
+
+	double MAX_TRANSACTIONS_PER_BYTE;
+
+	int64_t MIN_AVAILABLE_SPACE;
+	double MIN_AVAILABLE_SPACE_RATIO;
+	double TARGET_AVAILABLE_SPACE_RATIO;
+	double AVAILABLE_SPACE_UPDATE_DELAY;
+
+	double MAX_TL_SS_VERSION_DIFFERENCE; // spring starts at half this value
+	double MAX_TL_SS_VERSION_DIFFERENCE_BATCH;
+	int MAX_MACHINES_FALLING_BEHIND;
+
+	int MAX_TPS_HISTORY_SAMPLES;
+	int NEEDED_TPS_HISTORY_SAMPLES;
+	int64_t TARGET_DURABILITY_LAG_VERSIONS;
+	int64_t AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS;
+	int64_t TARGET_DURABILITY_LAG_VERSIONS_BATCH;
+	int64_t DURABILITY_LAG_UNLIMITED_THRESHOLD;
+	double INITIAL_DURABILITY_LAG_MULTIPLIER;
+	double DURABILITY_LAG_REDUCTION_RATE;
+	double DURABILITY_LAG_INCREASE_RATE;
+
+	double STORAGE_SERVER_LIST_FETCH_TIMEOUT;
+
+	// disk snapshot
+	int64_t MAX_FORKED_PROCESS_OUTPUT;
+	double SNAP_CREATE_MAX_TIMEOUT;
+
+	// Storage Metrics
+	double STORAGE_METRICS_AVERAGE_INTERVAL;
+	double STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
+	double SPLIT_JITTER_AMOUNT;
+	int64_t IOPS_UNITS_PER_SAMPLE;
+	int64_t BANDWIDTH_UNITS_PER_SAMPLE;
+	int64_t BYTES_READ_UNITS_PER_SAMPLE;
+	int64_t READ_HOT_SUB_RANGE_CHUNK_SIZE;
+	int64_t EMPTY_READ_PENALTY;
+	bool READ_SAMPLING_ENABLED;
+
+	// Storage Server
+	double STORAGE_LOGGING_DELAY;
+	double STORAGE_SERVER_POLL_METRICS_DELAY;
+	double FUTURE_VERSION_DELAY;
+	int STORAGE_LIMIT_BYTES;
+	int BUGGIFY_LIMIT_BYTES;
+	bool FETCH_USING_STREAMING;
+	int FETCH_BLOCK_BYTES;
+	int FETCH_KEYS_PARALLELISM_BYTES;
+	int FETCH_KEYS_PARALLELISM;
+	int FETCH_KEYS_LOWER_PRIORITY;
+	int BUGGIFY_BLOCK_BYTES;
+	double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD;
+	double STORAGE_DURABILITY_LAG_MIN_RATE;
+	int STORAGE_COMMIT_BYTES;
+	int STORAGE_FETCH_BYTES;
+	double STORAGE_COMMIT_INTERVAL;
+	double UPDATE_SHARD_VERSION_INTERVAL;
+	int BYTE_SAMPLING_FACTOR;
+	int BYTE_SAMPLING_OVERHEAD;
+	int MAX_STORAGE_SERVER_WATCH_BYTES;
+	int MAX_BYTE_SAMPLE_CLEAR_MAP_SIZE;
+	double LONG_BYTE_SAMPLE_RECOVERY_DELAY;
+	int BYTE_SAMPLE_LOAD_PARALLELISM;
+	double BYTE_SAMPLE_LOAD_DELAY;
+	double BYTE_SAMPLE_START_DELAY;
+	double UPDATE_STORAGE_PROCESS_STATS_INTERVAL;
+	double BEHIND_CHECK_DELAY;
+	int BEHIND_CHECK_COUNT;
+	int64_t BEHIND_CHECK_VERSIONS;
+	double WAIT_METRICS_WRONG_SHARD_CHANCE;
+	int64_t MIN_TAG_READ_PAGES_RATE;
+	int64_t MIN_TAG_WRITE_PAGES_RATE;
+	double TAG_MEASUREMENT_INTERVAL;
+	int64_t READ_COST_BYTE_FACTOR;
+	bool PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS;
+	bool REPORT_DD_METRICS;
+	double DD_METRICS_REPORT_INTERVAL;
+	double FETCH_KEYS_TOO_LONG_TIME_CRITERIA;
+	double MAX_STORAGE_COMMIT_TIME;
+	int64_t RANGESTREAM_LIMIT_BYTES;
+
+	// Wait Failure
+	int MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS;
+	double WAIT_FAILURE_DELAY_LIMIT;
+
+	// Worker
+	double WORKER_LOGGING_INTERVAL;
+	double HEAP_PROFILER_INTERVAL;
+	double UNKNOWN_CC_TIMEOUT;
+	double DEGRADED_RESET_INTERVAL;
+	double DEGRADED_WARNING_LIMIT;
+	double DEGRADED_WARNING_RESET_DELAY;
+	int64_t TRACE_LOG_FLUSH_FAILURE_CHECK_INTERVAL_SECONDS;
+	double TRACE_LOG_PING_TIMEOUT_SECONDS;
+	double MIN_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS; // Listen for a leader for N seconds, and if not heard, then try to
+	                                                 // become the leader.
+	double MAX_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS;
+	double DBINFO_FAILED_DELAY;
+	bool ENABLE_WORKER_HEALTH_MONITOR;
+	double WORKER_HEALTH_MONITOR_INTERVAL; // Interval between two health monitor health check.
+	int PEER_LATENCY_CHECK_MIN_POPULATION; // The minimum number of latency samples required to check a peer.
+	double PEER_LATENCY_DEGRADATION_PERCENTILE; // The percentile latency used to check peer health.
+	double PEER_LATENCY_DEGRADATION_THRESHOLD; // The latency threshold to consider a peer degraded.
+	double PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD; // The percentage of timeout to consider a peer degraded.
+
+	// Test harness
+	double WORKER_POLL_DELAY;
+
+	// Coordination
+	double COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL;
+	bool ENABLE_CROSS_CLUSTER_SUPPORT; // Allow a coordinator to serve requests whose connection string does not match
+	                                   // the local descriptor
+
+	// Buggification
+	double BUGGIFIED_EVENTUAL_CONSISTENCY;
+	bool BUGGIFY_ALL_COORDINATION;
+
+	// Status
+	double STATUS_MIN_TIME_BETWEEN_REQUESTS;
+	double MAX_STATUS_REQUESTS_PER_SECOND;
+	int CONFIGURATION_ROWS_TO_FETCH;
+	bool DISABLE_DUPLICATE_LOG_WARNING;
+	double HISTOGRAM_REPORT_INTERVAL;
+
+	// IPager
+	int PAGER_RESERVED_PAGES;
+
+	// IndirectShadowPager
+	int FREE_PAGE_VACUUM_THRESHOLD;
+	int VACUUM_QUEUE_SIZE;
+	int VACUUM_BYTES_PER_SECOND;
+
+	// Timekeeper
+	int64_t TIME_KEEPER_DELAY;
+	int64_t TIME_KEEPER_MAX_ENTRIES;
+
+	// Fast Restore
+	// TODO: After 6.3, review FR knobs, remove unneeded ones and change default value
+	int64_t FASTRESTORE_FAILURE_TIMEOUT;
+	int64_t FASTRESTORE_HEARTBEAT_INTERVAL;
+	double FASTRESTORE_SAMPLING_PERCENT;
+	int64_t FASTRESTORE_NUM_LOADERS;
+	int64_t FASTRESTORE_NUM_APPLIERS;
+	// FASTRESTORE_TXN_BATCH_MAX_BYTES is target txn size used by appliers to apply mutations
+	double FASTRESTORE_TXN_BATCH_MAX_BYTES;
+	// FASTRESTORE_VERSIONBATCH_MAX_BYTES is the maximum data size in each version batch
+	double FASTRESTORE_VERSIONBATCH_MAX_BYTES;
+	// FASTRESTORE_VB_PARALLELISM is the number of concurrently running version batches
+	int64_t FASTRESTORE_VB_PARALLELISM;
+	int64_t FASTRESTORE_VB_MONITOR_DELAY; // How quickly monitor finished version batch
+	double FASTRESTORE_VB_LAUNCH_DELAY;
+	int64_t FASTRESTORE_ROLE_LOGGING_DELAY;
+	int64_t FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL; // How quickly to update process metrics for restore
+	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
+	int64_t FASTRESTORE_APPLYING_PARALLELISM; // number of outstanding txns writing to dest. DB
+	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
+	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
+	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
+	bool FASTRESTORE_TRACK_LOADER_SEND_REQUESTS; // track requests of load send mutations to appliers?
+	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
+	int64_t FASTRESTORE_WAIT_FOR_MEMORY_LATENCY;
+	int64_t FASTRESTORE_HEARTBEAT_DELAY; // interval for master to ping loaders and appliers
+	int64_t
+	    FASTRESTORE_HEARTBEAT_MAX_DELAY; // master claim a node is down if no heart beat from the node for this delay
+	int64_t FASTRESTORE_APPLIER_FETCH_KEYS_SIZE; // number of keys to fetch in a txn on applier
+	int64_t FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES; // desired size of mutation message sent from loader to appliers
+	bool FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE; // parse each range file to get (range, version) it has?
+	int64_t FASTRESTORE_REQBATCH_PARALLEL; // number of requests to wait on for getBatchReplies()
+	bool FASTRESTORE_REQBATCH_LOG; // verbose log information for getReplyBatches
+	int FASTRESTORE_TXN_CLEAR_MAX; // threshold to start tracking each clear op in a txn
+	int FASTRESTORE_TXN_RETRY_MAX; // threshold to start output error on too many retries
+	double FASTRESTORE_TXN_EXTRA_DELAY; // extra delay to avoid overwhelming fdb
+	bool FASTRESTORE_NOT_WRITE_DB; // do not write result to DB. Only for dev testing
+	bool FASTRESTORE_USE_RANGE_FILE; // use range file in backup
+	bool FASTRESTORE_USE_LOG_FILE; // use log file in backup
+	int64_t FASTRESTORE_SAMPLE_MSG_BYTES; // sample message desired size
+	double FASTRESTORE_SCHED_UPDATE_DELAY; // delay in seconds in updating process metrics
+	int FASTRESTORE_SCHED_TARGET_CPU_PERCENT; // release as many requests as possible when cpu usage is below the knob
+	int FASTRESTORE_SCHED_MAX_CPU_PERCENT; // max cpu percent when scheduler shall not release non-urgent requests
+	int FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS; // number of inflight requests to load backup files
+	int FASTRESTORE_SCHED_INFLIGHT_SEND_REQS; // number of inflight requests for loaders to  send mutations to appliers
+	int FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE; // number of load request to release at once
+	int FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD; // we can send future VB requests if it is less than this knob
+	int FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH; // number of future VB sendLoadingParam requests to process at once
+	int FASTRESTORE_NUM_TRACE_EVENTS;
+	bool FASTRESTORE_EXPENSIVE_VALIDATION; // when set true, performance will be heavily affected
+	double FASTRESTORE_WRITE_BW_MB; // target aggregated write bandwidth from all appliers
+	double FASTRESTORE_RATE_UPDATE_SECONDS; // how long to update appliers target write rate
+
+	int REDWOOD_DEFAULT_PAGE_SIZE; // Page size for new Redwood files
+	int REDWOOD_DEFAULT_EXTENT_SIZE; // Extent size for new Redwood files
+	int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files
+	int REDWOOD_EXTENT_CONCURRENT_READS; // Max number of simultaneous extent disk reads in progress.
+	int REDWOOD_KVSTORE_CONCURRENT_READS; // Max number of simultaneous point or range reads in progress.
+	int REDWOOD_COMMIT_CONCURRENT_READS; // Max number of concurrent reads done to support commit operations
+	double REDWOOD_PAGE_REBUILD_MAX_SLACK; // When rebuilding pages, max slack to allow in page
+	int REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; // Number of pages to try to pop from the lazy delete queue and process at
+	                                         // once
+	int REDWOOD_LAZY_CLEAR_MIN_PAGES; // Minimum number of pages to free before ending a lazy clear cycle, unless the
+	                                  // queue is empty
+	int REDWOOD_LAZY_CLEAR_MAX_PAGES; // Maximum number of pages to free before ending a lazy clear cycle, unless the
+	                                  // queue is empty
+	int64_t REDWOOD_REMAP_CLEANUP_WINDOW; // Remap remover lag interval in which to coalesce page writes
+	double REDWOOD_REMAP_CLEANUP_LAG; // Maximum allowed remap remover lag behind the cleanup window as a multiple of
+	                                  // the window size
+	double REDWOOD_LOGGING_INTERVAL;
+
+	// Server request latency measurement
+	int LATENCY_SAMPLE_SIZE;
+	double LATENCY_METRICS_LOGGING_INTERVAL;
+
+	ServerKnobs(Randomize, ClientKnobs*, IsSimulated);
+	void initialize(Randomize, ClientKnobs*, IsSimulated);
+};

fdbclient/SimpleConfigTransaction.actor.cpp

@@ -0,0 +1,299 @@
+/*
+ * SimpleConfigTransaction.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+
+#include "fdbclient/CommitTransaction.h"
+#include "fdbclient/DatabaseContext.h"
+#include "fdbclient/IKnobCollection.h"
+#include "fdbclient/SimpleConfigTransaction.h"
+#include "fdbserver/Knobs.h"
+#include "flow/Arena.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+class SimpleConfigTransactionImpl {
+	ConfigTransactionCommitRequest toCommit;
+	Future<Version> getVersionFuture;
+	ConfigTransactionInterface cti;
+	int numRetries{ 0 };
+	bool committed{ false };
+	Optional<UID> dID;
+	Database cx;
+
+	ACTOR static Future<Version> getReadVersion(SimpleConfigTransactionImpl* self) {
+		if (self->dID.present()) {
+			TraceEvent("SimpleConfigTransactionGettingReadVersion", self->dID.get());
+		}
+		ConfigTransactionGetVersionRequest req;
+		ConfigTransactionGetVersionReply reply =
+		    wait(self->cti.getVersion.getReply(ConfigTransactionGetVersionRequest{}));
+		if (self->dID.present()) {
+			TraceEvent("SimpleConfigTransactionGotReadVersion", self->dID.get()).detail("Version", reply.version);
+		}
+		return reply.version;
+	}
+
+	ACTOR static Future<Optional<Value>> get(SimpleConfigTransactionImpl* self, KeyRef key) {
+		if (!self->getVersionFuture.isValid()) {
+			self->getVersionFuture = getReadVersion(self);
+		}
+		state ConfigKey configKey = ConfigKey::decodeKey(key);
+		Version version = wait(self->getVersionFuture);
+		if (self->dID.present()) {
+			TraceEvent("SimpleConfigTransactionGettingValue", self->dID.get())
+			    .detail("ConfigClass", configKey.configClass)
+			    .detail("KnobName", configKey.knobName);
+		}
+		ConfigTransactionGetReply reply =
+		    wait(self->cti.get.getReply(ConfigTransactionGetRequest{ version, configKey }));
+		if (self->dID.present()) {
+			TraceEvent("SimpleConfigTransactionGotValue", self->dID.get())
+			    .detail("Value", reply.value.get().toString());
+		}
+		if (reply.value.present()) {
+			return reply.value.get().toValue();
+		} else {
+			return {};
+		}
+	}
+
+	ACTOR static Future<Standalone<RangeResultRef>> getConfigClasses(SimpleConfigTransactionImpl* self) {
+		if (!self->getVersionFuture.isValid()) {
+			self->getVersionFuture = getReadVersion(self);
+		}
+		Version version = wait(self->getVersionFuture);
+		ConfigTransactionGetConfigClassesReply reply =
+		    wait(self->cti.getClasses.getReply(ConfigTransactionGetConfigClassesRequest{ version }));
+		Standalone<RangeResultRef> result;
+		for (const auto& configClass : reply.configClasses) {
+			result.push_back_deep(result.arena(), KeyValueRef(configClass, ""_sr));
+		}
+		return result;
+	}
+
+	ACTOR static Future<Standalone<RangeResultRef>> getKnobs(SimpleConfigTransactionImpl* self,
+	                                                         Optional<Key> configClass) {
+		if (!self->getVersionFuture.isValid()) {
+			self->getVersionFuture = getReadVersion(self);
+		}
+		Version version = wait(self->getVersionFuture);
+		ConfigTransactionGetKnobsReply reply =
+		    wait(self->cti.getKnobs.getReply(ConfigTransactionGetKnobsRequest{ version, configClass }));
+		Standalone<RangeResultRef> result;
+		for (const auto& knobName : reply.knobNames) {
+			result.push_back_deep(result.arena(), KeyValueRef(knobName, ""_sr));
+		}
+		return result;
+	}
+
+	ACTOR static Future<Void> commit(SimpleConfigTransactionImpl* self) {
+		if (!self->getVersionFuture.isValid()) {
+			self->getVersionFuture = getReadVersion(self);
+		}
+		wait(store(self->toCommit.version, self->getVersionFuture));
+		self->toCommit.annotation.timestamp = now();
+		wait(self->cti.commit.getReply(self->toCommit));
+		self->committed = true;
+		return Void();
+	}
+
+public:
+	SimpleConfigTransactionImpl(Database const& cx) : cx(cx) {
+		auto coordinators = cx->getConnectionFile()->getConnectionString().coordinators();
+		std::sort(coordinators.begin(), coordinators.end());
+		cti = ConfigTransactionInterface(coordinators[0]);
+	}
+
+	SimpleConfigTransactionImpl(ConfigTransactionInterface const& cti) : cti(cti) {}
+
+	void set(KeyRef key, ValueRef value) {
+		if (key == configTransactionDescriptionKey) {
+			toCommit.annotation.description = KeyRef(toCommit.arena, value);
+		} else {
+			ConfigKey configKey = ConfigKeyRef::decodeKey(key);
+			auto knobValue = IKnobCollection::parseKnobValue(
+			    configKey.knobName.toString(), value.toString(), IKnobCollection::Type::TEST);
+			toCommit.mutations.emplace_back_deep(toCommit.arena, configKey, knobValue.contents());
+		}
+	}
+
+	void clear(KeyRef key) {
+		if (key == configTransactionDescriptionKey) {
+			toCommit.annotation.description = ""_sr;
+		} else {
+			toCommit.mutations.emplace_back_deep(
+			    toCommit.arena, ConfigKeyRef::decodeKey(key), Optional<KnobValueRef>{});
+		}
+	}
+
+	Future<Optional<Value>> get(KeyRef key) { return get(this, key); }
+
+	Future<Standalone<RangeResultRef>> getRange(KeyRangeRef keys) {
+		if (keys == configClassKeys) {
+			return getConfigClasses(this);
+		} else if (keys == globalConfigKnobKeys) {
+			return getKnobs(this, {});
+		} else if (configKnobKeys.contains(keys) && keys.singleKeyRange()) {
+			const auto configClass = keys.begin.removePrefix(configKnobKeys.begin);
+			return getKnobs(this, configClass);
+		} else {
+			throw invalid_config_db_range_read();
+		}
+	}
+
+	Future<Void> commit() { return commit(this); }
+
+	Future<Void> onError(Error const& e) {
+		// TODO: Improve this:
+		if (e.code() == error_code_transaction_too_old) {
+			reset();
+			return delay((1 << numRetries++) * 0.01 * deterministicRandom()->random01());
+		}
+		throw e;
+	}
+
+	Future<Version> getReadVersion() {
+		if (!getVersionFuture.isValid())
+			getVersionFuture = getReadVersion(this);
+		return getVersionFuture;
+	}
+
+	Optional<Version> getCachedReadVersion() const {
+		if (getVersionFuture.isValid() && getVersionFuture.isReady() && !getVersionFuture.isError()) {
+			return getVersionFuture.get();
+		} else {
+			return {};
+		}
+	}
+
+	Version getCommittedVersion() const { return committed ? getVersionFuture.get() : ::invalidVersion; }
+
+	void reset() {
+		getVersionFuture = Future<Version>{};
+		toCommit = {};
+		committed = false;
+	}
+
+	void fullReset() {
+		numRetries = 0;
+		dID = {};
+		reset();
+	}
+
+	size_t getApproximateSize() const { return toCommit.expectedSize(); }
+
+	void debugTransaction(UID dID) {
+		this->dID = dID;
+	}
+
+	void checkDeferredError(Error const& deferredError) const {
+		if (deferredError.code() != invalid_error_code) {
+			throw deferredError;
+		}
+		if (cx.getPtr()) {
+			cx->checkDeferredError();
+		}
+	}
+}; // SimpleConfigTransactionImpl
+
+Future<Version> SimpleConfigTransaction::getReadVersion() {
+	return impl().getReadVersion();
+}
+
+Optional<Version> SimpleConfigTransaction::getCachedReadVersion() const {
+	return impl().getCachedReadVersion();
+}
+
+Future<Optional<Value>> SimpleConfigTransaction::get(Key const& key, bool snapshot) {
+	return impl().get(key);
+}
+
+Future<Standalone<RangeResultRef>> SimpleConfigTransaction::getRange(KeySelector const& begin,
+                                                                     KeySelector const& end,
+                                                                     int limit,
+                                                                     bool snapshot,
+                                                                     bool reverse) {
+	return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
+}
+
+Future<Standalone<RangeResultRef>> SimpleConfigTransaction::getRange(KeySelector begin,
+                                                                     KeySelector end,
+                                                                     GetRangeLimits limits,
+                                                                     bool snapshot,
+                                                                     bool reverse) {
+	return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey()));
+}
+
+void SimpleConfigTransaction::set(KeyRef const& key, ValueRef const& value) {
+	impl().set(key, value);
+}
+
+void SimpleConfigTransaction::clear(KeyRef const& key) {
+	impl().clear(key);
+}
+
+Future<Void> SimpleConfigTransaction::commit() {
+	return impl().commit();
+}
+
+Version SimpleConfigTransaction::getCommittedVersion() const {
+	return impl().getCommittedVersion();
+}
+
+int64_t SimpleConfigTransaction::getApproximateSize() const {
+	return impl().getApproximateSize();
+}
+
+void SimpleConfigTransaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
+	// TODO: Support using this option to determine atomicity
+}
+
+Future<Void> SimpleConfigTransaction::onError(Error const& e) {
+	return impl().onError(e);
+}
+
+void SimpleConfigTransaction::cancel() {
+	// TODO: Implement someday
+	throw client_invalid_operation();
+}
+
+void SimpleConfigTransaction::reset() {
+	return impl().reset();
+}
+
+void SimpleConfigTransaction::fullReset() {
+	return impl().fullReset();
+}
+
+void SimpleConfigTransaction::debugTransaction(UID dID) {
+	impl().debugTransaction(dID);
+}
+
+void SimpleConfigTransaction::checkDeferredError() const {
+	impl().checkDeferredError(deferredError);
+}
+
+SimpleConfigTransaction::SimpleConfigTransaction(Database const& cx)
+  : _impl(std::make_unique<SimpleConfigTransactionImpl>(cx)) {}
+
+SimpleConfigTransaction::SimpleConfigTransaction(ConfigTransactionInterface const& cti)
+  : _impl(std::make_unique<SimpleConfigTransactionImpl>(cti)) {}
+
+SimpleConfigTransaction::~SimpleConfigTransaction() = default;

fdbclient/SimpleConfigTransaction.h

@@ -0,0 +1,75 @@
+/*
+ * SimpleConfigTransaction.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "fdbclient/CommitTransaction.h"
+#include "fdbclient/ConfigTransactionInterface.h"
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbclient/IConfigTransaction.h"
+#include "flow/Error.h"
+#include "flow/flow.h"
+
+/*
+ * A configuration transaction implementation that interacts with a simple (test-only) implementation of
+ * the configuration database. All configuration database data is assumed to live on a single node
+ * (the lowest coordinator by IP address), so there is no fault tolerance.
+ */
+class SimpleConfigTransaction final : public IConfigTransaction, public FastAllocated<SimpleConfigTransaction> {
+	std::unique_ptr<class SimpleConfigTransactionImpl> _impl;
+	SimpleConfigTransactionImpl const& impl() const { return *_impl; }
+	SimpleConfigTransactionImpl& impl() { return *_impl; }
+
+public:
+	SimpleConfigTransaction(ConfigTransactionInterface const&);
+	SimpleConfigTransaction(Database const&);
+	~SimpleConfigTransaction();
+	Future<Version> getReadVersion() override;
+	Optional<Version> getCachedReadVersion() const override;
+
+	Future<Optional<Value>> get(Key const& key, bool snapshot = false) override;
+	Future<Standalone<RangeResultRef>> getRange(KeySelector const& begin,
+	                                            KeySelector const& end,
+	                                            int limit,
+	                                            bool snapshot = false,
+	                                            bool reverse = false) override;
+	Future<Standalone<RangeResultRef>> getRange(KeySelector begin,
+	                                            KeySelector end,
+	                                            GetRangeLimits limits,
+	                                            bool snapshot = false,
+	                                            bool reverse = false) override;
+	Future<Void> commit() override;
+	Version getCommittedVersion() const override;
+	void setOption(FDBTransactionOptions::Option option, Optional<StringRef> value = Optional<StringRef>()) override;
+	Future<Void> onError(Error const& e) override;
+	void cancel() override;
+	void reset() override;
+	void debugTransaction(UID dID) override;
+	void checkDeferredError() const override;
+	int64_t getApproximateSize() const override;
+	void set(KeyRef const&, ValueRef const&) override;
+	void clear(KeyRangeRef const&) override { throw client_invalid_operation(); }
+	void clear(KeyRef const&) override;
+
+	void fullReset();
+};

fdbclient/SystemData.cpp

@@ -1004,6 +1004,11 @@ const KeyRef writeRecoveryKey = LiteralStringRef("\xff/writeRecovery");
 const ValueRef writeRecoveryKeyTrue = LiteralStringRef("1");
 const KeyRef snapshotEndVersionKey = LiteralStringRef("\xff/snapshotEndVersion");
 
+const KeyRef configTransactionDescriptionKey = "\xff\xff/description"_sr;
+const KeyRange globalConfigKnobKeys = singleKeyRange("\xff\xff/globalKnobs"_sr);
+const KeyRangeRef configKnobKeys("\xff\xff/knobs/"_sr, "\xff\xff/knobs0"_sr);
+const KeyRangeRef configClassKeys("\xff\xff/configClasses/"_sr, "\xff\xff/configClasses0"_sr);
+
 // for tests
 void testSSISerdes(StorageServerInterface const& ssi, bool useFB) {
 	printf("ssi=\nid=%s\nlocality=%s\nisTss=%s\ntssId=%s\naddress=%s\ngetValue=%s\n\n\n",

fdbclient/SystemData.h

@@ -477,6 +477,12 @@ extern const ValueRef writeRecoveryKeyTrue;
 //	Allows incremental restore to read and set starting version for consistency.
 extern const KeyRef snapshotEndVersionKey;
 
+// Configuration database special keys
+extern const KeyRef configTransactionDescriptionKey;
+extern const KeyRange globalConfigKnobKeys;
+extern const KeyRangeRef configKnobKeys;
+extern const KeyRangeRef configClassKeys;
+
 #pragma clang diagnostic pop
 
 #endif

fdbclient/TestKnobCollection.cpp

@@ -0,0 +1,76 @@
+/*
+ * TestKnobCollection.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/TestKnobCollection.h"
+
+TestKnobCollection::TestKnobCollection(Randomize randomize, IsSimulated isSimulated)
+  : serverKnobCollection(randomize, isSimulated) {
+	initialize(randomize, isSimulated);
+}
+
+void TestKnobCollection::initialize(Randomize randomize, IsSimulated isSimulated) {
+	serverKnobCollection.initialize(randomize, isSimulated);
+	testKnobs.initialize();
+}
+
+void TestKnobCollection::reset(Randomize randomize, IsSimulated isSimulated) {
+	serverKnobCollection.reset(randomize, isSimulated);
+	testKnobs.reset();
+}
+
+Optional<KnobValue> TestKnobCollection::tryParseKnobValue(std::string const& knobName,
+                                                          std::string const& knobValue) const {
+	auto result = serverKnobCollection.tryParseKnobValue(knobName, knobValue);
+	if (result.present()) {
+		return result;
+	}
+	auto parsedKnobValue = testKnobs.parseKnobValue(knobName, knobValue);
+	if (!std::holds_alternative<NoKnobFound>(parsedKnobValue)) {
+		return KnobValueRef::create(parsedKnobValue);
+	}
+	return {};
+}
+
+bool TestKnobCollection::trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) {
+	return serverKnobCollection.trySetKnob(knobName, knobValue) || knobValue.visitSetKnob(knobName, testKnobs);
+}
+
+#define init(knob, value) initKnob(knob, value, #knob)
+
+TestKnobs::TestKnobs() {
+	initialize();
+}
+
+void TestKnobs::initialize() {
+	init(TEST_LONG, 0);
+	init(TEST_INT, 0);
+	init(TEST_DOUBLE, 0.0);
+	init(TEST_BOOL, false);
+	init(TEST_STRING, "");
+}
+
+bool TestKnobs::operator==(TestKnobs const& rhs) const {
+	return (TEST_LONG == rhs.TEST_LONG) && (TEST_INT == rhs.TEST_INT) && (TEST_DOUBLE == rhs.TEST_DOUBLE) &&
+	       (TEST_BOOL == rhs.TEST_BOOL) && (TEST_STRING == rhs.TEST_STRING);
+}
+
+bool TestKnobs::operator!=(TestKnobs const& rhs) const {
+	return !(*this == rhs);
+}

fdbclient/TestKnobCollection.h

@@ -0,0 +1,57 @@
+/*
+ * TestKnobCollection.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/IKnobCollection.h"
+#include "fdbclient/ServerKnobCollection.h"
+
+class TestKnobs : public KnobsImpl<TestKnobs> {
+public:
+	TestKnobs();
+	int64_t TEST_LONG;
+	int TEST_INT;
+	double TEST_DOUBLE;
+	bool TEST_BOOL;
+	std::string TEST_STRING;
+	bool operator==(TestKnobs const&) const;
+	bool operator!=(TestKnobs const&) const;
+	void initialize();
+};
+
+/*
+ * Stores both flow knobs, client knobs, server knobs, and test knobs. As the name implies, this class is only meant to
+ * be used for testing
+ */
+class TestKnobCollection : public IKnobCollection {
+	ServerKnobCollection serverKnobCollection;
+	TestKnobs testKnobs;
+
+public:
+	TestKnobCollection(Randomize randomize, IsSimulated isSimulated);
+	void initialize(Randomize randomize, IsSimulated isSimulated) override;
+	void reset(Randomize randomize, IsSimulated isSimulated) override;
+	FlowKnobs const& getFlowKnobs() const override { return serverKnobCollection.getFlowKnobs(); }
+	ClientKnobs const& getClientKnobs() const override { return serverKnobCollection.getClientKnobs(); }
+	ServerKnobs const& getServerKnobs() const override { return serverKnobCollection.getServerKnobs(); }
+	TestKnobs const& getTestKnobs() const override { return testKnobs; }
+	Optional<KnobValue> tryParseKnobValue(std::string const& knobName, std::string const& knobValue) const override;
+	bool trySetKnob(std::string const& knobName, KnobValueRef const& knobValue) override;
+};

fdbclient/ThreadSafeTransaction.cpp

@@ -19,7 +19,6 @@
  */
 
 #include "fdbclient/ThreadSafeTransaction.h"
-#include "fdbclient/ReadYourWrites.h"
 #include "fdbclient/DatabaseContext.h"
 #include "fdbclient/versions.h"
 #include "fdbclient/NativeAPI.actor.h"
@@ -46,7 +45,9 @@ ThreadFuture<Reference<IDatabase>> ThreadSafeDatabase::createFromExistingDatabas
 }
 
 Reference<ITransaction> ThreadSafeDatabase::createTransaction() {
-	return Reference<ITransaction>(new ThreadSafeTransaction(db));
+	auto type =
+		isConfigDB ? ISingleThreadTransaction::Type::SIMPLE_CONFIG : ISingleThreadTransaction::Type::RYW;
+	return Reference<ITransaction>(new ThreadSafeTransaction(db, type));
 }
 
 void ThreadSafeDatabase::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
@@ -57,6 +58,9 @@ void ThreadSafeDatabase::setOption(FDBDatabaseOptions::Option option, Optional<S
 		TraceEvent("UnknownDatabaseOption").detail("Option", option);
 		throw invalid_option();
 	}
+	if (itr->first == FDBDatabaseOptions::USE_CONFIG_DATABASE) {
+		isConfigDB = true;
+	}
 
 	DatabaseContext* db = this->db;
 	Standalone<Optional<StringRef>> passValue = value;
@@ -134,7 +138,7 @@ ThreadSafeDatabase::~ThreadSafeDatabase() {
 	onMainThreadVoid([db]() { db->delref(); }, nullptr);
 }
 
-ThreadSafeTransaction::ThreadSafeTransaction(DatabaseContext* cx) {
+ThreadSafeTransaction::ThreadSafeTransaction(DatabaseContext* cx, ISingleThreadTransaction::Type type) {
 	// Allocate memory for the transaction from this thread (so the pointer is known for subsequent method calls)
 	// but run its constructor on the main thread
 
@@ -142,12 +146,12 @@ ThreadSafeTransaction::ThreadSafeTransaction(DatabaseContext* cx) {
 	// because the reference count of the DatabaseContext is solely managed from the main thread.  If cx is destructed
 	// immediately after this call, it will defer the DatabaseContext::delref (and onMainThread preserves the order of
 	// these operations).
-	ReadYourWritesTransaction* tr = this->tr = ReadYourWritesTransaction::allocateOnForeignThread();
+	auto tr = this->tr = ISingleThreadTransaction::allocateOnForeignThread(type);
 	// No deferred error -- if the construction of the RYW transaction fails, we have no where to put it
 	onMainThreadVoid(
-	    [tr, cx]() {
+	    [tr, type, cx]() {
 		    cx->addref();
-		    new (tr) ReadYourWritesTransaction(Database(cx));
+		    ISingleThreadTransaction::create(tr, type, Database(cx));
 	    },
 	    nullptr);
 }
@@ -159,23 +163,23 @@ ThreadSafeTransaction::ThreadSafeTransaction(ReadYourWritesTransaction* ryw) : t
 }
 
 ThreadSafeTransaction::~ThreadSafeTransaction() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	if (tr)
 		onMainThreadVoid([tr]() { tr->delref(); }, nullptr);
 }
 
 void ThreadSafeTransaction::cancel() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr]() { tr->cancel(); }, nullptr);
 }
 
 void ThreadSafeTransaction::setVersion(Version v) {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, v]() { tr->setVersion(v); }, &tr->deferredError);
 }
 
 ThreadFuture<Version> ThreadSafeTransaction::getReadVersion() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr]() -> Future<Version> {
 		tr->checkDeferredError();
 		return tr->getReadVersion();
@@ -185,7 +189,7 @@ ThreadFuture<Version> ThreadSafeTransaction::getReadVersion() {
 ThreadFuture<Optional<Value>> ThreadSafeTransaction::get(const KeyRef& key, bool snapshot) {
 	Key k = key;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, k, snapshot]() -> Future<Optional<Value>> {
 		tr->checkDeferredError();
 		return tr->get(k, snapshot);
@@ -195,7 +199,7 @@ ThreadFuture<Optional<Value>> ThreadSafeTransaction::get(const KeyRef& key, bool
 ThreadFuture<Key> ThreadSafeTransaction::getKey(const KeySelectorRef& key, bool snapshot) {
 	KeySelector k = key;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, k, snapshot]() -> Future<Key> {
 		tr->checkDeferredError();
 		return tr->getKey(k, snapshot);
@@ -205,7 +209,7 @@ ThreadFuture<Key> ThreadSafeTransaction::getKey(const KeySelectorRef& key, bool
 ThreadFuture<int64_t> ThreadSafeTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) {
 	KeyRange r = keys;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, r]() -> Future<int64_t> {
 		tr->checkDeferredError();
 		return tr->getEstimatedRangeSizeBytes(r);
@@ -216,7 +220,7 @@ ThreadFuture<Standalone<VectorRef<KeyRef>>> ThreadSafeTransaction::getRangeSplit
                                                                                        int64_t chunkSize) {
 	KeyRange r = range;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, r, chunkSize]() -> Future<Standalone<VectorRef<KeyRef>>> {
 		tr->checkDeferredError();
 		return tr->getRangeSplitPoints(r, chunkSize);
@@ -231,7 +235,7 @@ ThreadFuture<RangeResult> ThreadSafeTransaction::getRange(const KeySelectorRef&
 	KeySelector b = begin;
 	KeySelector e = end;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, b, e, limit, snapshot, reverse]() -> Future<RangeResult> {
 		tr->checkDeferredError();
 		return tr->getRange(b, e, limit, snapshot, reverse);
@@ -246,7 +250,7 @@ ThreadFuture<RangeResult> ThreadSafeTransaction::getRange(const KeySelectorRef&
 	KeySelector b = begin;
 	KeySelector e = end;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, b, e, limits, snapshot, reverse]() -> Future<RangeResult> {
 		tr->checkDeferredError();
 		return tr->getRange(b, e, limits, snapshot, reverse);
@@ -256,7 +260,7 @@ ThreadFuture<RangeResult> ThreadSafeTransaction::getRange(const KeySelectorRef&
 ThreadFuture<Standalone<VectorRef<const char*>>> ThreadSafeTransaction::getAddressesForKey(const KeyRef& key) {
 	Key k = key;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, k]() -> Future<Standalone<VectorRef<const char*>>> {
 		tr->checkDeferredError();
 		return tr->getAddressesForKey(k);
@@ -266,12 +270,12 @@ ThreadFuture<Standalone<VectorRef<const char*>>> ThreadSafeTransaction::getAddre
 void ThreadSafeTransaction::addReadConflictRange(const KeyRangeRef& keys) {
 	KeyRange r = keys;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, r]() { tr->addReadConflictRange(r); }, &tr->deferredError);
 }
 
 void ThreadSafeTransaction::makeSelfConflicting() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr]() { tr->makeSelfConflicting(); }, &tr->deferredError);
 }
 
@@ -279,7 +283,7 @@ void ThreadSafeTransaction::atomicOp(const KeyRef& key, const ValueRef& value, u
 	Key k = key;
 	Value v = value;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, k, v, operationType]() { tr->atomicOp(k, v, operationType); }, &tr->deferredError);
 }
 
@@ -287,14 +291,14 @@ void ThreadSafeTransaction::set(const KeyRef& key, const ValueRef& value) {
 	Key k = key;
 	Value v = value;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, k, v]() { tr->set(k, v); }, &tr->deferredError);
 }
 
 void ThreadSafeTransaction::clear(const KeyRangeRef& range) {
 	KeyRange r = range;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, r]() { tr->clear(r); }, &tr->deferredError);
 }
 
@@ -302,7 +306,7 @@ void ThreadSafeTransaction::clear(const KeyRef& begin, const KeyRef& end) {
 	Key b = begin;
 	Key e = end;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid(
 	    [tr, b, e]() {
 		    if (b > e)
@@ -316,14 +320,14 @@ void ThreadSafeTransaction::clear(const KeyRef& begin, const KeyRef& end) {
 void ThreadSafeTransaction::clear(const KeyRef& key) {
 	Key k = key;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, k]() { tr->clear(k); }, &tr->deferredError);
 }
 
 ThreadFuture<Void> ThreadSafeTransaction::watch(const KeyRef& key) {
 	Key k = key;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, k]() -> Future<Void> {
 		tr->checkDeferredError();
 		return tr->watch(k);
@@ -333,12 +337,12 @@ ThreadFuture<Void> ThreadSafeTransaction::watch(const KeyRef& key) {
 void ThreadSafeTransaction::addWriteConflictRange(const KeyRangeRef& keys) {
 	KeyRange r = keys;
 
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr, r]() { tr->addWriteConflictRange(r); }, &tr->deferredError);
 }
 
 ThreadFuture<Void> ThreadSafeTransaction::commit() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr]() -> Future<Void> {
 		tr->checkDeferredError();
 		return tr->commit();
@@ -351,12 +355,12 @@ Version ThreadSafeTransaction::getCommittedVersion() {
 }
 
 ThreadFuture<int64_t> ThreadSafeTransaction::getApproximateSize() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr]() -> Future<int64_t> { return tr->getApproximateSize(); });
 }
 
 ThreadFuture<Standalone<StringRef>> ThreadSafeTransaction::getVersionstamp() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr]() -> Future<Standalone<StringRef>> { return tr->getVersionstamp(); });
 }
 
@@ -366,8 +370,7 @@ void ThreadSafeTransaction::setOption(FDBTransactionOptions::Option option, Opti
 		TraceEvent("UnknownTransactionOption").detail("Option", option);
 		throw invalid_option();
 	}
-
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	Standalone<Optional<StringRef>> passValue = value;
 
 	// ThreadSafeTransaction is not allowed to do anything with options except pass them through to RYW.
@@ -375,7 +378,7 @@ void ThreadSafeTransaction::setOption(FDBTransactionOptions::Option option, Opti
 }
 
 ThreadFuture<Void> ThreadSafeTransaction::checkDeferredError() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr]() {
 		try {
 			tr->checkDeferredError();
@@ -388,7 +391,7 @@ ThreadFuture<Void> ThreadSafeTransaction::checkDeferredError() {
 }
 
 ThreadFuture<Void> ThreadSafeTransaction::onError(Error const& e) {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	return onMainThread([tr, e]() { return tr->onError(e); });
 }
 
@@ -403,7 +406,7 @@ ThreadSafeTransaction::ThreadSafeTransaction(ThreadSafeTransaction&& r) noexcept
 }
 
 void ThreadSafeTransaction::reset() {
-	ReadYourWritesTransaction* tr = this->tr;
+	ISingleThreadTransaction* tr = this->tr;
 	onMainThreadVoid([tr]() { tr->reset(); }, nullptr);
 }
 

fdbclient/ThreadSafeTransaction.h

@@ -26,6 +26,7 @@
 #include "flow/ThreadHelper.actor.h"
 #include "fdbclient/ClusterInterface.h"
 #include "fdbclient/IClientApi.h"
+#include "fdbclient/ISingleThreadTransaction.h"
 
 // An implementation of IDatabase that serializes operations onto the network thread and interacts with the lower-level
 // client APIs exposed by NativeAPI and ReadYourWrites.
@@ -58,6 +59,7 @@ public:
 
 private:
 	friend class ThreadSafeTransaction;
+	bool isConfigDB { false };
 	DatabaseContext* db;
 
 public: // Internal use only
@@ -67,10 +69,10 @@ public: // Internal use only
 };
 
 // An implementation of ITransaction that serializes operations onto the network thread and interacts with the
-// lower-level client APIs exposed by NativeAPI and ReadYourWrites.
+// lower-level client APIs exposed by ISingleThreadTransaction
 class ThreadSafeTransaction : public ITransaction, ThreadSafeReferenceCounted<ThreadSafeTransaction>, NonCopyable {
 public:
-	explicit ThreadSafeTransaction(DatabaseContext* cx);
+	explicit ThreadSafeTransaction(DatabaseContext* cx, ISingleThreadTransaction::Type type);
 	~ThreadSafeTransaction() override;
 
 	// Note: used while refactoring fdbcli, need to be removed later
@@ -145,7 +147,7 @@ public:
 	void delref() override { ThreadSafeReferenceCounted<ThreadSafeTransaction>::delref(); }
 
 private:
-	ReadYourWritesTransaction* tr;
+	ISingleThreadTransaction* tr;
 };
 
 // An implementation of IClientApi that serializes operations onto the network thread and interacts with the lower-level

fdbclient/vexillographer/fdb.options

@@ -195,6 +195,8 @@ description is not currently required but encouraged.
     <Option name="transaction_bypass_unreadable" code="700"
             description="Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. This sets the ``bypass_unreadable`` option of each transaction created by this database. See the transaction option description for more information."
             defaultFor="1100"/>
+    <Option name="use_config_database" code="800"
+            description="Use configuration database." />
   </Scope>
   
   <Scope name="TransactionOption">

fdbrpc/FlowTransport.actor.cpp

@@ -51,7 +51,7 @@ constexpr UID WLTOKEN_PING_PACKET(-1, 1);
 constexpr int PACKET_LEN_WIDTH = sizeof(uint32_t);
 const uint64_t TOKEN_STREAM_FLAG = 1;
 
-const int WLTOKEN_COUNTS = 12; // number of wellKnownEndpoints
+static constexpr int WLTOKEN_COUNTS = 20; // number of wellKnownEndpoints
 
 class EndpointMap : NonCopyable {
 public:
@@ -158,7 +158,7 @@ const Endpoint& EndpointMap::insert(NetworkAddressList localAddresses,
 NetworkMessageReceiver* EndpointMap::get(Endpoint::Token const& token) {
 	uint32_t index = token.second();
 	if (index < wellKnownEndpointCount && data[index].receiver == nullptr) {
-		TraceEvent(SevWarnAlways, "WellKnownEndpointNotAdded").detail("Token", token);
+		TraceEvent(SevWarnAlways, "WellKnownEndpointNotAdded").detail("Token", token).detail("Index", index).backtrace();
 	}
 	if (index < data.size() && data[index].token().first() == token.first() &&
 	    ((data[index].token().second() & 0xffffffff00000000LL) | index) == token.second())

fdbserver/CMakeLists.txt

@@ -6,6 +6,11 @@ set(FDBSERVER_SRCS
   BackupProgress.actor.h
   BackupWorker.actor.cpp
   ClusterController.actor.cpp
+  ConfigBroadcaster.actor.cpp
+  ConfigBroadcaster.h
+  ConfigDatabaseUnitTests.actor.cpp
+  ConfigFollowerInterface.cpp
+  ConfigFollowerInterface.h
   ConflictSet.h
   CoordinatedState.actor.cpp
   CoordinatedState.h
@@ -23,6 +28,10 @@ set(FDBSERVER_SRCS
   FDBExecHelper.actor.cpp
   FDBExecHelper.actor.h
   GrvProxyServer.actor.cpp
+  IConfigDatabaseNode.cpp
+  IConfigDatabaseNode.h
+  IConfigConsumer.cpp
+  IConfigConsumer.h
   IDiskQueue.h
   IKeyValueContainer.h
   IKeyValueStore.h
@@ -31,12 +40,13 @@ set(FDBSERVER_SRCS
   KeyValueStoreMemory.actor.cpp
   KeyValueStoreRocksDB.actor.cpp
   KeyValueStoreSQLite.actor.cpp
-  Knobs.cpp
   Knobs.h
   LatencyBandConfig.cpp
   LatencyBandConfig.h
   LeaderElection.actor.cpp
   LeaderElection.h
+  LocalConfiguration.actor.cpp
+  LocalConfiguration.h
   LogProtocolMessage.h
   LogRouter.actor.cpp
   LogSystem.h
@@ -58,7 +68,13 @@ set(FDBSERVER_SRCS
   OldTLogServer_4_6.actor.cpp
   OldTLogServer_6_0.actor.cpp
   OldTLogServer_6_2.actor.cpp
+  OnDemandStore.actor.cpp
+  OnDemandStore.h
   Orderer.actor.h
+  PaxosConfigConsumer.actor.cpp
+  PaxosConfigConsumer.h
+  PaxosConfigDatabaseNode.actor.cpp
+  PaxosConfigDatabaseNode.h
   ProxyCommitData.actor.h
   pubsub.actor.cpp
   pubsub.h
@@ -88,6 +104,9 @@ set(FDBSERVER_SRCS
   ResolverInterface.h
   ServerDBInfo.actor.h
   ServerDBInfo.h
+  SimpleConfigConsumer.actor.cpp
+  SimpleConfigConsumer.h
+  SimpleConfigDatabaseNode.actor.cpp
   SimulatedCluster.actor.cpp
   SimulatedCluster.h
   SkipList.cpp

fdbserver/ClusterController.actor.cpp

@@ -31,6 +31,7 @@
 #include "fdbserver/CoordinationInterface.h"
 #include "fdbserver/DataDistributorInterface.h"
 #include "fdbserver/Knobs.h"
+#include "fdbserver/ConfigBroadcaster.h"
 #include "fdbserver/MoveKeys.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/LeaderElection.h"
@@ -2765,7 +2766,9 @@ public:
 	Counter registerMasterRequests;
 	Counter statusRequests;
 
-	ClusterControllerData(ClusterControllerFullInterface const& ccInterface, LocalityData const& locality)
+	ClusterControllerData(ClusterControllerFullInterface const& ccInterface,
+	                      LocalityData const& locality,
+	                      ServerCoordinators const& coordinators)
 	  : clusterControllerProcessId(locality.processId()), clusterControllerDcId(locality.dcId()), id(ccInterface.id()),
 	    ac(false), outstandingRequestChecker(Void()), outstandingRemoteRequestChecker(Void()), gotProcessClasses(false),
 	    gotFullyRecoveredConfig(false), startTime(now()), goodRecruitmentTime(Never()),
@@ -2853,6 +2856,7 @@ ACTOR Future<Void> clusterWatchDatabase(ClusterControllerData* cluster, ClusterC
 				dbInfo.distributor = db->serverInfo->get().distributor;
 				dbInfo.ratekeeper = db->serverInfo->get().ratekeeper;
 				dbInfo.latencyBandConfig = db->serverInfo->get().latencyBandConfig;
+				dbInfo.configBroadcaster = db->serverInfo->get().configBroadcaster;
 
 				TraceEvent("CCWDB", cluster->id)
 				    .detail("Lifetime", dbInfo.masterLifetime.toString())
@@ -3669,7 +3673,8 @@ ACTOR Future<Void> timeKeeper(ClusterControllerData* self) {
 
 ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
                                 ClusterControllerData* self,
-                                ServerCoordinators coordinators) {
+                                ServerCoordinators coordinators,
+                                ConfigBroadcaster const* configBroadcaster) {
 	// Seconds since the END of the last GetStatus executed
 	state double last_request_time = 0.0;
 
@@ -3736,7 +3741,8 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
 			                                                                  &self->db.clientStatus,
 			                                                                  coordinators,
 			                                                                  incompatibleConnections,
-			                                                                  self->datacenterVersionDifference)));
+			                                                                  self->datacenterVersionDifference,
+			                                                                  configBroadcaster)));
 
 			if (result.isError() && result.getError().code() == error_code_actor_cancelled)
 				throw result.getError();
@@ -4424,15 +4430,23 @@ ACTOR Future<Void> dbInfoUpdater(ClusterControllerData* self) {
 ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
                                          Future<Void> leaderFail,
                                          ServerCoordinators coordinators,
-                                         LocalityData locality) {
-	state ClusterControllerData self(interf, locality);
+                                         LocalityData locality,
+                                         UseConfigDB useConfigDB) {
+	state ClusterControllerData self(interf, locality, coordinators);
+	state ConfigBroadcaster configBroadcaster(coordinators, useConfigDB);
 	state Future<Void> coordinationPingDelay = delay(SERVER_KNOBS->WORKER_COORDINATION_PING_DELAY);
 	state uint64_t step = 0;
 	state Future<ErrorOr<Void>> error = errorOr(actorCollection(self.addActor.getFuture()));
 
+	if (useConfigDB != UseConfigDB::DISABLED) {
+		self.addActor.send(configBroadcaster.serve(self.db.serverInfo->get().configBroadcaster));
+	}
 	self.addActor.send(clusterWatchDatabase(&self, &self.db)); // Start the master database
 	self.addActor.send(self.updateWorkerList.init(self.db.db));
-	self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(), &self, coordinators));
+	self.addActor.send(statusServer(interf.clientInterface.databaseStatus.getFuture(),
+	                                &self,
+	                                coordinators,
+	                                (useConfigDB == UseConfigDB::DISABLED) ? nullptr : &configBroadcaster));
 	self.addActor.send(timeKeeper(&self));
 	self.addActor.send(monitorProcessClasses(&self));
 	self.addActor.send(monitorServerInfoConfig(&self.db));
@@ -4550,7 +4564,8 @@ ACTOR Future<Void> clusterController(ServerCoordinators coordinators,
                                      Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
                                      bool hasConnected,
                                      Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
-                                     LocalityData locality) {
+                                     LocalityData locality,
+                                     UseConfigDB useConfigDB) {
 	loop {
 		state ClusterControllerFullInterface cci;
 		state bool inRole = false;
@@ -4577,7 +4592,7 @@ ACTOR Future<Void> clusterController(ServerCoordinators coordinators,
 				startRole(Role::CLUSTER_CONTROLLER, cci.id(), UID());
 				inRole = true;
 
-				wait(clusterControllerCore(cci, leaderFail, coordinators, locality));
+				wait(clusterControllerCore(cci, leaderFail, coordinators, locality, useConfigDB));
 			}
 		} catch (Error& e) {
 			if (inRole)
@@ -4600,13 +4615,14 @@ ACTOR Future<Void> clusterController(Reference<ClusterConnectionFile> connFile,
                                      Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
                                      Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
                                      Future<Void> recoveredDiskFiles,
-                                     LocalityData locality) {
+                                     LocalityData locality,
+                                     UseConfigDB useConfigDB) {
 	wait(recoveredDiskFiles);
 	state bool hasConnected = false;
 	loop {
 		try {
 			ServerCoordinators coordinators(connFile);
-			wait(clusterController(coordinators, currentCC, hasConnected, asyncPriorityInfo, locality));
+			wait(clusterController(coordinators, currentCC, hasConnected, asyncPriorityInfo, locality, useConfigDB));
 		} catch (Error& e) {
 			if (e.code() != error_code_coordinators_changed)
 				throw; // Expected to terminate fdbserver

fdbserver/ConfigBroadcastFollowerInterface.h

@@ -0,0 +1,139 @@
+/*
+ * ConfigBroadcastFollowerInterface.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/ConfigFollowerInterface.h"
+
+class ConfigClassSet {
+	std::set<Key> classes;
+
+public:
+	static constexpr FileIdentifier file_identifier = 9854021;
+
+	bool operator==(ConfigClassSet const& rhs) const { return classes == rhs.classes; }
+	bool operator!=(ConfigClassSet const& rhs) const { return !(*this == rhs); }
+
+	ConfigClassSet() = default;
+	ConfigClassSet(VectorRef<KeyRef> configClasses) {
+		for (const auto& configClass : configClasses) {
+			classes.insert(configClass);
+		}
+	}
+
+	bool contains(KeyRef configClass) const { return classes.count(configClass); }
+	std::set<Key> const& getClasses() const { return classes; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, classes);
+	}
+};
+
+template <>
+struct Traceable<ConfigClassSet> : std::true_type {
+	static std::string toString(ConfigClassSet const& value) { return describe(value.getClasses()); }
+};
+
+struct ConfigBroadcastFollowerGetSnapshotReply {
+	static constexpr FileIdentifier file_identifier = 8701983;
+	Version version{ 0 };
+	std::map<ConfigKey, KnobValue> snapshot;
+
+	ConfigBroadcastFollowerGetSnapshotReply() = default;
+	template <class Snapshot>
+	explicit ConfigBroadcastFollowerGetSnapshotReply(Version version, Snapshot&& snapshot)
+	  : version(version), snapshot(std::forward<Snapshot>(snapshot)) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, snapshot);
+	}
+};
+
+struct ConfigBroadcastFollowerGetSnapshotRequest {
+	static constexpr FileIdentifier file_identifier = 10911924;
+	ConfigClassSet configClassSet;
+	ReplyPromise<ConfigBroadcastFollowerGetSnapshotReply> reply;
+
+	ConfigBroadcastFollowerGetSnapshotRequest() = default;
+	explicit ConfigBroadcastFollowerGetSnapshotRequest(ConfigClassSet const& configClassSet)
+	  : configClassSet(configClassSet) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, configClassSet, reply);
+	}
+};
+
+struct ConfigBroadcastFollowerGetChangesReply {
+	static constexpr FileIdentifier file_identifier = 4014927;
+	Version mostRecentVersion;
+	Standalone<VectorRef<VersionedConfigMutationRef>> changes;
+
+	ConfigBroadcastFollowerGetChangesReply()=default;
+	explicit ConfigBroadcastFollowerGetChangesReply(Version mostRecentVersion, Standalone<VectorRef<VersionedConfigMutationRef>> const& changes)
+		: mostRecentVersion(mostRecentVersion), changes(changes) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, mostRecentVersion, changes);
+	}
+};
+
+struct ConfigBroadcastFollowerGetChangesRequest {
+	static constexpr FileIdentifier file_identifier = 601280;
+	Version lastSeenVersion;
+	ConfigClassSet configClassSet;
+	ReplyPromise<ConfigBroadcastFollowerGetChangesReply> reply;
+
+	ConfigBroadcastFollowerGetChangesRequest() = default;
+	explicit ConfigBroadcastFollowerGetChangesRequest(Version lastSeenVersion, ConfigClassSet const& configClassSet)
+	  : lastSeenVersion(lastSeenVersion), configClassSet(configClassSet) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, lastSeenVersion, configClassSet, reply);
+	}
+};
+
+/*
+ * The ConfigBroadcaster serves a ConfigBroadcastFollowerInterface which all
+ * workers use to fetch updates.
+ */
+class ConfigBroadcastFollowerInterface {
+	UID _id;
+
+public:
+	static constexpr FileIdentifier file_identifier = 1984391;
+	RequestStream<ConfigBroadcastFollowerGetSnapshotRequest> getSnapshot;
+	RequestStream<ConfigBroadcastFollowerGetChangesRequest> getChanges;
+
+	ConfigBroadcastFollowerInterface() : _id(deterministicRandom()->randomUniqueID()) {}
+
+	bool operator==(ConfigBroadcastFollowerInterface const& rhs) const { return (_id == rhs._id); }
+	bool operator!=(ConfigBroadcastFollowerInterface const& rhs) const { return !(*this == rhs); }
+	UID id() const { return _id; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, _id, getSnapshot, getChanges);
+	}
+};

fdbserver/ConfigBroadcaster.actor.cpp

@@ -0,0 +1,478 @@
+/*
+ * ConfigBroadcaster.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+
+#include "fdbserver/ConfigBroadcaster.h"
+#include "fdbserver/Knobs.h"
+#include "fdbserver/IConfigConsumer.h"
+#include "flow/UnitTest.h"
+#include "flow/actorcompiler.h" // must be last include
+
+namespace {
+
+bool matchesConfigClass(ConfigClassSet const& configClassSet, Optional<KeyRef> configClass) {
+	return !configClass.present() || configClassSet.contains(configClass.get());
+}
+
+// Helper functions for STL containers, with flow-friendly error handling
+template <class MapContainer, class K>
+auto get(MapContainer& m, K const& k) -> decltype(m.at(k)) {
+	auto it = m.find(k);
+	ASSERT(it != m.end());
+	return it->second;
+}
+template <class Container, class K>
+void remove(Container& container, K const& k) {
+	auto it = container.find(k);
+	ASSERT(it != container.end());
+	container.erase(it);
+}
+
+} // namespace
+
+class ConfigBroadcasterImpl {
+	// PendingRequestStore stores a set of pending ConfigBroadcastFollowerGetChangesRequests,
+	// indexed by configuration class. When an update is received, replies are sent for all
+	// pending requests with affected configuration classes
+	class PendingRequestStore {
+		using Req = ConfigBroadcastFollowerGetChangesRequest;
+		std::map<Key, std::set<Endpoint::Token>> configClassToTokens;
+		std::map<Endpoint::Token, Req> tokenToRequest;
+
+	public:
+		void addRequest(Req const& req) {
+			auto token = req.reply.getEndpoint().token;
+			tokenToRequest[token] = req;
+			for (const auto& configClass : req.configClassSet.getClasses()) {
+				configClassToTokens[configClass].insert(token);
+			}
+		}
+
+		std::vector<Req> getRequestsToNotify(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes) const {
+			std::set<Endpoint::Token> tokenSet;
+			for (const auto& change : changes) {
+				if (!change.mutation.getConfigClass().present()) {
+					// Update everything
+					for (const auto& [token, req] : tokenToRequest) {
+						if (req.lastSeenVersion < change.version) {
+							tokenSet.insert(token);
+						}
+					}
+				} else {
+					Key configClass = change.mutation.getConfigClass().get();
+					if (configClassToTokens.count(configClass)) {
+						auto tokens = get(configClassToTokens, Key(change.mutation.getConfigClass().get()));
+						for (const auto& token : tokens) {
+							auto req = get(tokenToRequest, token);
+							if (req.lastSeenVersion < change.version) {
+								tokenSet.insert(token);
+							} else {
+								TEST(true); // Worker is ahead of config broadcaster
+							}
+						}
+					}
+				}
+			}
+			std::vector<Req> result;
+			for (const auto& token : tokenSet) {
+				result.push_back(get(tokenToRequest, token));
+			}
+			return result;
+		}
+
+		std::vector<Req> getOutdatedRequests(Version newSnapshotVersion) {
+			std::vector<Req> result;
+			for (const auto& [token, req] : tokenToRequest) {
+				if (req.lastSeenVersion < newSnapshotVersion) {
+					result.push_back(req);
+				}
+			}
+			return result;
+		}
+
+		void removeRequest(Req const& req) {
+			auto token = req.reply.getEndpoint().token;
+			for (const auto& configClass : req.configClassSet.getClasses()) {
+				remove(get(configClassToTokens, configClass), token);
+				// TODO: Don't leak config classes
+			}
+			remove(tokenToRequest, token);
+		}
+	} pending;
+	std::map<ConfigKey, KnobValue> snapshot;
+	std::deque<VersionedConfigMutation> mutationHistory;
+	std::deque<VersionedConfigCommitAnnotation> annotationHistory;
+	Version lastCompactedVersion;
+	Version mostRecentVersion;
+	std::unique_ptr<IConfigConsumer> consumer;
+	ActorCollection actors{ false };
+
+	UID id;
+	CounterCollection cc;
+	Counter compactRequest;
+	mutable Counter successfulChangeRequest;
+	Counter failedChangeRequest;
+	Counter snapshotRequest;
+	Future<Void> logger;
+
+	template <class Changes>
+	void sendChangesReply(ConfigBroadcastFollowerGetChangesRequest const& req, Changes const& changes) const {
+		ASSERT_LT(req.lastSeenVersion, mostRecentVersion);
+		ConfigBroadcastFollowerGetChangesReply reply;
+		reply.mostRecentVersion = mostRecentVersion;
+		for (const auto& versionedMutation : changes) {
+			if (versionedMutation.version > req.lastSeenVersion &&
+			    matchesConfigClass(req.configClassSet, versionedMutation.mutation.getConfigClass())) {
+				TraceEvent te(SevDebug, "ConfigBroadcasterSendingChangeMutation", id);
+				te.detail("Version", versionedMutation.version)
+				    .detail("ReqLastSeenVersion", req.lastSeenVersion)
+				    .detail("ConfigClass", versionedMutation.mutation.getConfigClass())
+				    .detail("KnobName", versionedMutation.mutation.getKnobName());
+				if (versionedMutation.mutation.isSet()) {
+					te.detail("Op", "Set").detail("KnobValue", versionedMutation.mutation.getValue().toString());
+				} else {
+					te.detail("Op", "Clear");
+				}
+
+				reply.changes.push_back_deep(reply.changes.arena(), versionedMutation);
+			}
+		}
+		req.reply.send(reply);
+		++successfulChangeRequest;
+	}
+
+	ACTOR static Future<Void> serve(ConfigBroadcaster* self,
+	                                ConfigBroadcasterImpl* impl,
+	                                ConfigBroadcastFollowerInterface cbfi) {
+		impl->actors.add(impl->consumer->consume(*self));
+		loop {
+			choose {
+				when(ConfigBroadcastFollowerGetSnapshotRequest req = waitNext(cbfi.getSnapshot.getFuture())) {
+					++impl->snapshotRequest;
+					ConfigBroadcastFollowerGetSnapshotReply reply;
+					for (const auto& [key, value] : impl->snapshot) {
+						if (matchesConfigClass(req.configClassSet, key.configClass)) {
+							reply.snapshot[key] = value;
+						}
+					}
+					reply.version = impl->mostRecentVersion;
+					TraceEvent(SevDebug, "ConfigBroadcasterGotSnapshotRequest", impl->id)
+					    .detail("Size", reply.snapshot.size())
+					    .detail("Version", reply.version);
+					req.reply.send(reply);
+				}
+				when(ConfigBroadcastFollowerGetChangesRequest req = waitNext(cbfi.getChanges.getFuture())) {
+					if (req.lastSeenVersion < impl->lastCompactedVersion) {
+						req.reply.sendError(version_already_compacted());
+						++impl->failedChangeRequest;
+						continue;
+					}
+					if (req.lastSeenVersion < impl->mostRecentVersion) {
+						impl->sendChangesReply(req, impl->mutationHistory);
+					} else {
+						TEST(req.lastSeenVersion > impl->mostRecentVersion); // Worker is ahead of ConfigBroadcaster
+						TraceEvent(SevDebug, "ConfigBroadcasterRegisteringChangeRequest", impl->id)
+						    .detail("Peer", req.reply.getEndpoint().getPrimaryAddress())
+						    .detail("MostRecentVersion", impl->mostRecentVersion)
+						    .detail("ReqLastSeenVersion", req.lastSeenVersion)
+						    .detail("ConfigClass", req.configClassSet);
+						impl->pending.addRequest(req);
+					}
+				}
+				when(wait(impl->actors.getResult())) { ASSERT(false); }
+			}
+		}
+	}
+
+	ConfigBroadcasterImpl()
+	  : id(deterministicRandom()->randomUniqueID()), lastCompactedVersion(0), mostRecentVersion(0),
+	    cc("ConfigBroadcaster"), compactRequest("CompactRequest", cc),
+	    successfulChangeRequest("SuccessfulChangeRequest", cc), failedChangeRequest("FailedChangeRequest", cc),
+	    snapshotRequest("SnapshotRequest", cc) {
+		logger = traceCounters(
+		    "ConfigBroadcasterMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ConfigBroadcasterMetrics");
+	}
+
+	void notifyFollowers(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes) {
+		auto toNotify = pending.getRequestsToNotify(changes);
+		TraceEvent(SevDebug, "ConfigBroadcasterNotifyingFollowers", id)
+		    .detail("ChangesSize", changes.size())
+		    .detail("ToNotify", toNotify.size());
+		for (auto& req : toNotify) {
+			sendChangesReply(req, changes);
+			pending.removeRequest(req);
+		}
+	}
+
+	void notifyOutdatedRequests() {
+		auto outdated = pending.getOutdatedRequests(mostRecentVersion);
+		for (auto& req : outdated) {
+			req.reply.sendError(version_already_compacted());
+			pending.removeRequest(req);
+		}
+	}
+
+	void addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                Version mostRecentVersion,
+	                Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations) {
+		this->mostRecentVersion = mostRecentVersion;
+		mutationHistory.insert(mutationHistory.end(), changes.begin(), changes.end());
+		annotationHistory.insert(annotationHistory.end(), annotations.begin(), annotations.end());
+		for (const auto& change : changes) {
+			const auto& mutation = change.mutation;
+			if (mutation.isSet()) {
+				snapshot[mutation.getKey()] = mutation.getValue();
+			} else {
+				snapshot.erase(mutation.getKey());
+			}
+		}
+	}
+
+	template <class Snapshot>
+	Future<Void> setSnapshot(Snapshot&& snapshot, Version snapshotVersion) {
+		this->snapshot = std::forward<Snapshot>(snapshot);
+		this->lastCompactedVersion = snapshotVersion;
+		return Void();
+	}
+
+public:
+	Future<Void> serve(ConfigBroadcaster* self, ConfigBroadcastFollowerInterface const& cbfi) {
+		return serve(self, this, cbfi);
+	}
+
+	void applyChanges(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                  Version mostRecentVersion,
+	                  Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations) {
+		TraceEvent(SevDebug, "ConfigBroadcasterApplyingChanges", id)
+		    .detail("ChangesSize", changes.size())
+		    .detail("CurrentMostRecentVersion", this->mostRecentVersion)
+		    .detail("NewMostRecentVersion", mostRecentVersion)
+		    .detail("AnnotationsSize", annotations.size());
+		addChanges(changes, mostRecentVersion, annotations);
+		notifyFollowers(changes);
+	}
+
+	template <class Snapshot>
+	void applySnapshotAndChanges(Snapshot&& snapshot,
+	                             Version snapshotVersion,
+	                             Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                             Version changesVersion,
+	                             Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations) {
+		TraceEvent(SevDebug, "ConfigBroadcasterApplyingSnapshotAndChanges", id)
+		    .detail("CurrentMostRecentVersion", this->mostRecentVersion)
+		    .detail("SnapshotSize", snapshot.size())
+		    .detail("SnapshotVersion", snapshotVersion)
+		    .detail("ChangesSize", changes.size())
+		    .detail("ChangesVersion", changesVersion)
+		    .detail("AnnotationsSize", annotations.size());
+		setSnapshot(std::forward<Snapshot>(snapshot), snapshotVersion);
+		addChanges(changes, changesVersion, annotations);
+		notifyOutdatedRequests();
+	}
+
+	ConfigBroadcasterImpl(ConfigFollowerInterface const& cfi) : ConfigBroadcasterImpl() {
+		consumer = IConfigConsumer::createTestSimple(cfi, 0.5, Optional<double>{});
+		TraceEvent(SevDebug, "ConfigBroadcasterStartingConsumer", id).detail("Consumer", consumer->getID());
+	}
+
+	ConfigBroadcasterImpl(ServerCoordinators const& coordinators, UseConfigDB useConfigDB) : ConfigBroadcasterImpl() {
+		if (useConfigDB != UseConfigDB::DISABLED) {
+			if (useConfigDB == UseConfigDB::SIMPLE) {
+				consumer = IConfigConsumer::createSimple(coordinators, 0.5, Optional<double>{});
+			} else {
+				consumer = IConfigConsumer::createPaxos(coordinators, 0.5, Optional<double>{});
+			}
+			TraceEvent(SevDebug, "BroadcasterStartingConsumer", id)
+			    .detail("Consumer", consumer->getID())
+			    .detail("UsingSimpleConsumer", useConfigDB == UseConfigDB::SIMPLE);
+		}
+	}
+
+	JsonBuilderObject getStatus() const {
+		JsonBuilderObject result;
+		JsonBuilderArray mutationsArray;
+		for (const auto& versionedMutation : mutationHistory) {
+			JsonBuilderObject mutationObject;
+			mutationObject["version"] = versionedMutation.version;
+			const auto& mutation = versionedMutation.mutation;
+			mutationObject["config_class"] = mutation.getConfigClass().orDefault("<global>"_sr);
+			mutationObject["knob_name"] = mutation.getKnobName();
+			mutationObject["knob_value"] = mutation.getValue().toString();
+			mutationsArray.push_back(std::move(mutationObject));
+		}
+		result["mutations"] = std::move(mutationsArray);
+		JsonBuilderArray commitsArray;
+		for (const auto& versionedAnnotation : annotationHistory) {
+			JsonBuilderObject commitObject;
+			commitObject["version"] = versionedAnnotation.version;
+			commitObject["description"] = versionedAnnotation.annotation.description;
+			commitObject["timestamp"] = versionedAnnotation.annotation.timestamp;
+			commitsArray.push_back(std::move(commitObject));
+		}
+		result["commits"] = std::move(commitsArray);
+		JsonBuilderObject snapshotObject;
+		std::map<Optional<Key>, std::vector<std::pair<Key, Value>>> snapshotMap;
+		for (const auto& [configKey, value] : snapshot) {
+			snapshotMap[configKey.configClass.castTo<Key>()].emplace_back(configKey.knobName, value.toString());
+		}
+		for (const auto& [configClass, kvs] : snapshotMap) {
+			JsonBuilderObject kvsObject;
+			for (const auto& [knobName, knobValue] : kvs) {
+				kvsObject[knobName] = knobValue;
+			}
+			snapshotObject[configClass.orDefault("<global>"_sr)] = std::move(kvsObject);
+		}
+		result["snapshot"] = std::move(snapshotObject);
+		result["last_compacted_version"] = lastCompactedVersion;
+		result["most_recent_version"] = mostRecentVersion;
+		return result;
+	}
+
+	void compact(Version compactionVersion) {
+		{
+			auto it = std::find_if(mutationHistory.begin(), mutationHistory.end(), [compactionVersion](const auto& vm) {
+				return vm.version > compactionVersion;
+			});
+			mutationHistory.erase(mutationHistory.begin(), it);
+		}
+		{
+			auto it = std::find_if(annotationHistory.begin(),
+			                       annotationHistory.end(),
+			                       [compactionVersion](const auto& va) { return va.version > compactionVersion; });
+			annotationHistory.erase(annotationHistory.begin(), it);
+		}
+	}
+
+	UID getID() const { return id; }
+
+	static void runPendingRequestStoreTest(bool includeGlobalMutation, int expectedMatches);
+};
+
+ConfigBroadcaster::ConfigBroadcaster(ConfigFollowerInterface const& cfi)
+  : _impl(std::make_unique<ConfigBroadcasterImpl>(cfi)) {}
+
+ConfigBroadcaster::ConfigBroadcaster(ServerCoordinators const& coordinators, UseConfigDB useConfigDB)
+  : _impl(std::make_unique<ConfigBroadcasterImpl>(coordinators, useConfigDB)) {}
+
+ConfigBroadcaster::ConfigBroadcaster(ConfigBroadcaster&&) = default;
+
+ConfigBroadcaster& ConfigBroadcaster::operator=(ConfigBroadcaster&&) = default;
+
+ConfigBroadcaster::~ConfigBroadcaster() = default;
+
+Future<Void> ConfigBroadcaster::serve(ConfigBroadcastFollowerInterface const& cbfi) {
+	return impl().serve(this, cbfi);
+}
+
+void ConfigBroadcaster::applyChanges(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+                                     Version mostRecentVersion,
+                                     Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations) {
+	impl().applyChanges(changes, mostRecentVersion, annotations);
+}
+
+void ConfigBroadcaster::applySnapshotAndChanges(
+    std::map<ConfigKey, KnobValue> const& snapshot,
+    Version snapshotVersion,
+    Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+    Version changesVersion,
+    Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations) {
+	impl().applySnapshotAndChanges(snapshot, snapshotVersion, changes, changesVersion, annotations);
+}
+
+void ConfigBroadcaster::applySnapshotAndChanges(
+    std::map<ConfigKey, KnobValue>&& snapshot,
+    Version snapshotVersion,
+    Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+    Version changesVersion,
+    Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations) {
+	impl().applySnapshotAndChanges(std::move(snapshot), snapshotVersion, changes, changesVersion, annotations);
+}
+
+UID ConfigBroadcaster::getID() const {
+	return impl().getID();
+}
+
+JsonBuilderObject ConfigBroadcaster::getStatus() const {
+	return impl().getStatus();
+}
+
+void ConfigBroadcaster::compact(Version compactionVersion) {
+	impl().compact(compactionVersion);
+}
+
+namespace {
+
+Standalone<VectorRef<VersionedConfigMutationRef>> getTestChanges(Version version, bool includeGlobalMutation) {
+	Standalone<VectorRef<VersionedConfigMutationRef>> changes;
+	if (includeGlobalMutation) {
+		ConfigKey key = ConfigKeyRef({}, "test_long"_sr);
+		auto value = KnobValue::create(int64_t{ 5 });
+		ConfigMutation mutation = ConfigMutationRef(key, value.contents());
+		changes.emplace_back_deep(changes.arena(), version, mutation);
+	}
+	{
+		ConfigKey key = ConfigKeyRef("class-A"_sr, "test_long"_sr);
+		auto value = KnobValue::create(int64_t{ 5 });
+		ConfigMutation mutation = ConfigMutationRef(key, value.contents());
+		changes.emplace_back_deep(changes.arena(), version, mutation);
+	}
+	return changes;
+}
+
+ConfigBroadcastFollowerGetChangesRequest getTestRequest(Version lastSeenVersion,
+                                                        std::vector<KeyRef> const& configClasses) {
+	Standalone<VectorRef<KeyRef>> configClassesVector;
+	for (const auto& configClass : configClasses) {
+		configClassesVector.push_back_deep(configClassesVector.arena(), configClass);
+	}
+	return ConfigBroadcastFollowerGetChangesRequest{ lastSeenVersion, ConfigClassSet{ configClassesVector } };
+}
+
+} // namespace
+
+void ConfigBroadcasterImpl::runPendingRequestStoreTest(bool includeGlobalMutation, int expectedMatches) {
+	PendingRequestStore pending;
+	for (Version v = 0; v < 5; ++v) {
+		pending.addRequest(getTestRequest(v, {}));
+		pending.addRequest(getTestRequest(v, { "class-A"_sr }));
+		pending.addRequest(getTestRequest(v, { "class-B"_sr }));
+		pending.addRequest(getTestRequest(v, { "class-A"_sr, "class-B"_sr }));
+	}
+	auto toNotify = pending.getRequestsToNotify(getTestChanges(0, includeGlobalMutation));
+	ASSERT_EQ(toNotify.size(), 0);
+	for (Version v = 1; v <= 5; ++v) {
+		auto toNotify = pending.getRequestsToNotify(getTestChanges(v, includeGlobalMutation));
+		ASSERT_EQ(toNotify.size(), expectedMatches);
+		for (const auto& req : toNotify) {
+			pending.removeRequest(req);
+		}
+	}
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ConfigBroadcaster/Internal/PendingRequestStore/Simple") {
+	ConfigBroadcasterImpl::runPendingRequestStoreTest(false, 2);
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ConfigBroadcaster/Internal/PendingRequestStore/GlobalMutation") {
+	ConfigBroadcasterImpl::runPendingRequestStoreTest(true, 4);
+	return Void();
+}

fdbserver/ConfigBroadcaster.h

@@ -0,0 +1,66 @@
+/*
+ * ConfigBroadcaster.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/JsonBuilder.h"
+#include "fdbserver/CoordinationInterface.h"
+#include "fdbserver/ConfigBroadcastFollowerInterface.h"
+#include "fdbserver/ConfigFollowerInterface.h"
+#include "flow/flow.h"
+#include <memory>
+
+/*
+ * The configuration broadcaster runs on the cluster controller. The broadcaster listens uses
+ * an IConfigConsumer instantitation to consume updates from the configuration database, and broadcasts
+ * these updates to all workers' local configurations
+ */
+class ConfigBroadcaster {
+	std::unique_ptr<class ConfigBroadcasterImpl> _impl;
+	ConfigBroadcasterImpl& impl() { return *_impl; }
+	ConfigBroadcasterImpl const& impl() const { return *_impl; }
+
+public:
+	explicit ConfigBroadcaster(ServerCoordinators const&, UseConfigDB);
+	ConfigBroadcaster(ConfigBroadcaster&&);
+	ConfigBroadcaster& operator=(ConfigBroadcaster&&);
+	~ConfigBroadcaster();
+	Future<Void> serve(ConfigBroadcastFollowerInterface const&);
+	void applyChanges(Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                  Version mostRecentVersion,
+	                  Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations);
+	void applySnapshotAndChanges(std::map<ConfigKey, KnobValue> const& snapshot,
+	                             Version snapshotVersion,
+	                             Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                             Version changesVersion,
+	                             Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations);
+	void applySnapshotAndChanges(std::map<ConfigKey, KnobValue>&& snapshot,
+	                             Version snapshotVersion,
+	                             Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                             Version changesVersion,
+	                             Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations);
+	UID getID() const;
+	JsonBuilderObject getStatus() const;
+	void compact(Version compactionVersion);
+
+public: // Testing
+	explicit ConfigBroadcaster(ConfigFollowerInterface const&);
+};

fdbserver/ConfigDatabaseUnitTests.actor.cpp

@@ -0,0 +1,766 @@
+/*
+ * ConfigDatabaseUnitTests.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbclient/IConfigTransaction.h"
+#include "fdbclient/TestKnobCollection.h"
+#include "fdbserver/ConfigBroadcaster.h"
+#include "fdbserver/IConfigDatabaseNode.h"
+#include "fdbserver/LocalConfiguration.h"
+#include "fdbclient/Tuple.h"
+#include "flow/UnitTest.h"
+#include "flow/actorcompiler.h" // must be last include
+
+namespace {
+
+Key encodeConfigKey(Optional<KeyRef> configClass, KeyRef knobName) {
+	Tuple tuple;
+	if (configClass.present()) {
+		tuple.append(configClass.get());
+	} else {
+		tuple.appendNull();
+	}
+	tuple << knobName;
+	return tuple.pack();
+}
+
+void appendVersionedMutation(Standalone<VectorRef<VersionedConfigMutationRef>>& versionedMutations,
+                             Version version,
+                             Optional<KeyRef> configClass,
+                             KeyRef knobName,
+                             Optional<KnobValueRef> knobValue) {
+	auto configKey = ConfigKeyRef(configClass, knobName);
+	auto mutation = ConfigMutationRef(configKey, knobValue);
+	versionedMutations.emplace_back_deep(versionedMutations.arena(), version, mutation);
+}
+
+class WriteToTransactionEnvironment {
+	std::string dataDir;
+	ConfigTransactionInterface cti;
+	ConfigFollowerInterface cfi;
+	Reference<IConfigDatabaseNode> node;
+	Future<Void> ctiServer;
+	Future<Void> cfiServer;
+	Version lastWrittenVersion{ 0 };
+
+	static Value longToValue(int64_t v) {
+		auto s = format("%ld", v);
+		return StringRef(reinterpret_cast<uint8_t const*>(s.c_str()), s.size());
+	}
+
+	ACTOR template <class T>
+	static Future<Void> set(WriteToTransactionEnvironment* self,
+	                        Optional<KeyRef> configClass,
+	                        T value,
+	                        KeyRef knobName) {
+		state Reference<IConfigTransaction> tr = IConfigTransaction::createTestSimple(self->cti);
+		auto configKey = encodeConfigKey(configClass, knobName);
+		tr->set(configKey, longToValue(value));
+		wait(tr->commit());
+		self->lastWrittenVersion = tr->getCommittedVersion();
+		return Void();
+	}
+
+	ACTOR static Future<Void> clear(WriteToTransactionEnvironment* self, Optional<KeyRef> configClass) {
+		state Reference<IConfigTransaction> tr = IConfigTransaction::createTestSimple(self->cti);
+		auto configKey = encodeConfigKey(configClass, "test_long"_sr);
+		tr->clear(configKey);
+		wait(tr->commit());
+		self->lastWrittenVersion = tr->getCommittedVersion();
+		return Void();
+	}
+
+	void setup() {
+		ctiServer = node->serve(cti);
+		cfiServer = node->serve(cfi);
+	}
+
+public:
+	WriteToTransactionEnvironment(std::string const& dataDir)
+	  : dataDir(dataDir), node(IConfigDatabaseNode::createSimple(dataDir)) {
+		platform::eraseDirectoryRecursive(dataDir);
+		setup();
+	}
+
+	template <class T>
+	Future<Void> set(Optional<KeyRef> configClass, T value, KeyRef knobName = "test_long"_sr) {
+		return set(this, configClass, value, knobName);
+	}
+
+	Future<Void> clear(Optional<KeyRef> configClass) { return clear(this, configClass); }
+
+	Future<Void> compact() { return cfi.compact.getReply(ConfigFollowerCompactRequest{ lastWrittenVersion }); }
+
+	void restartNode() {
+		cfiServer.cancel();
+		ctiServer.cancel();
+		node = IConfigDatabaseNode::createSimple(dataDir);
+		setup();
+	}
+
+	ConfigTransactionInterface getTransactionInterface() const { return cti; }
+
+	ConfigFollowerInterface getFollowerInterface() const { return cfi; }
+
+	Future<Void> getError() const { return cfiServer || ctiServer; }
+};
+
+class ReadFromLocalConfigEnvironment {
+	UID id;
+	std::string dataDir;
+	LocalConfiguration localConfiguration;
+	Reference<IDependentAsyncVar<ConfigBroadcastFollowerInterface> const> cbfi;
+	Future<Void> consumer;
+
+	ACTOR static Future<Void> checkEventually(LocalConfiguration const* localConfiguration,
+	                                          Optional<int64_t> expected) {
+		state double lastMismatchTime = now();
+		loop {
+			if (localConfiguration->getTestKnobs().TEST_LONG == expected.orDefault(0)) {
+				return Void();
+			}
+			if (now() > lastMismatchTime + 1.0) {
+				TraceEvent(SevWarn, "CheckEventuallyStillChecking")
+				    .detail("Expected", expected.present() ? expected.get() : 0)
+				    .detail("TestLong", localConfiguration->getTestKnobs().TEST_LONG);
+				lastMismatchTime = now();
+			}
+			wait(delayJittered(0.1));
+		}
+	}
+
+	ACTOR static Future<Void> setup(ReadFromLocalConfigEnvironment* self) {
+		wait(self->localConfiguration.initialize());
+		if (self->cbfi) {
+			self->consumer = self->localConfiguration.consume(self->cbfi);
+		}
+		return Void();
+	}
+
+public:
+	ReadFromLocalConfigEnvironment(std::string const& dataDir,
+	                               std::string const& configPath,
+	                               std::map<std::string, std::string> const& manualKnobOverrides)
+	  : dataDir(dataDir), localConfiguration(dataDir, configPath, manualKnobOverrides, IsTest::YES), consumer(Never()) {
+	}
+
+	Future<Void> setup() { return setup(this); }
+
+	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
+		localConfiguration = LocalConfiguration(dataDir, newConfigPath, {}, IsTest::YES);
+		return setup();
+	}
+
+	void connectToBroadcaster(Reference<IDependentAsyncVar<ConfigBroadcastFollowerInterface> const> const& cbfi) {
+		ASSERT(!this->cbfi);
+		this->cbfi = cbfi;
+		consumer = localConfiguration.consume(cbfi);
+	}
+
+	void checkImmediate(Optional<int64_t> expected) const {
+		if (expected.present()) {
+			ASSERT_EQ(localConfiguration.getTestKnobs().TEST_LONG, expected.get());
+		} else {
+			ASSERT_EQ(localConfiguration.getTestKnobs().TEST_LONG, 0);
+		}
+	}
+
+	Future<Void> checkEventually(Optional<int64_t> expected) const {
+		return checkEventually(&localConfiguration, expected);
+	}
+
+	LocalConfiguration& getMutableLocalConfiguration() { return localConfiguration; }
+
+	Future<Void> getError() const { return consumer; }
+};
+
+class LocalConfigEnvironment {
+	ReadFromLocalConfigEnvironment readFrom;
+	Version lastWrittenVersion{ 0 };
+
+	Future<Void> addMutation(Optional<KeyRef> configClass, Optional<KnobValueRef> value) {
+		Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations;
+		appendVersionedMutation(versionedMutations, ++lastWrittenVersion, configClass, "test_long"_sr, value);
+		return readFrom.getMutableLocalConfiguration().addChanges(versionedMutations, lastWrittenVersion);
+	}
+
+public:
+	LocalConfigEnvironment(std::string const& dataDir,
+	                       std::string const& configPath,
+	                       std::map<std::string, std::string> const& manualKnobOverrides = {})
+	  : readFrom(dataDir, configPath, manualKnobOverrides) {}
+	Future<Void> setup() { return readFrom.setup(); }
+	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
+		return readFrom.restartLocalConfig(newConfigPath);
+	}
+	Future<Void> getError() const { return Never(); }
+	Future<Void> clear(Optional<KeyRef> configClass) { return addMutation(configClass, {}); }
+	Future<Void> set(Optional<KeyRef> configClass, int64_t value) {
+		auto knobValue = KnobValueRef::create(value);
+		return addMutation(configClass, knobValue.contents());
+	}
+	void check(Optional<int64_t> value) const { return readFrom.checkImmediate(value); }
+};
+
+class BroadcasterToLocalConfigEnvironment {
+	ReadFromLocalConfigEnvironment readFrom;
+	Reference<AsyncVar<ConfigBroadcastFollowerInterface>> cbfi;
+	ConfigBroadcaster broadcaster;
+	Version lastWrittenVersion{ 0 };
+	Future<Void> broadcastServer;
+
+	ACTOR static Future<Void> setup(BroadcasterToLocalConfigEnvironment* self) {
+		wait(self->readFrom.setup());
+		self->readFrom.connectToBroadcaster(IDependentAsyncVar<ConfigBroadcastFollowerInterface>::create(self->cbfi));
+		self->broadcastServer = self->broadcaster.serve(self->cbfi->get());
+		return Void();
+	}
+
+	void addMutation(Optional<KeyRef> configClass, KnobValueRef value) {
+		Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations;
+		appendVersionedMutation(versionedMutations, ++lastWrittenVersion, configClass, "test_long"_sr, value);
+		broadcaster.applyChanges(versionedMutations, lastWrittenVersion, {});
+	}
+
+public:
+	BroadcasterToLocalConfigEnvironment(std::string const& dataDir, std::string const& configPath)
+	  : broadcaster(ConfigFollowerInterface{}), cbfi(makeReference<AsyncVar<ConfigBroadcastFollowerInterface>>()),
+	    readFrom(dataDir, configPath, {}) {}
+
+	Future<Void> setup() { return setup(this); }
+
+	void set(Optional<KeyRef> configClass, int64_t value) {
+		auto knobValue = KnobValueRef::create(value);
+		addMutation(configClass, knobValue.contents());
+	}
+
+	void clear(Optional<KeyRef> configClass) { addMutation(configClass, {}); }
+
+	Future<Void> check(Optional<int64_t> value) const { return readFrom.checkEventually(value); }
+
+	void changeBroadcaster() {
+		broadcastServer.cancel();
+		cbfi->set(ConfigBroadcastFollowerInterface{});
+		broadcastServer = broadcaster.serve(cbfi->get());
+	}
+
+	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
+		return readFrom.restartLocalConfig(newConfigPath);
+	}
+
+	void compact() { broadcaster.compact(lastWrittenVersion); }
+
+	Future<Void> getError() const { return readFrom.getError() || broadcastServer; }
+};
+
+class TransactionEnvironment {
+	WriteToTransactionEnvironment writeTo;
+
+	ACTOR static Future<Void> check(TransactionEnvironment* self,
+	                                Optional<KeyRef> configClass,
+	                                Optional<int64_t> expected) {
+		state Reference<IConfigTransaction> tr =
+		    IConfigTransaction::createTestSimple(self->writeTo.getTransactionInterface());
+		state Key configKey = encodeConfigKey(configClass, "test_long"_sr);
+		state Optional<Value> value = wait(tr->get(configKey));
+		if (expected.present()) {
+			ASSERT_EQ(BinaryReader::fromStringRef<int64_t>(value.get(), Unversioned()), expected.get());
+		} else {
+			ASSERT(!value.present());
+		}
+		return Void();
+	}
+
+	ACTOR static Future<Standalone<VectorRef<KeyRef>>> getConfigClasses(TransactionEnvironment* self) {
+		state Reference<IConfigTransaction> tr =
+		    IConfigTransaction::createTestSimple(self->writeTo.getTransactionInterface());
+		state KeySelector begin = firstGreaterOrEqual(configClassKeys.begin);
+		state KeySelector end = firstGreaterOrEqual(configClassKeys.end);
+		Standalone<RangeResultRef> range = wait(tr->getRange(begin, end, 1000));
+		Standalone<VectorRef<KeyRef>> result;
+		for (const auto& kv : range) {
+			result.push_back_deep(result.arena(), kv.key);
+			ASSERT(kv.value == ""_sr);
+		}
+		return result;
+	}
+
+	ACTOR static Future<Standalone<VectorRef<KeyRef>>> getKnobNames(TransactionEnvironment* self,
+	                                                                Optional<KeyRef> configClass) {
+		state Reference<IConfigTransaction> tr =
+		    IConfigTransaction::createTestSimple(self->writeTo.getTransactionInterface());
+		state KeyRange keys = globalConfigKnobKeys;
+		if (configClass.present()) {
+			keys = singleKeyRange(configClass.get().withPrefix(configKnobKeys.begin));
+		}
+		KeySelector begin = firstGreaterOrEqual(keys.begin);
+		KeySelector end = firstGreaterOrEqual(keys.end);
+		Standalone<RangeResultRef> range = wait(tr->getRange(begin, end, 1000));
+		Standalone<VectorRef<KeyRef>> result;
+		for (const auto& kv : range) {
+			result.push_back_deep(result.arena(), kv.key);
+			ASSERT(kv.value == ""_sr);
+		}
+		return result;
+	}
+
+	ACTOR static Future<Void> badRangeRead(TransactionEnvironment* self) {
+		state Reference<IConfigTransaction> tr =
+		    IConfigTransaction::createTestSimple(self->writeTo.getTransactionInterface());
+		KeySelector begin = firstGreaterOrEqual(normalKeys.begin);
+		KeySelector end = firstGreaterOrEqual(normalKeys.end);
+		wait(success(tr->getRange(begin, end, 1000)));
+		return Void();
+	}
+
+public:
+	TransactionEnvironment(std::string const& dataDir) : writeTo(dataDir) {}
+
+	Future<Void> setup() { return Void(); }
+
+	void restartNode() { writeTo.restartNode(); }
+	template <class T>
+	Future<Void> set(Optional<KeyRef> configClass, T value, KeyRef knobName = "test_long"_sr) {
+		return writeTo.set(configClass, value, knobName);
+	}
+	Future<Void> clear(Optional<KeyRef> configClass) { return writeTo.clear(configClass); }
+	Future<Void> check(Optional<KeyRef> configClass, Optional<int64_t> expected) {
+		return check(this, configClass, expected);
+	}
+	Future<Void> badRangeRead() { return badRangeRead(this); }
+
+	Future<Standalone<VectorRef<KeyRef>>> getConfigClasses() { return getConfigClasses(this); }
+	Future<Standalone<VectorRef<KeyRef>>> getKnobNames(Optional<KeyRef> configClass) {
+		return getKnobNames(this, configClass);
+	}
+
+	Future<Void> compact() { return writeTo.compact(); }
+	Future<Void> getError() const { return writeTo.getError(); }
+};
+
+class TransactionToLocalConfigEnvironment {
+	WriteToTransactionEnvironment writeTo;
+	ReadFromLocalConfigEnvironment readFrom;
+	Reference<AsyncVar<ConfigBroadcastFollowerInterface>> cbfi;
+	ConfigBroadcaster broadcaster;
+	Future<Void> broadcastServer;
+
+	ACTOR static Future<Void> setup(TransactionToLocalConfigEnvironment* self) {
+		wait(self->readFrom.setup());
+		self->readFrom.connectToBroadcaster(IDependentAsyncVar<ConfigBroadcastFollowerInterface>::create(self->cbfi));
+		self->broadcastServer = self->broadcaster.serve(self->cbfi->get());
+		return Void();
+	}
+
+public:
+	TransactionToLocalConfigEnvironment(std::string const& dataDir, std::string const& configPath)
+	  : writeTo(dataDir), readFrom(dataDir, configPath, {}), broadcaster(writeTo.getFollowerInterface()),
+	    cbfi(makeReference<AsyncVar<ConfigBroadcastFollowerInterface>>()) {}
+
+	Future<Void> setup() { return setup(this); }
+
+	void restartNode() { writeTo.restartNode(); }
+
+	void changeBroadcaster() {
+		broadcastServer.cancel();
+		cbfi->set(ConfigBroadcastFollowerInterface{});
+		broadcastServer = broadcaster.serve(cbfi->get());
+	}
+
+	Future<Void> restartLocalConfig(std::string const& newConfigPath) {
+		return readFrom.restartLocalConfig(newConfigPath);
+	}
+
+	Future<Void> compact() { return writeTo.compact(); }
+
+	template <class T>
+	Future<Void> set(Optional<KeyRef> configClass, T const& value) {
+		return writeTo.set(configClass, value);
+	}
+	Future<Void> clear(Optional<KeyRef> configClass) { return writeTo.clear(configClass); }
+	Future<Void> check(Optional<int64_t> value) const { return readFrom.checkEventually(value); }
+	Future<Void> getError() const { return writeTo.getError() || readFrom.getError() || broadcastServer; }
+};
+
+// These functions give a common interface to all environments, to improve code reuse
+template <class Env, class... Args>
+Future<Void> set(Env& env, Args&&... args) {
+	return waitOrError(env.set(std::forward<Args>(args)...), env.getError());
+}
+template <class... Args>
+Future<Void> set(BroadcasterToLocalConfigEnvironment& env, Args&&... args) {
+	env.set(std::forward<Args>(args)...);
+	return Void();
+}
+template <class Env, class... Args>
+Future<Void> clear(Env& env, Args&&... args) {
+	return waitOrError(env.clear(std::forward<Args>(args)...), env.getError());
+}
+template <class... Args>
+Future<Void> clear(BroadcasterToLocalConfigEnvironment& env, Args&&... args) {
+	env.clear(std::forward<Args>(args)...);
+	return Void();
+}
+template <class Env, class... Args>
+Future<Void> check(Env& env, Args&&... args) {
+	return waitOrError(env.check(std::forward<Args>(args)...), env.getError());
+}
+template <class... Args>
+Future<Void> check(LocalConfigEnvironment& env, Args&&... args) {
+	env.check(std::forward<Args>(args)...);
+	return Void();
+}
+template <class Env>
+Future<Void> compact(Env& env) {
+	return waitOrError(env.compact(), env.getError());
+}
+Future<Void> compact(BroadcasterToLocalConfigEnvironment& env) {
+	env.compact();
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testRestartLocalConfig(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(check(env, int64_t{ 1 }));
+	wait(env.restartLocalConfig("class-A"));
+	wait(check(env, int64_t{ 1 }));
+	wait(set(env, "class-A"_sr, 2));
+	wait(check(env, int64_t{ 2 }));
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testRestartLocalConfigAndChangeClass(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(check(env, int64_t{ 1 }));
+	wait(env.restartLocalConfig("class-B"));
+	wait(check(env, int64_t{ 0 }));
+	wait(set(env, "class-B"_sr, int64_t{ 2 }));
+	wait(check(env, int64_t{ 2 }));
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testSet(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(check(env, int64_t{ 1 }));
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testClear(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(clear(env, "class-A"_sr));
+	wait(check(env, Optional<int64_t>{}));
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testGlobalSet(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, Optional<KeyRef>{}, int64_t{ 1 }));
+	wait(check(env, int64_t{ 1 }));
+	wait(set(env, "class-A"_sr, int64_t{ 10 }));
+	wait(check(env, int64_t{ 10 }));
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testIgnore(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-B"_sr, int64_t{ 1 }));
+	choose {
+		when(wait(delay(5))) {}
+		when(wait(check(env, int64_t{ 1 }))) { ASSERT(false); }
+	}
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testCompact(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(compact(env));
+	wait(check(env, 1));
+	wait(set(env, "class-A"_sr, int64_t{ 2 }));
+	wait(check(env, 2));
+	return Void();
+}
+
+ACTOR template <class Env>
+Future<Void> testChangeBroadcaster(UnitTestParameters params) {
+	state Env env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(check(env, int64_t{ 1 }));
+	env.changeBroadcaster();
+	wait(set(env, "class-A"_sr, int64_t{ 2 }));
+	wait(check(env, int64_t{ 2 }));
+	return Void();
+}
+
+bool matches(Standalone<VectorRef<KeyRef>> const& vec, std::set<Key> const& compareTo) {
+	std::set<Key> s;
+	for (const auto& value : vec) {
+		s.insert(value);
+	}
+	return (s == compareTo);
+}
+
+ACTOR Future<Void> testGetConfigClasses(UnitTestParameters params, bool doCompact) {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(set(env, "class-B"_sr, int64_t{ 1 }));
+	if (doCompact) {
+		wait(compact(env));
+	}
+	Standalone<VectorRef<KeyRef>> configClasses = wait(env.getConfigClasses());
+	ASSERT(matches(configClasses, { "class-A"_sr, "class-B"_sr }));
+	return Void();
+}
+
+ACTOR Future<Void> testGetKnobs(UnitTestParameters params, bool global, bool doCompact) {
+	state TransactionEnvironment env(params.getDataDir());
+	state Optional<Key> configClass;
+	if (!global) {
+		configClass = "class-A"_sr;
+	}
+	wait(set(env, configClass.castTo<KeyRef>(), int64_t{ 1 }, "test_long"_sr));
+	wait(set(env, configClass.castTo<KeyRef>(), int{ 2 }, "test_int"_sr));
+	wait(set(env, "class-B"_sr, double{ 3.0 }, "test_double"_sr)); // ignored
+	if (doCompact) {
+		wait(compact(env));
+	}
+	Standalone<VectorRef<KeyRef>> knobNames =
+	    wait(waitOrError(env.getKnobNames(configClass.castTo<KeyRef>()), env.getError()));
+	ASSERT(matches(knobNames, { "test_long"_sr, "test_int"_sr }));
+	return Void();
+}
+
+} // namespace
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/Set") {
+	wait(testSet<LocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/Restart") {
+	wait(testRestartLocalConfig<LocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/RestartFresh") {
+	wait(testRestartLocalConfigAndChangeClass<LocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/Clear") {
+	wait(testClear<LocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/GlobalSet") {
+	wait(testGlobalSet<LocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/ConflictingOverrides") {
+	state LocalConfigEnvironment env(params.getDataDir(), "class-A/class-B", {});
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(set(env, "class-B"_sr, int64_t{ 10 }));
+	env.check(10);
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/LocalConfiguration/Manual") {
+	state LocalConfigEnvironment env(params.getDataDir(), "class-A", { { "test_long", "1000" } });
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	env.check(1000);
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/Set") {
+	wait(testSet<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/Clear") {
+	wait(testClear<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/Ignore") {
+	wait(testIgnore<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/GlobalSet") {
+	wait(testGlobalSet<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/ChangeBroadcaster") {
+	wait(testChangeBroadcaster<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/RestartLocalConfig") {
+	wait(testRestartLocalConfig<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/RestartLocalConfigAndChangeClass") {
+	wait(testRestartLocalConfigAndChangeClass<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/BroadcasterToLocalConfig/Compact") {
+	wait(testCompact<BroadcasterToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/Set") {
+	wait(testSet<TransactionToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/Clear") {
+	wait(testClear<TransactionToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/GlobalSet") {
+	wait(testGlobalSet<TransactionToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/RestartNode") {
+	state TransactionToLocalConfigEnvironment env(params.getDataDir(), "class-A");
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	env.restartNode();
+	wait(check(env, int64_t{ 1 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/ChangeBroadcaster") {
+	wait(testChangeBroadcaster<TransactionToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/RestartLocalConfigAndChangeClass") {
+	wait(testRestartLocalConfigAndChangeClass<TransactionToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/TransactionToLocalConfig/CompactNode") {
+	wait(testCompact<TransactionToLocalConfigEnvironment>(params));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/Set") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(check(env, "class-A"_sr, int64_t{ 1 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/Clear") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(env.setup());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(clear(env, "class-A"_sr));
+	wait(check(env, "class-A"_sr, Optional<int64_t>{}));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/Restart") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	env.restartNode();
+	wait(check(env, "class-A"_sr, int64_t{ 1 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/CompactNode") {
+	state TransactionEnvironment env(params.getDataDir());
+	wait(set(env, "class-A"_sr, int64_t{ 1 }));
+	wait(compact(env));
+	wait(check(env, "class-A"_sr, int64_t{ 1 }));
+	wait(set(env, "class-A"_sr, int64_t{ 2 }));
+	wait(check(env, "class-A"_sr, int64_t{ 2 }));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/GetConfigClasses") {
+	wait(testGetConfigClasses(params, false));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/CompactThenGetConfigClasses") {
+	wait(testGetConfigClasses(params, true));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/GetKnobs") {
+	wait(testGetKnobs(params, false, false));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/CompactThenGetKnobs") {
+	wait(testGetKnobs(params, false, true));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/GetGlobalKnobs") {
+	wait(testGetKnobs(params, true, false));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/CompactThenGetGlobalKnobs") {
+	wait(testGetKnobs(params, true, true));
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/Transaction/BadRangeRead") {
+	state TransactionEnvironment env(params.getDataDir());
+	try {
+		wait(env.badRangeRead() || env.getError());
+		ASSERT(false);
+	} catch (Error& e) {
+		ASSERT_EQ(e.code(), error_code_invalid_config_db_range_read);
+	}
+	return Void();
+}

fdbserver/ConfigFollowerInterface.cpp

@@ -0,0 +1,46 @@
+/*
+ * ConfigFollowerInterface.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flow/IRandom.h"
+#include "fdbserver/ConfigFollowerInterface.h"
+#include "fdbserver/CoordinationInterface.h"
+
+void ConfigFollowerInterface::setupWellKnownEndpoints() {
+	getSnapshotAndChanges.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES,
+	                                            TaskPriority::Coordination);
+	getChanges.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_GETCHANGES, TaskPriority::Coordination);
+	compact.makeWellKnownEndpoint(WLTOKEN_CONFIGFOLLOWER_COMPACT, TaskPriority::Coordination);
+}
+
+ConfigFollowerInterface::ConfigFollowerInterface() : _id(deterministicRandom()->randomUniqueID()) {}
+
+ConfigFollowerInterface::ConfigFollowerInterface(NetworkAddress const& remote)
+  : _id(deterministicRandom()->randomUniqueID()),
+    getSnapshotAndChanges(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES)),
+    getChanges(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_GETCHANGES)),
+    compact(Endpoint({ remote }, WLTOKEN_CONFIGFOLLOWER_COMPACT)) {}
+
+bool ConfigFollowerInterface::operator==(ConfigFollowerInterface const& rhs) const {
+	return _id == rhs._id;
+}
+
+bool ConfigFollowerInterface::operator!=(ConfigFollowerInterface const& rhs) const {
+	return !(*this == rhs);
+}

fdbserver/ConfigFollowerInterface.h

@@ -0,0 +1,175 @@
+/*
+ * ConfigFollowerInterface.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/CommitTransaction.h"
+#include "fdbclient/ConfigKnobs.h"
+#include "fdbclient/FDBTypes.h"
+#include "fdbrpc/fdbrpc.h"
+
+struct VersionedConfigMutationRef {
+	Version version;
+	ConfigMutationRef mutation;
+
+	VersionedConfigMutationRef() = default;
+	explicit VersionedConfigMutationRef(Arena& arena, Version version, ConfigMutationRef mutation)
+	  : version(version), mutation(arena, mutation) {}
+	explicit VersionedConfigMutationRef(Arena& arena, VersionedConfigMutationRef const& rhs)
+	  : version(rhs.version), mutation(arena, rhs.mutation) {}
+
+	size_t expectedSize() const { return mutation.expectedSize(); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, mutation);
+	}
+};
+using VersionedConfigMutation = Standalone<VersionedConfigMutationRef>;
+
+struct VersionedConfigCommitAnnotationRef {
+	Version version;
+	ConfigCommitAnnotationRef annotation;
+
+	VersionedConfigCommitAnnotationRef() = default;
+	explicit VersionedConfigCommitAnnotationRef(Arena& arena, Version version, ConfigCommitAnnotationRef annotation)
+	  : version(version), annotation(arena, annotation) {}
+	explicit VersionedConfigCommitAnnotationRef(Arena& arena, VersionedConfigCommitAnnotationRef rhs)
+	  : version(rhs.version), annotation(arena, rhs.annotation) {}
+
+	size_t expectedSize() const { return annotation.expectedSize(); }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, annotation);
+	}
+};
+using VersionedConfigCommitAnnotation = Standalone<VersionedConfigCommitAnnotationRef>;
+
+struct ConfigFollowerGetSnapshotAndChangesReply {
+	static constexpr FileIdentifier file_identifier = 1734095;
+	Version snapshotVersion;
+	Version changesVersion;
+	std::map<ConfigKey, KnobValue> snapshot;
+	// TODO: Share arena
+	Standalone<VectorRef<VersionedConfigMutationRef>> changes;
+	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+
+	ConfigFollowerGetSnapshotAndChangesReply() = default;
+	template <class Snapshot>
+	explicit ConfigFollowerGetSnapshotAndChangesReply(
+	    Version snapshotVersion,
+	    Version changesVersion,
+	    Snapshot&& snapshot,
+	    Standalone<VectorRef<VersionedConfigMutationRef>> changes,
+	    Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations)
+	  : snapshotVersion(snapshotVersion), changesVersion(changesVersion), snapshot(std::forward<Snapshot>(snapshot)),
+	    changes(changes), annotations(annotations) {
+		ASSERT_GE(changesVersion, snapshotVersion);
+	}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, snapshotVersion, changesVersion, snapshot, changes);
+	}
+};
+
+struct ConfigFollowerGetSnapshotAndChangesRequest {
+	static constexpr FileIdentifier file_identifier = 294811;
+	ReplyPromise<ConfigFollowerGetSnapshotAndChangesReply> reply;
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, reply);
+	}
+};
+
+struct ConfigFollowerGetChangesReply {
+	static constexpr FileIdentifier file_identifier = 234859;
+	Version mostRecentVersion;
+	// TODO: Share arena
+	Standalone<VectorRef<VersionedConfigMutationRef>> changes;
+	Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> annotations;
+
+	ConfigFollowerGetChangesReply() : mostRecentVersion(0) {}
+	explicit ConfigFollowerGetChangesReply(Version mostRecentVersion,
+	                                       Standalone<VectorRef<VersionedConfigMutationRef>> const& changes,
+	                                       Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> const& annotations)
+	  : mostRecentVersion(mostRecentVersion), changes(changes), annotations(annotations) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, mostRecentVersion, changes, annotations);
+	}
+};
+
+struct ConfigFollowerGetChangesRequest {
+	static constexpr FileIdentifier file_identifier = 178935;
+	Version lastSeenVersion{ 0 };
+	ReplyPromise<ConfigFollowerGetChangesReply> reply;
+
+	ConfigFollowerGetChangesRequest() = default;
+	explicit ConfigFollowerGetChangesRequest(Version lastSeenVersion) : lastSeenVersion(lastSeenVersion) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, lastSeenVersion, reply);
+	}
+};
+
+struct ConfigFollowerCompactRequest {
+	static constexpr FileIdentifier file_identifier = 568910;
+	Version version{ 0 };
+	ReplyPromise<Void> reply;
+
+	ConfigFollowerCompactRequest() = default;
+	explicit ConfigFollowerCompactRequest(Version version) : version(version) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, version, reply);
+	}
+};
+
+/*
+ * Configuration database nodes serve a ConfigFollowerInterface which contains well known endpoints,
+ * used by workers to receive configuration database updates
+ */
+class ConfigFollowerInterface {
+	UID _id;
+
+public:
+	static constexpr FileIdentifier file_identifier = 7721102;
+	RequestStream<ConfigFollowerGetSnapshotAndChangesRequest> getSnapshotAndChanges;
+	RequestStream<ConfigFollowerGetChangesRequest> getChanges;
+	RequestStream<ConfigFollowerCompactRequest> compact;
+
+	ConfigFollowerInterface();
+	void setupWellKnownEndpoints();
+	ConfigFollowerInterface(NetworkAddress const& remote);
+	bool operator==(ConfigFollowerInterface const& rhs) const;
+	bool operator!=(ConfigFollowerInterface const& rhs) const;
+	UID id() const { return _id; }
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, _id, getSnapshotAndChanges, getChanges, compact);
+	}
+};

fdbserver/Coordination.actor.cpp

@@ -18,9 +18,12 @@
  * limitations under the License.
  */
 
+#include "fdbclient/ConfigTransactionInterface.h"
 #include "fdbserver/CoordinationInterface.h"
+#include "fdbserver/IConfigDatabaseNode.h"
 #include "fdbserver/IKeyValueStore.h"
 #include "fdbserver/Knobs.h"
+#include "fdbserver/OnDemandStore.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/Status.h"
 #include "flow/ActorCollection.h"
@@ -70,56 +73,12 @@ LeaderElectionRegInterface::LeaderElectionRegInterface(INetwork* local) : Client
 ServerCoordinators::ServerCoordinators(Reference<ClusterConnectionFile> cf) : ClientCoordinators(cf) {
 	ClusterConnectionString cs = ccf->getConnectionString();
 	for (auto s = cs.coordinators().begin(); s != cs.coordinators().end(); ++s) {
-		leaderElectionServers.push_back(LeaderElectionRegInterface(*s));
-		stateServers.push_back(GenerationRegInterface(*s));
+		leaderElectionServers.emplace_back(*s);
+		stateServers.emplace_back(*s);
+		configServers.emplace_back(*s);
 	}
 }
 
-// The coordination server wants to create its key value store only if it is actually used
-struct OnDemandStore {
-public:
-	OnDemandStore(std::string folder, UID myID) : folder(folder), store(nullptr), myID(myID) {}
-	~OnDemandStore() {
-		if (store)
-			store->close();
-	}
-
-	IKeyValueStore* get() {
-		if (!store)
-			open();
-		return store;
-	}
-
-	bool exists() {
-		if (store)
-			return true;
-		return fileExists(joinPath(folder, "coordination-0.fdq")) ||
-		       fileExists(joinPath(folder, "coordination-1.fdq")) || fileExists(joinPath(folder, "coordination.fdb"));
-	}
-
-	IKeyValueStore* operator->() { return get(); }
-
-	Future<Void> getError() { return onErr(err.getFuture()); }
-
-private:
-	std::string folder;
-	UID myID;
-	IKeyValueStore* store;
-	Promise<Future<Void>> err;
-
-	ACTOR static Future<Void> onErr(Future<Future<Void>> e) {
-		Future<Void> f = wait(e);
-		wait(f);
-		return Void();
-	}
-
-	void open() {
-		platform::createDirectory(folder);
-		store = keyValueStoreMemory(joinPath(folder, "coordination-"), myID, 500e6);
-		err.send(store->getError());
-	}
-};
-
 ACTOR Future<Void> localGenerationReg(GenerationRegInterface interf, OnDemandStore* pstore) {
 	state GenerationRegVal v;
 	state OnDemandStore& store = *pstore;
@@ -176,8 +135,7 @@ ACTOR Future<Void> localGenerationReg(GenerationRegInterface interf, OnDemandSto
 
 TEST_CASE("/fdbserver/Coordination/localGenerationReg/simple") {
 	state GenerationRegInterface reg;
-	state OnDemandStore store("simfdb/unittests/", //< FIXME
-	                          deterministicRandom()->randomUniqueID());
+	state OnDemandStore store(params.getDataDir(), deterministicRandom()->randomUniqueID(), "coordination-");
 	state Future<Void> actor = localGenerationReg(reg, &store);
 	state Key the_key(deterministicRandom()->randomAlphaNumeric(deterministicRandom()->randomInt(0, 10)));
 
@@ -688,19 +646,36 @@ ACTOR Future<Void> leaderServer(LeaderElectionRegInterface interf,
 	}
 }
 
-ACTOR Future<Void> coordinationServer(std::string dataFolder, Reference<ClusterConnectionFile> ccf) {
+ACTOR Future<Void> coordinationServer(std::string dataFolder,
+                                      Reference<ClusterConnectionFile> ccf,
+                                      UseConfigDB useConfigDB) {
 	state UID myID = deterministicRandom()->randomUniqueID();
 	state LeaderElectionRegInterface myLeaderInterface(g_network);
 	state GenerationRegInterface myInterface(g_network);
-	state OnDemandStore store(dataFolder, myID);
-
+	state OnDemandStore store(dataFolder, myID, "coordination-");
+	state ConfigTransactionInterface configTransactionInterface;
+	state ConfigFollowerInterface configFollowerInterface;
+	state Reference<IConfigDatabaseNode> configDatabaseNode;
+	state Future<Void> configDatabaseServer = Never();
 	TraceEvent("CoordinationServer", myID)
 	    .detail("MyInterfaceAddr", myInterface.read.getEndpoint().getPrimaryAddress())
 	    .detail("Folder", dataFolder);
 
+	if (useConfigDB != UseConfigDB::DISABLED) {
+		configTransactionInterface.setupWellKnownEndpoints();
+		configFollowerInterface.setupWellKnownEndpoints();
+		if (useConfigDB == UseConfigDB::SIMPLE) {
+			configDatabaseNode = IConfigDatabaseNode::createSimple(dataFolder);
+		} else {
+			configDatabaseNode = IConfigDatabaseNode::createPaxos(dataFolder);
+		}
+		configDatabaseServer =
+		    configDatabaseNode->serve(configTransactionInterface) || configDatabaseNode->serve(configFollowerInterface);
+	}
+
 	try {
 		wait(localGenerationReg(myInterface, &store) || leaderServer(myLeaderInterface, &store, myID, ccf) ||
-		     store.getError());
+		     store.getError() || configDatabaseServer);
 		throw internal_error();
 	} catch (Error& e) {
 		TraceEvent("CoordinationServerError", myID).error(e, true);

fdbserver/CoordinationInterface.h

@@ -23,6 +23,7 @@
 #pragma once
 
 #include "fdbclient/CoordinationInterface.h"
+#include "fdbserver/ConfigFollowerInterface.h"
 
 constexpr UID WLTOKEN_LEADERELECTIONREG_CANDIDACY(-1, 4);
 constexpr UID WLTOKEN_LEADERELECTIONREG_ELECTIONRESULT(-1, 5);
@@ -31,6 +32,10 @@ constexpr UID WLTOKEN_LEADERELECTIONREG_FORWARD(-1, 7);
 constexpr UID WLTOKEN_GENERATIONREG_READ(-1, 8);
 constexpr UID WLTOKEN_GENERATIONREG_WRITE(-1, 9);
 
+constexpr UID WLTOKEN_CONFIGFOLLOWER_GETSNAPSHOTANDCHANGES(-1, 17);
+constexpr UID WLTOKEN_CONFIGFOLLOWER_GETCHANGES(-1, 18);
+constexpr UID WLTOKEN_CONFIGFOLLOWER_COMPACT(-1, 19);
+
 struct GenerationRegInterface {
 	constexpr static FileIdentifier file_identifier = 16726744;
 	RequestStream<struct GenerationRegReadRequest> read;
@@ -221,10 +226,13 @@ class ServerCoordinators : public ClientCoordinators {
 public:
 	explicit ServerCoordinators(Reference<ClusterConnectionFile>);
 
-	vector<LeaderElectionRegInterface> leaderElectionServers;
-	vector<GenerationRegInterface> stateServers;
+	std::vector<LeaderElectionRegInterface> leaderElectionServers;
+	std::vector<GenerationRegInterface> stateServers;
+	std::vector<ConfigFollowerInterface> configServers;
 };
 
-Future<Void> coordinationServer(std::string const& dataFolder, Reference<ClusterConnectionFile> const& ccf);
+Future<Void> coordinationServer(std::string const& dataFolder,
+                                Reference<ClusterConnectionFile> const& ccf,
+                                UseConfigDB const& useConfigDB);
 
 #endif

fdbserver/IConfigConsumer.cpp

@@ -0,0 +1,41 @@
+/*
+ * IConfigConsumer.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/IConfigConsumer.h"
+#include "fdbserver/PaxosConfigConsumer.h"
+#include "fdbserver/SimpleConfigConsumer.h"
+
+std::unique_ptr<IConfigConsumer> IConfigConsumer::createTestSimple(ConfigFollowerInterface const& cfi,
+                                                                   double pollingInterval,
+                                                                   Optional<double> compactionInterval) {
+	return std::make_unique<SimpleConfigConsumer>(cfi, pollingInterval, compactionInterval);
+}
+
+std::unique_ptr<IConfigConsumer> IConfigConsumer::createSimple(ServerCoordinators const& coordinators,
+                                                               double pollingInterval,
+                                                               Optional<double> compactionInterval) {
+	return std::make_unique<SimpleConfigConsumer>(coordinators, pollingInterval, compactionInterval);
+}
+
+std::unique_ptr<IConfigConsumer> IConfigConsumer::createPaxos(ServerCoordinators const& coordinators,
+                                                              double pollingInterval,
+                                                              Optional<double> compactionInterval) {
+	return std::make_unique<PaxosConfigConsumer>(coordinators, pollingInterval, compactionInterval);
+}

fdbserver/IConfigConsumer.h

@@ -0,0 +1,50 @@
+/*
+ * IConfigConsumer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/CoordinationInterface.h"
+#include "fdbserver/ConfigBroadcaster.h"
+#include "fdbserver/CoordinationInterface.h"
+#include "fdbserver/ConfigFollowerInterface.h"
+#include "flow/flow.h"
+#include <memory>
+
+/*
+ * An IConfigConsumer instantiation is used by the ConfigBroadcaster to consume
+ * updates from the configuration database nodes. A consumer sends mutations to a broadcaster
+ * once they are confirmed to be committed.
+ */
+class IConfigConsumer {
+public:
+	virtual ~IConfigConsumer() = default;
+	virtual Future<Void> consume(ConfigBroadcaster& broadcaster) = 0;
+	virtual UID getID() const = 0;
+
+	static std::unique_ptr<IConfigConsumer> createTestSimple(ConfigFollowerInterface const& cfi,
+	                                                         double pollingInterval,
+	                                                         Optional<double> compactionInterval);
+	static std::unique_ptr<IConfigConsumer> createSimple(ServerCoordinators const& coordinators,
+	                                                     double pollingInterval,
+	                                                     Optional<double> compactionInterval);
+	static std::unique_ptr<IConfigConsumer> createPaxos(ServerCoordinators const& coordinators,
+	                                                    double pollingInterval,
+	                                                    Optional<double> compactionInterval);
+};

fdbserver/IConfigDatabaseNode.cpp

@@ -0,0 +1,31 @@
+/*
+ * IConfigDatabaseNode.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/IConfigDatabaseNode.h"
+#include "fdbserver/PaxosConfigDatabaseNode.h"
+#include "fdbserver/SimpleConfigDatabaseNode.h"
+
+Reference<IConfigDatabaseNode> IConfigDatabaseNode::createSimple(std::string const& folder) {
+	return makeReference<SimpleConfigDatabaseNode>(folder);
+}
+
+Reference<IConfigDatabaseNode> IConfigDatabaseNode::createPaxos(std::string const& folder) {
+	return makeReference<PaxosConfigDatabaseNode>(folder);
+}

fdbserver/IConfigDatabaseNode.h

@@ -0,0 +1,40 @@
+/*
+ * IConfigDatabaseNode.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbclient/ConfigTransactionInterface.h"
+#include "fdbserver/ConfigFollowerInterface.h"
+#include "flow/FastRef.h"
+#include "flow/flow.h"
+
+#include <memory>
+
+/*
+ * Interface for a single node in the configuration database, run on coordinators
+ */
+class IConfigDatabaseNode : public ReferenceCounted<IConfigDatabaseNode> {
+public:
+	virtual Future<Void> serve(ConfigTransactionInterface const&) = 0;
+	virtual Future<Void> serve(ConfigFollowerInterface const&) = 0;
+
+	static Reference<IConfigDatabaseNode> createSimple(std::string const& folder);
+	static Reference<IConfigDatabaseNode> createPaxos(std::string const& folder);
+};

fdbserver/KeyValueStoreMemory.actor.cpp

@@ -18,6 +18,7 @@
  * limitations under the License.
  */
 
+#include "fdbclient/Knobs.h"
 #include "fdbclient/Notified.h"
 #include "fdbclient/SystemData.h"
 #include "fdbserver/DeltaTree.h"

fdbserver/Knobs.h

@@ -18,663 +18,8 @@
  * limitations under the License.
  */
 
-#ifndef FDBSERVER_KNOBS_H
-#define FDBSERVER_KNOBS_H
 #pragma once
 
-#include "flow/Knobs.h"
-#include "fdbrpc/fdbrpc.h"
-#include "fdbclient/Knobs.h"
+#include "fdbclient/IKnobCollection.h"
 
-// Disk queue
-static const int _PAGE_SIZE = 4096;
-
-class ServerKnobs : public Knobs {
-public:
-	// Versions
-	int64_t VERSIONS_PER_SECOND;
-	int64_t MAX_VERSIONS_IN_FLIGHT;
-	int64_t MAX_VERSIONS_IN_FLIGHT_FORCED;
-	int64_t MAX_READ_TRANSACTION_LIFE_VERSIONS;
-	int64_t MAX_WRITE_TRANSACTION_LIFE_VERSIONS;
-	double MAX_COMMIT_BATCH_INTERVAL; // Each commit proxy generates a CommitTransactionBatchRequest at least this
-	                                  // often, so that versions always advance smoothly
-
-	// TLogs
-	double TLOG_TIMEOUT; // tlog OR commit proxy failure - master's reaction time
-	double TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS; // Warns if a tlog takes too long to rejoin
-	double RECOVERY_TLOG_SMART_QUORUM_DELAY; // smaller might be better for bug amplification
-	double TLOG_STORAGE_MIN_UPDATE_INTERVAL;
-	double BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL;
-	int DESIRED_TOTAL_BYTES;
-	int DESIRED_UPDATE_BYTES;
-	double UPDATE_DELAY;
-	int MAXIMUM_PEEK_BYTES;
-	int APPLY_MUTATION_BYTES;
-	int RECOVERY_DATA_BYTE_LIMIT;
-	int BUGGIFY_RECOVERY_DATA_LIMIT;
-	double LONG_TLOG_COMMIT_TIME;
-	int64_t LARGE_TLOG_COMMIT_BYTES;
-	double BUGGIFY_RECOVER_MEMORY_LIMIT;
-	double BUGGIFY_WORKER_REMOVED_MAX_LAG;
-	int64_t UPDATE_STORAGE_BYTE_LIMIT;
-	int64_t REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT;
-	double TLOG_PEEK_DELAY;
-	int LEGACY_TLOG_UPGRADE_ENTRIES_PER_VERSION;
-	int VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS; // Multiplicative factor to bound total space used to store a version
-	                                              // message (measured in 1/1024ths, e.g. a value of 2048 yields a
-	                                              // factor of 2).
-	int64_t VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD;
-	double TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR;
-	int64_t TLOG_MESSAGE_BLOCK_BYTES;
-	int64_t MAX_MESSAGE_SIZE;
-	int LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE;
-	double PEEK_TRACKER_EXPIRATION_TIME;
-	int PARALLEL_GET_MORE_REQUESTS;
-	int MULTI_CURSOR_PRE_FETCH_LIMIT;
-	int64_t MAX_QUEUE_COMMIT_BYTES;
-	int DESIRED_OUTSTANDING_MESSAGES;
-	double DESIRED_GET_MORE_DELAY;
-	int CONCURRENT_LOG_ROUTER_READS;
-	int LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED; // 0==peek from primary, non-zero==peek from satellites
-	double DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME;
-	double DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME;
-	int64_t TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES;
-	int64_t TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK;
-	int64_t TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH;
-	int64_t DISK_QUEUE_FILE_EXTENSION_BYTES; // When we grow the disk queue, by how many bytes should it grow?
-	int64_t DISK_QUEUE_FILE_SHRINK_BYTES; // When we shrink the disk queue, by how many bytes should it shrink?
-	int64_t DISK_QUEUE_MAX_TRUNCATE_BYTES; // A truncate larger than this will cause the file to be replaced instead.
-	double TLOG_DEGRADED_DURATION;
-	int64_t MAX_CACHE_VERSIONS;
-	double TXS_POPPED_MAX_DELAY;
-	double TLOG_MAX_CREATE_DURATION;
-	int PEEK_LOGGING_AMOUNT;
-	double PEEK_LOGGING_DELAY;
-	double PEEK_RESET_INTERVAL;
-	double PEEK_MAX_LATENCY;
-	bool PEEK_COUNT_SMALL_MESSAGES;
-	double PEEK_STATS_INTERVAL;
-	double PEEK_STATS_SLOW_AMOUNT;
-	double PEEK_STATS_SLOW_RATIO;
-	double PUSH_RESET_INTERVAL;
-	double PUSH_MAX_LATENCY;
-	double PUSH_STATS_INTERVAL;
-	double PUSH_STATS_SLOW_AMOUNT;
-	double PUSH_STATS_SLOW_RATIO;
-	int TLOG_POP_BATCH_SIZE;
-
-	// Data distribution queue
-	double HEALTH_POLL_TIME;
-	double BEST_TEAM_STUCK_DELAY;
-	double BG_REBALANCE_POLLING_INTERVAL;
-	double BG_REBALANCE_SWITCH_CHECK_INTERVAL;
-	double DD_QUEUE_LOGGING_INTERVAL;
-	double RELOCATION_PARALLELISM_PER_SOURCE_SERVER;
-	int DD_QUEUE_MAX_KEY_SERVERS;
-	int DD_REBALANCE_PARALLELISM;
-	int DD_REBALANCE_RESET_AMOUNT;
-	double BG_DD_MAX_WAIT;
-	double BG_DD_MIN_WAIT;
-	double BG_DD_INCREASE_RATE;
-	double BG_DD_DECREASE_RATE;
-	double BG_DD_SATURATION_DELAY;
-	double INFLIGHT_PENALTY_HEALTHY;
-	double INFLIGHT_PENALTY_REDUNDANT;
-	double INFLIGHT_PENALTY_UNHEALTHY;
-	double INFLIGHT_PENALTY_ONE_LEFT;
-	bool USE_OLD_NEEDED_SERVERS;
-
-	// Higher priorities are executed first
-	// Priority/100 is the "priority group"/"superpriority".  Priority inversion
-	//   is possible within but not between priority groups; fewer priority groups
-	//   mean better worst case time bounds
-	// Maximum allowable priority is 999.
-	int PRIORITY_RECOVER_MOVE;
-	int PRIORITY_REBALANCE_UNDERUTILIZED_TEAM;
-	int PRIORITY_REBALANCE_OVERUTILIZED_TEAM;
-	int PRIORITY_PERPETUAL_STORAGE_WIGGLE;
-	int PRIORITY_TEAM_HEALTHY;
-	int PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER;
-	int PRIORITY_TEAM_REDUNDANT;
-	int PRIORITY_MERGE_SHARD;
-	int PRIORITY_POPULATE_REGION;
-	int PRIORITY_TEAM_UNHEALTHY;
-	int PRIORITY_TEAM_2_LEFT;
-	int PRIORITY_TEAM_1_LEFT;
-	int PRIORITY_TEAM_FAILED; // Priority when a server in the team is excluded as failed
-	int PRIORITY_TEAM_0_LEFT;
-	int PRIORITY_SPLIT_SHARD;
-
-	// Data distribution
-	double RETRY_RELOCATESHARD_DELAY;
-	double DATA_DISTRIBUTION_FAILURE_REACTION_TIME;
-	int MIN_SHARD_BYTES, SHARD_BYTES_RATIO, SHARD_BYTES_PER_SQRT_BYTES, MAX_SHARD_BYTES, KEY_SERVER_SHARD_BYTES;
-	int64_t SHARD_MAX_BYTES_PER_KSEC, // Shards with more than this bandwidth will be split immediately
-	    SHARD_MIN_BYTES_PER_KSEC, // Shards with more than this bandwidth will not be merged
-	    SHARD_SPLIT_BYTES_PER_KSEC; // When splitting a shard, it is split into pieces with less than this bandwidth
-	double SHARD_MAX_READ_DENSITY_RATIO;
-	int64_t SHARD_READ_HOT_BANDWITH_MIN_PER_KSECONDS;
-	double SHARD_MAX_BYTES_READ_PER_KSEC_JITTER;
-	double STORAGE_METRIC_TIMEOUT;
-	double METRIC_DELAY;
-	double ALL_DATA_REMOVED_DELAY;
-	double INITIAL_FAILURE_REACTION_DELAY;
-	double CHECK_TEAM_DELAY;
-	double LOG_ON_COMPLETION_DELAY;
-	int BEST_TEAM_MAX_TEAM_TRIES;
-	int BEST_TEAM_OPTION_COUNT;
-	int BEST_OF_AMT;
-	double SERVER_LIST_DELAY;
-	double RECRUITMENT_IDLE_DELAY;
-	double STORAGE_RECRUITMENT_DELAY;
-	bool TSS_HACK_IDENTITY_MAPPING;
-	double TSS_RECRUITMENT_TIMEOUT;
-	double TSS_DD_CHECK_INTERVAL;
-	double DATA_DISTRIBUTION_LOGGING_INTERVAL;
-	double DD_ENABLED_CHECK_DELAY;
-	double DD_STALL_CHECK_DELAY;
-	double DD_LOW_BANDWIDTH_DELAY;
-	double DD_MERGE_COALESCE_DELAY;
-	double STORAGE_METRICS_POLLING_DELAY;
-	double STORAGE_METRICS_RANDOM_DELAY;
-	double AVAILABLE_SPACE_RATIO_CUTOFF;
-	int DESIRED_TEAMS_PER_SERVER;
-	int MAX_TEAMS_PER_SERVER;
-	int64_t DD_SHARD_SIZE_GRANULARITY;
-	int64_t DD_SHARD_SIZE_GRANULARITY_SIM;
-	int DD_MOVE_KEYS_PARALLELISM;
-	int DD_FETCH_SOURCE_PARALLELISM;
-	int DD_MERGE_LIMIT;
-	double DD_SHARD_METRICS_TIMEOUT;
-	int64_t DD_LOCATION_CACHE_SIZE;
-	double MOVEKEYS_LOCK_POLLING_DELAY;
-	double DEBOUNCE_RECRUITING_DELAY;
-	int REBALANCE_MAX_RETRIES;
-	int DD_OVERLAP_PENALTY;
-	int DD_EXCLUDE_MIN_REPLICAS;
-	bool DD_VALIDATE_LOCALITY;
-	int DD_CHECK_INVALID_LOCALITY_DELAY;
-	bool DD_ENABLE_VERBOSE_TRACING;
-	int64_t
-	    DD_SS_FAILURE_VERSIONLAG; // Allowed SS version lag from the current read version before marking it as failed.
-	int64_t DD_SS_ALLOWED_VERSIONLAG; // SS will be marked as healthy if it's version lag goes below this value.
-	double DD_SS_STUCK_TIME_LIMIT; // If a storage server is not getting new versions for this amount of time, then it
-	                               // becomes undesired.
-	int DD_TEAMS_INFO_PRINT_INTERVAL;
-	int DD_TEAMS_INFO_PRINT_YIELD_COUNT;
-	int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY;
-	int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle
-
-	// TeamRemover to remove redundant teams
-	bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor
-	double TR_REMOVE_MACHINE_TEAM_DELAY; // wait for the specified time before try to remove next machine team
-	bool TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS; // guard to select which machineTeamRemover logic to use
-
-	bool TR_FLAG_DISABLE_SERVER_TEAM_REMOVER; // disable the serverTeamRemover actor
-	double TR_REMOVE_SERVER_TEAM_DELAY; // wait for the specified time before try to remove next server team
-	double TR_REMOVE_SERVER_TEAM_EXTRA_DELAY; // serverTeamRemover waits for the delay and check DD healthyness again to
-	                                          // ensure it runs after machineTeamRemover
-
-	// Remove wrong storage engines
-	double DD_REMOVE_STORE_ENGINE_DELAY; // wait for the specified time before remove the next batch
-
-	double DD_FAILURE_TIME;
-	double DD_ZERO_HEALTHY_TEAM_DELAY;
-
-	// KeyValueStore SQLITE
-	int CLEAR_BUFFER_SIZE;
-	double READ_VALUE_TIME_ESTIMATE;
-	double READ_RANGE_TIME_ESTIMATE;
-	double SET_TIME_ESTIMATE;
-	double CLEAR_TIME_ESTIMATE;
-	double COMMIT_TIME_ESTIMATE;
-	int CHECK_FREE_PAGE_AMOUNT;
-	double DISK_METRIC_LOGGING_INTERVAL;
-	int64_t SOFT_HEAP_LIMIT;
-
-	int SQLITE_PAGE_SCAN_ERROR_LIMIT;
-	int SQLITE_BTREE_PAGE_USABLE;
-	int SQLITE_BTREE_CELL_MAX_LOCAL;
-	int SQLITE_BTREE_CELL_MIN_LOCAL;
-	int SQLITE_FRAGMENT_PRIMARY_PAGE_USABLE;
-	int SQLITE_FRAGMENT_OVERFLOW_PAGE_USABLE;
-	double SQLITE_FRAGMENT_MIN_SAVINGS;
-	int SQLITE_CHUNK_SIZE_PAGES;
-	int SQLITE_CHUNK_SIZE_PAGES_SIM;
-	int SQLITE_READER_THREADS;
-	int SQLITE_WRITE_WINDOW_LIMIT;
-	double SQLITE_WRITE_WINDOW_SECONDS;
-
-	// KeyValueStoreSqlite spring cleaning
-	double SPRING_CLEANING_NO_ACTION_INTERVAL;
-	double SPRING_CLEANING_LAZY_DELETE_INTERVAL;
-	double SPRING_CLEANING_VACUUM_INTERVAL;
-	double SPRING_CLEANING_LAZY_DELETE_TIME_ESTIMATE;
-	double SPRING_CLEANING_VACUUM_TIME_ESTIMATE;
-	double SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE;
-	int SPRING_CLEANING_MIN_LAZY_DELETE_PAGES;
-	int SPRING_CLEANING_MAX_LAZY_DELETE_PAGES;
-	int SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE;
-	int SPRING_CLEANING_MIN_VACUUM_PAGES;
-	int SPRING_CLEANING_MAX_VACUUM_PAGES;
-
-	// KeyValueStoreMemory
-	int64_t REPLACE_CONTENTS_BYTES;
-
-	// KeyValueStoreRocksDB
-	int ROCKSDB_BACKGROUND_PARALLELISM;
-	int ROCKSDB_READ_PARALLELISM;
-	int64_t ROCKSDB_MEMTABLE_BYTES;
-	bool ROCKSDB_UNSAFE_AUTO_FSYNC;
-	int64_t ROCKSDB_PERIODIC_COMPACTION_SECONDS;
-	int ROCKSDB_PREFIX_LEN;
-	int64_t ROCKSDB_BLOCK_CACHE_SIZE;
-
-	// Leader election
-	int MAX_NOTIFICATIONS;
-	int MIN_NOTIFICATIONS;
-	double NOTIFICATION_FULL_CLEAR_TIME;
-	double CANDIDATE_MIN_DELAY;
-	double CANDIDATE_MAX_DELAY;
-	double CANDIDATE_GROWTH_RATE;
-	double POLLING_FREQUENCY;
-	double HEARTBEAT_FREQUENCY;
-
-	// Commit CommitProxy
-	double START_TRANSACTION_BATCH_INTERVAL_MIN;
-	double START_TRANSACTION_BATCH_INTERVAL_MAX;
-	double START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION;
-	double START_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA;
-	double START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL;
-	double START_TRANSACTION_MAX_TRANSACTIONS_TO_START;
-	int START_TRANSACTION_MAX_REQUESTS_TO_START;
-	double START_TRANSACTION_RATE_WINDOW;
-	double START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET;
-	int START_TRANSACTION_MAX_QUEUE_SIZE;
-	int KEY_LOCATION_MAX_QUEUE_SIZE;
-
-	double COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE;
-	double COMMIT_TRANSACTION_BATCH_INTERVAL_MIN;
-	double COMMIT_TRANSACTION_BATCH_INTERVAL_MAX;
-	double COMMIT_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION;
-	double COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA;
-	int COMMIT_TRANSACTION_BATCH_COUNT_MAX;
-	int COMMIT_TRANSACTION_BATCH_BYTES_MIN;
-	int COMMIT_TRANSACTION_BATCH_BYTES_MAX;
-	double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE;
-	double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER;
-	int64_t COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT;
-	double COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL;
-	double COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR;
-
-	double RESOLVER_COALESCE_TIME;
-	int BUGGIFIED_ROW_LIMIT;
-	double PROXY_SPIN_DELAY;
-	double UPDATE_REMOTE_LOG_VERSION_INTERVAL;
-	int MAX_TXS_POP_VERSION_HISTORY;
-	double MIN_CONFIRM_INTERVAL;
-	double ENFORCED_MIN_RECOVERY_DURATION;
-	double REQUIRED_MIN_RECOVERY_DURATION;
-	bool ALWAYS_CAUSAL_READ_RISKY;
-	int MAX_COMMIT_UPDATES;
-	double MAX_PROXY_COMPUTE;
-	double MAX_COMPUTE_PER_OPERATION;
-	int PROXY_COMPUTE_BUCKETS;
-	double PROXY_COMPUTE_GROWTH_RATE;
-	int TXN_STATE_SEND_AMOUNT;
-	double REPORT_TRANSACTION_COST_ESTIMATION_DELAY;
-	bool PROXY_REJECT_BATCH_QUEUED_TOO_LONG;
-
-	int RESET_MASTER_BATCHES;
-	int RESET_RESOLVER_BATCHES;
-	double RESET_MASTER_DELAY;
-	double RESET_RESOLVER_DELAY;
-
-	// Master Server
-	double COMMIT_SLEEP_TIME;
-	double MIN_BALANCE_TIME;
-	int64_t MIN_BALANCE_DIFFERENCE;
-	double SECONDS_BEFORE_NO_FAILURE_DELAY;
-	int64_t MAX_TXS_SEND_MEMORY;
-	int64_t MAX_RECOVERY_VERSIONS;
-	double MAX_RECOVERY_TIME;
-	double PROVISIONAL_START_DELAY;
-	double PROVISIONAL_DELAY_GROWTH;
-	double PROVISIONAL_MAX_DELAY;
-	double SECONDS_BEFORE_RECRUIT_BACKUP_WORKER;
-	double CC_INTERFACE_TIMEOUT;
-
-	// Resolver
-	int64_t KEY_BYTES_PER_SAMPLE;
-	int64_t SAMPLE_OFFSET_PER_KEY;
-	double SAMPLE_EXPIRATION_TIME;
-	double SAMPLE_POLL_TIME;
-	int64_t RESOLVER_STATE_MEMORY_LIMIT;
-
-	// Backup Worker
-	double BACKUP_TIMEOUT; // master's reaction time for backup failure
-	double BACKUP_NOOP_POP_DELAY;
-	int BACKUP_FILE_BLOCK_BYTES;
-	int64_t BACKUP_LOCK_BYTES;
-	double BACKUP_UPLOAD_DELAY;
-
-	// Cluster Controller
-	double CLUSTER_CONTROLLER_LOGGING_DELAY;
-	double MASTER_FAILURE_REACTION_TIME;
-	double MASTER_FAILURE_SLOPE_DURING_RECOVERY;
-	int WORKER_COORDINATION_PING_DELAY;
-	double SIM_SHUTDOWN_TIMEOUT;
-	double SHUTDOWN_TIMEOUT;
-	double MASTER_SPIN_DELAY;
-	double CC_CHANGE_DELAY;
-	double CC_CLASS_DELAY;
-	double WAIT_FOR_GOOD_RECRUITMENT_DELAY;
-	double WAIT_FOR_GOOD_REMOTE_RECRUITMENT_DELAY;
-	double ATTEMPT_RECRUITMENT_DELAY;
-	double WAIT_FOR_DISTRIBUTOR_JOIN_DELAY;
-	double WAIT_FOR_RATEKEEPER_JOIN_DELAY;
-	double WORKER_FAILURE_TIME;
-	double CHECK_OUTSTANDING_INTERVAL;
-	double INCOMPATIBLE_PEERS_LOGGING_INTERVAL;
-	double VERSION_LAG_METRIC_INTERVAL;
-	int64_t MAX_VERSION_DIFFERENCE;
-	double FORCE_RECOVERY_CHECK_DELAY;
-	double RATEKEEPER_FAILURE_TIME;
-	double REPLACE_INTERFACE_DELAY;
-	double REPLACE_INTERFACE_CHECK_DELAY;
-	double COORDINATOR_REGISTER_INTERVAL;
-	double CLIENT_REGISTER_INTERVAL;
-
-	// Knobs used to select the best policy (via monte carlo)
-	int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)
-	int POLICY_GENERATIONS; // number of policies to generate
-
-	int EXPECTED_MASTER_FITNESS;
-	int EXPECTED_TLOG_FITNESS;
-	int EXPECTED_LOG_ROUTER_FITNESS;
-	int EXPECTED_COMMIT_PROXY_FITNESS;
-	int EXPECTED_GRV_PROXY_FITNESS;
-	int EXPECTED_RESOLVER_FITNESS;
-	double RECRUITMENT_TIMEOUT;
-	int DBINFO_SEND_AMOUNT;
-	double DBINFO_BATCH_DELAY;
-
-	// Move Keys
-	double SHARD_READY_DELAY;
-	double SERVER_READY_QUORUM_INTERVAL;
-	double SERVER_READY_QUORUM_TIMEOUT;
-	double REMOVE_RETRY_DELAY;
-	int MOVE_KEYS_KRM_LIMIT;
-	int MOVE_KEYS_KRM_LIMIT_BYTES; // This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT
-	                               // (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an
-	                               // attempt to read a key range map
-	int MAX_SKIP_TAGS;
-	double MAX_ADDED_SOURCES_MULTIPLIER;
-
-	// FdbServer
-	double MIN_REBOOT_TIME;
-	double MAX_REBOOT_TIME;
-	std::string LOG_DIRECTORY;
-	int64_t SERVER_MEM_LIMIT;
-	double SYSTEM_MONITOR_FREQUENCY;
-
-	// Ratekeeper
-	double SMOOTHING_AMOUNT;
-	double SLOW_SMOOTHING_AMOUNT;
-	double METRIC_UPDATE_RATE;
-	double DETAILED_METRIC_UPDATE_RATE;
-	double LAST_LIMITED_RATIO;
-	double RATEKEEPER_DEFAULT_LIMIT;
-
-	int64_t TARGET_BYTES_PER_STORAGE_SERVER;
-	int64_t SPRING_BYTES_STORAGE_SERVER;
-	int64_t AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES;
-	int64_t TARGET_BYTES_PER_STORAGE_SERVER_BATCH;
-	int64_t SPRING_BYTES_STORAGE_SERVER_BATCH;
-	int64_t STORAGE_HARD_LIMIT_BYTES;
-	int64_t STORAGE_DURABILITY_LAG_HARD_MAX;
-	int64_t STORAGE_DURABILITY_LAG_SOFT_MAX;
-
-	int64_t LOW_PRIORITY_STORAGE_QUEUE_BYTES;
-	int64_t LOW_PRIORITY_DURABILITY_LAG;
-
-	int64_t TARGET_BYTES_PER_TLOG;
-	int64_t SPRING_BYTES_TLOG;
-	int64_t TARGET_BYTES_PER_TLOG_BATCH;
-	int64_t SPRING_BYTES_TLOG_BATCH;
-	int64_t TLOG_SPILL_THRESHOLD;
-	int64_t TLOG_HARD_LIMIT_BYTES;
-	int64_t TLOG_RECOVER_MEMORY_LIMIT;
-	double TLOG_IGNORE_POP_AUTO_ENABLE_DELAY;
-
-	int64_t MAX_MANUAL_THROTTLED_TRANSACTION_TAGS;
-	int64_t MAX_AUTO_THROTTLED_TRANSACTION_TAGS;
-	double MIN_TAG_COST;
-	double AUTO_THROTTLE_TARGET_TAG_BUSYNESS;
-	double AUTO_THROTTLE_RAMP_TAG_BUSYNESS;
-	double AUTO_TAG_THROTTLE_RAMP_UP_TIME;
-	double AUTO_TAG_THROTTLE_DURATION;
-	double TAG_THROTTLE_PUSH_INTERVAL;
-	double AUTO_TAG_THROTTLE_START_AGGREGATION_TIME;
-	double AUTO_TAG_THROTTLE_UPDATE_FREQUENCY;
-	double TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL;
-	bool AUTO_TAG_THROTTLING_ENABLED;
-
-	double MAX_TRANSACTIONS_PER_BYTE;
-
-	int64_t MIN_AVAILABLE_SPACE;
-	double MIN_AVAILABLE_SPACE_RATIO;
-	double TARGET_AVAILABLE_SPACE_RATIO;
-	double AVAILABLE_SPACE_UPDATE_DELAY;
-
-	double MAX_TL_SS_VERSION_DIFFERENCE; // spring starts at half this value
-	double MAX_TL_SS_VERSION_DIFFERENCE_BATCH;
-	int MAX_MACHINES_FALLING_BEHIND;
-
-	int MAX_TPS_HISTORY_SAMPLES;
-	int NEEDED_TPS_HISTORY_SAMPLES;
-	int64_t TARGET_DURABILITY_LAG_VERSIONS;
-	int64_t AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS;
-	int64_t TARGET_DURABILITY_LAG_VERSIONS_BATCH;
-	int64_t DURABILITY_LAG_UNLIMITED_THRESHOLD;
-	double INITIAL_DURABILITY_LAG_MULTIPLIER;
-	double DURABILITY_LAG_REDUCTION_RATE;
-	double DURABILITY_LAG_INCREASE_RATE;
-
-	double STORAGE_SERVER_LIST_FETCH_TIMEOUT;
-
-	// disk snapshot
-	int64_t MAX_FORKED_PROCESS_OUTPUT;
-	double SNAP_CREATE_MAX_TIMEOUT;
-
-	// Storage Metrics
-	double STORAGE_METRICS_AVERAGE_INTERVAL;
-	double STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS;
-	double SPLIT_JITTER_AMOUNT;
-	int64_t IOPS_UNITS_PER_SAMPLE;
-	int64_t BANDWIDTH_UNITS_PER_SAMPLE;
-	int64_t BYTES_READ_UNITS_PER_SAMPLE;
-	int64_t READ_HOT_SUB_RANGE_CHUNK_SIZE;
-	int64_t EMPTY_READ_PENALTY;
-	bool READ_SAMPLING_ENABLED;
-
-	// Storage Server
-	double STORAGE_LOGGING_DELAY;
-	double STORAGE_SERVER_POLL_METRICS_DELAY;
-	double FUTURE_VERSION_DELAY;
-	int STORAGE_LIMIT_BYTES;
-	int BUGGIFY_LIMIT_BYTES;
-	bool FETCH_USING_STREAMING;
-	int FETCH_BLOCK_BYTES;
-	int FETCH_KEYS_PARALLELISM_BYTES;
-	int FETCH_KEYS_PARALLELISM;
-	int FETCH_KEYS_LOWER_PRIORITY;
-	int BUGGIFY_BLOCK_BYTES;
-	double STORAGE_DURABILITY_LAG_REJECT_THRESHOLD;
-	double STORAGE_DURABILITY_LAG_MIN_RATE;
-	int STORAGE_COMMIT_BYTES;
-	int STORAGE_FETCH_BYTES;
-	double STORAGE_COMMIT_INTERVAL;
-	double UPDATE_SHARD_VERSION_INTERVAL;
-	int BYTE_SAMPLING_FACTOR;
-	int BYTE_SAMPLING_OVERHEAD;
-	int MAX_STORAGE_SERVER_WATCH_BYTES;
-	int MAX_BYTE_SAMPLE_CLEAR_MAP_SIZE;
-	double LONG_BYTE_SAMPLE_RECOVERY_DELAY;
-	int BYTE_SAMPLE_LOAD_PARALLELISM;
-	double BYTE_SAMPLE_LOAD_DELAY;
-	double BYTE_SAMPLE_START_DELAY;
-	double UPDATE_STORAGE_PROCESS_STATS_INTERVAL;
-	double BEHIND_CHECK_DELAY;
-	int BEHIND_CHECK_COUNT;
-	int64_t BEHIND_CHECK_VERSIONS;
-	double WAIT_METRICS_WRONG_SHARD_CHANCE;
-	int64_t MIN_TAG_READ_PAGES_RATE;
-	int64_t MIN_TAG_WRITE_PAGES_RATE;
-	double TAG_MEASUREMENT_INTERVAL;
-	int64_t READ_COST_BYTE_FACTOR;
-	bool PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS;
-	bool REPORT_DD_METRICS;
-	double DD_METRICS_REPORT_INTERVAL;
-	double FETCH_KEYS_TOO_LONG_TIME_CRITERIA;
-	double MAX_STORAGE_COMMIT_TIME;
-	int64_t RANGESTREAM_LIMIT_BYTES;
-
-	// Wait Failure
-	int MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS;
-	double WAIT_FAILURE_DELAY_LIMIT;
-
-	// Worker
-	double WORKER_LOGGING_INTERVAL;
-	double HEAP_PROFILER_INTERVAL;
-	double UNKNOWN_CC_TIMEOUT;
-	double DEGRADED_RESET_INTERVAL;
-	double DEGRADED_WARNING_LIMIT;
-	double DEGRADED_WARNING_RESET_DELAY;
-	int64_t TRACE_LOG_FLUSH_FAILURE_CHECK_INTERVAL_SECONDS;
-	double TRACE_LOG_PING_TIMEOUT_SECONDS;
-	double MIN_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS; // Listen for a leader for N seconds, and if not heard, then try to
-	                                                 // become the leader.
-	double MAX_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS;
-	double DBINFO_FAILED_DELAY;
-	bool ENABLE_WORKER_HEALTH_MONITOR;
-	double WORKER_HEALTH_MONITOR_INTERVAL; // Interval between two health monitor health check.
-	int PEER_LATENCY_CHECK_MIN_POPULATION; // The minimum number of latency samples required to check a peer.
-	double PEER_LATENCY_DEGRADATION_PERCENTILE; // The percentile latency used to check peer health.
-	double PEER_LATENCY_DEGRADATION_THRESHOLD; // The latency threshold to consider a peer degraded.
-	double PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD; // The percentage of timeout to consider a peer degraded.
-
-	// Test harness
-	double WORKER_POLL_DELAY;
-
-	// Coordination
-	double COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL;
-	bool ENABLE_CROSS_CLUSTER_SUPPORT; // Allow a coordinator to serve requests whose connection string does not match
-	                                   // the local descriptor
-
-	// Buggification
-	double BUGGIFIED_EVENTUAL_CONSISTENCY;
-	bool BUGGIFY_ALL_COORDINATION;
-
-	// Status
-	double STATUS_MIN_TIME_BETWEEN_REQUESTS;
-	double MAX_STATUS_REQUESTS_PER_SECOND;
-	int CONFIGURATION_ROWS_TO_FETCH;
-	bool DISABLE_DUPLICATE_LOG_WARNING;
-	double HISTOGRAM_REPORT_INTERVAL;
-
-	// IPager
-	int PAGER_RESERVED_PAGES;
-
-	// IndirectShadowPager
-	int FREE_PAGE_VACUUM_THRESHOLD;
-	int VACUUM_QUEUE_SIZE;
-	int VACUUM_BYTES_PER_SECOND;
-
-	// Timekeeper
-	int64_t TIME_KEEPER_DELAY;
-	int64_t TIME_KEEPER_MAX_ENTRIES;
-
-	// Fast Restore
-	// TODO: After 6.3, review FR knobs, remove unneeded ones and change default value
-	int64_t FASTRESTORE_FAILURE_TIMEOUT;
-	int64_t FASTRESTORE_HEARTBEAT_INTERVAL;
-	double FASTRESTORE_SAMPLING_PERCENT;
-	int64_t FASTRESTORE_NUM_LOADERS;
-	int64_t FASTRESTORE_NUM_APPLIERS;
-	// FASTRESTORE_TXN_BATCH_MAX_BYTES is target txn size used by appliers to apply mutations
-	double FASTRESTORE_TXN_BATCH_MAX_BYTES;
-	// FASTRESTORE_VERSIONBATCH_MAX_BYTES is the maximum data size in each version batch
-	double FASTRESTORE_VERSIONBATCH_MAX_BYTES;
-	// FASTRESTORE_VB_PARALLELISM is the number of concurrently running version batches
-	int64_t FASTRESTORE_VB_PARALLELISM;
-	int64_t FASTRESTORE_VB_MONITOR_DELAY; // How quickly monitor finished version batch
-	double FASTRESTORE_VB_LAUNCH_DELAY;
-	int64_t FASTRESTORE_ROLE_LOGGING_DELAY;
-	int64_t FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL; // How quickly to update process metrics for restore
-	int64_t FASTRESTORE_ATOMICOP_WEIGHT; // workload amplication factor for atomic op
-	int64_t FASTRESTORE_APPLYING_PARALLELISM; // number of outstanding txns writing to dest. DB
-	int64_t FASTRESTORE_MONITOR_LEADER_DELAY;
-	int64_t FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS;
-	bool FASTRESTORE_TRACK_REQUEST_LATENCY; // true to track reply latency of each request in a request batch
-	bool FASTRESTORE_TRACK_LOADER_SEND_REQUESTS; // track requests of load send mutations to appliers?
-	int64_t FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT; // threshold when pipelined actors should be delayed
-	int64_t FASTRESTORE_WAIT_FOR_MEMORY_LATENCY;
-	int64_t FASTRESTORE_HEARTBEAT_DELAY; // interval for master to ping loaders and appliers
-	int64_t
-	    FASTRESTORE_HEARTBEAT_MAX_DELAY; // master claim a node is down if no heart beat from the node for this delay
-	int64_t FASTRESTORE_APPLIER_FETCH_KEYS_SIZE; // number of keys to fetch in a txn on applier
-	int64_t FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES; // desired size of mutation message sent from loader to appliers
-	bool FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE; // parse each range file to get (range, version) it has?
-	int64_t FASTRESTORE_REQBATCH_PARALLEL; // number of requests to wait on for getBatchReplies()
-	bool FASTRESTORE_REQBATCH_LOG; // verbose log information for getReplyBatches
-	int FASTRESTORE_TXN_CLEAR_MAX; // threshold to start tracking each clear op in a txn
-	int FASTRESTORE_TXN_RETRY_MAX; // threshold to start output error on too many retries
-	double FASTRESTORE_TXN_EXTRA_DELAY; // extra delay to avoid overwhelming fdb
-	bool FASTRESTORE_NOT_WRITE_DB; // do not write result to DB. Only for dev testing
-	bool FASTRESTORE_USE_RANGE_FILE; // use range file in backup
-	bool FASTRESTORE_USE_LOG_FILE; // use log file in backup
-	int64_t FASTRESTORE_SAMPLE_MSG_BYTES; // sample message desired size
-	double FASTRESTORE_SCHED_UPDATE_DELAY; // delay in seconds in updating process metrics
-	int FASTRESTORE_SCHED_TARGET_CPU_PERCENT; // release as many requests as possible when cpu usage is below the knob
-	int FASTRESTORE_SCHED_MAX_CPU_PERCENT; // max cpu percent when scheduler shall not release non-urgent requests
-	int FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS; // number of inflight requests to load backup files
-	int FASTRESTORE_SCHED_INFLIGHT_SEND_REQS; // number of inflight requests for loaders to  send mutations to appliers
-	int FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE; // number of load request to release at once
-	int FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD; // we can send future VB requests if it is less than this knob
-	int FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH; // number of future VB sendLoadingParam requests to process at once
-	int FASTRESTORE_NUM_TRACE_EVENTS;
-	bool FASTRESTORE_EXPENSIVE_VALIDATION; // when set true, performance will be heavily affected
-	double FASTRESTORE_WRITE_BW_MB; // target aggregated write bandwidth from all appliers
-	double FASTRESTORE_RATE_UPDATE_SECONDS; // how long to update appliers target write rate
-
-	int REDWOOD_DEFAULT_PAGE_SIZE; // Page size for new Redwood files
-	int REDWOOD_DEFAULT_EXTENT_SIZE; // Extent size for new Redwood files
-	int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files
-	int REDWOOD_EXTENT_CONCURRENT_READS; // Max number of simultaneous extent disk reads in progress.
-	int REDWOOD_KVSTORE_CONCURRENT_READS; // Max number of simultaneous point or range reads in progress.
-	double REDWOOD_PAGE_REBUILD_MAX_SLACK; // When rebuilding pages, max slack to allow in page
-	int REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; // Number of pages to try to pop from the lazy delete queue and process at
-	                                         // once
-	int REDWOOD_LAZY_CLEAR_MIN_PAGES; // Minimum number of pages to free before ending a lazy clear cycle, unless the
-	                                  // queue is empty
-	int REDWOOD_LAZY_CLEAR_MAX_PAGES; // Maximum number of pages to free before ending a lazy clear cycle, unless the
-	                                  // queue is empty
-	int64_t REDWOOD_REMAP_CLEANUP_WINDOW; // Remap remover lag interval in which to coalesce page writes
-	double REDWOOD_REMAP_CLEANUP_LAG; // Maximum allowed remap remover lag behind the cleanup window as a multiple of
-	                                  // the window size
-	double REDWOOD_LOGGING_INTERVAL;
-
-	// Server request latency measurement
-	int LATENCY_SAMPLE_SIZE;
-	double LATENCY_METRICS_LOGGING_INTERVAL;
-
-	ServerKnobs();
-	void initialize(bool randomize = false, ClientKnobs* clientKnobs = nullptr, bool isSimulated = false);
-};
-
-extern std::unique_ptr<ServerKnobs> globalServerKnobs;
-extern ServerKnobs const* SERVER_KNOBS;
-
-#endif
+#define SERVER_KNOBS (&IKnobCollection::getGlobalKnobCollection().getServerKnobs())

fdbserver/LocalConfiguration.actor.cpp

@@ -0,0 +1,487 @@
+/*
+ * LocalConfiguration.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbclient/IKnobCollection.h"
+#include "fdbrpc/Stats.h"
+#include "fdbserver/ConfigBroadcastFollowerInterface.h"
+#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/LocalConfiguration.h"
+#include "fdbserver/OnDemandStore.h"
+#include "flow/UnitTest.h"
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+const KeyRef configPathKey = "configPath"_sr;
+const KeyRef lastSeenVersionKey = "lastSeenVersion"_sr;
+const KeyRangeRef knobOverrideKeys = KeyRangeRef("knobOverride/"_sr, "knobOverride0"_sr);
+
+KeyRef stringToKeyRef(std::string const& s) {
+	return StringRef(reinterpret_cast<uint8_t const*>(s.c_str()), s.size());
+}
+
+} // namespace
+
+class LocalConfigurationImpl {
+	UID id;
+	OnDemandStore kvStore;
+	Future<Void> initFuture;
+	Version lastSeenVersion{ 0 };
+	std::unique_ptr<IKnobCollection> testKnobCollection;
+
+	class ConfigKnobOverrides {
+		Standalone<VectorRef<KeyRef>> configPath;
+		std::map<Optional<Key>, std::map<Key, KnobValue>> configClassToKnobToValue;
+
+	public:
+		ConfigKnobOverrides() = default;
+		explicit ConfigKnobOverrides(std::string const& paramString) {
+			configClassToKnobToValue[{}] = {};
+			if (std::all_of(paramString.begin(), paramString.end(), [](char c) {
+				    return isalpha(c) || isdigit(c) || c == '/' || c == '-';
+			    })) {
+				StringRef s = stringToKeyRef(paramString);
+				while (s.size()) {
+					configPath.push_back_deep(configPath.arena(), s.eat("/"_sr));
+					configClassToKnobToValue[configPath.back()] = {};
+				}
+			} else {
+				TEST(true); // Invalid configuration path
+				if (!g_network->isSimulated()) {
+					fprintf(stderr, "WARNING: Invalid configuration path: `%s'\n", paramString.c_str());
+				}
+				throw invalid_config_path();
+			}
+		}
+		ConfigClassSet getConfigClassSet() const { return ConfigClassSet(configPath); }
+		void set(Optional<KeyRef> configClass, KeyRef knobName, KnobValueRef value) {
+			configClassToKnobToValue[configClass.castTo<Key>()][knobName] = value;
+		}
+		void remove(Optional<KeyRef> configClass, KeyRef knobName) {
+			configClassToKnobToValue[configClass.castTo<Key>()].erase(knobName);
+		}
+
+		void update(IKnobCollection& knobCollection) const {
+			// Apply global overrides
+			const auto& knobToValue = configClassToKnobToValue.at({});
+			for (const auto& [knobName, knobValue] : knobToValue) {
+				try {
+					knobCollection.setKnob(knobName.toString(), knobValue);
+				} catch (Error& e) {
+					if (e.code() == error_code_invalid_option_value) {
+						TEST(true); // invalid knob in configuration database
+						TraceEvent(SevWarnAlways, "InvalidKnobOptionValue")
+						    .detail("KnobName", knobName)
+						    .detail("KnobValue", knobValue.toString());
+					} else {
+						throw e;
+					}
+				}
+			}
+			// Apply specific overrides
+			for (const auto& configClass : configPath) {
+				const auto& knobToValue = configClassToKnobToValue.at(configClass);
+				for (const auto& [knobName, knobValue] : knobToValue) {
+					knobCollection.setKnob(knobName.toString(), knobValue);
+				}
+			}
+		}
+
+		bool hasSameConfigPath(ConfigKnobOverrides const& other) const { return configPath == other.configPath; }
+
+		template <class Ar>
+		void serialize(Ar& ar) {
+			serializer(ar, configPath);
+		}
+	} configKnobOverrides;
+
+	class ManualKnobOverrides {
+		std::map<Key, KnobValue> overrides;
+
+	public:
+		explicit ManualKnobOverrides(std::map<std::string, std::string> const& overrides) {
+			for (const auto& [knobName, knobValueString] : overrides) {
+				try {
+					auto knobValue =
+					    IKnobCollection::parseKnobValue(knobName, knobValueString, IKnobCollection::Type::TEST);
+					this->overrides[stringToKeyRef(knobName)] = knobValue;
+				} catch (Error& e) {
+					if (e.code() == error_code_invalid_option) {
+						TEST(true); // Attempted to manually set invalid knob option
+						if (!g_network->isSimulated()) {
+							fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", knobName.c_str());
+						}
+						TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(knobName));
+					} else if (e.code() == error_code_invalid_option_value) {
+						TEST(true); // Invalid manually set knob value
+						if (!g_network->isSimulated()) {
+							fprintf(stderr,
+							        "WARNING: Invalid value '%s' for knob option '%s'\n",
+							        knobValueString.c_str(),
+							        knobName.c_str());
+						}
+						TraceEvent(SevWarnAlways, "InvalidKnobValue")
+						    .detail("Knob", printable(knobName))
+						    .detail("Value", printable(knobValueString));
+					} else {
+						throw e;
+					}
+				}
+			}
+		}
+
+		void update(IKnobCollection& knobCollection) {
+			for (const auto& [knobName, knobValue] : overrides) {
+				knobCollection.setKnob(knobName.toString(), knobValue);
+			}
+		}
+	} manualKnobOverrides;
+
+	IKnobCollection& getKnobs() {
+		return testKnobCollection ? *testKnobCollection : IKnobCollection::getMutableGlobalKnobCollection();
+	}
+
+	IKnobCollection const& getKnobs() const {
+		return testKnobCollection ? *testKnobCollection : IKnobCollection::getGlobalKnobCollection();
+	}
+
+	CounterCollection cc;
+	Counter broadcasterChanges;
+	Counter snapshots;
+	Counter changeRequestsFetched;
+	Counter mutations;
+	Future<Void> logger;
+
+	ACTOR static Future<Void> saveConfigPath(LocalConfigurationImpl* self) {
+		self->kvStore->set(
+		    KeyValueRef(configPathKey, BinaryWriter::toValue(self->configKnobOverrides, IncludeVersion())));
+		wait(self->kvStore->commit());
+		return Void();
+	}
+
+	ACTOR static Future<Void> clearKVStore(LocalConfigurationImpl* self) {
+		self->kvStore->clear(singleKeyRange(configPathKey));
+		self->kvStore->clear(knobOverrideKeys);
+		wait(self->kvStore->commit());
+		return Void();
+	}
+
+	ACTOR static Future<Version> getLastSeenVersion(LocalConfigurationImpl* self) {
+		state Version result = 0;
+		state Optional<Value> lastSeenVersionValue = wait(self->kvStore->readValue(lastSeenVersionKey));
+		if (!lastSeenVersionValue.present()) {
+			self->kvStore->set(KeyValueRef(lastSeenVersionKey, BinaryWriter::toValue(result, IncludeVersion())));
+			wait(self->kvStore->commit());
+		} else {
+			result = BinaryReader::fromStringRef<Version>(lastSeenVersionValue.get(), IncludeVersion());
+		}
+		return result;
+	}
+
+	ACTOR static Future<Void> initialize(LocalConfigurationImpl* self) {
+		state Version lastSeenVersion = wait(getLastSeenVersion(self));
+		state Optional<Value> storedConfigPathValue = wait(self->kvStore->readValue(configPathKey));
+		if (!storedConfigPathValue.present()) {
+			wait(saveConfigPath(self));
+			self->updateInMemoryState(lastSeenVersion);
+			return Void();
+		}
+		state ConfigKnobOverrides storedConfigPath =
+		    BinaryReader::fromStringRef<ConfigKnobOverrides>(storedConfigPathValue.get(), IncludeVersion());
+		if (!storedConfigPath.hasSameConfigPath(self->configKnobOverrides)) {
+			TEST(true); // All local information is outdated
+			wait(clearKVStore(self));
+			wait(saveConfigPath(self));
+			self->updateInMemoryState(lastSeenVersion);
+			return Void();
+		}
+		Standalone<RangeResultRef> range = wait(self->kvStore->readRange(knobOverrideKeys));
+		for (const auto& kv : range) {
+			auto configKey =
+			    BinaryReader::fromStringRef<ConfigKey>(kv.key.removePrefix(knobOverrideKeys.begin), IncludeVersion());
+			self->configKnobOverrides.set(configKey.configClass,
+			                              configKey.knobName,
+			                              ObjectReader::fromStringRef<KnobValue>(kv.value, IncludeVersion()));
+		}
+		self->updateInMemoryState(lastSeenVersion);
+		return Void();
+	}
+
+	void updateInMemoryState(Version lastSeenVersion) {
+		this->lastSeenVersion = lastSeenVersion;
+		// TODO: Support randomization?
+		getKnobs().reset(Randomize::NO, g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO);
+		configKnobOverrides.update(getKnobs());
+		manualKnobOverrides.update(getKnobs());
+		// Must reinitialize in order to update dependent knobs
+		getKnobs().initialize(Randomize::NO, g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO);
+	}
+
+	ACTOR static Future<Void> setSnapshot(LocalConfigurationImpl* self,
+	                                      std::map<ConfigKey, KnobValue> snapshot,
+	                                      Version snapshotVersion) {
+		// TODO: Concurrency control?
+		ASSERT(self->initFuture.isValid() && self->initFuture.isReady());
+		++self->snapshots;
+		self->kvStore->clear(knobOverrideKeys);
+		for (const auto& [configKey, knobValue] : snapshot) {
+			self->configKnobOverrides.set(configKey.configClass, configKey.knobName, knobValue);
+			self->kvStore->set(
+			    KeyValueRef(BinaryWriter::toValue(configKey, IncludeVersion()).withPrefix(knobOverrideKeys.begin),
+			                ObjectWriter::toValue(knobValue, IncludeVersion())));
+		}
+		ASSERT_GE(snapshotVersion, self->lastSeenVersion);
+		self->kvStore->set(KeyValueRef(lastSeenVersionKey, BinaryWriter::toValue(snapshotVersion, IncludeVersion())));
+		wait(self->kvStore->commit());
+		self->updateInMemoryState(snapshotVersion);
+		return Void();
+	}
+
+	ACTOR static Future<Void> addChanges(LocalConfigurationImpl* self,
+	                                     Standalone<VectorRef<VersionedConfigMutationRef>> changes,
+	                                     Version mostRecentVersion) {
+		// TODO: Concurrency control?
+		ASSERT(self->initFuture.isValid() && self->initFuture.isReady());
+		++self->changeRequestsFetched;
+		for (const auto& versionedMutation : changes) {
+			ASSERT_GT(versionedMutation.version, self->lastSeenVersion);
+			++self->mutations;
+			const auto& mutation = versionedMutation.mutation;
+			auto serializedKey = BinaryWriter::toValue(mutation.getKey(), IncludeVersion());
+			if (mutation.isSet()) {
+				self->kvStore->set(KeyValueRef(serializedKey.withPrefix(knobOverrideKeys.begin),
+				                               ObjectWriter::toValue(mutation.getValue(), IncludeVersion())));
+				self->configKnobOverrides.set(mutation.getConfigClass(), mutation.getKnobName(), mutation.getValue());
+			} else {
+				self->kvStore->clear(singleKeyRange(serializedKey.withPrefix(knobOverrideKeys.begin)));
+				self->configKnobOverrides.remove(mutation.getConfigClass(), mutation.getKnobName());
+			}
+		}
+		self->kvStore->set(KeyValueRef(lastSeenVersionKey, BinaryWriter::toValue(mostRecentVersion, IncludeVersion())));
+		wait(self->kvStore->commit());
+		self->updateInMemoryState(mostRecentVersion);
+		return Void();
+	}
+
+	ACTOR static Future<Void> consumeInternal(LocalConfigurationImpl* self,
+	                                          ConfigBroadcastFollowerInterface broadcaster) {
+		loop {
+			try {
+				state ConfigBroadcastFollowerGetChangesReply changesReply =
+				    wait(broadcaster.getChanges.getReply(ConfigBroadcastFollowerGetChangesRequest{
+				        self->lastSeenVersion, self->configKnobOverrides.getConfigClassSet() }));
+				TraceEvent(SevDebug, "LocalConfigGotChanges", self->id)
+				    .detail("Size", changesReply.changes.size())
+				    .detail("Version", changesReply.mostRecentVersion);
+				wait(self->addChanges(changesReply.changes, changesReply.mostRecentVersion));
+			} catch (Error& e) {
+				if (e.code() == error_code_version_already_compacted) {
+					state ConfigBroadcastFollowerGetSnapshotReply snapshotReply = wait(broadcaster.getSnapshot.getReply(
+					    ConfigBroadcastFollowerGetSnapshotRequest{ self->configKnobOverrides.getConfigClassSet() }));
+					ASSERT_GT(snapshotReply.version, self->lastSeenVersion);
+					++self->snapshots;
+					wait(setSnapshot(self, std::move(snapshotReply.snapshot), snapshotReply.version));
+				} else {
+					throw e;
+				}
+			}
+			wait(yield()); // Necessary to not immediately trigger retry?
+		}
+	}
+
+	ACTOR static Future<Void> consume(
+	    LocalConfigurationImpl* self,
+	    Reference<IDependentAsyncVar<ConfigBroadcastFollowerInterface> const> broadcaster) {
+		ASSERT(self->initFuture.isValid() && self->initFuture.isReady());
+		loop {
+			choose {
+				when(wait(brokenPromiseToNever(consumeInternal(self, broadcaster->get())))) { ASSERT(false); }
+				when(wait(broadcaster->onChange())) { ++self->broadcasterChanges; }
+				when(wait(self->kvStore->getError())) { ASSERT(false); }
+			}
+		}
+	}
+
+public:
+	LocalConfigurationImpl(std::string const& dataFolder,
+	                       std::string const& configPath,
+	                       std::map<std::string, std::string> const& manualKnobOverrides,
+	                       IsTest isTest)
+	  : id(deterministicRandom()->randomUniqueID()), kvStore(dataFolder, id, "localconf-"), cc("LocalConfiguration"),
+	    broadcasterChanges("BroadcasterChanges", cc), snapshots("Snapshots", cc),
+	    changeRequestsFetched("ChangeRequestsFetched", cc), mutations("Mutations", cc), configKnobOverrides(configPath),
+	    manualKnobOverrides(manualKnobOverrides) {
+		if (isTest == IsTest::YES) {
+			testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST,
+			                                             Randomize::NO,
+			                                             g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO);
+		}
+		logger = traceCounters(
+		    "LocalConfigurationMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "LocalConfigurationMetrics");
+	}
+
+	Future<Void> initialize() {
+		ASSERT(!initFuture.isValid());
+		initFuture = initialize(this);
+		return initFuture;
+	}
+
+	Future<Void> addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> changes, Version mostRecentVersion) {
+		return addChanges(this, changes, mostRecentVersion);
+	}
+
+	FlowKnobs const& getFlowKnobs() const {
+		ASSERT(initFuture.isValid() && initFuture.isReady());
+		return getKnobs().getFlowKnobs();
+	}
+
+	ClientKnobs const& getClientKnobs() const {
+		ASSERT(initFuture.isValid() && initFuture.isReady());
+		return getKnobs().getClientKnobs();
+	}
+
+	ServerKnobs const& getServerKnobs() const {
+		ASSERT(initFuture.isValid() && initFuture.isReady());
+		return getKnobs().getServerKnobs();
+	}
+
+	TestKnobs const& getTestKnobs() const {
+		ASSERT(initFuture.isValid() && initFuture.isReady());
+		return getKnobs().getTestKnobs();
+	}
+
+	Future<Void> consume(Reference<IDependentAsyncVar<ConfigBroadcastFollowerInterface> const> const& broadcaster) {
+		return consume(this, broadcaster);
+	}
+
+	UID getID() const { return id; }
+
+	static void testManualKnobOverridesInvalidName() {
+		std::map<std::string, std::string> invalidOverrides;
+		invalidOverrides["knob_name_that_does_not_exist"] = "";
+		// Should only trace and not throw an error:
+		ManualKnobOverrides manualKnobOverrides(invalidOverrides);
+	}
+
+	static void testManualKnobOverridesInvalidValue() {
+		std::map<std::string, std::string> invalidOverrides;
+		invalidOverrides["test_int"] = "not_an_int";
+		// Should only trace and not throw an error:
+		ManualKnobOverrides manualKnobOverrides(invalidOverrides);
+	}
+
+	static void testConfigKnobOverridesInvalidConfigPath() {
+		try {
+			ConfigKnobOverrides configKnobOverrides("#invalid_config_path");
+			ASSERT(false);
+		} catch (Error& e) {
+			ASSERT_EQ(e.code(), error_code_invalid_config_path);
+		}
+	}
+
+	static void testConfigKnobOverridesInvalidName() {
+		ConfigKnobOverrides configKnobOverrides;
+		configKnobOverrides.set(
+		    {}, "knob_name_that_does_not_exist"_sr, KnobValueRef::create(ParsedKnobValue(int{ 1 })));
+		auto testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::NO, IsSimulated::NO);
+		// Should only trace and not throw an error:
+		configKnobOverrides.update(*testKnobCollection);
+	}
+
+	static void testConfigKnobOverridesInvalidValue() {
+		ConfigKnobOverrides configKnobOverrides;
+		configKnobOverrides.set({}, "test_int"_sr, KnobValueRef::create(ParsedKnobValue("not_an_int")));
+		auto testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::NO, IsSimulated::NO);
+		// Should only trace and not throw an error:
+		configKnobOverrides.update(*testKnobCollection);
+	}
+};
+
+LocalConfiguration::LocalConfiguration(std::string const& dataFolder,
+                                       std::string const& configPath,
+                                       std::map<std::string, std::string> const& manualKnobOverrides,
+                                       IsTest isTest)
+  : _impl(std::make_unique<LocalConfigurationImpl>(dataFolder, configPath, manualKnobOverrides, isTest)) {}
+
+LocalConfiguration::LocalConfiguration(LocalConfiguration&&) = default;
+
+LocalConfiguration& LocalConfiguration::operator=(LocalConfiguration&&) = default;
+
+LocalConfiguration::~LocalConfiguration() = default;
+
+Future<Void> LocalConfiguration::initialize() {
+	return impl().initialize();
+}
+
+FlowKnobs const& LocalConfiguration::getFlowKnobs() const {
+	return impl().getFlowKnobs();
+}
+
+ClientKnobs const& LocalConfiguration::getClientKnobs() const {
+	return impl().getClientKnobs();
+}
+
+ServerKnobs const& LocalConfiguration::getServerKnobs() const {
+	return impl().getServerKnobs();
+}
+
+TestKnobs const& LocalConfiguration::getTestKnobs() const {
+	return impl().getTestKnobs();
+}
+
+Future<Void> LocalConfiguration::consume(
+    Reference<IDependentAsyncVar<ConfigBroadcastFollowerInterface> const> const& broadcaster) {
+	return impl().consume(broadcaster);
+}
+
+Future<Void> LocalConfiguration::addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> changes,
+                                            Version mostRecentVersion) {
+	return impl().addChanges(changes, mostRecentVersion);
+}
+
+UID LocalConfiguration::getID() const {
+	return impl().getID();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ManualKnobOverrides/InvalidName") {
+	LocalConfigurationImpl::testManualKnobOverridesInvalidName();
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ManualKnobOverrides/InvalidValue") {
+	LocalConfigurationImpl::testManualKnobOverridesInvalidValue();
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ConfigKnobOverrides/InvalidConfigPath") {
+	LocalConfigurationImpl::testConfigKnobOverridesInvalidConfigPath();
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ConfigKnobOverrides/InvalidName") {
+	LocalConfigurationImpl::testConfigKnobOverridesInvalidName();
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/ConfigKnobOverrides/InvalidValue") {
+	LocalConfigurationImpl::testConfigKnobOverridesInvalidValue();
+	return Void();
+}

fdbserver/LocalConfiguration.h

@@ -0,0 +1,70 @@
+/*
+ * LocalConfiguration.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "fdbclient/ConfigKnobs.h"
+#include "fdbclient/IKnobCollection.h"
+#include "fdbserver/ConfigBroadcastFollowerInterface.h"
+#include "fdbserver/Knobs.h"
+#include "flow/Arena.h"
+#include "flow/Knobs.h"
+
+// To be used effectively as a boolean parameter with added type safety
+enum class IsTest { NO, YES };
+
+/*
+ * Each worker maintains a LocalConfiguration object used to update its knob collection.
+ * When a worker starts, the following steps are executed:
+ *    - Apply manual knob updates
+ *    - Read the local configuration file (with "localconf" prefix)
+ *      - If the stored configuration path does not match the current configuration path, delete the local configuration
+ * file
+ *      - Otherwise, apply knob updates from the local configuration file (without overriding manual knob overrides)
+ *    - Register with the broadcaster to receive new updates for the relevant configuration classes
+ *      - Persist these updates when received, and restart if necessary
+ */
+class LocalConfiguration {
+	std::unique_ptr<class LocalConfigurationImpl> _impl;
+	LocalConfigurationImpl& impl() { return *_impl; }
+	LocalConfigurationImpl const& impl() const { return *_impl; }
+
+public:
+	LocalConfiguration(std::string const& dataFolder,
+	                   std::string const& configPath,
+	                   std::map<std::string, std::string> const& manualKnobOverrides,
+	                   IsTest isTest = IsTest::NO);
+	LocalConfiguration(LocalConfiguration&&);
+	LocalConfiguration& operator=(LocalConfiguration&&);
+	~LocalConfiguration();
+	Future<Void> initialize();
+	FlowKnobs const& getFlowKnobs() const;
+	ClientKnobs const& getClientKnobs() const;
+	ServerKnobs const& getServerKnobs() const;
+	TestKnobs const& getTestKnobs() const;
+	Future<Void> consume(Reference<IDependentAsyncVar<ConfigBroadcastFollowerInterface> const> const& broadcaster);
+	UID getID() const;
+
+public: // Testing
+	Future<Void> addChanges(Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations,
+	                        Version mostRecentVersion);
+};

fdbserver/OnDemandStore.actor.cpp

@@ -0,0 +1,63 @@
+/*
+ * OnDemandStore.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/OnDemandStore.h"
+#include "flow/actorcompiler.h" // must be last include
+
+ACTOR static Future<Void> onErr(Future<Future<Void>> e) {
+	Future<Void> f = wait(e);
+	wait(f);
+	return Void();
+}
+
+void OnDemandStore::open() {
+	platform::createDirectory(folder);
+	store = keyValueStoreMemory(joinPath(folder, prefix), myID, 500e6);
+	err.send(store->getError());
+}
+
+OnDemandStore::OnDemandStore(std::string const& folder, UID myID, std::string const& prefix)
+  : folder(folder), prefix(prefix), store(nullptr), myID(myID) {}
+
+OnDemandStore::~OnDemandStore() {
+	if (store) {
+		store->close();
+	}
+}
+
+IKeyValueStore* OnDemandStore::get() {
+	if (!store) {
+		open();
+	}
+	return store;
+}
+
+bool OnDemandStore::exists() const {
+	return store || fileExists(joinPath(folder, prefix + "0.fdq")) || fileExists(joinPath(folder, prefix + "1.fdq")) ||
+	       fileExists(joinPath(folder, prefix + ".fdb"));
+}
+
+IKeyValueStore* OnDemandStore::operator->() {
+	return get();
+}
+
+Future<Void> OnDemandStore::getError() const {
+	return onErr(err.getFuture());
+}

fdbserver/OnDemandStore.h

@@ -0,0 +1,44 @@
+/*
+ * OnDemandStore.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "flow/Arena.h"
+#include "flow/IRandom.h"
+#include "flow/Platform.h"
+#include "fdbserver/IKeyValueStore.h"
+
+// Create a key value store if and only if it is actually used
+class OnDemandStore : NonCopyable {
+	std::string folder;
+	UID myID;
+	IKeyValueStore* store;
+	Promise<Future<Void>> err;
+	std::string prefix;
+	void open();
+
+public:
+	OnDemandStore(std::string const& folder, UID myID, std::string const& prefix);
+	~OnDemandStore();
+	IKeyValueStore* get();
+	bool exists() const;
+	IKeyValueStore* operator->();
+	Future<Void> getError() const;
+};

fdbserver/PaxosConfigConsumer.actor.cpp

@@ -0,0 +1,44 @@
+/*
+ * PaxosConfigConsumer.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/PaxosConfigConsumer.h"
+
+class PaxosConfigConsumerImpl {};
+
+PaxosConfigConsumer::PaxosConfigConsumer(ServerCoordinators const& cfi,
+                                         Optional<double> pollingInterval,
+                                         Optional<double> compactionInterval) {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+PaxosConfigConsumer::~PaxosConfigConsumer() = default;
+
+Future<Void> PaxosConfigConsumer::consume(ConfigBroadcaster& broadcaster) {
+	// TODO: Implement
+	ASSERT(false);
+	return Void();
+}
+
+UID PaxosConfigConsumer::getID() const {
+	// TODO: Implement
+	ASSERT(false);
+	return {};
+}

fdbserver/PaxosConfigConsumer.h

@@ -0,0 +1,40 @@
+/*
+ * PaxosConfigConsumer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/IConfigConsumer.h"
+
+/*
+ * A fault-tolerant configuration database consumer implementation
+ */
+class PaxosConfigConsumer : public IConfigConsumer {
+	std::unique_ptr<class PaxosConfigConsumerImpl> _impl;
+	PaxosConfigConsumerImpl const& impl() const { return *_impl; }
+	PaxosConfigConsumerImpl& impl() { return *_impl; }
+
+public:
+	PaxosConfigConsumer(ServerCoordinators const& cfi,
+	                    Optional<double> pollingInterval,
+	                    Optional<double> compactionInterval);
+	~PaxosConfigConsumer();
+	Future<Void> consume(ConfigBroadcaster& broadcaster) override;
+	UID getID() const override;
+};

fdbserver/PaxosConfigDatabaseNode.actor.cpp

@@ -0,0 +1,42 @@
+/*
+ * PaxosConfigDatabaseNode.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/PaxosConfigDatabaseNode.h"
+
+class PaxosConfigDatabaseNodeImpl {};
+
+PaxosConfigDatabaseNode::PaxosConfigDatabaseNode(std::string const& folder) {
+	// TODO: Implement
+	ASSERT(false);
+}
+
+PaxosConfigDatabaseNode::~PaxosConfigDatabaseNode() = default;
+
+Future<Void> PaxosConfigDatabaseNode::serve(ConfigTransactionInterface const& cti) {
+	// TODO: Implement
+	ASSERT(false);
+	return Void();
+}
+
+Future<Void> PaxosConfigDatabaseNode::serve(ConfigFollowerInterface const& cfi) {
+	// TODO: Implement
+	ASSERT(false);
+	return Void();
+}

fdbserver/PaxosConfigDatabaseNode.h

@@ -0,0 +1,36 @@
+/*
+ * PaxosConfigDatabaseNode.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/IConfigDatabaseNode.h"
+
+/*
+ * Fault-tolerant configuration database node implementation
+ */
+class PaxosConfigDatabaseNode : public IConfigDatabaseNode {
+	std::unique_ptr<class PaxosConfigDatabaseNodeImpl> impl;
+
+public:
+	PaxosConfigDatabaseNode(std::string const& folder);
+	~PaxosConfigDatabaseNode();
+	Future<Void> serve(ConfigTransactionInterface const&) override;
+	Future<Void> serve(ConfigFollowerInterface const&) override;
+};

fdbserver/ServerDBInfo.actor.h

@@ -26,6 +26,7 @@
 #define FDBSERVER_SERVERDBINFO_H
 #pragma once
 
+#include "fdbserver/ConfigBroadcastFollowerInterface.h"
 #include "fdbserver/DataDistributorInterface.h"
 #include "fdbserver/MasterInterface.h"
 #include "fdbserver/LogSystemConfig.h"
@@ -62,6 +63,7 @@ struct ServerDBInfo {
 	                                           // which need to stay alive in case this recovery fails
 	Optional<LatencyBandConfig> latencyBandConfig;
 	int64_t infoGeneration;
+	ConfigBroadcastFollowerInterface configBroadcaster;
 
 	ServerDBInfo()
 	  : recoveryCount(0), recoveryState(RecoveryState::UNINITIALIZED), logSystemConfig(0), infoGeneration(0) {}
@@ -85,7 +87,8 @@ struct ServerDBInfo {
 		           logSystemConfig,
 		           priorCommittedLogServers,
 		           latencyBandConfig,
-		           infoGeneration);
+		           infoGeneration,
+		           configBroadcaster);
 	}
 };
 

fdbserver/SimpleConfigConsumer.actor.cpp

@@ -0,0 +1,150 @@
+/*
+ * SimpleConfigConsumer.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/ConfigBroadcastFollowerInterface.h"
+#include "fdbserver/SimpleConfigConsumer.h"
+
+class SimpleConfigConsumerImpl {
+	ConfigFollowerInterface cfi;
+	Version lastSeenVersion{ 0 };
+	double pollingInterval;
+	Optional<double> compactionInterval;
+
+	UID id;
+	CounterCollection cc;
+	Counter compactRequest;
+	Counter successfulChangeRequest;
+	Counter failedChangeRequest;
+	Counter snapshotRequest;
+	Future<Void> logger;
+
+	ACTOR static Future<Void> compactor(SimpleConfigConsumerImpl* self, ConfigBroadcaster* broadcaster) {
+		if (!self->compactionInterval.present()) {
+			wait(Never());
+			return Void();
+		}
+		loop {
+			state Version compactionVersion = self->lastSeenVersion;
+			wait(delayJittered(self->compactionInterval.get()));
+			wait(self->cfi.compact.getReply(ConfigFollowerCompactRequest{ compactionVersion }));
+			++self->compactRequest;
+			broadcaster->compact(compactionVersion);
+		}
+	}
+
+	ACTOR static Future<Void> fetchChanges(SimpleConfigConsumerImpl* self, ConfigBroadcaster* broadcaster) {
+		wait(getSnapshotAndChanges(self, broadcaster));
+		loop {
+			try {
+				ConfigFollowerGetChangesReply reply =
+				    wait(self->cfi.getChanges.getReply(ConfigFollowerGetChangesRequest{ self->lastSeenVersion }));
+				++self->successfulChangeRequest;
+				for (const auto& versionedMutation : reply.changes) {
+					TraceEvent te(SevDebug, "ConsumerFetchedMutation", self->id);
+					te.detail("Version", versionedMutation.version)
+					    .detail("ConfigClass", versionedMutation.mutation.getConfigClass())
+					    .detail("KnobName", versionedMutation.mutation.getKnobName());
+					if (versionedMutation.mutation.isSet()) {
+						te.detail("Op", "Set").detail("KnobValue", versionedMutation.mutation.getValue().toString());
+					} else {
+						te.detail("Op", "Clear");
+					}
+				}
+				ASSERT_GE(reply.mostRecentVersion, self->lastSeenVersion);
+				if (reply.mostRecentVersion > self->lastSeenVersion) {
+					self->lastSeenVersion = reply.mostRecentVersion;
+					broadcaster->applyChanges(reply.changes, reply.mostRecentVersion, reply.annotations);
+				}
+				wait(delayJittered(self->pollingInterval));
+			} catch (Error& e) {
+				++self->failedChangeRequest;
+				if (e.code() == error_code_version_already_compacted) {
+					TEST(true); // SimpleConfigConsumer get version_already_compacted error
+					wait(getSnapshotAndChanges(self, broadcaster));
+				} else {
+					throw e;
+				}
+			}
+		}
+	}
+
+	ACTOR static Future<Void> getSnapshotAndChanges(SimpleConfigConsumerImpl* self, ConfigBroadcaster* broadcaster) {
+		ConfigFollowerGetSnapshotAndChangesReply reply =
+		    wait(self->cfi.getSnapshotAndChanges.getReply(ConfigFollowerGetSnapshotAndChangesRequest{}));
+		++self->snapshotRequest;
+		TraceEvent(SevDebug, "ConfigConsumerGotSnapshotAndChanges", self->id)
+		    .detail("SnapshotVersion", reply.snapshotVersion)
+		    .detail("SnapshotSize", reply.snapshot.size())
+		    .detail("ChangesVersion", reply.changesVersion)
+		    .detail("ChangesSize", reply.changes.size())
+		    .detail("AnnotationsSize", reply.annotations.size());
+		broadcaster->applySnapshotAndChanges(
+		    std::move(reply.snapshot), reply.snapshotVersion, reply.changes, reply.changesVersion, reply.annotations);
+		ASSERT_GE(reply.changesVersion, self->lastSeenVersion);
+		self->lastSeenVersion = reply.changesVersion;
+		return Void();
+	}
+
+	static ConfigFollowerInterface getConfigFollowerInterface(ConfigFollowerInterface const& cfi) { return cfi; }
+
+	static ConfigFollowerInterface getConfigFollowerInterface(ServerCoordinators const& coordinators) {
+		return ConfigFollowerInterface(coordinators.configServers[0]);
+	}
+
+public:
+	template <class ConfigSource>
+	SimpleConfigConsumerImpl(ConfigSource const& configSource,
+	                         double const& pollingInterval,
+	                         Optional<double> const& compactionInterval)
+	  : pollingInterval(pollingInterval), compactionInterval(compactionInterval),
+	    id(deterministicRandom()->randomUniqueID()), cc("ConfigConsumer"), compactRequest("CompactRequest", cc),
+	    successfulChangeRequest("SuccessfulChangeRequest", cc), failedChangeRequest("FailedChangeRequest", cc),
+	    snapshotRequest("SnapshotRequest", cc) {
+		cfi = getConfigFollowerInterface(configSource);
+		logger = traceCounters(
+		    "ConfigConsumerMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ConfigConsumerMetrics");
+	}
+
+	Future<Void> consume(ConfigBroadcaster& broadcaster) {
+		return fetchChanges(this, &broadcaster) || compactor(this, &broadcaster);
+	}
+
+	UID getID() const { return id; }
+};
+
+SimpleConfigConsumer::SimpleConfigConsumer(ConfigFollowerInterface const& cfi,
+                                           double pollingInterval,
+                                           Optional<double> compactionInterval)
+  : _impl(std::make_unique<SimpleConfigConsumerImpl>(cfi, pollingInterval, compactionInterval)) {}
+
+SimpleConfigConsumer::SimpleConfigConsumer(ServerCoordinators const& coordinators,
+                                           double pollingInterval,
+                                           Optional<double> compactionInterval)
+  : _impl(std::make_unique<SimpleConfigConsumerImpl>(coordinators, pollingInterval, compactionInterval)) {}
+
+Future<Void> SimpleConfigConsumer::consume(ConfigBroadcaster& broadcaster) {
+	return impl().consume(broadcaster);
+}
+
+SimpleConfigConsumer::~SimpleConfigConsumer() = default;
+
+UID SimpleConfigConsumer::getID() const {
+	return impl().getID();
+}

fdbserver/SimpleConfigConsumer.h

@@ -0,0 +1,49 @@
+/*
+ * SimpleConfigConsumer.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/IConfigConsumer.h"
+#include "fdbserver/LocalConfiguration.h"
+#include <memory>
+
+/*
+ * A test-only configuration database consumer implementation that interacts with a single-node
+ * configuration database. It assumed that a single coordinator (the lowest coordinator by IP address)
+ * stores all data, so there is no fault tolerance.
+ */
+class SimpleConfigConsumer : public IConfigConsumer {
+	std::unique_ptr<class SimpleConfigConsumerImpl> _impl;
+	SimpleConfigConsumerImpl const& impl() const { return *_impl; }
+	SimpleConfigConsumerImpl& impl() { return *_impl; }
+
+public:
+	SimpleConfigConsumer(ServerCoordinators const& coordinators,
+	                     double pollingInterval,
+	                     Optional<double> compactionInterval);
+	~SimpleConfigConsumer();
+	Future<Void> consume(ConfigBroadcaster& broadcaster) override;
+	UID getID() const override;
+
+public: // Testing
+	SimpleConfigConsumer(ConfigFollowerInterface const& cfi,
+	                     double pollingInterval,
+	                     Optional<double> compactionInterval);
+};

fdbserver/SimpleConfigDatabaseNode.actor.cpp

@@ -0,0 +1,495 @@
+/*
+ * SimpleConfigDatabaseNode.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <map>
+
+#include "fdbclient/SystemData.h"
+#include "fdbserver/SimpleConfigDatabaseNode.h"
+#include "fdbserver/IKeyValueStore.h"
+#include "fdbserver/OnDemandStore.h"
+#include "flow/Arena.h"
+#include "flow/genericactors.actor.h"
+#include "flow/UnitTest.h"
+
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+namespace {
+
+const KeyRef lastCompactedVersionKey = "lastCompactedVersion"_sr;
+const KeyRef liveTransactionVersionKey = "liveTransactionVersion"_sr;
+const KeyRef committedVersionKey = "committedVersion"_sr;
+const KeyRangeRef kvKeys = KeyRangeRef("kv/"_sr, "kv0"_sr);
+const KeyRangeRef mutationKeys = KeyRangeRef("mutation/"_sr, "mutation0"_sr);
+const KeyRangeRef annotationKeys = KeyRangeRef("annotation/"_sr, "annotation0"_sr);
+
+Key versionedAnnotationKey(Version version) {
+	ASSERT_GE(version, 0);
+	return BinaryWriter::toValue(bigEndian64(version), IncludeVersion()).withPrefix(annotationKeys.begin);
+}
+
+Version getVersionFromVersionedAnnotationKey(KeyRef versionedAnnotationKey) {
+	return fromBigEndian64(BinaryReader::fromStringRef<uint64_t>(
+	    versionedAnnotationKey.removePrefix(annotationKeys.begin), IncludeVersion()));
+}
+
+Key versionedMutationKey(Version version, uint32_t index) {
+	ASSERT_GE(version, 0);
+	BinaryWriter bw(IncludeVersion());
+	bw << bigEndian64(version);
+	bw << bigEndian32(index);
+	return bw.toValue().withPrefix(mutationKeys.begin);
+}
+
+Version getVersionFromVersionedMutationKey(KeyRef versionedMutationKey) {
+	uint64_t bigEndianResult;
+	ASSERT(versionedMutationKey.startsWith(mutationKeys.begin));
+	BinaryReader br(versionedMutationKey.removePrefix(mutationKeys.begin), IncludeVersion());
+	br >> bigEndianResult;
+	return fromBigEndian64(bigEndianResult);
+}
+
+} //namespace
+
+TEST_CASE("/fdbserver/ConfigDB/SimpleConfigDatabaseNode/Internal/versionedMutationKeys") {
+	std::vector<Key> keys;
+	for (Version version = 0; version < 1000; ++version) {
+		for (int index = 0; index < 5; ++index) {
+			keys.push_back(versionedMutationKey(version, index));
+		}
+	}
+	for (int i = 0; i < 5000; ++i) {
+		ASSERT(getVersionFromVersionedMutationKey(keys[i]) == i / 5);
+	}
+	return Void();
+}
+
+TEST_CASE("/fdbserver/ConfigDB/SimpleConfigDatabaseNode/Internal/versionedMutationKeyOrdering") {
+	Standalone<VectorRef<KeyRef>> keys;
+	for (Version version = 0; version < 1000; ++version) {
+		for (auto index = 0; index < 5; ++index) {
+			keys.push_back_deep(keys.arena(), versionedMutationKey(version, index));
+		}
+	}
+	for (auto index = 0; index < 1000; ++index) {
+		keys.push_back_deep(keys.arena(), versionedMutationKey(1000, index));
+	}
+	ASSERT(std::is_sorted(keys.begin(), keys.end()));
+	return Void();
+}
+
+class SimpleConfigDatabaseNodeImpl {
+	UID id;
+	OnDemandStore kvStore;
+	CounterCollection cc;
+
+	// Follower counters
+	Counter compactRequests;
+	Counter successfulChangeRequests;
+	Counter failedChangeRequests;
+	Counter snapshotRequests;
+
+	// Transaction counters
+	Counter successfulCommits;
+	Counter failedCommits;
+	Counter setMutations;
+	Counter clearMutations;
+	Counter getValueRequests;
+	Counter newVersionRequests;
+	Future<Void> logger;
+
+	ACTOR static Future<Version> getLiveTransactionVersion(SimpleConfigDatabaseNodeImpl *self) {
+		Optional<Value> value = wait(self->kvStore->readValue(liveTransactionVersionKey));
+		state Version liveTransactionVersion = 0;
+		if (value.present()) {
+			liveTransactionVersion = BinaryReader::fromStringRef<Version>(value.get(), IncludeVersion());
+		} else {
+			self->kvStore->set(KeyValueRef(liveTransactionVersionKey, BinaryWriter::toValue(liveTransactionVersion, IncludeVersion())));
+			wait(self->kvStore->commit());
+		}
+		return liveTransactionVersion;
+	}
+
+	ACTOR static Future<Version> getCommittedVersion(SimpleConfigDatabaseNodeImpl *self) {
+		Optional<Value> value = wait(self->kvStore->readValue(committedVersionKey));
+		state Version committedVersion = 0;
+		if (value.present()) {
+			committedVersion = BinaryReader::fromStringRef<Version>(value.get(), IncludeVersion());
+		} else {
+			self->kvStore->set(KeyValueRef(committedVersionKey, BinaryWriter::toValue(committedVersion, IncludeVersion())));
+			wait(self->kvStore->commit());
+		}
+		return committedVersion;
+	}
+
+	ACTOR static Future<Version> getLastCompactedVersion(SimpleConfigDatabaseNodeImpl* self) {
+		Optional<Value> value = wait(self->kvStore->readValue(lastCompactedVersionKey));
+		state Version lastCompactedVersion = 0;
+		if (value.present()) {
+			lastCompactedVersion = BinaryReader::fromStringRef<Version>(value.get(), IncludeVersion());
+		} else {
+			self->kvStore->set(
+			    KeyValueRef(lastCompactedVersionKey, BinaryWriter::toValue(lastCompactedVersion, IncludeVersion())));
+			wait(self->kvStore->commit());
+		}
+		return lastCompactedVersion;
+	}
+
+	// Returns all commit annotations between for commits with version in [startVersion, endVersion]
+	ACTOR static Future<Standalone<VectorRef<VersionedConfigCommitAnnotationRef>>>
+	getAnnotations(SimpleConfigDatabaseNodeImpl* self, Version startVersion, Version endVersion) {
+		Key startKey = versionedAnnotationKey(startVersion);
+		Key endKey = versionedAnnotationKey(endVersion + 1);
+		state KeyRangeRef keys(startKey, endKey);
+		Standalone<RangeResultRef> range = wait(self->kvStore->readRange(keys));
+		Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> result;
+		for (const auto& kv : range) {
+			auto version = getVersionFromVersionedAnnotationKey(kv.key);
+			ASSERT_LE(version, endVersion);
+			auto annotation = BinaryReader::fromStringRef<ConfigCommitAnnotation>(kv.value, IncludeVersion());
+			result.emplace_back_deep(result.arena(), version, annotation);
+		}
+		return result;
+	}
+
+	// Returns all mutations with version in [startVersion, endVersion]
+	ACTOR static Future<Standalone<VectorRef<VersionedConfigMutationRef>>>
+	getMutations(SimpleConfigDatabaseNodeImpl* self, Version startVersion, Version endVersion) {
+		Key startKey = versionedMutationKey(startVersion, 0);
+		Key endKey = versionedMutationKey(endVersion + 1, 0);
+		state KeyRangeRef keys(startKey, endKey);
+		Standalone<RangeResultRef> range = wait(self->kvStore->readRange(keys));
+		Standalone<VectorRef<VersionedConfigMutationRef>> result;
+		for (const auto &kv : range) {
+			auto version = getVersionFromVersionedMutationKey(kv.key);
+			ASSERT_LE(version, endVersion);
+			auto mutation = ObjectReader::fromStringRef<ConfigMutation>(kv.value, IncludeVersion());
+			result.emplace_back_deep(result.arena(), version, mutation);
+		}
+		return result;
+	}
+
+	ACTOR static Future<Void> getChanges(SimpleConfigDatabaseNodeImpl *self, ConfigFollowerGetChangesRequest req) {
+		Version lastCompactedVersion = wait(getLastCompactedVersion(self));
+		if (req.lastSeenVersion < lastCompactedVersion) {
+			++self->failedChangeRequests;
+			req.reply.sendError(version_already_compacted());
+			return Void();
+		}
+		state Version committedVersion = wait(getCommittedVersion(self));
+		state Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations =
+		    wait(getMutations(self, req.lastSeenVersion + 1, committedVersion));
+		state Standalone<VectorRef<VersionedConfigCommitAnnotationRef>> versionedAnnotations =
+		    wait(getAnnotations(self, req.lastSeenVersion + 1, committedVersion));
+		TraceEvent(SevDebug, "ConfigDatabaseNodeSendingChanges")
+		    .detail("ReqLastSeenVersion", req.lastSeenVersion)
+		    .detail("CommittedVersion", committedVersion)
+		    .detail("NumMutations", versionedMutations.size())
+		    .detail("NumCommits", versionedAnnotations.size());
+		++self->successfulChangeRequests;
+		req.reply.send(ConfigFollowerGetChangesReply{ committedVersion, versionedMutations, versionedAnnotations });
+		return Void();
+	}
+
+	// New transactions increment the database's current live version. This effectively serves as a lock, providing
+	// serializability
+	ACTOR static Future<Void> getNewVersion(SimpleConfigDatabaseNodeImpl* self, ConfigTransactionGetVersionRequest req) {
+		state Version currentVersion = wait(getLiveTransactionVersion(self));
+		self->kvStore->set(KeyValueRef(liveTransactionVersionKey, BinaryWriter::toValue(++currentVersion, IncludeVersion())));
+		wait(self->kvStore->commit());
+		req.reply.send(ConfigTransactionGetVersionReply(currentVersion));
+		return Void();
+	}
+
+	ACTOR static Future<Void> get(SimpleConfigDatabaseNodeImpl* self, ConfigTransactionGetRequest req) {
+		Version currentVersion = wait(getLiveTransactionVersion(self));
+		if (req.version != currentVersion) {
+			req.reply.sendError(transaction_too_old());
+			return Void();
+		}
+		state Optional<Value> serializedValue =
+		    wait(self->kvStore->readValue(BinaryWriter::toValue(req.key, IncludeVersion()).withPrefix(kvKeys.begin)));
+		state Optional<KnobValue> value;
+		if (serializedValue.present()) {
+			value = ObjectReader::fromStringRef<KnobValue>(serializedValue.get(), IncludeVersion());
+		}
+		Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations = wait(getMutations(self, 0, req.version));
+		for (const auto &versionedMutation : versionedMutations) {
+			const auto &mutation = versionedMutation.mutation;
+			if (mutation.getKey() == req.key) {
+				if (mutation.isSet()) {
+					value = mutation.getValue();
+				} else {
+					value = {};
+				}
+			}
+		}
+		req.reply.send(ConfigTransactionGetReply{ value });
+		return Void();
+	}
+
+	// Retrieve all configuration classes that contain explicitly defined knobs
+	// TODO: Currently it is possible that extra configuration classes may be returned, we
+	// may want to fix this to clean up the contract
+	ACTOR static Future<Void> getConfigClasses(SimpleConfigDatabaseNodeImpl* self,
+	                                           ConfigTransactionGetConfigClassesRequest req) {
+		Version currentVersion = wait(getLiveTransactionVersion(self));
+		if (req.version != currentVersion) {
+			req.reply.sendError(transaction_too_old());
+			return Void();
+		}
+		state Standalone<RangeResultRef> snapshot = wait(self->kvStore->readRange(kvKeys));
+		state std::set<Key> configClassesSet;
+		for (const auto& kv : snapshot) {
+			auto configKey =
+			    BinaryReader::fromStringRef<ConfigKey>(kv.key.removePrefix(kvKeys.begin), IncludeVersion());
+			if (configKey.configClass.present()) {
+				configClassesSet.insert(configKey.configClass.get());
+			}
+		}
+		state Version lastCompactedVersion = wait(getLastCompactedVersion(self));
+		state Standalone<VectorRef<VersionedConfigMutationRef>> mutations =
+		    wait(getMutations(self, lastCompactedVersion + 1, req.version));
+		for (const auto& versionedMutation : mutations) {
+			auto configClass = versionedMutation.mutation.getConfigClass();
+			if (configClass.present()) {
+				configClassesSet.insert(configClass.get());
+			}
+		}
+		Standalone<VectorRef<KeyRef>> configClasses;
+		for (const auto& configClass : configClassesSet) {
+			configClasses.push_back_deep(configClasses.arena(), configClass);
+		}
+		req.reply.send(ConfigTransactionGetConfigClassesReply{ configClasses });
+		return Void();
+	}
+
+	// Retrieve all knobs explicitly defined for the specified configuration class
+	ACTOR static Future<Void> getKnobs(SimpleConfigDatabaseNodeImpl* self, ConfigTransactionGetKnobsRequest req) {
+		Version currentVersion = wait(getLiveTransactionVersion(self));
+		if (req.version != currentVersion) {
+			req.reply.sendError(transaction_too_old());
+			return Void();
+		}
+		// FIXME: Filtering after reading from disk is very inefficient
+		state Standalone<RangeResultRef> snapshot = wait(self->kvStore->readRange(kvKeys));
+		state std::set<Key> knobSet;
+		for (const auto& kv : snapshot) {
+			auto configKey =
+			    BinaryReader::fromStringRef<ConfigKey>(kv.key.removePrefix(kvKeys.begin), IncludeVersion());
+			if (configKey.configClass.castTo<Key>() == req.configClass) {
+				knobSet.insert(configKey.knobName);
+			}
+		}
+		state Version lastCompactedVersion = wait(getLastCompactedVersion(self));
+		state Standalone<VectorRef<VersionedConfigMutationRef>> mutations =
+		    wait(getMutations(self, lastCompactedVersion + 1, req.version));
+		for (const auto& versionedMutation : mutations) {
+			if (versionedMutation.mutation.getConfigClass().castTo<Key>() == req.configClass) {
+				if (versionedMutation.mutation.isSet()) {
+					knobSet.insert(versionedMutation.mutation.getKnobName());
+				} else {
+					knobSet.erase(versionedMutation.mutation.getKnobName());
+				}
+			}
+		}
+		Standalone<VectorRef<KeyRef>> knobNames;
+		for (const auto& knobName : knobSet) {
+			knobNames.push_back_deep(knobNames.arena(), knobName);
+		}
+		req.reply.send(ConfigTransactionGetKnobsReply{ knobNames });
+		return Void();
+	}
+
+	ACTOR static Future<Void> commit(SimpleConfigDatabaseNodeImpl* self, ConfigTransactionCommitRequest req) {
+		Version currentVersion = wait(getLiveTransactionVersion(self));
+		if (req.version != currentVersion) {
+			++self->failedCommits;
+			req.reply.sendError(transaction_too_old());
+			return Void();
+		}
+		int index = 0;
+		for (const auto &mutation : req.mutations) {
+			Key key = versionedMutationKey(req.version, index++);
+			Value value = ObjectWriter::toValue(mutation, IncludeVersion());
+			if (mutation.isSet()) {
+				TraceEvent("SimpleConfigDatabaseNodeSetting")
+				    .detail("ConfigClass", mutation.getConfigClass())
+				    .detail("KnobName", mutation.getKnobName())
+				    .detail("Value", mutation.getValue().toString())
+				    .detail("Version", req.version);
+				++self->setMutations;
+			} else {
+				++self->clearMutations;
+			}
+			self->kvStore->set(KeyValueRef(key, value));
+		}
+		self->kvStore->set(
+		    KeyValueRef(versionedAnnotationKey(req.version), BinaryWriter::toValue(req.annotation, IncludeVersion())));
+		self->kvStore->set(KeyValueRef(committedVersionKey, BinaryWriter::toValue(req.version, IncludeVersion())));
+		wait(self->kvStore->commit());
+		++self->successfulCommits;
+		req.reply.send(Void());
+		return Void();
+	}
+
+	ACTOR static Future<Void> serve(SimpleConfigDatabaseNodeImpl* self, ConfigTransactionInterface const* cti) {
+		loop {
+			choose {
+				when(ConfigTransactionGetVersionRequest req = waitNext(cti->getVersion.getFuture())) {
+					++self->newVersionRequests;
+					wait(getNewVersion(self, req));
+				}
+				when(ConfigTransactionGetRequest req = waitNext(cti->get.getFuture())) {
+					++self->getValueRequests;
+					wait(get(self, req));
+				}
+				when(ConfigTransactionCommitRequest req = waitNext(cti->commit.getFuture())) {
+					wait(commit(self, req));
+				}
+				when(ConfigTransactionGetConfigClassesRequest req = waitNext(cti->getClasses.getFuture())) {
+					wait(getConfigClasses(self, req));
+				}
+				when(ConfigTransactionGetKnobsRequest req = waitNext(cti->getKnobs.getFuture())) {
+					wait(getKnobs(self, req));
+				}
+				when(wait(self->kvStore->getError())) { ASSERT(false); }
+			}
+		}
+	}
+
+	ACTOR static Future<Void> getSnapshotAndChanges(SimpleConfigDatabaseNodeImpl* self,
+	                                                ConfigFollowerGetSnapshotAndChangesRequest req) {
+		state ConfigFollowerGetSnapshotAndChangesReply reply;
+		Standalone<RangeResultRef> data = wait(self->kvStore->readRange(kvKeys));
+		for (const auto& kv : data) {
+			reply
+			    .snapshot[BinaryReader::fromStringRef<ConfigKey>(kv.key.removePrefix(kvKeys.begin), IncludeVersion())] =
+			    ObjectReader::fromStringRef<KnobValue>(kv.value, IncludeVersion());
+		}
+		wait(store(reply.snapshotVersion, getLastCompactedVersion(self)));
+		wait(store(reply.changesVersion, getCommittedVersion(self)));
+		wait(store(reply.changes, getMutations(self, reply.snapshotVersion + 1, reply.changesVersion)));
+		wait(store(reply.annotations, getAnnotations(self, reply.snapshotVersion + 1, reply.changesVersion)));
+		TraceEvent(SevDebug, "ConfigDatabaseNodeGettingSnapshot", self->id)
+		    .detail("SnapshotVersion", reply.snapshotVersion)
+		    .detail("ChangesVersion", reply.changesVersion)
+		    .detail("SnapshotSize", reply.snapshot.size())
+		    .detail("ChangesSize", reply.changes.size())
+		    .detail("AnnotationsSize", reply.annotations.size());
+		req.reply.send(reply);
+		return Void();
+	}
+
+	// Apply mutations from the WAL in mutationKeys into the kvKeys key space.
+	// Periodic compaction prevents the database from growing too large, and improve read performance.
+	// However, commit annotations for compacted mutations are lost
+	ACTOR static Future<Void> compact(SimpleConfigDatabaseNodeImpl* self, ConfigFollowerCompactRequest req) {
+		state Version lastCompactedVersion = wait(getLastCompactedVersion(self));
+		TraceEvent(SevDebug, "ConfigDatabaseNodeCompacting", self->id)
+		    .detail("Version", req.version)
+		    .detail("LastCompacted", lastCompactedVersion);
+		if (req.version <= lastCompactedVersion) {
+			req.reply.send(Void());
+			return Void();
+		}
+		Standalone<VectorRef<VersionedConfigMutationRef>> versionedMutations =
+		    wait(getMutations(self, lastCompactedVersion + 1, req.version));
+		self->kvStore->clear(
+		    KeyRangeRef(versionedMutationKey(lastCompactedVersion + 1, 0), versionedMutationKey(req.version + 1, 0)));
+		self->kvStore->clear(
+		    KeyRangeRef(versionedAnnotationKey(lastCompactedVersion + 1), versionedAnnotationKey(req.version + 1)));
+		for (const auto& versionedMutation : versionedMutations) {
+			const auto& version = versionedMutation.version;
+			const auto& mutation = versionedMutation.mutation;
+			if (version > req.version) {
+				break;
+			} else {
+				TraceEvent(SevDebug, "ConfigDatabaseNodeCompactionApplyingMutation", self->id)
+				    .detail("IsSet", mutation.isSet())
+				    .detail("MutationVersion", version)
+				    .detail("LastCompactedVersion", lastCompactedVersion)
+				    .detail("ReqVersion", req.version);
+				auto serializedKey = BinaryWriter::toValue(mutation.getKey(), IncludeVersion());
+				if (mutation.isSet()) {
+					self->kvStore->set(KeyValueRef(serializedKey.withPrefix(kvKeys.begin),
+					                               ObjectWriter::toValue(mutation.getValue(), IncludeVersion())));
+				} else {
+					self->kvStore->clear(singleKeyRange(serializedKey.withPrefix(kvKeys.begin)));
+				}
+				lastCompactedVersion = version;
+			}
+		}
+		self->kvStore->set(
+		    KeyValueRef(lastCompactedVersionKey, BinaryWriter::toValue(lastCompactedVersion, IncludeVersion())));
+		wait(self->kvStore->commit());
+		req.reply.send(Void());
+		return Void();
+	}
+
+	ACTOR static Future<Void> serve(SimpleConfigDatabaseNodeImpl* self, ConfigFollowerInterface const* cfi) {
+		loop {
+			choose {
+				when(ConfigFollowerGetSnapshotAndChangesRequest req =
+				         waitNext(cfi->getSnapshotAndChanges.getFuture())) {
+					++self->snapshotRequests;
+					wait(getSnapshotAndChanges(self, req));
+				}
+				when(ConfigFollowerGetChangesRequest req = waitNext(cfi->getChanges.getFuture())) {
+					wait(getChanges(self, req));
+				}
+				when(ConfigFollowerCompactRequest req = waitNext(cfi->compact.getFuture())) {
+					++self->compactRequests;
+					wait(compact(self, req));
+				}
+				when(wait(self->kvStore->getError())) { ASSERT(false); }
+			}
+		}
+	}
+
+public:
+	SimpleConfigDatabaseNodeImpl(std::string const& folder)
+	  : id(deterministicRandom()->randomUniqueID()), kvStore(folder, id, "globalconf-"), cc("ConfigDatabaseNode"),
+	    compactRequests("CompactRequests", cc), successfulChangeRequests("SuccessfulChangeRequests", cc),
+	    failedChangeRequests("FailedChangeRequests", cc), snapshotRequests("SnapshotRequests", cc),
+	    successfulCommits("SuccessfulCommits", cc), failedCommits("FailedCommits", cc),
+	    setMutations("SetMutations", cc), clearMutations("ClearMutations", cc),
+	    getValueRequests("GetValueRequests", cc), newVersionRequests("NewVersionRequests", cc) {
+		logger = traceCounters(
+		    "ConfigDatabaseNodeMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ConfigDatabaseNode");
+		TraceEvent(SevDebug, "StartingSimpleConfigDatabaseNode", id).detail("KVStoreAlreadyExists", kvStore.exists());
+	}
+
+	Future<Void> serve(ConfigTransactionInterface const& cti) { return serve(this, &cti); }
+
+	Future<Void> serve(ConfigFollowerInterface const& cfi) { return serve(this, &cfi); }
+};
+
+SimpleConfigDatabaseNode::SimpleConfigDatabaseNode(std::string const& folder)
+  : _impl(std::make_unique<SimpleConfigDatabaseNodeImpl>(folder)) {}
+
+SimpleConfigDatabaseNode::~SimpleConfigDatabaseNode() = default;
+
+Future<Void> SimpleConfigDatabaseNode::serve(ConfigTransactionInterface const& cti) {
+	return impl().serve(cti);
+}
+
+Future<Void> SimpleConfigDatabaseNode::serve(ConfigFollowerInterface const& cfi) {
+	return impl().serve(cfi);
+}

fdbserver/SimpleConfigDatabaseNode.h

@@ -0,0 +1,40 @@
+/*
+ * SimpleConfigDatabaseNode.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "fdbserver/IConfigDatabaseNode.h"
+
+/*
+ * A test-only configuration database node implementation that assumes all data is stored on a single coordinator.
+ * As such, there is no need to handle rolling forward or rolling back mutations, because this one node is considered
+ * the source of truth.
+ */
+class SimpleConfigDatabaseNode : public IConfigDatabaseNode {
+	std::unique_ptr<class SimpleConfigDatabaseNodeImpl> _impl;
+	SimpleConfigDatabaseNodeImpl const& impl() const { return *_impl; }
+	SimpleConfigDatabaseNodeImpl& impl() { return *_impl; }
+
+public:
+	SimpleConfigDatabaseNode(std::string const& folder);
+	~SimpleConfigDatabaseNode();
+	Future<Void> serve(ConfigTransactionInterface const&) override;
+	Future<Void> serve(ConfigFollowerInterface const&) override;
+};

fdbserver/SimulatedCluster.actor.cpp

@@ -503,7 +503,10 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(Reference<ClusterConnec
 					                       "",
 					                       "",
 					                       -1,
-					                       whitelistBinPaths));
+					                       whitelistBinPaths,
+					                       "",
+					                       {},
+					                       UseConfigDB::DISABLED));
 				}
 				if (runBackupAgents != AgentNone) {
 					futures.push_back(runBackup(connFile));
@@ -1125,6 +1128,7 @@ ACTOR Future<Void> restartSimulatedSystem(vector<Future<Void>>* systemActors,
 struct SimulationConfig {
 	explicit SimulationConfig(const TestConfig& testConfig);
 	int extraDB;
+	bool generateFearless;
 
 	DatabaseConfiguration db;
 
@@ -1132,11 +1136,23 @@ struct SimulationConfig {
 
 	// Simulation layout
 	int datacenters;
+	int replication_type;
 	int machine_count; // Total, not per DC.
 	int processes_per_machine;
 	int coordinators;
 
 private:
+	void setRandomConfig();
+	void setSimpleConfig();
+	void setSpecificConfig(const TestConfig& testConfig);
+	void setDatacenters(const TestConfig& testConfig);
+	void setStorageEngine(const TestConfig& testConfig);
+	void setRegions(const TestConfig& testConfig);
+	void setReplicationType(const TestConfig& testConfig);
+	void setMachineCount(const TestConfig& testConfig);
+	void setCoordinators(const TestConfig& testConfig);
+	void setProcessesPerMachine(const TestConfig& testConfig);
+	void setTss(const TestConfig& testConfig);
 	void generateNormalConfig(const TestConfig& testConfig);
 };
 
@@ -1156,16 +1172,68 @@ void SimulationConfig::set_config(std::string config) {
 StringRef StringRefOf(const char* s) {
 	return StringRef((uint8_t*)s, strlen(s));
 }
-// Generates and sets an appropriate configuration for the database according to
-// the provided testConfig. Some attributes are randomly generated for more coverage
-// of different combinations
-void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
-	set_config("new");
-	// generateMachineTeamTestConfig set up the number of servers per machine and the number of machines such that
-	// if we do not remove the surplus server and machine teams, the simulation test will report error.
-	// This is needed to make sure the number of server (and machine) teams is no larger than the desired number.
-	bool generateMachineTeamTestConfig = BUGGIFY_WITH_PROB(0.1) ? true : false;
-	bool generateFearless =
+
+// Set the randomly generated options of the config. Compiled here to easily observe and trace random options
+void SimulationConfig::setRandomConfig() {
+	if (deterministicRandom()->random01() < 0.25) {
+		db.desiredTLogCount = deterministicRandom()->randomInt(1, 7);
+	}
+	if (deterministicRandom()->random01() < 0.25) {
+		db.commitProxyCount = deterministicRandom()->randomInt(1, 7);
+	}
+	if (deterministicRandom()->random01() < 0.25) {
+		db.grvProxyCount = deterministicRandom()->randomInt(1, 4);
+	}
+	if (deterministicRandom()->random01() < 0.25) {
+		db.resolverCount = deterministicRandom()->randomInt(1, 7);
+	}
+	// TraceEvent("SimulatedConfigRandom")
+	// 	.detail("DesiredTLogCount", db.desiredTLogCount)
+	// 	.detail("CommitProxyCount", db.commitProxyCount)
+	// 	.detail("GRVProxyCount", db.grvProxyCount)
+	// 	.detail("ResolverCount", db.resolverCount);
+
+	if (deterministicRandom()->random01() < 0.5) {
+		// TraceEvent("SimulatedConfigRandom").detail("PerpetualWiggle", 0);
+		set_config("perpetual_storage_wiggle=0");
+	} else {
+		// TraceEvent("SimulatedConfigRandom").detail("PerpetualWiggle", 1);
+		set_config("perpetual_storage_wiggle=1");
+	}
+
+	if (deterministicRandom()->random01() < 0.5) {
+		set_config("backup_worker_enabled:=1");
+	}
+}
+
+// Overwrite DB with simple options, used when simpleConfig is true in the TestConfig
+void SimulationConfig::setSimpleConfig() {
+	db.desiredTLogCount = 1;
+	db.commitProxyCount = 1;
+	db.grvProxyCount = 1;
+	db.resolverCount = 1;
+}
+
+// Overwrite previous options with ones specified by TestConfig
+void SimulationConfig::setSpecificConfig(const TestConfig& testConfig) {
+	if (testConfig.desiredTLogCount.present()) {
+		db.desiredTLogCount = testConfig.desiredTLogCount.get();
+	}
+	if (testConfig.commitProxyCount.present()) {
+		db.commitProxyCount = testConfig.commitProxyCount.get();
+	}
+	if (testConfig.grvProxyCount.present()) {
+		db.grvProxyCount = testConfig.grvProxyCount.get();
+	}
+	if (testConfig.resolverCount.present()) {
+		db.resolverCount = testConfig.resolverCount.get();
+	}
+}
+
+// Sets generateFearless and number of dataCenters based on testConfig details
+// The number of datacenters may be overwritten in setRegions
+void SimulationConfig::setDatacenters(const TestConfig& testConfig) {
+	generateFearless =
 	    testConfig.simpleConfig ? false : (testConfig.minimumRegions > 1 || deterministicRandom()->random01() < 0.5);
 	if (testConfig.generateFearless.present()) {
 		// overwrite whatever decision we made before
@@ -1176,32 +1244,15 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 	        ? 1
 	        : (generateFearless ? (testConfig.minimumReplication > 0 || deterministicRandom()->random01() < 0.5 ? 4 : 6)
 	                            : deterministicRandom()->randomInt(1, 4));
+
+	// Overwrite with specific option if present
 	if (testConfig.datacenters.present()) {
 		datacenters = testConfig.datacenters.get();
 	}
-	if (testConfig.desiredTLogCount.present()) {
-		db.desiredTLogCount = testConfig.desiredTLogCount.get();
-	} else if (deterministicRandom()->random01() < 0.25) {
-		db.desiredTLogCount = deterministicRandom()->randomInt(1, 7);
 }
 
-	if (testConfig.commitProxyCount.present()) {
-		db.commitProxyCount = testConfig.commitProxyCount.get();
-	} else if (deterministicRandom()->random01() < 0.25) {
-		db.commitProxyCount = deterministicRandom()->randomInt(1, 7);
-	}
-
-	if (testConfig.grvProxyCount.present()) {
-		db.grvProxyCount = testConfig.grvProxyCount.get();
-	} else if (deterministicRandom()->random01() < 0.25) {
-		db.grvProxyCount = deterministicRandom()->randomInt(1, 4);
-	}
-
-	if (testConfig.resolverCount.present()) {
-		db.resolverCount = testConfig.resolverCount.get();
-	} else if (deterministicRandom()->random01() < 0.25) {
-		db.resolverCount = deterministicRandom()->randomInt(1, 7);
-	}
+// Sets storage engine based on testConfig details
+void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 	int storage_engine_type = deterministicRandom()->randomInt(0, 4);
 	if (testConfig.storageEngineType.present()) {
 		storage_engine_type = testConfig.storageEngineType.get();
@@ -1238,34 +1289,11 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 	default:
 		ASSERT(false); // Programmer forgot to adjust cases.
 	}
-
-	int tssCount = 0;
-	if (!testConfig.simpleConfig && !testConfig.disableTss && deterministicRandom()->random01() < 0.25) {
-		// 1 or 2 tss
-		tssCount = deterministicRandom()->randomInt(1, 3);
 }
 
-	//	if (deterministicRandom()->random01() < 0.5) {
-	//		set_config("ssd");
-	//	} else {
-	//		set_config("memory");
-	//	}
-	//	set_config("memory");
-	//  set_config("memory-radixtree-beta");
-
-    if (deterministicRandom()->random01() < 0.5) {
-        set_config("perpetual_storage_wiggle=0");
-    } else {
-        set_config("perpetual_storage_wiggle=1");
-    }
-// 	set_config("perpetual_storage_wiggle=1");
-	if (testConfig.simpleConfig) {
-		db.desiredTLogCount = 1;
-		db.commitProxyCount = 1;
-		db.grvProxyCount = 1;
-		db.resolverCount = 1;
-	}
-	int replication_type = testConfig.simpleConfig
+// Sets replication type and TLogSpillType and Version
+void SimulationConfig::setReplicationType(const TestConfig& testConfig) {
+	replication_type = testConfig.simpleConfig
 	                       ? 1
 	                       : (std::max(testConfig.minimumReplication,
 	                                   datacenters > 4 ? deterministicRandom()->randomInt(1, 3)
@@ -1330,13 +1358,12 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 			if (deterministicRandom()->random01() < 0.5)
 				set_config(format("log_spill:=%d", TLogSpillType::DEFAULT));
 		}
-
-		if (deterministicRandom()->random01() < 0.5) {
-			set_config("backup_worker_enabled:=1");
 	}
 }
 
-	if (generateFearless || (datacenters == 2 && deterministicRandom()->random01() < 0.5)) {
+// Set the regions of the config, including the primary and remote options
+// This will also determine the replication types used for satellite and remote.
+void SimulationConfig::setRegions(const TestConfig& testConfig) {
 	// The kill region workload relies on the fact that all "0", "2", and "4" are all of the possible primary dcids.
 	StatusObject primaryObj;
 	StatusObject primaryDcObj;
@@ -1515,8 +1542,8 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 	}
 
 	if (needsRemote) {
-			g_simulator.originalRegions = "regions=" + json_spirit::write_string(json_spirit::mValue(regionArr),
-			                                                                     json_spirit::Output_options::none);
+		g_simulator.originalRegions =
+		    "regions=" + json_spirit::write_string(json_spirit::mValue(regionArr), json_spirit::Output_options::none);
 
 		StatusArray disablePrimary = regionArr;
 		disablePrimary[0].get_obj()["datacenters"].get_array()[0].get_obj()["priority"] = -1;
@@ -1535,6 +1562,9 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 	}
 }
 
+// Sets the machine count based on the testConfig. May be overwritten later
+// if the end result is not a viable config.
+void SimulationConfig::setMachineCount(const TestConfig& testConfig) {
 	if (testConfig.machineCount.present()) {
 		machine_count = testConfig.machineCount.get();
 	} else if (generateFearless && testConfig.minimumReplication > 1) {
@@ -1551,6 +1581,10 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 		                         ((db.minDatacentersRequired() > 0) ? datacenters : 1) *
 		                             std::max(3, db.minZonesRequiredPerDatacenter()));
 		machine_count = deterministicRandom()->randomInt(machine_count, std::max(machine_count + 1, extraDB ? 6 : 10));
+		// generateMachineTeamTestConfig set up the number of servers per machine and the number of machines such that
+		// if we do not remove the surplus server and machine teams, the simulation test will report error.
+		// This is needed to make sure the number of server (and machine) teams is no larger than the desired number.
+		bool generateMachineTeamTestConfig = BUGGIFY_WITH_PROB(0.1) ? true : false;
 		if (generateMachineTeamTestConfig) {
 			// When DESIRED_TEAMS_PER_SERVER is set to 1, the desired machine team number is 5
 			// while the max possible machine team number is 10.
@@ -1559,7 +1593,11 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 			machine_count = std::max(machine_count, deterministicRandom()->randomInt(5, extraDB ? 6 : 10));
 		}
 	}
+}
 
+// Sets the coordinator count based on the testConfig. May be overwritten later
+// if the end result is not a viable config.
+void SimulationConfig::setCoordinators(const TestConfig& testConfig) {
 	if (testConfig.coordinators.present()) {
 		coordinators = testConfig.coordinators.get();
 	} else {
@@ -1569,14 +1607,10 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 		                   ? deterministicRandom()->randomInt(1, std::max(machine_count, 2))
 		                   : 1;
 	}
-
-	if (testConfig.minimumReplication > 1 && datacenters == 3) {
-		// low latency tests in 3 data hall mode need 2 other data centers with 2 machines each to avoid waiting for
-		// logs to recover.
-		machine_count = std::max(machine_count, 6);
-		coordinators = 3;
 }
 
+// Sets the processes per machine based on the testConfig.
+void SimulationConfig::setProcessesPerMachine(const TestConfig& testConfig) {
 	if (testConfig.processesPerMachine.present()) {
 		processes_per_machine = testConfig.processesPerMachine.get();
 	} else if (generateFearless) {
@@ -1584,6 +1618,16 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 	} else {
 		processes_per_machine = deterministicRandom()->randomInt(1, (extraDB ? 14 : 28) / machine_count + 2);
 	}
+}
+
+// Sets the TSS configuration based on the testConfig.
+// Also configures the cluster behaviour through setting some flags on the simulator.
+void SimulationConfig::setTss(const TestConfig& testConfig) {
+	int tssCount = 0;
+	if (!testConfig.simpleConfig && !testConfig.disableTss && deterministicRandom()->random01() < 0.25) {
+		// 1 or 2 tss
+		tssCount = deterministicRandom()->randomInt(1, 3);
+	}
 
 	// reduce tss to half of extra non-seed servers that can be recruited in usable regions.
 	tssCount =
@@ -1608,6 +1652,43 @@ void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
 	}
 }
 
+// Generates and sets an appropriate configuration for the database according to
+// the provided testConfig. Some attributes are randomly generated for more coverage
+// of different combinations
+void SimulationConfig::generateNormalConfig(const TestConfig& testConfig) {
+	set_config("new");
+	// Some of these options will overwrite one another so the ordering is important.
+	// This is a bit inefficient but separates the different types of option setting paths for better readability.
+	setDatacenters(testConfig);
+
+	// These 3 sets will only change the settings with trivial logic and low coupling with
+	// other portions of the configuration. The parameters that are more involved and use
+	// complex logic will be found in their respective "set----" methods following after.
+	setRandomConfig();
+	if (testConfig.simpleConfig) {
+		setSimpleConfig();
+	}
+	setSpecificConfig(testConfig);
+
+	setStorageEngine(testConfig);
+	setReplicationType(testConfig);
+	if (generateFearless || (datacenters == 2 && deterministicRandom()->random01() < 0.5)) {
+		setRegions(testConfig);
+	}
+	setMachineCount(testConfig);
+	setCoordinators(testConfig);
+
+	if (testConfig.minimumReplication > 1 && datacenters == 3) {
+		// low latency tests in 3 data hall mode need 2 other data centers with 2 machines each to avoid waiting for
+		// logs to recover.
+		machine_count = std::max(machine_count, 6);
+		coordinators = 3;
+	}
+
+	setProcessesPerMachine(testConfig);
+	setTss(testConfig);
+}
+
 // Configures the system according to the given specifications in order to run
 // simulation under the correct conditions
 void setupSimulatedSystem(vector<Future<Void>>* systemActors,

fdbserver/Status.actor.cpp

@@ -2675,7 +2675,8 @@ ACTOR Future<StatusReply> clusterGetStatus(
     std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* clientStatus,
     ServerCoordinators coordinators,
     std::vector<NetworkAddress> incompatibleConnections,
-    Version datacenterVersionDifference) {
+    Version datacenterVersionDifference,
+    ConfigBroadcaster const* configBroadcaster) {
 	state double tStart = timer();
 
 	state JsonBuilderArray messages;
@@ -2907,6 +2908,10 @@ ACTOR Future<StatusReply> clusterGetStatus(
 				statusObj["workload"] = workerStatuses[1];
 
 			statusObj["layers"] = workerStatuses[2];
+			if (configBroadcaster) {
+				// TODO: Read from coordinators for more up-to-date config database status?
+				statusObj["configuration_database"] = configBroadcaster->getStatus();
+			}
 
 			// Add qos section if it was populated
 			if (!qos.empty())

fdbserver/Status.h

@@ -23,6 +23,7 @@
 #pragma once
 
 #include "fdbrpc/fdbrpc.h"
+#include "fdbserver/ConfigBroadcaster.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/MasterInterface.h"
 #include "fdbclient/ClusterInterface.h"
@@ -42,6 +43,7 @@ Future<StatusReply> clusterGetStatus(
     std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* const& clientStatus,
     ServerCoordinators const& coordinators,
     std::vector<NetworkAddress> const& incompatibleConnections,
-    Version const& datacenterVersionDifference);
+    Version const& datacenterVersionDifference,
+    ConfigBroadcaster const* const& conifgBroadcaster);
 
 #endif

fdbserver/WorkerInterface.actor.h

@@ -156,7 +156,7 @@ struct ClusterControllerFullInterface {
 	bool operator==(ClusterControllerFullInterface const& r) const { return id() == r.id(); }
 	bool operator!=(ClusterControllerFullInterface const& r) const { return id() != r.id(); }
 
-	bool hasMessage() {
+	bool hasMessage() const {
 		return clientInterface.hasMessage() || recruitFromConfiguration.getFuture().isReady() ||
 		       recruitRemoteFromConfiguration.getFuture().isReady() || recruitStorage.getFuture().isReady() ||
 		       registerWorker.getFuture().isReady() || getWorkers.getFuture().isReady() ||
@@ -828,13 +828,17 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> ccf,
                         std::string metricsConnFile,
                         std::string metricsPrefix,
                         int64_t memoryProfilingThreshold,
-                        std::string whitelistBinPaths);
+                        std::string whitelistBinPaths,
+                        std::string configPath,
+                        std::map<std::string, std::string> manualKnobOverrides,
+                        UseConfigDB useConfigDB);
 
 ACTOR Future<Void> clusterController(Reference<ClusterConnectionFile> ccf,
                                      Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC,
                                      Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
                                      Future<Void> recoveredDiskFiles,
-                                     LocalityData locality);
+                                     LocalityData locality,
+                                     UseConfigDB useConfigDB);
 
 // These servers are started by workerServer
 class IKeyValueStore;

fdbserver/fdbserver.actor.cpp

@@ -35,6 +35,7 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 
+#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/versions.h"
@@ -91,7 +92,7 @@ enum {
 	OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
 	OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
 	OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
-	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE
+	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB,
 };
 
 CSimpleOpt::SOption g_rgOptions[] = {
@@ -174,6 +175,8 @@ CSimpleOpt::SOption g_rgOptions[] = {
 	{ OPT_TRACE_FORMAT      ,    "--trace_format",              SO_REQ_SEP },
 	{ OPT_WHITELIST_BINPATH,     "--whitelist_binpath",         SO_REQ_SEP },
 	{ OPT_BLOB_CREDENTIAL_FILE,  "--blob_credential_file",      SO_REQ_SEP },
+	{ OPT_CONFIG_PATH,           "--config_path",               SO_REQ_SEP },
+	{ OPT_USE_TEST_CONFIG_DB,    "--use_test_config_db",        SO_NONE },
 
 #ifndef TLS_DISABLED
 	TLS_OPTION_FLAGS
@@ -954,7 +957,7 @@ struct CLIOptions {
 	NetworkAddressList publicAddresses, listenAddresses;
 
 	const char* targetKey = nullptr;
-	uint64_t memLimit =
+	int64_t memLimit =
 	    8LL << 30; // Nice to maintain the same default value for memLimit and SERVER_KNOBS->SERVER_MEM_LIMIT and
 	               // SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT
 	uint64_t storageMemLimit = 1LL << 30;
@@ -965,6 +968,7 @@ struct CLIOptions {
 	bool useNet2 = true;
 	bool useThreadPool = false;
 	std::vector<std::pair<std::string, std::string>> knobs;
+	std::map<std::string, std::string> manualKnobOverrides;
 	LocalityData localities;
 	int minTesterCount = 1;
 	bool testOnServers = false;
@@ -976,6 +980,9 @@ struct CLIOptions {
 	std::vector<std::string> blobCredentials; // used for fast restore workers & backup workers
 	const char* blobCredsFromENV = nullptr;
 
+	std::string configPath;
+	UseConfigDB useConfigDB{ UseConfigDB::DISABLED };
+
 	Reference<ClusterConnectionFile> connectionFile;
 	Standalone<StringRef> machineId;
 	UnitTestParameters testParams;
@@ -1051,6 +1058,7 @@ private:
 				}
 				syn = syn.substr(7);
 				knobs.emplace_back(syn, args.OptionArg());
+				manualKnobOverrides[syn] = args.OptionArg();
 				break;
 			}
 			case OPT_UNITTESTPARAM: {
@@ -1429,6 +1437,12 @@ private:
 					} while (t.size() != 0);
 				}
 				break;
+			case OPT_CONFIG_PATH:
+				configPath = args.OptionArg();
+				break;
+			case OPT_USE_TEST_CONFIG_DB:
+				useConfigDB = UseConfigDB::SIMPLE;
+				break;
 
 #ifndef TLS_DISABLED
 			case TLSConfig::OPT_TLS_PLUGIN:
@@ -1626,46 +1640,44 @@ int main(int argc, char* argv[]) {
 
 		enableBuggify(opts.buggifyEnabled, BuggifyType::General);
 
-		if (!globalServerKnobs->setKnob("log_directory", opts.logFolder))
-			ASSERT(false);
+		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::SERVER,
+		                                         Randomize::YES,
+		                                         role == ServerRole::Simulation ? IsSimulated::YES : IsSimulated::NO);
+		IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder));
 		if (role != ServerRole::Simulation) {
-			if (!globalServerKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(opts.memLimit)))
-				ASSERT(false);
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit",
+			                                                          KnobValue::create(int64_t{ opts.memLimit }));
 		}
-		for (auto k = opts.knobs.begin(); k != opts.knobs.end(); ++k) {
+
+		for (const auto& [knobName, knobValueString] : opts.knobs) {
 			try {
-				if (!globalFlowKnobs->setKnob(k->first, k->second) &&
-				    !globalClientKnobs->setKnob(k->first, k->second) &&
-				    !globalServerKnobs->setKnob(k->first, k->second)) {
-					fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
-					TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
-				}
+				auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
+				auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
+				g_knobs.setKnob(knobName, knobValue);
 			} catch (Error& e) {
 				if (e.code() == error_code_invalid_option_value) {
 					fprintf(stderr,
 					        "WARNING: Invalid value '%s' for knob option '%s'\n",
-					        k->second.c_str(),
-					        k->first.c_str());
+					        knobName.c_str(),
+					        knobValueString.c_str());
 					TraceEvent(SevWarnAlways, "InvalidKnobValue")
-					    .detail("Knob", printable(k->first))
-					    .detail("Value", printable(k->second));
+					    .detail("Knob", printable(knobName))
+					    .detail("Value", printable(knobValueString));
 				} else {
-					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what());
+					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
 					TraceEvent(SevError, "FailedToSetKnob")
-					    .detail("Knob", printable(k->first))
-					    .detail("Value", printable(k->second))
+					    .detail("Knob", printable(knobName))
+					    .detail("Value", printable(knobValueString))
 					    .error(e);
 					throw;
 				}
 			}
 		}
-		if (!globalServerKnobs->setKnob("server_mem_limit", std::to_string(opts.memLimit)))
-			ASSERT(false);
-
+		IKnobCollection::getMutableGlobalKnobCollection().setKnob("server_mem_limit",
+		                                                          KnobValue::create(int64_t{ opts.memLimit }));
 		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
-		globalFlowKnobs->initialize(true, role == ServerRole::Simulation);
-		globalClientKnobs->initialize(true);
-		globalServerKnobs->initialize(true, globalClientKnobs.get(), role == ServerRole::Simulation);
+		IKnobCollection::getMutableGlobalKnobCollection().initialize(
+		    Randomize::YES, role == ServerRole::Simulation ? IsSimulated::YES : IsSimulated::NO);
 
 		// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
 		EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
@@ -1786,21 +1798,6 @@ int main(int argc, char* argv[]) {
 		    .detail("MemoryLimit", opts.memLimit)
 		    .trackLatest("ProgramStart");
 
-		// Test for TraceEvent length limits
-		/*std::string foo(4096, 'x');
-		TraceEvent("TooLongDetail").detail("Contents", foo);
-
-		TraceEvent("TooLongEvent")
-		    .detail("Contents1", foo)
-		    .detail("Contents2", foo)
-		    .detail("Contents3", foo)
-		    .detail("Contents4", foo)
-		    .detail("Contents5", foo)
-		    .detail("Contents6", foo)
-		    .detail("Contents7", foo)
-		    .detail("Contents8", foo)
-		    .detail("ExtraTest", 1776);*/
-
 		Error::init();
 		std::set_new_handler(&platform::outOfMemory);
 		setMemoryQuota(opts.memLimit);
@@ -1980,7 +1977,10 @@ int main(int argc, char* argv[]) {
 				                      opts.metricsConnFile,
 				                      opts.metricsPrefix,
 				                      opts.rsssize,
-				                      opts.whitelistBinPaths));
+				                      opts.whitelistBinPaths,
+				                      opts.configPath,
+				                      opts.manualKnobOverrides,
+				                      opts.useConfigDB));
 				actors.push_back(histogramReport());
 				// actors.push_back( recurring( []{}, .001 ) );  // for ASIO latency measurement
 

fdbserver/masterserver.actor.cpp

@@ -267,7 +267,7 @@ struct MasterData : NonCopyable, ReferenceCounted<MasterData> {
 	    safeLocality(tagLocalityInvalid), primaryLocality(tagLocalityInvalid), neverCreated(false),
 	    lastEpochEnd(invalidVersion), liveCommittedVersion(invalidVersion), databaseLocked(false),
 	    minKnownCommittedVersion(invalidVersion), recoveryTransactionVersion(invalidVersion), lastCommitTime(0),
-	    registrationCount(0), version(invalidVersion), lastVersionTime(0), txnStateStore(0), memoryLimit(2e9),
+	    registrationCount(0), version(invalidVersion), lastVersionTime(0), txnStateStore(nullptr), memoryLimit(2e9),
 	    addActor(addActor), hasConfiguration(false), recruitmentStalled(makeReference<AsyncVar<bool>>(false)),
 	    cc("Master", dbgid.toString()), changeCoordinatorsRequests("ChangeCoordinatorsRequests", cc),
 	    getCommitVersionRequests("GetCommitVersionRequests", cc),
@@ -294,7 +294,9 @@ ACTOR Future<Void> newCommitProxies(Reference<MasterData> self, RecruitFromConfi
 		req.recoveryCount = self->cstate.myDBState.recoveryCount + 1;
 		req.recoveryTransactionVersion = self->recoveryTransactionVersion;
 		req.firstProxy = i == 0;
-		TraceEvent("CommitProxyReplies", self->dbgid).detail("WorkerID", recr.commitProxies[i].id());
+		TraceEvent("CommitProxyReplies", self->dbgid)
+			.detail("WorkerID", recr.commitProxies[i].id())
+			.detail("FirstProxy", req.firstProxy ? "True" : "False");
 		initializationReplies.push_back(
 		    transformErrors(throwErrorOr(recr.commitProxies[i].commitProxy.getReplyUnlessFailedFor(
 		                        req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)),
@@ -953,6 +955,10 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<MasterData> self) {
 		wait(yield());
 	}
 	wait(waitForAll(txnReplies));
+	TraceEvent("RecoveryInternal", self->dbgid)
+		.detail("StatusCode", RecoveryStatus::recovery_transaction)
+		.detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
+		.detail("Step", "SentTxnStateStoreToCommitProxies");
 
 	vector<Future<ResolveTransactionBatchReply>> replies;
 	for (auto& r : self->resolvers) {
@@ -965,6 +971,10 @@ ACTOR Future<Void> sendInitialCommitToResolvers(Reference<MasterData> self) {
 	}
 
 	wait(waitForAll(replies));
+	TraceEvent("RecoveryInternal", self->dbgid)
+		.detail("StatusCode", RecoveryStatus::recovery_transaction)
+		.detail("Status", RecoveryStatus::names[RecoveryStatus::recovery_transaction])
+		.detail("Step", "InitializedAllResolvers");
 	return Void();
 }
 

fdbserver/worker.actor.cpp

@@ -43,6 +43,7 @@
 #include "fdbserver/ServerDBInfo.h"
 #include "fdbserver/FDBExecHelper.actor.h"
 #include "fdbserver/CoordinationInterface.h"
+#include "fdbserver/LocalConfiguration.h"
 #include "fdbclient/MonitorLeader.h"
 #include "fdbclient/ClientWorkerInterface.h"
 #include "flow/Profiler.h"
@@ -473,8 +474,9 @@ std::vector<DiskStore> getDiskStores(std::string folder,
 			store.tLogOptions.version = TLogVersion::V2;
 			store.tLogOptions.spillType = TLogSpillType::VALUE;
 			prefix = fileLogDataPrefix;
-		} else
+		} else {
 			continue;
+		}
 
 		store.storeID = UID::fromString(files[idx].substr(prefix.size(), 32));
 		store.filename = filenameFromSample(type, folder, files[idx]);
@@ -2194,7 +2196,8 @@ ACTOR Future<Void> monitorLeaderRemotelyWithDelayedCandidacy(
     Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
     Future<Void> recoveredDiskFiles,
     LocalityData locality,
-    Reference<AsyncVar<ServerDBInfo>> dbInfo) {
+    Reference<AsyncVar<ServerDBInfo>> dbInfo,
+    UseConfigDB useConfigDB) {
 	state Future<Void> monitor = monitorLeaderRemotely(connFile, currentCC);
 	state Future<Void> timeout;
 
@@ -2220,7 +2223,8 @@ ACTOR Future<Void> monitorLeaderRemotelyWithDelayedCandidacy(
 			                                     : Never())) {}
 			when(wait(timeout.isValid() ? timeout : Never())) {
 				monitor.cancel();
-				wait(clusterController(connFile, currentCC, asyncPriorityInfo, recoveredDiskFiles, locality));
+				wait(clusterController(
+				    connFile, currentCC, asyncPriorityInfo, recoveredDiskFiles, locality, useConfigDB));
 				return Void();
 			}
 		}
@@ -2246,9 +2250,17 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
                         std::string metricsConnFile,
                         std::string metricsPrefix,
                         int64_t memoryProfileThreshold,
-                        std::string whitelistBinPaths) {
+                        std::string whitelistBinPaths,
+                        std::string configPath,
+                        std::map<std::string, std::string> manualKnobOverrides,
+                        UseConfigDB useConfigDB) {
 	state vector<Future<Void>> actors;
 	state Promise<Void> recoveredDiskFiles;
+	state LocalConfiguration localConfig(dataFolder, configPath, manualKnobOverrides);
+
+	if (useConfigDB != UseConfigDB::DISABLED) {
+		wait(localConfig.initialize());
+	}
 
 	actors.push_back(serveProtocolInfo());
 
@@ -2270,7 +2282,7 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
 		if (coordFolder.size()) {
 			// SOMEDAY: remove the fileNotFound wrapper and make DiskQueue construction safe from errors setting up
 			// their files
-			actors.push_back(fileNotFoundToNever(coordinationServer(coordFolder, coordinators.ccf)));
+			actors.push_back(fileNotFoundToNever(coordinationServer(coordFolder, coordinators.ccf, useConfigDB)));
 		}
 
 		state UID processIDUid = wait(createAndLockProcessIdFile(dataFolder));
@@ -2284,19 +2296,26 @@ ACTOR Future<Void> fdbd(Reference<ClusterConnectionFile> connFile,
 		    makeReference<AsyncVar<ClusterControllerPriorityInfo>>(getCCPriorityInfo(fitnessFilePath, processClass));
 		auto dbInfo = makeReference<AsyncVar<ServerDBInfo>>();
 
+		if (useConfigDB != UseConfigDB::DISABLED) {
+			actors.push_back(
+			    reportErrors(localConfig.consume(IDependentAsyncVar<ConfigBroadcastFollowerInterface>::create(
+			                     dbInfo, [](auto const& info) { return info.configBroadcaster; })),
+			                 "LocalConfiguration"));
+		}
 		actors.push_back(reportErrors(monitorAndWriteCCPriorityInfo(fitnessFilePath, asyncPriorityInfo),
 		                              "MonitorAndWriteCCPriorityInfo"));
 		if (processClass.machineClassFitness(ProcessClass::ClusterController) == ProcessClass::NeverAssign) {
 			actors.push_back(reportErrors(monitorLeader(connFile, cc), "ClusterController"));
 		} else if (processClass.machineClassFitness(ProcessClass::ClusterController) == ProcessClass::WorstFit &&
 		           SERVER_KNOBS->MAX_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS > 0) {
-			actors.push_back(
-			    reportErrors(monitorLeaderRemotelyWithDelayedCandidacy(
-			                     connFile, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities, dbInfo),
+			actors.push_back(reportErrors(
+			    monitorLeaderRemotelyWithDelayedCandidacy(
+			        connFile, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities, dbInfo, useConfigDB),
 			    "ClusterController"));
 		} else {
 			actors.push_back(reportErrors(
-			    clusterController(connFile, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities),
+			    clusterController(
+			        connFile, cc, asyncPriorityInfo, recoveredDiskFiles.getFuture(), localities, useConfigDB),
 			    "ClusterController"));
 		}
 		actors.push_back(reportErrors(extractClusterInterface(cc, ci), "ExtractClusterInterface"));

fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp

@@ -218,7 +218,8 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override {
 		if (clientId == 0) {
-			globalClientKnobs->CSI_STATUS_DELAY = 2.0; // 2 seconds
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob("csi_status_delay",
+			                                                          KnobValueRef::create(double{ 2.0 })); // 2 seconds
 			return changeProfilingParameters(cx, trInfoSizeLimit, samplingProbability);
 		}
 		return Void();

fdbserver/workloads/ConfigureDatabase.actor.cpp

@@ -213,12 +213,15 @@ std::string generateRegions() {
 struct ConfigureDatabaseWorkload : TestWorkload {
 	double testDuration;
 	int additionalDBs;
-
+	bool allowDescriptorChange;
 	vector<Future<Void>> clients;
 	PerfIntCounter retries;
 
 	ConfigureDatabaseWorkload(WorkloadContext const& wcx) : TestWorkload(wcx), retries("Retries") {
 		testDuration = getOption(options, LiteralStringRef("testDuration"), 200.0);
+		allowDescriptorChange =
+		    getOption(options, LiteralStringRef("allowDescriptorChange"), SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT);
+
 		g_simulator.usableRegions = 1;
 	}
 
@@ -316,10 +319,10 @@ struct ConfigureDatabaseWorkload : TestWorkload {
 
 				//TraceEvent("ConfigureTestConfigureEnd").detail("NewConfig", newConfig);
 			} else if (randomChoice == 4) {
-				//TraceEvent("ConfigureTestQuorumBegin").detail("NewQuorum", s);
+				//TraceEvent("ConfigureTestQuorumBegin");
 				auto ch = autoQuorumChange();
 				std::string desiredClusterName = "NewName%d";
-				if (!SERVER_KNOBS->ENABLE_CROSS_CLUSTER_SUPPORT) {
+				if (!self->allowDescriptorChange) {
 					// if configuration does not allow changing the descriptor, pass empty string (keep old descriptor)
 					desiredClusterName = "";
 				}

fdbserver/workloads/ConsistencyCheck.actor.cpp

@@ -1498,7 +1498,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 								    .error(e);
 
 								// All shards should be available in quiscence
-								if (self->performQuiescentChecks) {
+								if (self->performQuiescentChecks && !storageServerInterfaces[j].isTss()) {
 									self->testFailure("Storage server unavailable");
 									return false;
 								}

fdbserver/workloads/LowLatency.actor.cpp

@@ -19,6 +19,7 @@
  */
 
 #include "fdbrpc/ContinuousSample.h"
+#include "fdbclient/IKnobCollection.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbclient/ReadYourWrites.h"
@@ -50,8 +51,10 @@ struct LowLatencyWorkload : TestWorkload {
 
 	Future<Void> setup(Database const& cx) override {
 		if (g_network->isSimulated()) {
-			ASSERT(const_cast<ServerKnobs*>(SERVER_KNOBS)->setKnob("min_delay_cc_worst_fit_candidacy_seconds", "5"));
-			ASSERT(const_cast<ServerKnobs*>(SERVER_KNOBS)->setKnob("max_delay_cc_worst_fit_candidacy_seconds", "10"));
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob("min_delay_cc_worst_fit_candidacy_seconds",
+			                                                          KnobValueRef::create(double{ 5.0 }));
+			IKnobCollection::getMutableGlobalKnobCollection().setKnob("max_delay_cc_worst_fit_candidacy_seconds",
+			                                                          KnobValueRef::create(double{ 10.0 }));
 		}
 		return Void();
 	}

fdbserver/workloads/UnitTests.actor.cpp

@@ -47,6 +47,7 @@ struct UnitTestWorkload : TestWorkload {
 		enabled = !clientId; // only do this on the "first" client
 		testPattern = getOption(options, LiteralStringRef("testsMatching"), Value()).toString();
 		testRunLimit = getOption(options, LiteralStringRef("maxTestCases"), -1);
+		testParams.setDataDir(getOption(options, LiteralStringRef("dataDir"), "simfdb/unittests/"_sr).toString());
 
 		// Consume all remaining options as testParams which the unit test can access
 		for (auto& kv : options) {
@@ -66,7 +67,10 @@ struct UnitTestWorkload : TestWorkload {
 	}
 
 	std::string description() const override { return "UnitTests"; }
-	Future<Void> setup(Database const& cx) override { return Void(); }
+	Future<Void> setup(Database const& cx) override {
+		platform::eraseDirectoryRecursive(testParams.getDataDir());
+		return Void();
+	}
 	Future<Void> start(Database const& cx) override {
 		if (enabled)
 			return runUnitTests(this);
@@ -104,12 +108,14 @@ struct UnitTestWorkload : TestWorkload {
 			state double start_now = now();
 			state double start_timer = timer();
 
+			platform::createDirectory(self->testParams.getDataDir());
 			try {
 				wait(test->func(self->testParams));
 			} catch (Error& e) {
 				++self->testsFailed;
 				result = e;
 			}
+			platform::eraseDirectoryRecursive(self->testParams.getDataDir());
 			++self->testsExecuted;
 			double wallTime = timer() - start_timer;
 			double simTime = now() - start_now;

flow/ActorCollection.h

@@ -72,7 +72,7 @@ public:
 	}
 
 	void add(Future<Void> a) { m_add.send(a); }
-	Future<Void> getResult() { return m_out; }
+	Future<Void> getResult() const { return m_out; }
 	void clear(bool returnWhenEmptied) {
 		m_out.cancel();
 		m_out = actorCollection(m_add.getFuture(), nullptr, nullptr, nullptr, nullptr, returnWhenEmptied);

flow/Knobs.cpp

@@ -23,17 +23,19 @@
 #include <cmath>
 #include <cinttypes>
 
-std::unique_ptr<FlowKnobs> globalFlowKnobs = std::make_unique<FlowKnobs>();
-FlowKnobs const* FLOW_KNOBS = globalFlowKnobs.get();
+FlowKnobs::FlowKnobs(Randomize randomize, IsSimulated isSimulated) {
+	initialize(randomize, isSimulated);
+}
+
+FlowKnobs bootstrapGlobalFlowKnobs(Randomize::NO, IsSimulated::NO);
+FlowKnobs const* FLOW_KNOBS = &bootstrapGlobalFlowKnobs;
 
 #define init(knob, value) initKnob(knob, value, #knob)
 
-FlowKnobs::FlowKnobs() {
-	initialize();
-}
-
 // clang-format off
-void FlowKnobs::initialize(bool randomize, bool isSimulated) {
+void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) {
+	bool const randomize = _randomize == Randomize::YES;
+	bool const isSimulated = _isSimulated == IsSimulated::YES;
 	init( AUTOMATIC_TRACE_DUMP,                                  1 );
 	init( PREVENT_FAST_SPIN_DELAY,                             .01 );
 	init( CACHE_REFRESH_INTERVAL_WHEN_ALL_ALTERNATIVES_FAILED, 1.0 );
@@ -254,60 +256,88 @@ static std::string toLower(std::string const& name) {
 	return lower_name;
 }
 
-bool Knobs::setKnob(std::string const& knob, std::string const& value) {
-	explicitlySetKnobs.insert(toLower(knob));
+template <class T>
+static T parseIntegral(std::string const& value) {
+	T v;
+	int n = 0;
+	if (StringRef(value).startsWith(LiteralStringRef("0x"))) {
+		if (sscanf(value.c_str(), "0x%" SCNx64 "%n", &v, &n) != 1 || n != value.size())
+			throw invalid_option_value();
+	} else {
+		if (sscanf(value.c_str(), "%" SCNd64 "%n", &v, &n) != 1 || n != value.size())
+			throw invalid_option_value();
+	}
+	return v;
+}
+
+ParsedKnobValue Knobs::parseKnobValue(std::string const& knob, std::string const& value) const {
 	if (double_knobs.count(knob)) {
 		double v;
 		int n = 0;
 		if (sscanf(value.c_str(), "%lf%n", &v, &n) != 1 || n != value.size())
 			throw invalid_option_value();
-		*double_knobs[knob] = v;
-		return true;
-	}
-	if (bool_knobs.count(knob)) {
+		return v;
+	} else if (bool_knobs.count(knob)) {
 		if (toLower(value) == "true") {
-			*bool_knobs[knob] = true;
+			return true;
 		} else if (toLower(value) == "false") {
-			*bool_knobs[knob] = false;
-		} else {
-			int64_t v;
-			int n = 0;
-			if (StringRef(value).startsWith(LiteralStringRef("0x"))) {
-				if (sscanf(value.c_str(), "0x%" SCNx64 "%n", &v, &n) != 1 || n != value.size())
-					throw invalid_option_value();
-			} else {
-				if (sscanf(value.c_str(), "%" SCNd64 "%n", &v, &n) != 1 || n != value.size())
-					throw invalid_option_value();
-			}
-			*bool_knobs[knob] = v;
-		}
-		return true;
-	}
-	if (int64_knobs.count(knob) || int_knobs.count(knob)) {
-		int64_t v;
-		int n = 0;
-		if (StringRef(value).startsWith(LiteralStringRef("0x"))) {
-			if (sscanf(value.c_str(), "0x%" SCNx64 "%n", &v, &n) != 1 || n != value.size())
-				throw invalid_option_value();
-		} else {
-			if (sscanf(value.c_str(), "%" SCNd64 "%n", &v, &n) != 1 || n != value.size())
-				throw invalid_option_value();
-		}
-		if (int64_knobs.count(knob))
-			*int64_knobs[knob] = v;
-		else {
-			if (v < std::numeric_limits<int>::min() || v > std::numeric_limits<int>::max())
-				throw invalid_option_value();
-			*int_knobs[knob] = v;
-		}
-		return true;
-	}
-	if (string_knobs.count(knob)) {
-		*string_knobs[knob] = value;
-		return true;
-	}
-	explicitlySetKnobs.erase(toLower(knob)); // don't store knobs that don't exist
 			return false;
+		} else {
+			return parseIntegral<bool>(value);
+		}
+	} else if (int64_knobs.count(knob)) {
+		return parseIntegral<int64_t>(value);
+	} else if (int_knobs.count(knob)) {
+		return parseIntegral<int>(value);
+	} else if (string_knobs.count(knob)) {
+		return value;
+	}
+	return NoKnobFound{};
+}
+
+bool Knobs::setKnob(std::string const& knob, int value) {
+	if (!int_knobs.count(knob)) {
+		return false;
+	}
+	*int_knobs[knob] = value;
+	explicitlySetKnobs.insert(toLower(knob));
+	return true;
+}
+
+bool Knobs::setKnob(std::string const& knob, int64_t value) {
+	if (!int64_knobs.count(knob)) {
+		return false;
+	}
+	*int64_knobs[knob] = value;
+	explicitlySetKnobs.insert(toLower(knob));
+	return true;
+}
+
+bool Knobs::setKnob(std::string const& knob, bool value) {
+	if (!bool_knobs.count(knob)) {
+		return false;
+	}
+	*bool_knobs[knob] = value;
+	explicitlySetKnobs.insert(toLower(knob));
+	return true;
+}
+
+bool Knobs::setKnob(std::string const& knob, double value) {
+	if (!double_knobs.count(knob)) {
+		return false;
+	}
+	*double_knobs[knob] = value;
+	explicitlySetKnobs.insert(toLower(knob));
+	return true;
+}
+
+bool Knobs::setKnob(std::string const& knob, std::string const& value) {
+	if (!string_knobs.count(knob)) {
+		return false;
+	}
+	*string_knobs[knob] = value;
+	explicitlySetKnobs.insert(toLower(knob));
+	return true;
 }
 
 void Knobs::initKnob(double& knob, double value, std::string const& name) {

flow/Knobs.h

@@ -28,15 +28,23 @@
 #include <set>
 #include <string>
 #include <stdint.h>
+#include <variant>
+#include <optional>
+
+// NOTE: Directly using KnobValueRef as the return type for Knobs::parseKnobValue would result
+// in a cyclic dependency, so we use this intermediate ParsedKnobValue type
+struct NoKnobFound {};
+using ParsedKnobValue = std::variant<NoKnobFound, int, double, int64_t, bool, std::string>;
+
+// To be used as effectively boolean parameters with added type safety
+enum class IsSimulated { NO, YES };
+enum class Randomize { NO, YES };
 
 class Knobs {
-public:
-	bool setKnob(std::string const& name,
-	             std::string const& value); // Returns true if the knob name is known, false if it is unknown
-	void trace() const;
-
 protected:
 	Knobs() = default;
+	Knobs(Knobs const&) = delete;
+	Knobs& operator=(Knobs const&) = delete;
 	void initKnob(double& knob, double value, std::string const& name);
 	void initKnob(int64_t& knob, int64_t value, std::string const& name);
 	void initKnob(int& knob, int value, std::string const& name);
@@ -49,9 +57,28 @@ protected:
 	std::map<std::string, std::string*> string_knobs;
 	std::map<std::string, bool*> bool_knobs;
 	std::set<std::string> explicitlySetKnobs;
+
+public:
+	bool setKnob(std::string const& name, int value);
+	bool setKnob(std::string const& name, bool value);
+	bool setKnob(std::string const& name, int64_t value);
+	bool setKnob(std::string const& name, double value);
+	bool setKnob(std::string const& name, std::string const& value);
+	ParsedKnobValue parseKnobValue(std::string const& name, std::string const& value) const;
+	void trace() const;
 };
 
-class FlowKnobs : public Knobs {
+template <class T>
+class KnobsImpl : public Knobs {
+public:
+	template <class... Args>
+	void reset(Args&&... args) {
+		explicitlySetKnobs.clear();
+		static_cast<T*>(this)->initialize(std::forward<Args>(args)...);
+	}
+};
+
+class FlowKnobs : public KnobsImpl<FlowKnobs> {
 public:
 	int AUTOMATIC_TRACE_DUMP;
 	double PREVENT_FAST_SPIN_DELAY;
@@ -257,12 +284,12 @@ public:
 	bool HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS;
 	int HEALTH_MONITOR_CLIENT_REQUEST_INTERVAL_SECS;
 	int HEALTH_MONITOR_CONNECTION_MAX_CLOSED;
-
-	FlowKnobs();
-	void initialize(bool randomize = false, bool isSimulated = false);
+	FlowKnobs(Randomize, IsSimulated);
+	void initialize(Randomize, IsSimulated);
 };
 
-extern std::unique_ptr<FlowKnobs> globalFlowKnobs;
+// Flow knobs are needed before the knob collections are available, so a global FlowKnobs object is used to bootstrap
+extern FlowKnobs bootstrapGlobalFlowKnobs;
 extern FlowKnobs const* FLOW_KNOBS;
 
 #endif

flow/ObjectSerializer.h

@@ -120,6 +120,14 @@ public:
 		vo.read(*this);
 	}
 
+	template <class T, class VersionOptions>
+	static T fromStringRef(StringRef sr, VersionOptions vo) {
+		T t;
+		ObjectReader reader(sr.begin(), vo);
+		reader.deserialize(t);
+		return t;
+	}
+
 	const uint8_t* data() { return _data; }
 
 	Arena& arena() { return _arena; }

flow/Platform.h

@@ -580,10 +580,16 @@ inline static int64_t flowInterlockedAnd64(int64_t* p, int64_t a) {
 #define bigEndian16(value) uint16_t(_byteswap_ushort(value))
 #define bigEndian32(value) uint32_t(_byteswap_ulong(value))
 #define bigEndian64(value) uint64_t(_byteswap_uint64(value))
+#define fromBigEndian16(value) uint16_t(_byteswap_ushort(value))
+#define fromBigEndian32(value) uint32_t(_byteswap_ulong(value))
+#define fromBigEndian64(value) uint64_t(_byteswap_uint64(value))
 #elif __GNUG__
 #define bigEndian16(value) uint16_t((value >> 8) | (value << 8))
 #define bigEndian32(value) uint32_t(__builtin_bswap32(value))
 #define bigEndian64(value) uint64_t(__builtin_bswap64(value))
+#define fromBigEndian16(value) uint16_t((value >> 8) | (value << 8))
+#define fromBigEndian32(value) uint32_t(__builtin_bswap32(value))
+#define fromBigEndian64(value) uint64_t(__builtin_bswap64(value))
 #else
 #error Missing byte swap methods
 #endif

flow/UnitTest.cpp

@@ -63,3 +63,11 @@ Optional<double> UnitTestParameters::getDouble(const std::string& name) const {
 	}
 	return {};
 }
+
+std::string UnitTestParameters::getDataDir() const {
+	return dataDir.get();
+}
+
+void UnitTestParameters::setDataDir(std::string const& dataDir) {
+	this->dataDir = dataDir;
+}

flow/UnitTest.h

@@ -47,7 +47,10 @@
 
 #include <cinttypes>
 
-struct UnitTestParameters {
+class UnitTestParameters {
+	Optional<std::string> dataDir;
+
+public:
 	// Map of named case-sensitive parameters
 	std::map<std::string, std::string> params;
 
@@ -68,6 +71,12 @@ struct UnitTestParameters {
 
 	// Get a parameter's value parsed as a double, will return !present() if parameter was not set
 	Optional<double> getDouble(const std::string& name) const;
+
+	// This is separate because it assumes data directory has already been set, and doesn't return an optional
+	std::string getDataDir() const;
+
+	// Set the data directory used to persist data for this test
+	void setDataDir(std::string const&);
 };
 
 // Unit test definition structured as a linked list item

flow/error_definitions.h

@@ -75,6 +75,8 @@ ERROR( batch_transaction_throttled, 1051, "Batch GRV request rate limit exceeded
 ERROR( dd_cancelled, 1052, "Data distribution components cancelled")
 ERROR( dd_not_found, 1053, "Data distributor not found")
 ERROR( wrong_connection_file, 1054, "Connection file mismatch")
+ERROR( version_already_compacted, 1055, "The requested changes have been compacted away")
+ERROR( local_config_changed, 1056, "Local configuration file has changed. Restart and apply these changes" )
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )
@@ -148,6 +150,9 @@ ERROR( transaction_read_only, 2023, "Attempted to commit a transaction specified
 ERROR( invalid_cache_eviction_policy, 2024, "Invalid cache eviction policy, only random and lru are supported" )
 ERROR( network_cannot_be_restarted, 2025, "Network can only be started once" )
 ERROR( blocked_from_network_thread, 2026, "Detected a deadlock in a callback called from the network thread" )
+ERROR( invalid_config_db_range_read, 2027, "Invalid configuration database range read" )
+ERROR( invalid_config_db_key, 2028, "Invalid configuration database key provided" )
+ERROR( invalid_config_path, 2029, "Invalid configuration path" )
 
 ERROR( incompatible_protocol_version, 2100, "Incompatible protocol version" )
 ERROR( transaction_too_large, 2101, "Transaction exceeds byte limit" )

flow/flat_buffers.h

@@ -33,8 +33,6 @@
 #include <vector>
 #include <cstring>
 #include <array>
-#include <typeinfo>
-#include <typeindex>
 #include <unordered_map>
 #include <deque>
 #include "flow/FileIdentifier.h"

flow/genericactors.actor.cpp

@@ -19,6 +19,7 @@
  */
 
 #include "flow/flow.h"
+#include "flow/UnitTest.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 ACTOR Future<bool> allTrue(std::vector<Future<bool>> all) {
@@ -136,3 +137,46 @@ ACTOR Future<Void> lowPriorityDelay(double waitTime) {
 	}
 	return Void();
 }
+
+namespace {
+
+struct DummyState {
+	int changed{ 0 };
+	int unchanged{ 0 };
+	bool operator==(DummyState const& rhs) const { return changed == rhs.changed && unchanged == rhs.unchanged; }
+	bool operator!=(DummyState const& rhs) const { return !(*this == rhs); }
+};
+
+ACTOR Future<Void> testPublisher(Reference<AsyncVar<DummyState>> input) {
+	state int i = 0;
+	for (; i < 100; ++i) {
+		wait(delay(deterministicRandom()->random01()));
+		auto var = input->get();
+		++var.changed;
+		input->set(var);
+	}
+	return Void();
+}
+
+ACTOR Future<Void> testSubscriber(Reference<IDependentAsyncVar<int>> output, Optional<int> expected) {
+	loop {
+		wait(output->onChange());
+		ASSERT(expected.present());
+		if (output->get() == expected.get()) {
+			return Void();
+		}
+	}
+}
+
+} // namespace
+
+TEST_CASE("/flow/genericactors/DependentAsyncVar") {
+	auto input = makeReference<AsyncVar<DummyState>>();
+	state Future<Void> subscriber1 =
+	    testSubscriber(IDependentAsyncVar<int>::create(input, [](auto const& var) { return var.changed; }), 100);
+	state Future<Void> subscriber2 =
+	    testSubscriber(IDependentAsyncVar<int>::create(input, [](auto const& var) { return var.unchanged; }), {});
+	wait(subscriber1 && testPublisher(input));
+	ASSERT(!subscriber2.isReady());
+	return Void();
+}

flow/genericactors.actor.h

@@ -1946,9 +1946,9 @@ Future<U> runAfter(Future<T> lhs, Future<U> rhs) {
 	return res;
 }
 
-template <class T, class Res>
-Future<Res> operator>>=(Future<T> lhs, std::function<Future<Res>(T const&)> rhs) {
-	return runAfter(lhs, rhs);
+template <class T, class Fun>
+auto operator>>=(Future<T> lhs, Fun&& rhs) -> Future<decltype(rhs(std::declval<T>()))> {
+	return runAfter(lhs, std::forward<Fun>(rhs));
 }
 
 template <class T, class U>
@@ -1956,6 +1956,52 @@ Future<U> operator>>(Future<T> const& lhs, Future<U> const& rhs) {
 	return runAfter(lhs, rhs);
 }
 
+/*
+ * IDependentAsyncVar is similar to AsyncVar, but it decouples the input and output, so the translation unit
+ * responsible for handling the output does not need to have knowledge of how the output is generated
+ */
+template <class Output>
+class IDependentAsyncVar : public ReferenceCounted<IDependentAsyncVar<Output>> {
+public:
+	virtual ~IDependentAsyncVar() = default;
+	virtual Output const& get() const = 0;
+	virtual Future<Void> onChange() const = 0;
+	template <class Input, class F>
+	static Reference<IDependentAsyncVar> create(Reference<AsyncVar<Input>> const& input, F const& f);
+	static Reference<IDependentAsyncVar> create(Reference<AsyncVar<Output>> const& output);
+};
+
+template <class Input, class Output, class F>
+class DependentAsyncVar final : public IDependentAsyncVar<Output> {
+	Reference<AsyncVar<Output>> output;
+	Future<Void> monitorActor;
+	ACTOR static Future<Void> monitor(Reference<AsyncVar<Input>> input, Reference<AsyncVar<Output>> output, F f) {
+		loop {
+			wait(input->onChange());
+			output->set(f(input->get()));
+		}
+	}
+
+public:
+	DependentAsyncVar(Reference<AsyncVar<Input>> const& input, F const& f)
+	  : output(makeReference<AsyncVar<Output>>(f(input->get()))), monitorActor(monitor(input, output, f)) {}
+	Output const& get() const override { return output->get(); }
+	Future<Void> onChange() const override { return output->onChange(); }
+};
+
+template <class Output>
+template <class Input, class F>
+Reference<IDependentAsyncVar<Output>> IDependentAsyncVar<Output>::create(Reference<AsyncVar<Input>> const& input,
+                                                                         F const& f) {
+	return makeReference<DependentAsyncVar<Input, Output, F>>(input, f);
+}
+
+template <class Output>
+Reference<IDependentAsyncVar<Output>> IDependentAsyncVar<Output>::create(Reference<AsyncVar<Output>> const& input) {
+	auto identity = [](const auto& x) { return x; };
+	return makeReference<DependentAsyncVar<Output, Output, decltype(identity)>>(input, identity);
+}
+
 // A weak reference type to wrap a future Reference<T> object.
 // Once the future is complete, this object holds a pointer to the referenced object but does
 // not contribute to its reference count.

tests/restarting/from_7.0.0/ConfigureTestRestart-1.txt

@@ -3,6 +3,7 @@ clearAfterTest=false
 
     testName=ConfigureDatabase
     testDuration=30.0
+    allowDescriptorChange=false
 
     testName=RandomClogging
     testDuration=30.0

tests/restarting/from_7.0.0/ConfigureTestRestart-2.txt

@@ -3,6 +3,7 @@ runSetup=false
 
     testName=ConfigureDatabase
     testDuration=300.0
+    allowDescriptorChange=false
 
     testName=RandomClogging
     testDuration=300.0