Merge pull request #5144 from Daniel-B-Smith/rocksdb-in-simulation

RocksDB in simulation

fdbclient/ServerKnobs.cpp

@@ -335,7 +335,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// KeyValueStoreRocksDB
 	init( ROCKSDB_BACKGROUND_PARALLELISM,                          0 );
 	init( ROCKSDB_READ_PARALLELISM,                                4 );
-	init( ROCKSDB_MEMTABLE_BYTES,                  512 * 1024 * 1024 );
+	// Use a smaller memtable in simulation to avoid OOMs.
+	int64_t memtableBytes = isSimulated ? 32 * 1024 : 512 * 1024 * 1024;
+	init( ROCKSDB_MEMTABLE_BYTES,                      memtableBytes );
 	init( ROCKSDB_UNSAFE_AUTO_FSYNC,                           false );
 	init( ROCKSDB_PERIODIC_COMPACTION_SECONDS,                     0 );
 	init( ROCKSDB_PREFIX_LEN,                                      0 );

fdbserver/KeyValueStoreRocksDB.actor.cpp

@@ -7,6 +7,7 @@
 #include <rocksdb/slice_transform.h>
 #include <rocksdb/table.h>
 #include <rocksdb/utilities/table_properties_collectors.h>
+#include "fdbserver/CoroFlow.h"
 #include "flow/flow.h"
 #include "flow/IThreadPool.h"
 
@@ -283,7 +284,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				                              std::min(value.size(), size_t(a.maxLength)))));
 			} else {
 				if (!s.IsNotFound()) {
-					TraceEvent(SevError, "RocksDBError").detail("Error", s.ToString()).detail("Method", "ReadValuePrefix");
+					TraceEvent(SevError, "RocksDBError")
+					    .detail("Error", s.ToString())
+					    .detail("Method", "ReadValuePrefix");
 				}
 				a.result.send(Optional<Value>());
 			}
@@ -367,8 +370,23 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
 
 	explicit RocksDBKeyValueStore(const std::string& path, UID id) : path(path), id(id) {
-		writeThread = createGenericThreadPool();
-		readThreads = createGenericThreadPool();
+		// In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage engine
+		// is still multi-threaded as background compaction threads are still present. Reads/writes to disk will also
+		// block the network thread in a way that would be unacceptable in production but is a necessary evil here. When
+		// performing the reads in background threads in simulation, the event loop thinks there is no work to do and
+		// advances time faster than 1 sec/sec. By the time the blocking read actually finishes, simulation has advanced
+		// time by more than 5 seconds, so every read fails with a transaction_too_old error. Doing blocking IO on the
+		// main thread solves this issue. There are almost certainly better fixes, but my goal was to get a less
+		// invasive change merged first and work on a more realistic version if/when we think that would provide
+		// substantially more confidence in the correctness.
+		// TODO: Adapt the simulation framework to not advance time quickly when background reads/writes are occurring.
+		if (g_network->isSimulated()) {
+			writeThread = CoroThreadPool::createThreadPool();
+			readThreads = CoroThreadPool::createThreadPool();
+		} else {
+			writeThread = createGenericThreadPool();
+			readThreads = createGenericThreadPool();
+		}
 		writeThread->addThread(new Writer(db, id), "fdb-rocksdb-wr");
 		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
 			readThreads->addThread(new Reader(db), "fdb-rocksdb-re");

fdbserver/SimulatedCluster.actor.cpp

@@ -232,6 +232,7 @@ public:
 	//	1 = "memory"
 	//	2 = "memory-radixtree-beta"
 	//	3 = "ssd-redwood-experimental"
+	//	4 = "ssd-rocksdb-experimental"
 	// Requires a comma-separated list of numbers WITHOUT whitespaces
 	std::vector<int> storageEngineExcludeTypes;
 	// Set the maximum TLog version that can be selected for a test
@@ -1252,7 +1253,7 @@ void SimulationConfig::setDatacenters(const TestConfig& testConfig) {
 
 // Sets storage engine based on testConfig details
 void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
-	int storage_engine_type = deterministicRandom()->randomInt(0, 4);
+	int storage_engine_type = deterministicRandom()->randomInt(0, 5);
 	if (testConfig.storageEngineType.present()) {
 		storage_engine_type = testConfig.storageEngineType.get();
 	} else {
@@ -1260,7 +1261,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		while (std::find(testConfig.storageEngineExcludeTypes.begin(),
 		                 testConfig.storageEngineExcludeTypes.end(),
 		                 storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) {
-			storage_engine_type = deterministicRandom()->randomInt(0, 4);
+			storage_engine_type = deterministicRandom()->randomInt(0, 5);
 		}
 	}
 
@@ -1285,6 +1286,16 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		set_config("ssd-redwood-experimental");
 		break;
 	}
+	case 4: {
+		TEST(true); // Simulated cluster using RocksDB storage engine
+		set_config("ssd-rocksdb-experimental");
+		// Tests using the RocksDB engine are necessarily non-deterministic because of RocksDB
+		// background threads.
+		TraceEvent(SevWarn, "RocksDBNonDeterminism")
+		    .detail("Explanation", "The RocksDB storage engine is threaded and non-deterministic");
+		noUnseed = true;
+		break;
+	}
 	default:
 		ASSERT(false); // Programmer forgot to adjust cases.
 	}

tests/restarting/from_6.2.33/SnapCycleRestart-1.txt

@@ -1,3 +1,5 @@
+storageEngineExcludeTypes=4
+
 ;Take snap and do cycle test
 testTitle=SnapCyclePre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapCycleRestart-2.txt

@@ -1,4 +1,5 @@
 buggify=off
+storageEngineExcludeTypes=4
 
 testTitle=SnapCycleRestore
 runSetup=false

tests/restarting/from_6.2.33/SnapTestAttrition-1.txt

@@ -1,3 +1,5 @@
+storageEngineExcludeTypes=4
+
 ;write 1000 Keys ending with even numbers
 testTitle=SnapTestPre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapTestAttrition-2.txt

@@ -1,4 +1,5 @@
 buggify=off
+storageEngineExcludeTypes=4
 
 ; verify all keys are even numbered
 testTitle=SnapTestVerify

tests/restarting/from_6.2.33/SnapTestRestart-1.txt

@@ -1,3 +1,5 @@
+storageEngineExcludeTypes=4
+
 ;write 1000 Keys ending with even numbers
 testTitle=SnapTestPre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapTestRestart-2.txt

@@ -1,4 +1,5 @@
 buggify=off
+storageEngineExcludeTypes=4
 
 ; verify all keys are even numbered
 testTitle=SnapTestVerify

tests/restarting/from_6.2.33/SnapTestSimpleRestart-1.txt

@@ -1,3 +1,5 @@
+storageEngineExcludeTypes=4
+
 ;write 1000 Keys ending with even number
 testTitle=SnapSimplePre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapTestSimpleRestart-2.txt

@@ -1,4 +1,5 @@
 buggify=off
+storageEngineExcludeTypes=4
 
 ; verify all keys are even numbered
 testTitle=SnapSimpleVerify

tests/restarting/from_7.0.0/SnapIncrementalRestore-1.toml

@@ -1,7 +1,8 @@
 [configuration]
 logAntiQuorum = 0
+storageEngineExcludeTypes = [4]
 
-[[test]] 
+[[test]]
 testTitle = 'SubmitBackup'
 simBackupAgents = 'BackupToFile'
 clearAfterTest = false

tests/restarting/from_7.0.0/SnapIncrementalRestore-2.toml

@@ -1,3 +1,6 @@
+[configuration]
+storageEngineExcludeTypes = [4]
+
 [[test]]
 testTitle = 'RestoreBackup'
 simBackupAgents = 'BackupToFile'