Added new test that combines atomic backup/restore. Added randomization to delays in AtomicRestore workload.

fdbserver/workloads/AtomicRestore.actor.cpp

@@ -26,14 +26,14 @@
 
 //A workload which test the correctness of backup and restore process
 struct AtomicRestoreWorkload : TestWorkload {
-	double startAfter, switch1After;
+	double startAfter, restoreAfter;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
 
 	AtomicRestoreWorkload(WorkloadContext const& wcx)
 		: TestWorkload(wcx) {
 
 		startAfter = getOption(options, LiteralStringRef("startAfter"), 10.0);
-		switch1After = getOption(options, LiteralStringRef("switch1After"), 20.0);
+		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 20.0);
 		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 	}
 
@@ -60,10 +60,10 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 	ACTOR static Future<Void> _start(Database cx, AtomicRestoreWorkload* self) {
 		state FileBackupAgent backupAgent;
-		state Future<Void> switch1After = delay(self->switch1After);
 		state Future<Void> disabler = disableConnectionFailuresAfter(300, "atomicRestore");
-		Void _ = wait( delay(self->startAfter) );
-		TraceEvent("AR_Start");
+
+		Void _ = wait( delay(self->startAfter * g_random->random01()) );
+		TraceEvent("AtomicRestore_Start");
 
 		state std::string backupContainer = "file://simfdb/backups/";
 		try {
@@ -74,11 +74,11 @@ struct AtomicRestoreWorkload : TestWorkload {
 				throw;
 		}
 
-		TraceEvent("AS_Wait1");
+		TraceEvent("AtomicRestore_Wait");
 		int _ = wait( backupAgent.waitBackup(cx, BackupAgentBase::getDefaultTagName(), false) );
-		TraceEvent("AS_Ready1");
-		Void _ = wait( switch1After );
-		TraceEvent("AS_Switch1");
+		TraceEvent("AtomicRestore_BackupStart");
+		Void _ = wait( delay(self->restoreAfter * g_random->random01()) );
+		TraceEvent("AtomicRestore_RestoreStart");
 
 		loop {
 			std::vector<Future<Version>> restores;
@@ -102,7 +102,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 			g_simulator.backupAgents = ISimulator::NoBackupAgents;
 		}
 
-		TraceEvent("AS_Done");
+		TraceEvent("AtomicRestore_Done");
 		return Void();
 	}
 };

tests/slow/ApiCorrectnessAtomicRestore.txt

@@ -0,0 +1,28 @@
+testTitle=ApiCorrectnessTest
+testName=ApiCorrectness
+runSetup=true
+clearAfterTest=true
+numKeys=5000
+onlyLowerCase=true
+shortKeysRatio=0.5
+minShortKeyLength=1
+maxShortKeyLength=3
+minLongKeyLength=1
+maxLongKeyLength=128
+minValueLength=1
+maxValueLength=1000
+numGets=1000
+numGetRanges=100
+numGetRangeSelectors=100
+numGetKeys=100
+numClears=100
+numClearRanges=10
+maxTransactionBytes=500000
+randomTestDuration=60
+timeout=2100
+
+testName=AtomicRestore
+startAfter=10.0
+restoreAfter=50.0
+clearAfterTest=false
+simBackupAgents=BackupToFile

tests/slow/WriteDuringReadAtomicRestore.txt

@@ -8,7 +8,7 @@ testTitle=WriteDuringReadTest
 
     testName=AtomicRestore
     startAfter=10.0
-    switch1After=50.0
+    restoreAfter=50.0
     clearAfterTest=false
     simBackupAgents=BackupToFile