Add usePartitionedLog option for Backup workload

This allows us to specify old or new style of backup to be used. Added two
tests that switch between them and randomly choose one backup to restore.

20250314-034057-jzhou-c27ca23b6c69cecf

fdbserver/workloads/Backup.actor.cpp

@@ -36,14 +36,17 @@ struct BackupWorkload : TestWorkload {
 	double minBackupAfter;
 	double backupStartAt, restoreStartAfterBackupFinished, stopDifferentialAfter;
 	Key backupTag;
-	bool differentialBackup, agentRequest;
+	bool differentialBackup;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
 	LockDB locked{ false };
+	UsePartitionedLog usePartitionedLog{ true };
 	bool allowPauses;
 	Optional<std::string> encryptionKeyFileName;
 
 	BackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
+		bool partitioned = getOption(options, "usePartitionedLog"_sr, true);
+		usePartitionedLog.set(partitioned);
 		backupAfter = getOption(options, "backupAfter"_sr, 10.0);
 		double minBackupAfter = getOption(options, "minBackupAfter"_sr, backupAfter);
 		if (backupAfter > minBackupAfter) {
@@ -167,7 +170,7 @@ struct BackupWorkload : TestWorkload {
 			                               backupRanges,
 			                               false,
 			                               StopWhenDone{ !stopDifferentialDelay },
-			                               UsePartitionedLog::True, // enable partitioned log here
+			                               self->usePartitionedLog,
 			                               IncrementalBackupOnly::False,
 			                               self->encryptionKeyFileName));
 		} catch (Error& e) {

fdbserver/workloads/Restore.actor.cpp

@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ * Copyright 2013-2025 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,22 +35,23 @@
 // A workload which test the correctness of backup and restore process
 struct RestoreWorkload : TestWorkload {
 	static constexpr auto NAME = "Restore";
-	Key backupTag;
-	bool differentialBackup, performRestore, agentRequest;
+	Key backupTag, backupTag1, backupTag2;
+	bool performRestore, agentRequest;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges, restoreRanges;
 	static int backupAgentRequests;
 	LockDB locked{ false };
 	bool allowPauses;
 	bool shareLogRange;
 	bool shouldSkipRestoreRanges;
-	bool defaultBackup;
 	Optional<std::string> encryptionKeyFileName;
 	UID randomID;
 
 	RestoreWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
 		locked.set(sharedRandomNumber % 2);
 		performRestore = getOption(options, "performRestore"_sr, true);
-		backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
+		backupTag1 = getOption(options, "backupTag1"_sr, BackupAgentBase::getDefaultTag());
+		backupTag2 = getOption(options, "backupTag2"_sr, BackupAgentBase::getDefaultTag());
+		backupTag = deterministicRandom()->coinflip() ? backupTag1 : backupTag2;
 		agentRequest = getOption(options, "simBackupAgents"_sr, true);
 		allowPauses = getOption(options, "allowPauses"_sr, true);
 		shareLogRange = getOption(options, "shareLogRange"_sr, false);
@@ -108,7 +109,6 @@ struct RestoreWorkload : TestWorkload {
 
 	ACTOR static Future<Void> _start(Database cx, RestoreWorkload* self) {
 		state FileBackupAgent backupAgent;
-		state bool extraTasks = false;
 		state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
 		TraceEvent("RW_Arguments")
 		    .detail("BackupTag", printable(self->backupTag))

tests/CMakeLists.txt

@@ -485,6 +485,8 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES slow/ApiCorrectnessWithConsistencyCheck.toml)
   add_fdb_test(TEST_FILES slow/BackupAndRestore.toml)
   add_fdb_test(TEST_FILES slow/BackupCorrectnessPartitioned.toml)
+  add_fdb_test(TEST_FILES slow/BackupNewAndOldRestore.toml)
+  add_fdb_test(TEST_FILES slow/BackupOldAndNewRestore.toml)
   add_fdb_test(TEST_FILES slow/ClogWithRollbacks.toml)
   add_fdb_test(TEST_FILES slow/CloggedCycleTest.toml)
   add_fdb_test(TEST_FILES slow/CloggedStorefront.toml)

tests/slow/BackupAndRestore.toml

@@ -19,6 +19,7 @@ simBackupAgents = 'BackupToFile'
 
     [[test.workload]]
     testName = 'Backup'
+    usePartitionedLog = true
     backupAfter = 10.0
     restoreAfter = 60.0
 

tests/slow/BackupNewAndOldRestore.toml

@@ -0,0 +1,68 @@
+testClass = "Backup"
+
+[configuration]
+buggify = false
+tenantModes = ['disabled'] # Do not support tenant
+encryptModes = ['disabled'] # Do not support encryption
+
+[[test]]
+testTitle = 'NewBackup'
+clearAfterTest = false
+simBackupAgents = 'BackupToFile'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 2500.0
+    testDuration = 30.0
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'Backup'
+    usePartitionedLog = true
+    backupTag = 'newBackup'
+    backupAfter = 10.0
+    restoreAfter = 60.0
+
+[[test]]
+testTitle = 'OldBackup'
+clearAfterTest = false
+simBackupAgents = 'BackupToFile'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 2500.0
+    testDuration = 30.0
+    skipSetup = true
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'Backup'
+    usePartitionedLog = false
+    backupTag = 'oldBackup'
+    backupAfter = 10.0
+    restoreAfter = 60.0
+
+# Randomly pick one of the backup tag and restore it.
+[[test]]
+testTitle = 'RestoreRandomBackup'
+simBackupAgents = 'BackupToFile'
+clearAfterTest = false
+
+    [[test.workload]]
+    testName = 'Restore'
+    backupTag1 = 'newBackup'
+    backupTag2 = 'oldBackup'
+
+# check consistency after restore
+[[test]]
+testTitle = 'CycleAfterRestore'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 2500.0
+    testDuration = 10.0
+    skipSetup = true
+    expectedRate = 0

tests/slow/BackupOldAndNewRestore.toml

@@ -0,0 +1,68 @@
+testClass = "Backup"
+
+[configuration]
+buggify = false
+tenantModes = ['disabled'] # Do not support tenant
+encryptModes = ['disabled'] # Do not support encryption
+
+[[test]]
+testTitle = 'OldBackup'
+clearAfterTest = false
+simBackupAgents = 'BackupToFile'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 2500.0
+    testDuration = 30.0
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'Backup'
+    usePartitionedLog = false
+    backupTag = 'oldBackup'
+    backupAfter = 10.0
+    restoreAfter = 60.0
+
+[[test]]
+testTitle = 'NewBackup'
+clearAfterTest = false
+simBackupAgents = 'BackupToFile'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 2500.0
+    testDuration = 30.0
+    skipSetup = true
+    expectedRate = 0
+
+    [[test.workload]]
+    testName = 'Backup'
+    usePartitionedLog = true
+    backupTag = 'newBackup'
+    backupAfter = 10.0
+    restoreAfter = 60.0
+
+# Randomly pick one of the backup tag and restore it.
+[[test]]
+testTitle = 'RestoreRandomBackup'
+simBackupAgents = 'BackupToFile'
+clearAfterTest = false
+
+    [[test.workload]]
+    testName = 'Restore'
+    backupTag1 = 'oldBackup'
+    backupTag2 = 'newBackup'
+
+# check consistency after restore
+[[test]]
+testTitle = 'CycleAfterRestore'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 3000
+    transactionsPerSecond = 2500.0
+    testDuration = 10.0
+    skipSetup = true
+    expectedRate = 0