From 5d2deddb7dee53f10ece037be18a4f5d313cb049 Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Fri, 2 Aug 2024 17:15:59 -0700 Subject: [PATCH 1/4] Reduce the chance to run some rare tests E.g., StatusBuilderPerf and TLogVersionMessagesOverheadFactor are more like performance tests, which shouldn't be running so many times. Without the change, a 100k-run has this many for these tests: 1318 tests/rare/CycleWithKills.toml 1591 tests/rare/TLogVersionMessagesOverheadFactor.toml 1647 tests/rare/ConfigDBUnitTest.toml 1839 tests/rare/StatusBuilderPerf.toml After the change, a 100k-run has: 129 tests/rare/TLogVersionMessagesOverheadFactor.toml 151 tests/rare/CycleWithKills.toml 160 tests/rare/StatusBuilderPerf.toml 375 tests/rare/ConfigDBUnitTest.toml --- fdbclient/BackupAgentBase.actor.cpp | 3 +-- tests/rare/ConfigDBUnitTest.toml | 2 +- tests/rare/CycleWithKills.toml | 4 ++++ tests/rare/StatusBuilderPerf.toml | 4 ++++ tests/rare/TLogVersionMessagesOverheadFactor.toml | 4 ++++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp index 9a80999799..45a7c6912e 100644 --- a/fdbclient/BackupAgentBase.actor.cpp +++ b/fdbclient/BackupAgentBase.actor.cpp @@ -847,8 +847,7 @@ ACTOR Future kvMutationLogToTransactions(Database cx, mutationSize = 0; } - state int i; - for (i = 0; i < curReq.transaction.mutations.size(); i++) { + for (int i = 0; i < curReq.transaction.mutations.size(); i++) { req.transaction.mutations.push_back_deep(req.arena, curReq.transaction.mutations[i]); req.transaction.encryptedMutations.push_back_deep(req.arena, curReq.transaction.encryptedMutations[i]); diff --git a/tests/rare/ConfigDBUnitTest.toml b/tests/rare/ConfigDBUnitTest.toml index c0e9a931ff..0cd45f4660 100644 --- a/tests/rare/ConfigDBUnitTest.toml +++ b/tests/rare/ConfigDBUnitTest.toml @@ -1,5 +1,5 @@ # This test can't run as part of RandomUnitTests.toml, so run it less frequently -testPriority = '10' +testPriority = '50' [[test]] testTitle = "ConfigDBUnitTest" diff --git a/tests/rare/CycleWithKills.toml b/tests/rare/CycleWithKills.toml index 949c30eaa2..f5c4efac28 100644 --- a/tests/rare/CycleWithKills.toml +++ b/tests/rare/CycleWithKills.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [[test]] testTitle = 'CycleTestWithKills' diff --git a/tests/rare/StatusBuilderPerf.toml b/tests/rare/StatusBuilderPerf.toml index 2a9bbdda5d..6c843777bc 100644 --- a/tests/rare/StatusBuilderPerf.toml +++ b/tests/rare/StatusBuilderPerf.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [[test]] testTitle = 'StatusBuilderPerf' useDB = false diff --git a/tests/rare/TLogVersionMessagesOverheadFactor.toml b/tests/rare/TLogVersionMessagesOverheadFactor.toml index c87e0721a5..7bc0279eae 100644 --- a/tests/rare/TLogVersionMessagesOverheadFactor.toml +++ b/tests/rare/TLogVersionMessagesOverheadFactor.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [[test]] testTitle = 'TLogVersionMessagesOverheadFactor' useDB = false From 6d580e16b2e63face9a1ebdda542e8b43a4ceafc Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Sat, 3 Aug 2024 15:42:12 -0700 Subject: [PATCH 2/4] Make rare tests run less in Joshua By increasing the priority value --- tests/rare/CloggedCycleWithKills.toml | 4 ++++ tests/rare/CycleRollbackClogged.toml | 4 ++++ tests/rare/InventoryTestHeavyWrites.toml | 4 ++++ tests/rare/PerpetualWiggleStorageMigration.toml | 4 ++++ tests/rare/RYWDisable.toml | 4 ++++ tests/rare/RedwoodDeltaTree.toml | 4 ++++ tests/rare/RestoreMultiRanges.toml | 4 ++++ tests/rare/Throttling.toml | 4 ++++ tests/rare/TransactionCost.toml | 4 ++++ 9 files changed, 36 insertions(+) diff --git a/tests/rare/CloggedCycleWithKills.toml b/tests/rare/CloggedCycleWithKills.toml index c31355fa39..c7aafbd46a 100644 --- a/tests/rare/CloggedCycleWithKills.toml +++ b/tests/rare/CloggedCycleWithKills.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [[test]] testTitle = 'CloggedCycleTestWithKills' diff --git a/tests/rare/CycleRollbackClogged.toml b/tests/rare/CycleRollbackClogged.toml index 1644a28a89..5a515a0ab0 100644 --- a/tests/rare/CycleRollbackClogged.toml +++ b/tests/rare/CycleRollbackClogged.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '10' + [[test]] testTitle = 'RollbackCycleTest' diff --git a/tests/rare/InventoryTestHeavyWrites.toml b/tests/rare/InventoryTestHeavyWrites.toml index f6fa5a6cb0..c400824edb 100644 --- a/tests/rare/InventoryTestHeavyWrites.toml +++ b/tests/rare/InventoryTestHeavyWrites.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [[test]] testTitle = 'InventoryTestHeavyWrites' diff --git a/tests/rare/PerpetualWiggleStorageMigration.toml b/tests/rare/PerpetualWiggleStorageMigration.toml index 0d2f7185f9..e1be5d361b 100644 --- a/tests/rare/PerpetualWiggleStorageMigration.toml +++ b/tests/rare/PerpetualWiggleStorageMigration.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [configuration] config = 'triple' storageEngineType = 0 diff --git a/tests/rare/RYWDisable.toml b/tests/rare/RYWDisable.toml index a1c9731455..fe6a905473 100644 --- a/tests/rare/RYWDisable.toml +++ b/tests/rare/RYWDisable.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [[test]] testTitle = 'RYWDisableTest' diff --git a/tests/rare/RedwoodDeltaTree.toml b/tests/rare/RedwoodDeltaTree.toml index ce79fb2289..536528dc48 100644 --- a/tests/rare/RedwoodDeltaTree.toml +++ b/tests/rare/RedwoodDeltaTree.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '50' + [[test]] testTitle = 'RedwoodDeltaTree' useDB = false diff --git a/tests/rare/RestoreMultiRanges.toml b/tests/rare/RestoreMultiRanges.toml index befdfa02ea..cc2e89d52a 100644 --- a/tests/rare/RestoreMultiRanges.toml +++ b/tests/rare/RestoreMultiRanges.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '10' + [configuration] tenantModes = ['disabled'] diff --git a/tests/rare/Throttling.toml b/tests/rare/Throttling.toml index 9fd94f30ea..48f2ff9bf3 100644 --- a/tests/rare/Throttling.toml +++ b/tests/rare/Throttling.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [configuration] storageEngineExcludeTypes = [5] diff --git a/tests/rare/TransactionCost.toml b/tests/rare/TransactionCost.toml index 158e52b435..eda1fefc00 100644 --- a/tests/rare/TransactionCost.toml +++ b/tests/rare/TransactionCost.toml @@ -1,3 +1,7 @@ +# High priority number decreases the chance to run this test. +# Priority 1.0 is the default in TestHarness2 +testPriority = '20' + [configuration] storageEngineExcludeTypes=[5] From 1ac8f39ccd1c34627346ae69ee7ba17499df7ca2 Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Sat, 3 Aug 2024 15:43:56 -0700 Subject: [PATCH 3/4] Set rare test priority to be 10 as default This will decrease the chance of running them. --- contrib/Joshua/scripts/correctnessTest.sh | 1 + contrib/TestHarness2/test_harness/run.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/contrib/Joshua/scripts/correctnessTest.sh b/contrib/Joshua/scripts/correctnessTest.sh index bee09acf25..4759d5e53a 100755 --- a/contrib/Joshua/scripts/correctnessTest.sh +++ b/contrib/Joshua/scripts/correctnessTest.sh @@ -6,4 +6,5 @@ export ASAN_OPTIONS="detect_leaks=0" OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}" #mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false +# export RARE_PRIORITY=20 python3 -m test_harness.app -s ${JOSHUA_SEED} --old-binaries-path ${OLDBINDIR} diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index 17e3ccbd24..0c60934dd7 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -70,6 +70,7 @@ class TestPicker: self.restart_test: Pattern = re.compile(r".*-\d+\.(txt|toml)") self.follow_test: Pattern = re.compile(r".*-[2-9]\d*\.(txt|toml)") self.old_binaries: OrderedDict[Version, Path] = binaries + self.rare_priority: int = int(os.getenv("RARE_PRIORITY", 10)) for subdir in self.test_dir.iterdir(): if subdir.is_dir() and subdir.name in config.test_dirs: @@ -195,6 +196,8 @@ class TestPicker: test_class = test_name if priority is None: priority = 1.0 + if is_rare(path) and priority <= 1.0: + priority = self.rare_priority if ( self.include_tests_regex.search(test_class) is None or self.exclude_tests_regex.search(test_class) is not None @@ -326,6 +329,9 @@ def is_no_sim(test_file: Path): return test_file.parts[-2] == "noSim" +def is_rare(test_file: Path): + return test_file.parts[-2] == "rare" + class ResourceMonitor(threading.Thread): def __init__(self): super().__init__() From 3ff6c01a2cbadf578fbc722db989c234370b1115 Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Sat, 3 Aug 2024 15:57:29 -0700 Subject: [PATCH 4/4] Move BlobGranule and Metacluster tests to rare directory To reduce the number of runs for these experimental features in Joshua. --- tests/CMakeLists.txt | 22 +++++++++---------- .../BlobGranuleApiCorrectness.toml | 0 .../BlobGranuleCorrectness.toml | 0 .../BlobGranuleCorrectnessClean.toml | 0 .../BlobGranuleVerifyBalance.toml | 0 .../BlobGranuleVerifyBalanceClean.toml | 0 .../BlobGranuleVerifyLarge.toml | 0 .../BlobGranuleVerifyLargeClean.toml | 0 .../{slow => rare}/MetaclusterManagement.toml | 0 .../MetaclusterManagementConcurrency.toml | 0 tests/{slow => rare}/MetaclusterRecovery.toml | 0 11 files changed, 11 insertions(+), 11 deletions(-) rename tests/{slow => rare}/BlobGranuleApiCorrectness.toml (100%) rename tests/{slow => rare}/BlobGranuleCorrectness.toml (100%) rename tests/{slow => rare}/BlobGranuleCorrectnessClean.toml (100%) rename tests/{slow => rare}/BlobGranuleVerifyBalance.toml (100%) rename tests/{slow => rare}/BlobGranuleVerifyBalanceClean.toml (100%) rename tests/{slow => rare}/BlobGranuleVerifyLarge.toml (100%) rename tests/{slow => rare}/BlobGranuleVerifyLargeClean.toml (100%) rename tests/{slow => rare}/MetaclusterManagement.toml (100%) rename tests/{slow => rare}/MetaclusterManagementConcurrency.toml (100%) rename tests/{slow => rare}/MetaclusterRecovery.toml (100%) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7f039091da..b58498993a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -258,9 +258,16 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/MockDDReadWrite.toml) endif() + add_fdb_test(TEST_FILES rare/BlobGranuleApiCorrectness.toml) + add_fdb_test(TEST_FILES rare/BlobGranuleCorrectnessClean.toml) + add_fdb_test(TEST_FILES rare/BlobGranuleCorrectness.toml IGNORE) + add_fdb_test(TEST_FILES rare/BlobGranuleMergeBoundaries.toml) add_fdb_test(TEST_FILES rare/BlobGranuleRanges.toml) add_fdb_test(TEST_FILES rare/BlobGranuleRangesChangeLog.toml) - add_fdb_test(TEST_FILES rare/BlobGranuleMergeBoundaries.toml) + add_fdb_test(TEST_FILES rare/BlobGranuleVerifyBalance.toml) + add_fdb_test(TEST_FILES rare/BlobGranuleVerifyBalanceClean.toml) + add_fdb_test(TEST_FILES rare/BlobGranuleVerifyLarge.toml IGNORE) + add_fdb_test(TEST_FILES rare/BlobGranuleVerifyLargeClean.toml IGNORE) add_fdb_test(TEST_FILES rare/CheckRelocation.toml) add_fdb_test(TEST_FILES rare/ClogTlog.toml) add_fdb_test(TEST_FILES rare/ClogUnclog.toml) @@ -279,6 +286,9 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml) add_fdb_test(TEST_FILES rare/LargeApiCorrectness.toml) add_fdb_test(TEST_FILES rare/LargeApiCorrectnessStatus.toml) + add_fdb_test(TEST_FILES rare/MetaclusterManagement.toml) + add_fdb_test(TEST_FILES rare/MetaclusterManagementConcurrency.toml) + add_fdb_test(TEST_FILES rare/MetaclusterRecovery.toml) add_fdb_test(TEST_FILES rare/RYWDisable.toml) add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml) add_fdb_test(TEST_FILES rare/ReadSkewReadWrite.toml) @@ -464,13 +474,6 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES slow/ApiCorrectnessAtomicRestore.toml) add_fdb_test(TEST_FILES slow/ApiCorrectnessSwitchover.toml) add_fdb_test(TEST_FILES slow/ApiCorrectnessWithConsistencyCheck.toml) - add_fdb_test(TEST_FILES slow/BlobGranuleVerifyLarge.toml IGNORE) - add_fdb_test(TEST_FILES slow/BlobGranuleVerifyLargeClean.toml IGNORE) - add_fdb_test(TEST_FILES slow/BlobGranuleVerifyBalance.toml) - add_fdb_test(TEST_FILES slow/BlobGranuleVerifyBalanceClean.toml) - add_fdb_test(TEST_FILES slow/BlobGranuleCorrectnessClean.toml) - add_fdb_test(TEST_FILES slow/BlobGranuleCorrectness.toml IGNORE) - add_fdb_test(TEST_FILES slow/BlobGranuleApiCorrectness.toml) add_fdb_test(TEST_FILES slow/ClogWithRollbacks.toml) add_fdb_test(TEST_FILES slow/CloggedCycleTest.toml) add_fdb_test(TEST_FILES slow/CloggedStorefront.toml) @@ -487,9 +490,6 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES slow/GcGenerations.toml) add_fdb_test(TEST_FILES slow/LongRunning.toml LONG_RUNNING) add_fdb_test(TEST_FILES slow/LowLatencyWithFailures.toml) - add_fdb_test(TEST_FILES slow/MetaclusterManagement.toml) - add_fdb_test(TEST_FILES slow/MetaclusterManagementConcurrency.toml) - add_fdb_test(TEST_FILES slow/MetaclusterRecovery.toml) add_fdb_test(TEST_FILES slow/MoveKeysClean.toml) add_fdb_test(TEST_FILES slow/MoveKeysSideband.toml) add_fdb_test(TEST_FILES slow/RyowCorrectness.toml) diff --git a/tests/slow/BlobGranuleApiCorrectness.toml b/tests/rare/BlobGranuleApiCorrectness.toml similarity index 100% rename from tests/slow/BlobGranuleApiCorrectness.toml rename to tests/rare/BlobGranuleApiCorrectness.toml diff --git a/tests/slow/BlobGranuleCorrectness.toml b/tests/rare/BlobGranuleCorrectness.toml similarity index 100% rename from tests/slow/BlobGranuleCorrectness.toml rename to tests/rare/BlobGranuleCorrectness.toml diff --git a/tests/slow/BlobGranuleCorrectnessClean.toml b/tests/rare/BlobGranuleCorrectnessClean.toml similarity index 100% rename from tests/slow/BlobGranuleCorrectnessClean.toml rename to tests/rare/BlobGranuleCorrectnessClean.toml diff --git a/tests/slow/BlobGranuleVerifyBalance.toml b/tests/rare/BlobGranuleVerifyBalance.toml similarity index 100% rename from tests/slow/BlobGranuleVerifyBalance.toml rename to tests/rare/BlobGranuleVerifyBalance.toml diff --git a/tests/slow/BlobGranuleVerifyBalanceClean.toml b/tests/rare/BlobGranuleVerifyBalanceClean.toml similarity index 100% rename from tests/slow/BlobGranuleVerifyBalanceClean.toml rename to tests/rare/BlobGranuleVerifyBalanceClean.toml diff --git a/tests/slow/BlobGranuleVerifyLarge.toml b/tests/rare/BlobGranuleVerifyLarge.toml similarity index 100% rename from tests/slow/BlobGranuleVerifyLarge.toml rename to tests/rare/BlobGranuleVerifyLarge.toml diff --git a/tests/slow/BlobGranuleVerifyLargeClean.toml b/tests/rare/BlobGranuleVerifyLargeClean.toml similarity index 100% rename from tests/slow/BlobGranuleVerifyLargeClean.toml rename to tests/rare/BlobGranuleVerifyLargeClean.toml diff --git a/tests/slow/MetaclusterManagement.toml b/tests/rare/MetaclusterManagement.toml similarity index 100% rename from tests/slow/MetaclusterManagement.toml rename to tests/rare/MetaclusterManagement.toml diff --git a/tests/slow/MetaclusterManagementConcurrency.toml b/tests/rare/MetaclusterManagementConcurrency.toml similarity index 100% rename from tests/slow/MetaclusterManagementConcurrency.toml rename to tests/rare/MetaclusterManagementConcurrency.toml diff --git a/tests/slow/MetaclusterRecovery.toml b/tests/rare/MetaclusterRecovery.toml similarity index 100% rename from tests/slow/MetaclusterRecovery.toml rename to tests/rare/MetaclusterRecovery.toml