wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add option to set perpetual_storage_wiggle_engine to none

This commit is contained in:
Zhe Wu 2023-08-24 13:43:58 -07:00
parent 7e8f326277
commit 6610a228c7
7 changed files with 35 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fdbclient/DatabaseConfiguration.cpp
View File

@ -37,8 +37,8 @@ void DatabaseConfiguration::resetInternal() {
commitProxyCount = grvProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor =
storageTeamSize = desiredLogRouterCount = -1;
tLogVersion = TLogVersion::DEFAULT;
tLogDataStoreType = storageServerStoreType = testingStorageServerStoreType = perpetualStoreType =
KeyValueStoreType::END;
tLogDataStoreType = storageServerStoreType = testingStorageServerStoreType = KeyValueStoreType::END;
perpetualStoreType = KeyValueStoreType::NONE;
desiredTSSCount = 0;
tLogSpillType = TLogSpillType::DEFAULT;
autoCommitProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_COMMIT_PROXIES;

7
fdbclient/FDBTypes.cpp
View File

@ -227,6 +227,8 @@ std::string KeyValueStoreType::getStoreTypeStr(const StoreType& storeType) {
return "memory";
case MEMORY_RADIXTREE:
return "memory-radixtree-beta";
case NONE:
return "none";
default:
return "unknown";
}
@ -242,10 +244,11 @@ KeyValueStoreType KeyValueStoreType::fromString(const std::string& str) {
{ "ssd-rocksdb-v1", SSD_ROCKSDB_V1 },
{ "ssd-sharded-rocksdb", SSD_SHARDED_ROCKSDB },
{ "memory", MEMORY },
{ "memory-radixtree-beta", MEMORY_RADIXTREE } };
{ "memory-radixtree-beta", MEMORY_RADIXTREE },
{ "none", NONE } };
auto it = names.find(str);
if (it == names.end()) {
throw unknown_storage_engine();
}
return it->second;
}
}

4
fdbclient/include/fdbclient/FDBTypes.h
View File

@ -976,6 +976,7 @@ struct KeyValueStoreType {
MEMORY_RADIXTREE = 4,
SSD_ROCKSDB_V1 = 5,
SSD_SHARDED_ROCKSDB = 6,
NONE = 7,
END
};
@ -1001,6 +1002,9 @@ struct KeyValueStoreType {
static KeyValueStoreType fromString(const std::string& str);
std::string toString() const { return getStoreTypeStr((StoreType)type); }
// Whether the storage type is a valid storage type.
bool isValid() const { return type != NONE && type != END; }
private:
uint32_t type;
};

2
fdbserver/DDTeamCollection.actor.cpp
View File

@ -2407,7 +2407,7 @@ public:
// perpetualStoreType for all new SSes that match perpetualStorageWiggleLocality.
// Note that this only applies to regular storage servers, not TSS.
if (!recruitTss && self->configuration.storageMigrationType == StorageMigrationType::GRADUAL &&
self->configuration.perpetualStoreType.storeType() != KeyValueStoreType::END) {
self->configuration.perpetualStoreType.isValid()) {
if (self->configuration.perpetualStorageWiggleLocality == "0") {
isr.storeType = self->configuration.perpetualStoreType;
} else {

2
fdbserver/workloads/ConsistencyCheck.actor.cpp
View File

@ -949,7 +949,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
if (!keyValueStoreType.present()) {
TraceEvent("ConsistencyCheck_ServerUnavailable").detail("ServerID", storageServers[i].id());
self->testFailure("Storage server unavailable");
} else if (configuration.perpetualStoreType.storeType() != KeyValueStoreType::END) {
} else if (configuration.perpetualStoreType.isValid()) {
// Perpetual storage wiggle is used to migrate storage. Check that the matched storage servers are
// correctly migrated.
if (wiggleLocalityKeyValue == "0" ||

23
fdbserver/workloads/PerpetualWiggleStorageMigrationWorkload.actor.cpp
View File

@ -105,6 +105,23 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
TraceEvent("Test_ConfigChangeDone").detail("Success", change);
ASSERT(change);
wait(excludeIncludeServer(cx, ssToExcludeInclude));
wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, "ssd-rocksdb-v1"));
if (deterministicRandom()->coinflip()) {
TraceEvent("Test_ClearPerpetualStorageWiggleEngine").log();
bool change = wait(IssueConfigurationChange(cx, "perpetual_storage_wiggle_engine=none", true));
TraceEvent("Test_ClearPerpetualStorageWiggleEngineDone").detail("Success", change);
ASSERT(change);
wait(excludeIncludeServer(cx, ssToWiggle));
wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle, "ssd-2"));
}
return Void();
}
ACTOR static Future<Void> excludeIncludeServer(Database cx, StorageServerInterface ssToExcludeInclude) {
// Now, let's exclude `ssToExcludeInclude` process and include it again. The new SS created on this process
// should always uses `storage_engine` config, which is `ssd-2`.
state std::vector<AddressExclusion> servers;
@ -132,13 +149,13 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
wait(includeServers(cx, std::vector<AddressExclusion>(1)));
TraceEvent("Test_IncludeServerDone").log();
wait(validateDatabase(cx, ssToExcludeInclude, ssToWiggle));
return Void();
}
ACTOR static Future<Void> validateDatabase(Database cx,
StorageServerInterface ssToExcludeInclude,
StorageServerInterface ssToWiggle) {
StorageServerInterface ssToWiggle,
std::string wiggleStorageType) {
// Wait until `ssToExcludeInclude` to be recruited as storage server again.
state int missingTargetCount = 0;
loop {
@ -191,7 +208,7 @@ struct PerpetualWiggleStorageMigrationWorkload : public TestWorkload {
if (ssInterface.address() == ssToWiggle.address()) {
// If `ssToWiggle` exists, we wait until it is migrate to `perpetual_storage_wiggle_engine`.
containWiggleStorage = true;
if (keyValueStoreType.get().toString() == "ssd-rocksdb-v1") {
if (keyValueStoreType.get().toString() == wiggleStorageType) {
doneCheckingWiggleStorage = true;
}
}

3
tests/CMakeLists.txt
View File

@ -242,6 +242,7 @@ if(WITH_PYTHON)
# Mock DD Tests
add_fdb_test(TEST_FILES fast/IDDTxnProcessorMoveKeys.toml IGNORE)
add_fdb_test(TEST_FILES fast/MockDDReadWrite.toml IGNORE)
add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml)
else()
add_fdb_test(TEST_FILES fast/ValidateStorage.toml IGNORE)
add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml IGNORE)
@ -249,6 +250,7 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES noSim/PerfShardedRocksDBTest.toml IGNORE)
add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml IGNORE)
add_fdb_test(TEST_FILES fast/StorageServerCheckpointRestore.toml IGNORE)
add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml IGNORE)
# Mock DD Tests
add_fdb_test(TEST_FILES fast/IDDTxnProcessorMoveKeys.toml)
@ -275,7 +277,6 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES rare/InventoryTestHeavyWrites.toml)
add_fdb_test(TEST_FILES rare/LargeApiCorrectness.toml)
add_fdb_test(TEST_FILES rare/LargeApiCorrectnessStatus.toml)
add_fdb_test(TEST_FILES rare/PerpetualWiggleStorageMigration.toml)
add_fdb_test(TEST_FILES rare/RYWDisable.toml)
add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml)
add_fdb_test(TEST_FILES rare/ReadSkewReadWrite.toml)