Add a way to configure log spill type.

This commit is contained in:
Alex Miller 2019-02-07 17:02:46 -08:00
parent df61bd07db
commit 0cf3ee9f99
4 changed files with 41 additions and 0 deletions

View File

@ -31,6 +31,7 @@ void DatabaseConfiguration::resetInternal() {
initialized = false; initialized = false;
masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1; masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1;
tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END; tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
tLogSpillType = TLogSpillType::DEFAULT;
autoMasterProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES; autoMasterProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES;
autoResolverCount = CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS; autoResolverCount = CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS;
autoDesiredTLogCount = CLIENT_KNOBS->DEFAULT_AUTO_LOGS; autoDesiredTLogCount = CLIENT_KNOBS->DEFAULT_AUTO_LOGS;
@ -166,6 +167,7 @@ bool DatabaseConfiguration::isValid() const {
getDesiredLogs() >= 1 && getDesiredLogs() >= 1 &&
getDesiredResolvers() >= 1 && getDesiredResolvers() >= 1 &&
tLogDataStoreType != KeyValueStoreType::END && tLogDataStoreType != KeyValueStoreType::END &&
tLogSpillType != TLogSpillType::UNSET &&
storageServerStoreType != KeyValueStoreType::END && storageServerStoreType != KeyValueStoreType::END &&
autoMasterProxyCount >= 1 && autoMasterProxyCount >= 1 &&
autoResolverCount >= 1 && autoResolverCount >= 1 &&
@ -255,6 +257,10 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
result["storage_engine"] = "custom"; result["storage_engine"] = "custom";
} }
if ( tLogSpillType != TLogSpillType::DEFAULT ) {
result["log_spill"] = (int)tLogSpillType;
}
if( remoteTLogReplicationFactor == 1 ) { if( remoteTLogReplicationFactor == 1 ) {
result["remote_redundancy_mode"] = "remote_single"; result["remote_redundancy_mode"] = "remote_single";
} else if( remoteTLogReplicationFactor == 2 ) { } else if( remoteTLogReplicationFactor == 2 ) {
@ -371,6 +377,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1)
tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2; tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2;
} }
else if (ck == LiteralStringRef("log_spill")) { parse((&type), value); tLogSpillType = (TLogSpillType::SpillType)type; }
else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; } else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; }
else if (ck == LiteralStringRef("auto_proxies")) parse(&autoMasterProxyCount, value); else if (ck == LiteralStringRef("auto_proxies")) parse(&autoMasterProxyCount, value);
else if (ck == LiteralStringRef("auto_resolvers")) parse(&autoResolverCount, value); else if (ck == LiteralStringRef("auto_resolvers")) parse(&autoResolverCount, value);

View File

@ -162,6 +162,7 @@ struct DatabaseConfiguration {
int32_t tLogWriteAntiQuorum; int32_t tLogWriteAntiQuorum;
int32_t tLogReplicationFactor; int32_t tLogReplicationFactor;
KeyValueStoreType tLogDataStoreType; KeyValueStoreType tLogDataStoreType;
TLogSpillType tLogSpillType;
// Storage Servers // Storage Servers
IRepPolicyRef storagePolicy; IRepPolicyRef storagePolicy;

View File

@ -513,6 +513,37 @@ private:
uint32_t type; uint32_t type;
}; };
struct TLogSpillType {
// These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones just before END.
enum SpillType {
DEFAULT = 2,
VALUE = 1,
REFERENCE = 2,
UNSET,
};
TLogSpillType() : type(DEFAULT) {}
TLogSpillType( SpillType type ) : type(type) {
if ((uint32_t)type > UNSET)
this->type = UNSET;
}
operator SpillType() const { return SpillType(type); }
template <class Ar>
void serialize(Ar& ar) { serializer(ar, type); }
std::string toString() const {
switch( type ) {
case VALUE: return ""; // For backwards compatiblity.
case REFERENCE: return "reference";
default: return "unknown";
}
}
private:
uint32_t type;
};
//Contains the amount of free and total space for a storage server, in bytes //Contains the amount of free and total space for a storage server, in bytes
struct StorageBytes { struct StorageBytes {
int64_t free; int64_t free;

View File

@ -1953,6 +1953,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
InitializeTLogRequest &req = reqs[i]; InitializeTLogRequest &req = reqs[i];
req.recruitmentID = logSystem->recruitmentID; req.recruitmentID = logSystem->recruitmentID;
req.storeType = configuration.tLogDataStoreType; req.storeType = configuration.tLogDataStoreType;
req.spillType = configuration.tLogSpillType;
req.recoverFrom = oldLogSystem->getLogSystemConfig(); req.recoverFrom = oldLogSystem->getLogSystemConfig();
req.recoverAt = oldLogSystem->recoverAt.get(); req.recoverAt = oldLogSystem->recoverAt.get();
req.knownCommittedVersion = oldLogSystem->knownCommittedVersion; req.knownCommittedVersion = oldLogSystem->knownCommittedVersion;
@ -1996,6 +1997,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
InitializeTLogRequest &req = sreqs[i]; InitializeTLogRequest &req = sreqs[i];
req.recruitmentID = logSystem->recruitmentID; req.recruitmentID = logSystem->recruitmentID;
req.storeType = configuration.tLogDataStoreType; req.storeType = configuration.tLogDataStoreType;
req.spillType = configuration.tLogSpillType;
req.recoverFrom = oldLogSystem->getLogSystemConfig(); req.recoverFrom = oldLogSystem->getLogSystemConfig();
req.recoverAt = oldLogSystem->recoverAt.get(); req.recoverAt = oldLogSystem->recoverAt.get();
req.knownCommittedVersion = oldLogSystem->knownCommittedVersion; req.knownCommittedVersion = oldLogSystem->knownCommittedVersion;