wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1314 from etschannen/feature-ssd-memory-spill 

configure memory now selects the ssd engine for transaction log spilling
This commit is contained in:
A.J. Beamon 2019-03-19 08:22:22 -07:00 committed by GitHub
parent 10a30fed42 eb54a700ba
commit 7f4adcc338
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 49 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
fdbcli/fdbcli.actor.cpp
View File

@ -900,6 +900,7 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
// If there is a configuration message then there is no configuration information to display
outputString += "\nConfiguration:";
std::string outputStringCache = outputString;
bool isOldMemory = false;
try {
// Configuration section
// FIXME: Should we suppress this if there are cluster messages implying that the database has no configuration?
@ -914,6 +915,9 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
outputString += "\n Storage engine - ";
if (statusObjConfig.get("storage_engine", strVal)){
if(strVal == "memory-1") {
isOldMemory = true;
}
outputString += strVal;
} else
outputString += "unknown";
@ -1173,6 +1177,7 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
// Workload section
outputString += "\n\nWorkload:";
outputStringCache = outputString;
bool foundLogAndStorage = false;
try {
// Determine which rates are unknown
StatusObjectReader statusObjWorkload;
@ -1225,6 +1230,22 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
std::vector<std::string> messagesAddrs;
for (auto proc : processesMap.obj()){
StatusObjectReader process(proc.second);
if(process.has("roles")) {
StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
bool storageRole = false;
bool logRole = false;
for (StatusObjectReader role : rolesArray) {
if (role["role"].get_str() == "storage") {
storageRole = true;
}
else if (role["role"].get_str() == "log") {
logRole = true;
}
}
if(storageRole && logRole) {
foundLogAndStorage = true;
}
}
if (process.has("messages")) {
StatusArray processMessagesArr = process.last().get_array();
if (processMessagesArr.size()){
@ -1408,6 +1429,14 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
if (clientTime != ""){
outputString += "\n\nClient time: " + clientTime;
}
if(processesMap.obj().size() > 1 && isOldMemory) {
outputString += "\n\nWARNING: type `configure memory' to switch to a safer method of persisting data on the transaction logs.";
}
if(processesMap.obj().size() > 9 && foundLogAndStorage) {
outputString += "\n\nWARNING: A single process is both a transaction log and a storage server.\n For best performance use dedicated disks for the transaction logs by setting process classes.";
}
printf("%s\n", outputString.c_str());
}
@ -2185,7 +2214,7 @@ void onoff_generator(const char* text, const char *line, std::vector<std::string
}
void configure_generator(const char* text, const char *line, std::vector<std::string>& lc) {
const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "proxies=", "logs=", "resolvers=", NULL};
const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "proxies=", "logs=", "resolvers=", NULL};
array_generator(text, line, opts, lc);
}

4
fdbclient/DatabaseConfiguration.cpp
View File

@ -262,7 +262,9 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
} else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::SSD_REDWOOD_V1 ) {
result["storage_engine"] = "ssd-redwood-experimental";
} else if( tLogDataStoreType == KeyValueStoreType::MEMORY && storageServerStoreType == KeyValueStoreType::MEMORY ) {
result["storage_engine"] = "memory";
result["storage_engine"] = "memory-1";
} else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::MEMORY ) {
result["storage_engine"] = "memory-2";
} else {
result["storage_engine"] = "custom";
}

15
fdbclient/ManagementAPI.actor.cpp
View File

@ -81,20 +81,29 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
return out;
}
Optional<KeyValueStoreType> logType;
Optional<KeyValueStoreType> storeType;
if (mode == "ssd-1") {
storeType= KeyValueStoreType::SSD_BTREE_V1;
logType = KeyValueStoreType::SSD_BTREE_V1;
storeType = KeyValueStoreType::SSD_BTREE_V1;
} else if (mode == "ssd" || mode == "ssd-2") {
logType = KeyValueStoreType::SSD_BTREE_V2;
storeType = KeyValueStoreType::SSD_BTREE_V2;
} else if (mode == "ssd-redwood-experimental") {
logType = KeyValueStoreType::SSD_BTREE_V2;
storeType = KeyValueStoreType::SSD_REDWOOD_V1;
} else if (mode == "memory") {
} else if (mode == "memory" || mode == "memory-2") {
logType = KeyValueStoreType::SSD_BTREE_V2;
storeType= KeyValueStoreType::MEMORY;
} else if (mode == "memory-1") {
logType = KeyValueStoreType::MEMORY;
storeType= KeyValueStoreType::MEMORY;
}
// Add any new store types to fdbserver/workloads/ConfigureDatabase, too
if (storeType.present()) {
out[p+"log_engine"] = out[p+"storage_engine"] = format("%d", storeType.get());
out[p+"log_engine"] = format("%d", logType.get());
out[p+"storage_engine"] = format("%d", storeType.get());
return out;
}

4
fdbclient/Schemas.cpp
View File

@ -501,7 +501,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
"ssd-1",
"ssd-2",
"ssd-redwood-experimental",
"memory"
"memory",
"memory-1",
"memory-2"
]},
"coordinators_count":1,
"excluded_servers":[

7
fdbserver/KeyValueStoreMemory.actor.cpp
View File

@ -139,13 +139,6 @@ public:
virtual Future<Void> commit(bool sequential) {
if(getAvailableSize() <= 0) {
if(g_network->isSimulated()) { //FIXME: known bug in simulation we are supressing
int unseed = noUnseed ? 0 : g_random->randomInt(0, 100001);
TraceEvent(SevWarnAlways, "KeyValueStoreMemory_OutOfSpace", id);
TraceEvent("ElapsedTime").detail("SimTime", now()).detail("RealTime", 0)
.detail("RandomUnseed", unseed);
flushAndExit(0);
}
TraceEvent(SevError, "KeyValueStoreMemory_OutOfSpace", id);
return Never();
}

2
fdbserver/workloads/ConfigureDatabase.actor.cpp
View File

@ -26,7 +26,7 @@
#include "flow/actorcompiler.h" // This must be the last #include.
// "ssd" is an alias to the preferred type which skews the random distribution toward it but that's okay.
static const char* storeTypes[] = { "ssd", "ssd-1", "ssd-2", "memory" };
static const char* storeTypes[] = { "ssd", "ssd-1", "ssd-2", "memory", "memory-1", "memory-2" };
static const char* logTypes[] = { "log_engine:=1", "log_engine:=2", "log_spill:=1", "log_spill:=2", "log_version:=2", "log_version:=3" };
static const char* redundancies[] = { "single", "double", "triple" };