wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added configurable memory limits for backup and dr executables 

added a default memory limit of 8GB for fdbcli
This commit is contained in:
Evan Tschannen 2017-09-29 10:35:40 -07:00
parent 333fb65a91
commit e2b65e86ed
5 changed files with 90 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
fdbbackup/backup.actor.cpp
View File

@ -99,7 +99,7 @@ enum {
OPT_CLUSTERFILE, OPT_QUIET, OPT_DRYRUN, OPT_FORCE,
OPT_HELP, OPT_DEVHELP, OPT_VERSION, OPT_PARENTPID, OPT_CRASHONERROR,
OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACE, OPT_TRACE_DIR,
OPT_KNOB, OPT_TRACE_LOG_GROUP,
OPT_KNOB, OPT_TRACE_LOG_GROUP, OPT_MEMLIMIT,
//DB constants
OPT_SOURCE_CLUSTER,
@ -122,6 +122,8 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -157,6 +159,8 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -183,6 +187,8 @@ CSimpleOpt::SOption g_rgBackupStatusOptions[] = {
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -206,6 +212,8 @@ CSimpleOpt::SOption g_rgBackupAbortOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -231,6 +239,8 @@ CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -256,6 +266,8 @@ CSimpleOpt::SOption g_rgBackupWaitOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -291,6 +303,8 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
{ OPT_FORCE, "-f", SO_NONE },
{ OPT_FORCE, "--force", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -316,6 +330,8 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -343,6 +359,8 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -370,6 +388,8 @@ CSimpleOpt::SOption g_rgDBStatusOptions[] = {
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -395,6 +415,8 @@ CSimpleOpt::SOption g_rgDBSwitchOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -421,6 +443,8 @@ CSimpleOpt::SOption g_rgDBAbortOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -434,6 +458,8 @@ CSimpleOpt::SOption g_rgBlobOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -1995,6 +2021,8 @@ int main(int argc, char* argv[]) {
std::string traceLogGroup;
ESOError lastError;
bool partial = true;
uint64_t memLimit = 8LL << 30;
Optional<uint64_t> ti;
std::vector<std::string> blobArgs;
@ -2182,6 +2210,15 @@ int main(int argc, char* argv[]) {
case OPT_CRASHONERROR:
g_crashOnError = true;
break;
case OPT_MEMLIMIT:
ti = parse_with_suffix(args->OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
memLimit = ti.get();
break;
}
}
@ -2303,6 +2340,7 @@ int main(int argc, char* argv[]) {
setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
Error::init();
std::set_new_handler( &platform::outOfMemory );
setMemoryQuota( memLimit );
int total = 0;
for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i)
@ -2363,6 +2401,7 @@ int main(int argc, char* argv[]) {
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
.detail("CommandLine", commandLine)
.detail("MemoryLimit", memLimit)
.trackLatest("ProgramStart");
db = cluster->createDatabase(databaseKey).get();

3
fdbcli/fdbcli.actor.cpp
View File

@ -2844,6 +2844,9 @@ int main(int argc, char **argv) {
platformInit();
initSignalSafeUnwind();
Error::init();
std::set_new_handler( &platform::outOfMemory );
uint64_t memLimit = 8LL << 30;
setMemoryQuota( memLimit );
registerCrashHandler();

46
fdbserver/fdbserver.actor.cpp
View File

@ -705,52 +705,6 @@ static void printUsage( const char *name, bool devhelp ) {
"KiB=2^10, MB=10^6, MiB=2^20, GB=10^9, GiB=2^30, TB=10^12, or TiB=2^40.\n");
}
Optional<uint64_t> parse_with_suffix(std::string toparse, std::string default_unit = "") {
char *endptr;
uint64_t ret = strtoull(toparse.c_str(), &endptr, 10);
if (endptr == toparse.c_str()) {
return Optional<uint64_t>();
}
std::string unit;
if (*endptr == '\0') {
if (!default_unit.empty()) {
unit = default_unit;
} else {
return Optional<uint64_t>();
}
} else {
unit = endptr;
}
if (!unit.compare("B")) {
// Nothing to do
} else if (!unit.compare("KB")) {
ret *= int64_t(1e3);
} else if (!unit.compare("KiB")) {
ret *= 1LL << 10;
} else if (!unit.compare("MB")) {
ret *= int64_t(1e6);
} else if (!unit.compare("MiB")) {
ret *= 1LL << 20;
} else if (!unit.compare("GB")) {
ret *= int64_t(1e9);
} else if (!unit.compare("GiB")) {
ret *= 1LL << 30;
} else if (!unit.compare("TB")) {
ret *= int64_t(1e12);
} else if (!unit.compare("TiB")) {
ret *= 1LL << 40;
} else {
return Optional<uint64_t>();
}
return ret;
}
extern bool g_crashOnError;
#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)

46
flow/flow.cpp
View File

@ -47,6 +47,52 @@ std::string UID::shortString() const {
void detectFailureAfter( int const& address, double const& delay );
Optional<uint64_t> parse_with_suffix(std::string toparse, std::string default_unit) {
char *endptr;
uint64_t ret = strtoull(toparse.c_str(), &endptr, 10);
if (endptr == toparse.c_str()) {
return Optional<uint64_t>();
}
std::string unit;
if (*endptr == '\0') {
if (!default_unit.empty()) {
unit = default_unit;
} else {
return Optional<uint64_t>();
}
} else {
unit = endptr;
}
if (!unit.compare("B")) {
// Nothing to do
} else if (!unit.compare("KB")) {
ret *= int64_t(1e3);
} else if (!unit.compare("KiB")) {
ret *= 1LL << 10;
} else if (!unit.compare("MB")) {
ret *= int64_t(1e6);
} else if (!unit.compare("MiB")) {
ret *= 1LL << 20;
} else if (!unit.compare("GB")) {
ret *= int64_t(1e9);
} else if (!unit.compare("GiB")) {
ret *= 1LL << 30;
} else if (!unit.compare("TB")) {
ret *= int64_t(1e12);
} else if (!unit.compare("TiB")) {
ret *= 1LL << 40;
} else {
return Optional<uint64_t>();
}
return ret;
}
std::string format( const char* form, ... ) {
char buf[200];
va_list args;

1
flow/flow.h
View File

@ -66,6 +66,7 @@ bool validationIsEnabled();
#define BUGGIFY BUGGIFY_WITH_PROB(P_BUGGIFIED_SECTION_FIRES)
#define EXPENSIVE_VALIDATION (validationIsEnabled() && g_random->random01() < P_EXPENSIVE_VALIDATION)
extern Optional<uint64_t> parse_with_suffix(std::string toparse, std::string default_unit = "");
extern std::string format(const char* form, ...);
extern Standalone<StringRef> strinc(StringRef const& str);
extern StringRef strinc(StringRef const& str, Arena& arena);