wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13770 from tclinkenbeard-oai/dev/tclinkenbeard/foundationdb-complexity-simplifications-20260722 

Reduce duplicated logic
This commit is contained in:
Trevor Clinkenbeard 2026-07-31 14:22:25 -07:00 committed by GitHub
parent dadc0fe643 20241eefbd
commit c1d388a06f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 777 additions and 1394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/c/test/fdb_c_client_config_tests.py
View File

@ -432,7 +432,8 @@ class ClientConfigTests(unittest.TestCase):
)
test.check_current_client(CURRENT_VERSION)
def test_no_external_client_support_api_version_ignore(self):
@unittest.skip("API functions are present in the two supported previous releases")
def test_no_external_client_support_api_version(self):
# Multiple external clients, API version supported by none of them
# Note: Ignored because API function won't be missing in last 2 releases.
test = ClientConfigTest(self)
@ -452,7 +453,8 @@ class ClientConfigTests(unittest.TestCase):
test.expected_error = 2124 # All external clients failed
test.exec()
def test_one_external_client_wrong_api_version_ignore(self):
@unittest.skip("API functions are present in the two supported previous releases")
def test_one_external_client_wrong_api_version(self):
# Multiple external clients, API version unsupported by one of them.
# Note: Ignored because API function won't be missing in last 2 releases.
test = ClientConfigTest(self)

492
fdbbackup/backup.cpp
View File

@ -199,6 +199,37 @@ enum {
OPT_TRACE_FORMAT,
};
#define BACKUP_LOG_OPTIONS \
{ OPT_TRACE, "--log", SO_NONE }, { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, \
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, { \
OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP \
}
#define BACKUP_QUIET_OPTIONS \
{ OPT_QUIET, "-q", SO_NONE }, { \
OPT_QUIET, "--quiet", SO_NONE \
}
#define BACKUP_MEMORY_OPTIONS \
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP }, { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, { \
OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP \
}
#define BACKUP_HELP_OPTIONS \
{ OPT_HELP, "-?", SO_NONE }, { OPT_HELP, "-h", SO_NONE }, { OPT_HELP, "--help", SO_NONE }, { \
OPT_DEVHELP, "--dev-help", SO_NONE \
}
#define BACKUP_CLUSTER_FILE_OPTIONS \
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, { \
OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP \
}
#define BACKUP_TAG_OPTIONS \
{ OPT_TAGNAME, "-t", SO_REQ_SEP }, { \
OPT_TAGNAME, "--tagname", SO_REQ_SEP \
}
// Top level binary commands.
CSimpleOpt::SOption g_rgOptions[] = { { OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
@ -213,27 +244,17 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build-flags", SO_NONE },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
BACKUP_QUIET_OPTIONS,
BACKUP_LOG_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_LOCALITY, "--locality-", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
TLS_OPTION_FLAGS,
@ -244,8 +265,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_WAITFORDONE, "-w", SO_NONE },
{ OPT_WAITFORDONE, "--waitfordone", SO_NONE },
{ OPT_NOSTOPWHENDONE, "-z", SO_NONE },
@ -257,27 +277,17 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
{ OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP },
{ OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP },
{ OPT_INITIAL_SNAPSHOT_INTERVAL, "--initial-snapshot-interval", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
BACKUP_TAG_OPTIONS,
{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
{ OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP },
{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
{ OPT_DRYRUN, "-n", SO_NONE },
{ OPT_DRYRUN, "--dryrun", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_INCREMENTALONLY, "--incremental", SO_NONE },
@ -292,26 +302,15 @@ CSimpleOpt::SOption g_rgBackupModifyOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_TAG_OPTIONS,
{ OPT_MOD_VERIFY_UID, "--verify-uid", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
@ -328,26 +327,15 @@ CSimpleOpt::SOption g_rgBackupStatusOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_ERRORLIMIT, "-e", SO_REQ_SEP },
{ OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_TAG_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_JSON, "--json", SO_NONE },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
@ -358,24 +346,13 @@ CSimpleOpt::SOption g_rgBackupAbortOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_TAG_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -385,22 +362,12 @@ CSimpleOpt::SOption g_rgBackupCleanupOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_DELETE_DATA, "--delete-data", SO_NONE },
{ OPT_MIN_CLEANUP_SECONDS, "--min-cleanup-seconds", SO_REQ_SEP },
@ -412,26 +379,15 @@ CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_TAG_OPTIONS,
{ OPT_WAITFORDONE, "-w", SO_NONE },
{ OPT_WAITFORDONE, "--waitfordone", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -441,26 +397,15 @@ CSimpleOpt::SOption g_rgBackupWaitOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_TAG_OPTIONS,
{ OPT_NOSTOPWHENDONE, "-z", SO_NONE },
{ OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -470,22 +415,12 @@ CSimpleOpt::SOption g_rgBackupPauseOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -495,25 +430,15 @@ CSimpleOpt::SOption g_rgBackupExpireOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_FORCE, "-f", SO_NONE },
@ -535,20 +460,11 @@ CSimpleOpt::SOption g_rgBackupDeleteOptions[] = {
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
@ -559,25 +475,15 @@ CSimpleOpt::SOption g_rgBackupDescribeOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_DESCRIBE_DEEP, "--deep", SO_NONE },
@ -591,24 +497,17 @@ CSimpleOpt::SOption g_rgBackupDumpOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP },
@ -621,14 +520,9 @@ CSimpleOpt::SOption g_rgBackupTagsOptions[] = {
#ifdef _WIN32
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_CLUSTER_FILE_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
};
@ -640,20 +534,11 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
{ OPT_BASEURL, "-b", SO_REQ_SEP },
{ OPT_BASEURL, "--base-url", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
@ -665,8 +550,7 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#endif
{ OPT_RESTORE_TIMESTAMP, "--query-restore-timestamp", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
{ OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP },
BACKUP_CLUSTER_FILE_OPTIONS,
{ OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
{ OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
@ -675,22 +559,13 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
{ OPT_RESTORE_SNAPSHOT_VERSION, "--query-restore-snapshot-version", SO_REQ_SEP },
{ OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP },
{ OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
@ -712,8 +587,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
{ OPT_PROXY, "--proxy", SO_REQ_SEP },
{ OPT_PREFIX_ADD, "--add-prefix", SO_REQ_SEP },
{ OPT_PREFIX_REMOVE, "--remove-prefix", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
BACKUP_TAG_OPTIONS,
{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
{ OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP },
{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
@ -724,23 +598,14 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
{ OPT_MODE, "--mode", SO_REQ_SEP },
{ OPT_RESTORE_VERSION, "--version", SO_REQ_SEP },
{ OPT_RESTORE_VERSION, "-v", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_DRYRUN, "-n", SO_NONE },
{ OPT_DRYRUN, "--dryrun", SO_NONE },
{ OPT_FORCE, "-f", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP },
{ OPT_INCREMENTALONLY, "--incremental", SO_NONE },
{ OPT_RESTORE_BEGIN_VERSION, "--begin-version", SO_REQ_SEP },
@ -762,21 +627,12 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_BUILD_FLAGS, "--build-flags", SO_NONE },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
BACKUP_QUIET_OPTIONS,
BACKUP_LOG_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_LOCALITY, "--locality-", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
};
@ -789,25 +645,15 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = {
{ OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
{ OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
{ OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
BACKUP_TAG_OPTIONS,
{ OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
{ OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP },
{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -823,22 +669,12 @@ CSimpleOpt::SOption g_rgDBStatusOptions[] = {
{ OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
{ OPT_ERRORLIMIT, "-e", SO_REQ_SEP },
{ OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_TAG_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -852,23 +688,13 @@ CSimpleOpt::SOption g_rgDBSwitchOptions[] = {
{ OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
{ OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
{ OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_TAG_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_FORCE, "-f", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -884,22 +710,12 @@ CSimpleOpt::SOption g_rgDBAbortOptions[] = {
{ OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
{ OPT_CLEANUP, "--cleanup", SO_NONE },
{ OPT_DSTONLY, "--dstonly", SO_NONE },
{ OPT_TAGNAME, "-t", SO_REQ_SEP },
{ OPT_TAGNAME, "--tagname", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_TAG_OPTIONS,
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
@ -913,25 +729,23 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
{ OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
{ OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
{ OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
BACKUP_LOG_OPTIONS,
BACKUP_QUIET_OPTIONS,
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
BACKUP_MEMORY_OPTIONS,
BACKUP_HELP_OPTIONS,
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
TLS_OPTION_FLAGS,
SO_END_OF_OPTIONS
};
#undef BACKUP_LOG_OPTIONS
#undef BACKUP_QUIET_OPTIONS
#undef BACKUP_MEMORY_OPTIONS
#undef BACKUP_HELP_OPTIONS
#undef BACKUP_CLUSTER_FILE_OPTIONS
#undef BACKUP_TAG_OPTIONS
const KeyRef exeAgent = "backup_agent"_sr;
const KeyRef exeBackup = "fdbbackup"_sr;
const KeyRef exeRestore = "fdbrestore"_sr;
@ -3839,7 +3653,7 @@ int main(int argc, char* argv[]) {
break;
case OPT_BACKUPKEYS_FILE:
try {
std::string line = readFileBytes(args->OptionArg(), 64 * 1024 * 1024);
std::string line = readFileBytes(args->OptionArg(), 64ULL * 1024 * 1024);
addKeyRange(line, backupKeys);
} catch (Error&) {
printHelpTeaser(newArgV[0]);

108
fdbcli/ExcludeCommand.cpp
View File

@ -86,100 +86,22 @@ Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
}
}
AsyncResult<std::vector<std::string>> getExcludedServers(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
while (true) {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(fdb_cli::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> exclusions;
for (const auto& i : r) {
auto addr = i.key.removePrefix(fdb_cli::excludedServersSpecialKeyRange.begin).toString();
exclusions.push_back(addr);
}
co_return exclusions;
} catch (Error& e) {
err = e;
}
TraceEvent(SevWarn, "GetExcludedServersError").error(err);
co_await safeThreadFutureToFuture(tr->onError(err));
}
Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> db) {
return getManagementApiSpecialKeyValues(db, fdb_cli::excludedServersSpecialKeyRange, "GetExcludedServersError");
}
// Get the list of excluded localities by reading the keys.
AsyncResult<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
while (true) {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(fdb_cli::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> excludedLocalities;
for (const auto& i : r) {
auto locality = i.key.removePrefix(fdb_cli::excludedLocalitySpecialKeyRange.begin).toString();
excludedLocalities.push_back(locality);
}
co_return excludedLocalities;
} catch (Error& e) {
err = e;
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase> db) {
return getManagementApiSpecialKeyValues(db, fdb_cli::excludedLocalitySpecialKeyRange);
}
AsyncResult<std::vector<std::string>> getFailedServers(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
while (true) {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(fdb_cli::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> exclusions;
for (const auto& i : r) {
auto addr = i.key.removePrefix(fdb_cli::failedServersSpecialKeyRange.begin).toString();
exclusions.push_back(addr);
}
co_return exclusions;
} catch (Error& e) {
err = e;
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
Future<std::vector<std::string>> getFailedServers(Reference<IDatabase> db) {
return getManagementApiSpecialKeyValues(db, fdb_cli::failedServersSpecialKeyRange);
}
// Get the list of failed localities by reading the keys.
AsyncResult<std::vector<std::string>> getFailedLocalities(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
while (true) {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(fdb_cli::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> excludedLocalities;
for (const auto& i : r) {
auto locality = i.key.removePrefix(fdb_cli::failedLocalitySpecialKeyRange.begin).toString();
excludedLocalities.push_back(locality);
}
co_return excludedLocalities;
} catch (Error& e) {
err = e;
}
TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(err);
co_await safeThreadFutureToFuture(tr->onError(err));
}
Future<std::vector<std::string>> getFailedLocalities(Reference<IDatabase> db) {
return getManagementApiSpecialKeyValues(db, fdb_cli::failedLocalitySpecialKeyRange, "GetExcludedLocalitiesError");
}
Future<std::set<NetworkAddress>> getInProgressExclusion(Reference<ITransaction> tr) {
@ -286,20 +208,6 @@ Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<AddressExclu
namespace fdb_cli {
const KeyRangeRef excludedServersSpecialKeyRange("\xff\xff/management/excluded/"_sr,
"\xff\xff/management/excluded0"_sr);
const KeyRangeRef failedServersSpecialKeyRange("\xff\xff/management/failed/"_sr, "\xff\xff/management/failed0"_sr);
const KeyRangeRef excludedLocalitySpecialKeyRange("\xff\xff/management/excluded_locality/"_sr,
"\xff\xff/management/excluded_locality0"_sr);
const KeyRangeRef failedLocalitySpecialKeyRange("\xff\xff/management/failed_locality/"_sr,
"\xff\xff/management/failed_locality0"_sr);
const KeyRef excludedForceOptionSpecialKey = "\xff\xff/management/options/excluded/force"_sr;
const KeyRef failedForceOptionSpecialKey = "\xff\xff/management/options/failed/force"_sr;
const KeyRef excludedLocalityForceOptionSpecialKey = "\xff\xff/management/options/excluded_locality/force"_sr;
const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/failed_locality/force"_sr;
const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr,
"\xff\xff/management/in_progress_exclusion0"_sr);
Future<bool> excludeCommandActor(Reference<IDatabase> db, std::vector<StringRef> tokens, Future<Void> _warn) {
auto warn = std::move(_warn);
if (tokens.size() <= 1) {

18
fdbcli/include/fdbcli/fdbcli.h
View File

@ -98,15 +98,15 @@ extern const KeyRef coordinatorsProcessSpecialKey;
extern const KeyRef ddModeSpecialKey;
extern const KeyRef ddIgnoreRebalanceSpecialKey;
extern const KeyRangeRef excludedServersSpecialKeyRange;
extern const KeyRangeRef failedServersSpecialKeyRange;
extern const KeyRangeRef excludedLocalitySpecialKeyRange;
extern const KeyRangeRef failedLocalitySpecialKeyRange;
extern const KeyRef excludedForceOptionSpecialKey;
extern const KeyRef failedForceOptionSpecialKey;
extern const KeyRef excludedLocalityForceOptionSpecialKey;
extern const KeyRef failedLocalityForceOptionSpecialKey;
extern const KeyRangeRef exclusionInProgressSpecialKeyRange;
using management_api::excludedForceOptionSpecialKey;
using management_api::excludedLocalityForceOptionSpecialKey;
using management_api::excludedLocalitySpecialKeyRange;
using management_api::excludedServersSpecialKeyRange;
using management_api::exclusionInProgressSpecialKeyRange;
using management_api::failedForceOptionSpecialKey;
using management_api::failedLocalityForceOptionSpecialKey;
using management_api::failedLocalitySpecialKeyRange;
using management_api::failedServersSpecialKeyRange;
extern const KeyRef lockSpecialKey;

187
fdbclient/DatabaseBackupAgent.cpp
View File

@ -108,6 +108,19 @@ bool copyDefaultParameters(Reference<Task> source, Reference<Task> dest) {
return false;
}
Future<Key> addTaskWithOptionalDependency(Reference<ReadYourWritesTransaction> tr,
Reference<TaskBucket> taskBucket,
Reference<Task> task,
Key validationKey,
Reference<TaskFuture> waitFor) {
if (!waitFor) {
co_return taskBucket->addTask(tr, task, validationKey, task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr, taskBucket, task, validationKey, task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
}
template <class Tr>
Future<Void> checkTaskVersion(Tr tr, Reference<Task> task, StringRef name, uint32_t version) {
uint32_t taskVersion = task->getVersion();
@ -188,19 +201,8 @@ struct BackupRangeTaskFunc : TaskFuncBase {
task->params[BackupAgentBase::keyBeginKey] = begin;
task->params[BackupAgentBase::keyEndKey] = end;
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
static Future<Void> _execute(Database cx,
@ -582,19 +584,8 @@ struct FinishFullBackupTaskFunc : TaskFuncBase {
copyDefaultParameters(parentTask, task);
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
StringRef getName() const override { return name; };
@ -683,19 +674,8 @@ struct EraseLogRangeTaskFunc : TaskFuncBase {
BinaryWriter::toValue(1, Unversioned()); // FIXME: remove in 6.X, only needed for 5.2 backward compatibility
task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@ -965,19 +945,8 @@ struct CopyLogRangeTaskFunc : TaskFuncBase {
task->params[DatabaseBackupAgent::keyBeginVersion] = BinaryWriter::toValue(beginVersion, Unversioned());
task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@ -1127,19 +1096,8 @@ struct CopyLogsTaskFunc : TaskFuncBase {
task->params[BackupAgentBase::keyBeginVersion] = BinaryWriter::toValue(beginVersion, Unversioned());
task->params[DatabaseBackupAgent::keyPrevBeginVersion] = BinaryWriter::toValue(prevBeginVersion, Unversioned());
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
StringRef getName() const override { return name; };
@ -1250,19 +1208,8 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase {
copyDefaultParameters(parentTask, task);
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@ -1409,19 +1356,8 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase {
task->params[DatabaseBackupAgent::keyBeginVersion] = BinaryWriter::toValue(beginVersion, Unversioned());
task->params[DatabaseBackupAgent::keyPrevBeginVersion] = BinaryWriter::toValue(prevBeginVersion, Unversioned());
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
StringRef getName() const override { return name; };
@ -1670,19 +1606,8 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase {
task->params[DatabaseBackupAgent::keyBeginVersion] = BinaryWriter::toValue(beginVersion, Unversioned());
task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr,
@ -1772,19 +1697,8 @@ struct AbortOldBackupTaskFunc : TaskFuncBase {
copyDefaultParameters(parentTask, task);
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
StringRef getName() const override { return name; };
@ -2066,19 +1980,8 @@ struct BackupRestorableTaskFunc : TaskFuncBase {
copyDefaultParameters(parentTask, task);
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
parentTask->params[Task::reservedTaskParamValidKey],
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
co_return co_await addTaskWithOptionalDependency(
tr, taskBucket, task, parentTask->params[Task::reservedTaskParamValidKey], waitFor);
}
StringRef getName() const override { return name; };
@ -2351,25 +2254,11 @@ struct StartFullBackupTaskFunc : TaskFuncBase {
task->params[DatabaseBackupAgent::keyDatabasesInSync] =
backupAction == DatabaseBackupAgent::PreBackupAction::NONE ? "t"_sr : "f"_sr;
if (!waitFor) {
co_return taskBucket->addTask(tr,
task,
Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keyConfig)
.get(logUid)
.pack(BackupAgentBase::keyFolderId),
task->params[BackupAgentBase::keyFolderId]);
}
co_await waitFor->onSetAddTask(tr,
taskBucket,
task,
Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keyConfig)
.get(logUid)
.pack(BackupAgentBase::keyFolderId),
task->params[BackupAgentBase::keyFolderId]);
co_return "OnSetAddTask"_sr;
Key validationKey = Subspace(databaseBackupPrefixRange.begin)
.get(BackupAgentBase::keyConfig)
.get(logUid)
.pack(BackupAgentBase::keyFolderId);
co_return co_await addTaskWithOptionalDependency(tr, taskBucket, task, validationKey, waitFor);
}
StringRef getName() const override { return name; };

258
fdbclient/ManagementAPI.cpp
View File

@ -27,6 +27,7 @@
#include "fdbclient/BulkLoading.h"
#include "fdbclient/GenericManagementAPI.h"
#include "fdbclient/KeyRangeMap.h"
#include "fdbclient/IClientApi.h"
#include "fdbclient/RangeLock.h"
#include "flow/Error.h"
#include "fmt/format.h"
@ -36,6 +37,7 @@
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/RunRYWTransaction.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbclient/SystemData.h"
@ -1422,6 +1424,37 @@ Reference<IQuorumChange> autoQuorumChange(int desired) {
return makeReference<AutoQuorumChange>(desired);
}
Future<std::vector<std::string>> getManagementApiSpecialKeyValues(Reference<IDatabase> db,
KeyRange range,
const char* errorEvent) {
Reference<ITransaction> tr = db->createTransaction();
while (true) {
Error err;
try {
ThreadFuture<RangeResult> resultFuture = tr->getRange(range, CLIENT_KNOBS->TOO_MANY);
RangeResult result = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> values;
values.reserve(result.size());
for (const auto& entry : result) {
values.push_back(entry.key.removePrefix(range.begin).toString());
}
co_return values;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw;
}
err = e;
}
if (errorEvent != nullptr) {
TraceEvent(SevWarn, errorEvent).error(err);
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
}
Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> servers, bool failed) {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
@ -1457,32 +1490,28 @@ Future<Void> excludeServers(Transaction* tr, std::vector<AddressExclusion> serve
Future<Void> excludeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
if (cx->apiVersionAtLeast(700)) {
ReadYourWritesTransaction ryw(cx);
while (true) {
Error err;
try {
ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
ryw.set(
SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "excluded", "force"),
ValueRef());
for (auto& s : servers) {
Key addr = failed
? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());
ryw.set(addr, ValueRef());
}
TraceEvent("ExcludeServersSpecialKeySpaceCommit")
.detail("Servers", describe(servers))
.detail("ExcludeFailed", failed);
co_await ryw.commit();
co_return;
} catch (Error& e) {
err = e;
}
TraceEvent("ExcludeServersError").errorUnsuppressed(err);
co_await ryw.onError(err);
}
co_await runRYWTransactionVoid(
cx,
[&servers, failed](Reference<ReadYourWritesTransaction> ryw) -> Future<Void> {
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
ryw->set(
SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "excluded", "force"),
ValueRef());
for (const auto& server : servers) {
Key address =
failed
? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(server.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(server.toString());
ryw->set(address, ValueRef());
}
TraceEvent("ExcludeServersSpecialKeySpaceCommit")
.detail("Servers", describe(servers))
.detail("ExcludeFailed", failed);
return Void();
},
"ExcludeServersError");
co_return;
} else {
Transaction tr(cx);
while (true) {
@ -1536,33 +1565,28 @@ Future<Void> excludeLocalities(Transaction* tr, std::unordered_set<std::string>
// excludes localities by setting the keys.
Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> localities, bool failed) {
if (cx->apiVersionAtLeast(700)) {
ReadYourWritesTransaction ryw(cx);
while (true) {
Error err;
try {
ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
ryw.set(SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
failed ? "failed_locality" : "excluded_locality", "force"),
ValueRef());
for (const auto& l : localities) {
Key addr = failed
? SpecialKeySpace::getManagementApiCommandPrefix("failedlocality").withSuffix(l)
: SpecialKeySpace::getManagementApiCommandPrefix("excludedlocality").withSuffix(l);
ryw.set(addr, ValueRef());
}
TraceEvent("ExcludeLocalitiesSpecialKeySpaceCommit")
.detail("Localities", describe(localities))
.detail("ExcludeFailed", failed);
co_await ryw.commit();
co_return;
} catch (Error& e) {
err = e;
}
TraceEvent("ExcludeLocalitiesError").errorUnsuppressed(err);
co_await ryw.onError(err);
}
co_await runRYWTransactionVoid(
cx,
[&localities, failed](Reference<ReadYourWritesTransaction> ryw) -> Future<Void> {
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
ryw->set(SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
failed ? "failed_locality" : "excluded_locality", "force"),
ValueRef());
for (const auto& locality : localities) {
Key address =
failed
? SpecialKeySpace::getManagementApiCommandPrefix("failedlocality").withSuffix(locality)
: SpecialKeySpace::getManagementApiCommandPrefix("excludedlocality").withSuffix(locality);
ryw->set(address, ValueRef());
}
TraceEvent("ExcludeLocalitiesSpecialKeySpaceCommit")
.detail("Localities", describe(localities))
.detail("ExcludeFailed", failed);
return Void();
},
"ExcludeLocalitiesError");
co_return;
} else {
Transaction tr(cx);
while (true) {
@ -1583,46 +1607,35 @@ Future<Void> excludeLocalities(Database cx, std::unordered_set<std::string> loca
Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers, bool failed) {
std::string versionKey = deterministicRandom()->randomUniqueID().toString();
if (cx->apiVersionAtLeast(700)) {
ReadYourWritesTransaction ryw(cx);
while (true) {
Error err;
try {
ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (auto& s : servers) {
if (!s.isValid()) {
if (failed) {
ryw.clear(SpecialKeySpace::getManagementApiCommandRange("failed"));
} else {
ryw.clear(SpecialKeySpace::getManagementApiCommandRange("exclude"));
}
} else {
Key addr =
failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());
ryw.clear(addr);
// Eliminate both any ip-level exclusion (1.2.3.4) and any
// port-level exclusions (1.2.3.4:5)
// The range ['IP', 'IP;'] was originally deleted. ';' is
// char(':' + 1). This does not work, as other for all
// x between 0 and 9, 'IPx' will also be in this range.
//
// This is why we now make two clears: first only of the ip
// address, the second will delete all ports.
if (s.isWholeMachine())
ryw.clear(KeyRangeRef(addr.withSuffix(":"_sr), addr.withSuffix(";"_sr)));
}
}
TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
co_await ryw.commit();
co_return;
} catch (Error& e) {
err = e;
}
TraceEvent("IncludeServersError").errorUnsuppressed(err);
co_await ryw.onError(err);
}
co_await runRYWTransactionVoid(
cx,
[&servers, failed](Reference<ReadYourWritesTransaction> ryw) -> Future<Void> {
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
for (const auto& server : servers) {
if (!server.isValid()) {
ryw->clear(SpecialKeySpace::getManagementApiCommandRange(failed ? "failed" : "exclude"));
} else {
Key address =
failed
? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(server.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(
server.toString());
ryw->clear(address);
// Eliminate both any ip-level exclusion (1.2.3.4) and any
// port-level exclusions (1.2.3.4:5). Clearing ['IP', 'IP;']
// would also include 'IPx', so clear the IP and port range
// separately.
if (server.isWholeMachine()) {
ryw->clear(KeyRangeRef(address.withSuffix(":"_sr), address.withSuffix(";"_sr)));
}
}
}
TraceEvent("IncludeServersCommit").detail("Servers", describe(servers)).detail("Failed", failed);
return Void();
},
"IncludeServersError");
co_return;
} else {
Transaction tr(cx);
while (true) {
@ -1690,39 +1703,32 @@ Future<Void> includeServers(Database cx, std::vector<AddressExclusion> servers,
Future<Void> includeLocalities(Database cx, std::vector<std::string> localities, bool failed, bool includeAll) {
std::string versionKey = deterministicRandom()->randomUniqueID().toString();
if (cx->apiVersionAtLeast(700)) {
ReadYourWritesTransaction ryw(cx);
while (true) {
Error err;
try {
ryw.setOption(FDBTransactionOptions::RAW_ACCESS);
ryw.setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
if (includeAll) {
if (failed) {
ryw.clear(SpecialKeySpace::getManagementApiCommandRange("failedlocality"));
} else {
ryw.clear(SpecialKeySpace::getManagementApiCommandRange("excludedlocality"));
}
} else {
for (const auto& l : localities) {
Key locality =
failed ? SpecialKeySpace::getManagementApiCommandPrefix("failedlocality").withSuffix(l)
: SpecialKeySpace::getManagementApiCommandPrefix("excludedlocality").withSuffix(l);
ryw.clear(locality);
}
}
TraceEvent("IncludeLocalitiesCommit")
.detail("Localities", describe(localities))
.detail("Failed", failed)
.detail("IncludeAll", includeAll);
co_await ryw.commit();
co_return;
} catch (Error& e) {
err = e;
}
TraceEvent("IncludeLocalitiesError").errorUnsuppressed(err);
co_await ryw.onError(err);
}
co_await runRYWTransactionVoid(
cx,
[&localities, failed, includeAll](Reference<ReadYourWritesTransaction> ryw) -> Future<Void> {
ryw->setOption(FDBTransactionOptions::RAW_ACCESS);
ryw->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
if (includeAll) {
ryw->clear(
SpecialKeySpace::getManagementApiCommandRange(failed ? "failedlocality" : "excludedlocality"));
} else {
for (const auto& locality : localities) {
Key key =
failed
? SpecialKeySpace::getManagementApiCommandPrefix("failedlocality").withSuffix(locality)
: SpecialKeySpace::getManagementApiCommandPrefix("excludedlocality")
.withSuffix(locality);
ryw->clear(key);
}
}
TraceEvent("IncludeLocalitiesCommit")
.detail("Localities", describe(localities))
.detail("Failed", failed)
.detail("IncludeAll", includeAll);
return Void();
},
"IncludeLocalitiesError");
co_return;
} else {
Transaction tr(cx);
while (true) {

53
fdbclient/StorageServerInterface.cpp
View File

@ -235,19 +235,25 @@ static std::string hexStringRef(const StringRef& s) {
return result;
}
template <class Request, class Reply>
static void traceKeyValuesDiff(TraceEvent& event,
const KeySelectorRef& begin,
const KeySelectorRef& end,
Version version,
int limit,
int limitBytes,
const VectorRef<KeyValueRef>& ssKV,
bool ssMore,
const VectorRef<KeyValueRef>& tssKV,
bool tssMore,
const Request& request,
const Reply& source,
const Reply& target,
const ComparisonType& type) {
traceKeyValuesSummary(
event, begin, end, version, limit, limitBytes, ssKV.size(), ssMore, tssKV.size(), tssMore, type);
const auto& ssKV = source.data;
const auto& tssKV = target.data;
traceKeyValuesSummary(event,
request.begin,
request.end,
request.version,
request.limit,
request.limitBytes,
ssKV.size(),
source.more,
tssKV.size(),
target.more,
type);
bool mismatchFound = false;
for (int i = 0; i < std::max(ssKV.size(), tssKV.size()); i++) {
if (i >= ssKV.size() || i >= tssKV.size() || ssKV[i] != tssKV[i]) {
@ -279,17 +285,7 @@ void TSS_traceMismatch(TraceEvent& event,
const GetKeyValuesReply& src,
const GetKeyValuesReply& tss,
const ComparisonType& type) {
traceKeyValuesDiff(event,
req.begin,
req.end,
req.version,
req.limit,
req.limitBytes,
src.data,
src.more,
tss.data,
tss.more,
type);
traceKeyValuesDiff(event, req, src, tss, type);
}
// range reads and flat map
@ -334,24 +330,13 @@ const char* LB_mismatchTraceName(const GetKeyValuesStreamRequest& req, const Com
return type == TSS_COMPARISON ? "TSSMismatchGetKeyValuesStream" : "ReplicaMismatchGetKeyValuesStream";
}
// TODO this is all duplicated from above, simplify?
template <>
void TSS_traceMismatch(TraceEvent& event,
const GetKeyValuesStreamRequest& req,
const GetKeyValuesStreamReply& src,
const GetKeyValuesStreamReply& tss,
const ComparisonType& type) {
traceKeyValuesDiff(event,
req.begin,
req.end,
req.version,
req.limit,
req.limitBytes,
src.data,
src.more,
tss.data,
tss.more,
type);
traceKeyValuesDiff(event, req, src, tss, type);
}
template <>

24
fdbclient/include/fdbclient/ManagementAPI.h
View File

@ -36,6 +36,25 @@ standard API and some knowledge of the contents of the system key space.
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/MonitorLeader.h"
class IDatabase;
namespace management_api {
inline const KeyRangeRef excludedServersSpecialKeyRange("\xff\xff/management/excluded/"_sr,
"\xff\xff/management/excluded0"_sr);
inline const KeyRangeRef failedServersSpecialKeyRange("\xff\xff/management/failed/"_sr,
"\xff\xff/management/failed0"_sr);
inline const KeyRangeRef excludedLocalitySpecialKeyRange("\xff\xff/management/excluded_locality/"_sr,
"\xff\xff/management/excluded_locality0"_sr);
inline const KeyRangeRef failedLocalitySpecialKeyRange("\xff\xff/management/failed_locality/"_sr,
"\xff\xff/management/failed_locality0"_sr);
inline const KeyRef excludedForceOptionSpecialKey = "\xff\xff/management/options/excluded/force"_sr;
inline const KeyRef failedForceOptionSpecialKey = "\xff\xff/management/options/failed/force"_sr;
inline const KeyRef excludedLocalityForceOptionSpecialKey = "\xff\xff/management/options/excluded_locality/force"_sr;
inline const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/failed_locality/force"_sr;
inline const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr,
"\xff\xff/management/in_progress_exclusion0"_sr);
} // namespace management_api
Future<DatabaseConfiguration> getDatabaseConfiguration(Transaction* tr, bool useSystemPriority = false);
Future<DatabaseConfiguration> getDatabaseConfiguration(Database cx, bool useSystemPriority = false);
Future<Void> waitForFullReplication(Database cx);
@ -57,6 +76,11 @@ Future<CoordinatorsResult> changeQuorum(Database cx, Reference<IQuorumChange> ch
Reference<IQuorumChange> autoQuorumChange(int desired = -1);
Reference<IQuorumChange> nameQuorumChange(std::string const& name, Reference<IQuorumChange> const& other);
// Return the suffixes of all keys in a management special-key range.
Future<std::vector<std::string>> getManagementApiSpecialKeyValues(Reference<IDatabase> db,
KeyRange range,
const char* errorEvent = nullptr);
// Exclude the given set of servers from use as state servers. Returns as soon as the change is durable, without
// necessarily waiting for the servers to be evacuated. A NetworkAddress with a port of 0 means all servers on the
// given IP.

5
fdbclient/include/fdbclient/RunRYWTransaction.h
View File

@ -91,7 +91,7 @@ Future<RunRYWTransactionResult<Function>> runRYWTransactionDebug(Database cx,
// The supplied function should be idempotent. Otherwise, outcome of this function will depend on how many times the
// transaction is retried.
template <class Function>
Future<Void> runRYWTransactionVoid(Database cx, Function func) {
Future<Void> runRYWTransactionVoid(Database cx, Function func, const char* errorEvent = nullptr) {
Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
while (true) {
Error err;
@ -102,6 +102,9 @@ Future<Void> runRYWTransactionVoid(Database cx, Function func) {
} catch (Error& e) {
err = e;
}
if (errorEvent != nullptr) {
TraceEvent(errorEvent).errorUnsuppressed(err);
}
co_await tr->onError(err);
}
}

112
fdbctl/ExcludeCommand.cpp
View File

@ -36,90 +36,17 @@ namespace fdbctl {
namespace utils {
Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
loop {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(special_keys::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> exclusions;
for (const auto& i : r) {
auto addr = i.key.removePrefix(special_keys::excludedServersSpecialKeyRange.begin).toString();
exclusions.push_back(addr);
}
co_return exclusions;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw e;
}
TraceEvent(SevWarn, "GetExcludedServersError").error(e);
err = e;
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
return getManagementApiSpecialKeyValues(
db, special_keys::excludedServersSpecialKeyRange, "GetExcludedServersError");
}
Future<std::vector<std::string>> getFailedServers(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
loop {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(special_keys::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> exclusions;
for (const auto& i : r) {
auto addr = i.key.removePrefix(special_keys::failedServersSpecialKeyRange.begin).toString();
exclusions.push_back(addr);
}
co_return exclusions;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw e;
}
TraceEvent(SevWarn, "GetExcludedServersError").error(e);
err = e;
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
return getManagementApiSpecialKeyValues(db, special_keys::failedServersSpecialKeyRange, "GetExcludedServersError");
}
Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
loop {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(special_keys::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> excludedLocalities;
for (const auto& i : r) {
auto locality = i.key.removePrefix(special_keys::excludedLocalitySpecialKeyRange.begin).toString();
excludedLocalities.push_back(locality);
}
co_return excludedLocalities;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw e;
}
TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(e);
err = e;
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
return getManagementApiSpecialKeyValues(
db, special_keys::excludedLocalitySpecialKeyRange, "GetExcludedLocalitiesError");
}
Future<std::set<NetworkAddress>> getInProgressExclusion(Reference<ITransaction> tr) {
@ -136,32 +63,8 @@ Future<std::set<NetworkAddress>> getInProgressExclusion(Reference<ITransaction>
}
Future<std::vector<std::string>> getFailedLocalities(Reference<IDatabase> db) {
Reference<ITransaction> tr = db->createTransaction();
loop {
Error err;
try {
ThreadFuture<RangeResult> resultFuture =
tr->getRange(special_keys::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY);
RangeResult r = co_await safeThreadFutureToFuture(resultFuture);
ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY);
std::vector<std::string> excludedLocalities;
for (const auto& i : r) {
auto locality = i.key.removePrefix(special_keys::failedLocalitySpecialKeyRange.begin).toString();
excludedLocalities.push_back(locality);
}
co_return excludedLocalities;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw e;
}
TraceEvent(SevWarn, "GetFailedLocalitiesError").error(e);
err = e;
}
co_await safeThreadFutureToFuture(tr->onError(err));
}
return getManagementApiSpecialKeyValues(
db, special_keys::failedLocalitySpecialKeyRange, "GetFailedLocalitiesError");
}
} // namespace utils
@ -263,6 +166,7 @@ Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatabase> d
break;
co_await delayJittered(1.0); // SOMEDAY: watches!
continue;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw;

26
fdbctl/include/fdbctl/ControlCommands.h
View File

@ -23,6 +23,7 @@
#define FDB_CTL_LIB_FDB_CONTROL_COMMANDS_H
#include "fdbclient/IClientApi.h"
#include "fdbclient/ManagementAPI.h"
#include "fdbclient/StorageServerInterface.h"
#include "fdbrpc/FlowGrpc.h"
#include "fdbctl/control_service/control_service.pb.h"
@ -73,26 +74,19 @@ Future<bool> getWorkersProcessData(Reference<IDatabase> db, std::vector<ProcessD
namespace special_keys {
// TODO: Point fdbcli ones to this.
const KeyRef clusterDescriptionSpecialKey = "\xff\xff/configuration/coordinators/cluster_description"_sr;
const KeyRef coordinatorsAutoSpecialKey = "\xff\xff/management/auto_coordinators"_sr;
const KeyRef coordinatorsProcessSpecialKey = "\xff\xff/configuration/coordinators/processes"_sr;
// Special key ranges for include/exclude functionality
const KeyRange excludedServersSpecialKeyRange =
KeyRangeRef("\xff\xff/management/excluded/"_sr, "\xff\xff/management/excluded0"_sr);
const KeyRange failedServersSpecialKeyRange =
KeyRangeRef("\xff\xff/management/failed/"_sr, "\xff\xff/management/failed0"_sr);
const KeyRange excludedLocalitySpecialKeyRange =
KeyRangeRef("\xff\xff/management/excluded_locality/"_sr, "\xff\xff/management/excluded_locality0"_sr);
const KeyRange failedLocalitySpecialKeyRange =
KeyRangeRef("\xff\xff/management/failed_locality/"_sr, "\xff\xff/management/failed_locality0"_sr);
const KeyRef excludedForceOptionSpecialKey = "\xff\xff/management/options/excluded/force"_sr;
const KeyRef failedForceOptionSpecialKey = "\xff\xff/management/options/failed/force"_sr;
const KeyRef excludedLocalityForceOptionSpecialKey = "\xff\xff/management/options/excluded_locality/force"_sr;
const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/failed_locality/force"_sr;
const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr,
"\xff\xff/management/in_progress_exclusion0"_sr);
using management_api::excludedForceOptionSpecialKey;
using management_api::excludedLocalityForceOptionSpecialKey;
using management_api::excludedLocalitySpecialKeyRange;
using management_api::excludedServersSpecialKeyRange;
using management_api::exclusionInProgressSpecialKeyRange;
using management_api::failedForceOptionSpecialKey;
using management_api::failedLocalityForceOptionSpecialKey;
using management_api::failedLocalitySpecialKeyRange;
using management_api::failedServersSpecialKeyRange;
const KeyRef workerInterfacesVerifyOptionSpecialKey = "\xff\xff/management/options/worker_interfaces/verify"_sr;
} // namespace special_keys

89
fdbserver/commitproxy/CommitProxyServer.cpp
View File

@ -112,38 +112,50 @@ struct ResolutionRequestBuilder {
return *out;
}
std::vector<int> getResolversForRange(const KeyRangeRef& range, Optional<Version> readSnapshot) const {
std::vector<int> resolvers;
resolvers.reserve(self->resolvers.size());
std::vector<unsigned char> seen(self->resolvers.size(), 0);
for (auto& intersectingRange : self->keyResolvers.intersectingRanges(range)) {
auto& versionResolvers = intersectingRange.value();
if (readSnapshot.present()) {
for (int i = versionResolvers.size() - 1; i >= 0; --i) {
const int resolver = versionResolvers[i].second;
if (!seen[resolver]) {
seen[resolver] = 1;
resolvers.push_back(resolver);
}
if (versionResolvers[i].first < readSnapshot.get()) {
break;
}
}
} else if (!versionResolvers.empty()) {
const int resolver = versionResolvers.back().second;
if (!seen[resolver]) {
seen[resolver] = 1;
resolvers.push_back(resolver);
}
}
}
if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS && systemKeys.intersects(range)) {
resolvers.clear();
for (int resolver = 0; resolver < self->resolvers.size(); ++resolver) {
resolvers.push_back(resolver);
}
}
ASSERT(!resolvers.empty());
return resolvers;
}
// Returns a read conflict index map: [resolver_index][read_conflict_range_index_on_the_resolver]
// -> read_conflict_range's original index
std::vector<std::vector<int>> addReadConflictRanges(CommitTransactionRef& trIn) {
std::vector<std::vector<int>> rCRIndexMap(requests.size());
for (int idx = 0; idx < trIn.read_conflict_ranges.size(); ++idx) {
const auto& r = trIn.read_conflict_ranges[idx];
auto ranges = self->keyResolvers.intersectingRanges(r);
std::vector<int> resolvers;
resolvers.reserve(self->resolvers.size());
// O(1) de-dup keyed by resolver id (deterministic)
std::vector<unsigned char> seen(self->resolvers.size(), 0);
for (auto& ir : ranges) {
auto& version_resolver = ir.value();
for (int i = version_resolver.size() - 1; i >= 0; i--) {
const int resolver_id = version_resolver[i].second;
if (!seen[resolver_id]) {
seen[resolver_id] = 1;
resolvers.push_back(resolver_id);
}
if (version_resolver[i].first < trIn.read_snapshot)
break;
}
}
if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS && systemKeys.intersects(r)) {
// All resolvers are eligible; skip per-id de-dup and just fill 0..N-1.
resolvers.clear();
for (int k = 0; k < self->resolvers.size(); ++k) {
resolvers.push_back(k);
}
}
ASSERT(!resolvers.empty());
for (int resolver : resolvers) {
for (int resolver : getResolversForRange(r, trIn.read_snapshot)) {
getOutTransaction(resolver, trIn.read_snapshot)
.read_conflict_ranges.push_back(requests[resolver].arena, r);
rCRIndexMap[resolver].push_back(idx);
@ -154,31 +166,10 @@ struct ResolutionRequestBuilder {
void addWriteConflictRanges(CommitTransactionRef& trIn) {
for (auto& r : trIn.write_conflict_ranges) {
auto ranges = self->keyResolvers.intersectingRanges(r);
std::vector<int> resolvers;
resolvers.reserve(self->resolvers.size());
std::vector<unsigned char> seen(self->resolvers.size(), 0);
for (auto& ir : ranges) {
auto& version_resolver = ir.value();
if (!version_resolver.empty()) {
const int resolver_id = version_resolver.back().second;
if (!seen[resolver_id]) {
seen[resolver_id] = 1;
resolvers.push_back(resolver_id);
}
}
}
if (SERVER_KNOBS->PROXY_USE_RESOLVER_PRIVATE_MUTATIONS && systemKeys.intersects(r)) {
// All resolvers are eligible.
resolvers.clear();
for (int k = 0; k < self->resolvers.size(); ++k) {
resolvers.push_back(k);
}
}
ASSERT(!resolvers.empty());
for (int resolver : resolvers)
for (int resolver : getResolversForRange(r, Optional<Version>())) {
getOutTransaction(resolver, trIn.read_snapshot)
.write_conflict_ranges.push_back(requests[resolver].arena, r);
}
}
}

57
fdbserver/core/include/fdbserver/core/BusyTagCollector.h Normal file
View File

@ -0,0 +1,57 @@
/*
* BusyTagCollector.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <functional>
#include <queue>
#include <vector>
#include "fdbclient/StorageServerInterface.h"
class BusyTagCollector {
std::priority_queue<BusyTagInfo, std::vector<BusyTagInfo>, std::greater<BusyTagInfo>> busiestTags;
size_t maxTagsTracked;
double minRateTracked;
public:
BusyTagCollector(int maxTagsTracked, double minRateTracked)
: maxTagsTracked(maxTagsTracked), minRateTracked(minRateTracked) {}
bool add(const TransactionTag& tag, double rate, double fractionalBusyness) {
if (rate < minRateTracked) {
return false;
}
if (busiestTags.size() < maxTagsTracked) {
busiestTags.emplace(tag, rate, fractionalBusyness);
} else if (busiestTags.top().rate < rate) {
busiestTags.pop();
busiestTags.emplace(tag, rate, fractionalBusyness);
}
return true;
}
void drainInto(std::vector<BusyTagInfo>& result) {
while (!busiestTags.empty()) {
result.push_back(busiestTags.top());
busiestTags.pop();
}
}
};

30
fdbserver/datadistributor/DDTeamCollection.actor.cpp
View File

@ -759,43 +759,19 @@ public:
for (; idx < largeOrBadTeams.size(); idx++) {
servers.clear();
serverIds.clear();
for (const auto& server : largeOrBadTeams[idx]->getServers()) {
if (server->isInDesiredDC() && !self->server_status.get(server->getId()).isUnhealthy()) {
servers.push_back(server);
serverIds.push_back(server->getId());
}
}
// For the bad team that is too big (too many servers), we will try to find a subset of servers in the
// team to construct a new healthy team, so that moving data to the new healthy team will not cause too
// much data movement overhead
// FIXME: This code logic can be simplified.
if (servers.size() >= self->configuration.storageTeamSize) {
bool foundTeam = false;
for (int j = 0; j < servers.size() - self->configuration.storageTeamSize + 1 && !foundTeam; j++) {
auto const& serverTeams = servers[j]->getTeams();
for (int k = 0; k < serverTeams.size(); k++) {
auto& testTeam = serverTeams[k]->getServerIDs();
bool allInTeam = true; // All servers in testTeam belong to the healthy servers
for (int l = 0; l < testTeam.size(); l++) {
bool foundServer = false;
for (auto it : servers) {
if (it->getId() == testTeam[l]) {
foundServer = true;
break;
}
}
if (!foundServer) {
allInTeam = false;
break;
}
}
if (allInTeam) {
foundTeam = true;
break;
}
}
}
if (!foundTeam) {
if (!self->findTeamFromServers(serverIds, /*wantHealthy=*/false).present()) {
if (self->satisfiesPolicy(servers)) {
if (servers.size() == self->configuration.storageTeamSize ||
self->satisfiesPolicy(servers, self->configuration.storageTeamSize)) {

111
fdbserver/ratekeeper/Ratekeeper.cpp
View File

@ -21,6 +21,7 @@
#include <algorithm>
#include "fdbclient/Knobs.h"
#include "fdbserver/core/BusyTagCollector.h"
#include "fdbserver/core/Knobs.h"
#include "fdbserver/core/ServerDBInfo.h"
#include "fdbserver/core/WaitFailure.h"
@ -1163,16 +1164,41 @@ Future<Void> ratekeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerD
co_await Ratekeeper::run(rkInterf, dbInfo);
}
StorageQueueInfo::StorageQueueInfo(const UID& ratekeeperID_, const UID& id_, const LocalityData& locality_)
: valid(false), ratekeeperID(ratekeeperID_), id(id_), locality(locality_), acceptingRequests(false),
smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT),
smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT),
smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), limitReason(limitReason_t::unlimited) {
// FIXME: this is a tacky workaround for a potential uninitialized use in trackStorageServerQueueInfo
lastReply.instanceID = -1;
QueueMetricsSmoother::QueueMetricsSmoother()
: smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT), smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT),
smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT) {}
bool QueueMetricsSmoother::update(int64_t newInstanceID,
int64_t newDurableBytes,
int64_t inputBytes,
const StorageBytes& storageBytes,
Smoother& smoothTotalDurableBytes) {
const bool reset = !instanceID.present() || instanceID.get() != newInstanceID;
if (reset) {
smoothDurableBytes.reset(newDurableBytes);
verySmoothDurableBytes.reset(newDurableBytes);
smoothInputBytes.reset(inputBytes);
smoothFreeSpace.reset(storageBytes.available);
smoothTotalSpace.reset(storageBytes.total);
} else {
smoothTotalDurableBytes.addDelta(newDurableBytes - durableBytes);
smoothDurableBytes.setTotal(newDurableBytes);
verySmoothDurableBytes.setTotal(newDurableBytes);
smoothInputBytes.setTotal(inputBytes);
smoothFreeSpace.setTotal(storageBytes.available);
smoothTotalSpace.setTotal(storageBytes.total);
}
instanceID = newInstanceID;
durableBytes = newDurableBytes;
return reset;
}
StorageQueueInfo::StorageQueueInfo(const UID& ratekeeperID_, const UID& id_, const LocalityData& locality_)
: ratekeeperID(ratekeeperID_), smoothDurableVersion(SERVER_KNOBS->SMOOTHING_AMOUNT),
smoothLatestVersion(SERVER_KNOBS->SMOOTHING_AMOUNT), valid(false), id(id_), locality(locality_),
acceptingRequests(false), limitReason(limitReason_t::unlimited) {}
StorageQueueInfo::StorageQueueInfo(const UID& id_, const LocalityData& locality_)
: StorageQueueInfo(UID(), id_, locality_) {}
@ -1184,23 +1210,12 @@ void StorageQueueInfo::addCommitCost(TransactionTagRef tagName, TransactionCommi
void StorageQueueInfo::update(StorageQueuingMetricsReply const& reply, Smoother& smoothTotalDurableBytes) {
valid = true;
auto prevReply = std::move(lastReply);
lastReply = reply;
if (prevReply.instanceID != reply.instanceID) {
smoothDurableBytes.reset(reply.bytesDurable);
verySmoothDurableBytes.reset(reply.bytesDurable);
smoothInputBytes.reset(reply.bytesInput);
smoothFreeSpace.reset(reply.storageBytes.available);
smoothTotalSpace.reset(reply.storageBytes.total);
if (queueMetrics.update(
reply.instanceID, reply.bytesDurable, reply.bytesInput, reply.storageBytes, smoothTotalDurableBytes)) {
smoothDurableVersion.reset(reply.durableVersion);
smoothLatestVersion.reset(reply.version);
} else {
smoothTotalDurableBytes.addDelta(reply.bytesDurable - prevReply.bytesDurable);
smoothDurableBytes.setTotal(reply.bytesDurable);
verySmoothDurableBytes.setTotal(reply.bytesDurable);
smoothInputBytes.setTotal(reply.bytesInput);
smoothFreeSpace.setTotal(reply.storageBytes.available);
smoothTotalSpace.setTotal(reply.storageBytes.total);
smoothDurableVersion.setTotal(reply.durableVersion);
smoothLatestVersion.setTotal(reply.version);
}
@ -1213,19 +1228,14 @@ UpdateCommitCostRequest StorageQueueInfo::refreshCommitCost(double elapsed) {
TransactionTag busiestTag;
TransactionCommitCostEstimation maxCost;
double maxRate = 0;
std::priority_queue<BusyTagInfo, std::vector<BusyTagInfo>, std::greater<BusyTagInfo>> topKWriters;
BusyTagCollector busiestWriters(SERVER_KNOBS->SS_THROTTLE_TAGS_TRACKED,
SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE);
for (const auto& [tag, cost] : tagCostEst) {
double rate = cost.getCostSum() / elapsed;
double busyness = static_cast<double>(maxCost.getCostSum()) / totalWriteCosts;
if (rate < SERVER_KNOBS->MIN_TAG_WRITE_PAGES_RATE * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE) {
if (!busiestWriters.add(tag, rate, busyness)) {
continue;
}
if (topKWriters.size() < SERVER_KNOBS->SS_THROTTLE_TAGS_TRACKED) {
topKWriters.emplace(tag, rate, busyness);
} else if (topKWriters.top().rate < rate) {
topKWriters.pop();
topKWriters.emplace(tag, rate, busyness);
}
if (rate > maxRate) {
busiestTag = tag;
@ -1234,10 +1244,7 @@ UpdateCommitCostRequest StorageQueueInfo::refreshCommitCost(double elapsed) {
}
}
while (!topKWriters.empty()) {
busiestWriteTags.push_back(topKWriters.top());
topKWriters.pop();
}
busiestWriters.drainInto(busiestWriteTags);
UpdateCommitCostRequest updateCommitCostRequest{ ratekeeperID,
now(),
@ -1257,43 +1264,11 @@ UpdateCommitCostRequest StorageQueueInfo::refreshCommitCost(double elapsed) {
return updateCommitCostRequest;
}
Optional<double> StorageQueueInfo::getTagThrottlingRatio(int64_t storageTargetBytes, int64_t storageSpringBytes) const {
auto const storageQueue = getStorageQueueBytes();
// TODO: Remove duplicate calculation from Ratekeeper::updateRate
double inverseResult = std::min(
2.0, (storageQueue - storageTargetBytes + storageSpringBytes) / static_cast<double>(storageSpringBytes));
if (inverseResult > 0) {
return 1.0 / inverseResult;
} else {
return {};
}
}
TLogQueueInfo::TLogQueueInfo(UID id)
: valid(false), id(id), smoothDurableBytes(SERVER_KNOBS->SMOOTHING_AMOUNT),
smoothInputBytes(SERVER_KNOBS->SMOOTHING_AMOUNT), verySmoothDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
smoothFreeSpace(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalSpace(SERVER_KNOBS->SMOOTHING_AMOUNT) {
// FIXME: this is a tacky workaround for a potential uninitialized use in trackTLogQueueInfo (copied
// from storageQueueInfO)
lastReply.instanceID = -1;
}
TLogQueueInfo::TLogQueueInfo(UID id) : valid(false), id(id) {}
void TLogQueueInfo::update(TLogQueuingMetricsReply const& reply, Smoother& smoothTotalDurableBytes) {
valid = true;
auto prevReply = lastReply;
lastReply = reply;
if (prevReply.instanceID != reply.instanceID) {
smoothDurableBytes.reset(reply.bytesDurable);
verySmoothDurableBytes.reset(reply.bytesDurable);
smoothInputBytes.reset(reply.bytesInput);
smoothFreeSpace.reset(reply.storageBytes.available);
smoothTotalSpace.reset(reply.storageBytes.total);
} else {
smoothTotalDurableBytes.addDelta(reply.bytesDurable - prevReply.bytesDurable);
smoothDurableBytes.setTotal(reply.bytesDurable);
verySmoothDurableBytes.setTotal(reply.bytesDurable);
smoothInputBytes.setTotal(reply.bytesInput);
smoothFreeSpace.setTotal(reply.storageBytes.available);
smoothTotalSpace.setTotal(reply.storageBytes.total);
}
queueMetrics.update(
reply.instanceID, reply.bytesDurable, reply.bytesInput, reply.storageBytes, smoothTotalDurableBytes);
}

54
fdbserver/ratekeeper/Ratekeeper.h
View File

@ -34,6 +34,30 @@
struct ServerDBInfo;
class QueueMetricsSmoother {
Optional<int64_t> instanceID;
int64_t durableBytes{ 0 };
Smoother smoothDurableBytes;
Smoother smoothInputBytes;
Smoother verySmoothDurableBytes;
Smoother smoothFreeSpace;
Smoother smoothTotalSpace;
public:
QueueMetricsSmoother();
bool update(int64_t newInstanceID,
int64_t newDurableBytes,
int64_t inputBytes,
const StorageBytes& storageBytes,
Smoother& smoothTotalDurableBytes);
double getSmoothFreeSpace() const { return smoothFreeSpace.smoothTotal(); }
double getSmoothTotalSpace() const { return smoothTotalSpace.smoothTotal(); }
double getSmoothDurableBytes() const { return smoothDurableBytes.smoothTotal(); }
double getSmoothInputBytesRate() const { return smoothInputBytes.smoothRate(); }
double getVerySmoothDurableBytesRate() const { return verySmoothDurableBytes.smoothRate(); }
};
class StorageQueueInfo {
uint64_t totalWriteCosts{ 0 };
int totalWriteOps{ 0 };
@ -41,8 +65,7 @@ class StorageQueueInfo {
TransactionTagMap<TransactionCommitCostEstimation> tagCostEst;
UID ratekeeperID;
Smoother smoothFreeSpace, smoothTotalSpace;
Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
QueueMetricsSmoother queueMetrics;
Smoother smoothDurableVersion, smoothLatestVersion;
public:
@ -57,36 +80,33 @@ public:
StorageQueueInfo(const UID& id, const LocalityData& locality);
StorageQueueInfo(const UID& rateKeeperID, const UID& id, const LocalityData& locality);
UpdateCommitCostRequest refreshCommitCost(double elapsed);
int64_t getStorageQueueBytes() const { return lastReply.bytesInput - smoothDurableBytes.smoothTotal(); }
int64_t getStorageQueueBytes() const { return lastReply.bytesInput - queueMetrics.getSmoothDurableBytes(); }
int64_t getDurabilityLag() const { return smoothLatestVersion.smoothTotal() - smoothDurableVersion.smoothTotal(); }
void update(StorageQueuingMetricsReply const&, Smoother& smoothTotalDurableBytes);
void addCommitCost(TransactionTagRef tagName, TransactionCommitCostEstimation const& cost);
double getSmoothFreeSpace() const { return smoothFreeSpace.smoothTotal(); }
double getSmoothTotalSpace() const { return smoothTotalSpace.smoothTotal(); }
double getSmoothDurableBytes() const { return smoothDurableBytes.smoothTotal(); }
double getSmoothInputBytesRate() const { return smoothInputBytes.smoothRate(); }
double getVerySmoothDurableBytesRate() const { return verySmoothDurableBytes.smoothRate(); }
double getSmoothFreeSpace() const { return queueMetrics.getSmoothFreeSpace(); }
double getSmoothTotalSpace() const { return queueMetrics.getSmoothTotalSpace(); }
double getSmoothDurableBytes() const { return queueMetrics.getSmoothDurableBytes(); }
double getSmoothInputBytesRate() const { return queueMetrics.getSmoothInputBytesRate(); }
double getVerySmoothDurableBytesRate() const { return queueMetrics.getVerySmoothDurableBytesRate(); }
Version getLatestVersion() const { return lastReply.version; }
Optional<double> getTagThrottlingRatio(int64_t storageTargetBytes, int64_t storageSpringBytes) const;
};
class TLogQueueInfo {
Smoother smoothDurableBytes, smoothInputBytes, verySmoothDurableBytes;
Smoother smoothFreeSpace;
Smoother smoothTotalSpace;
QueueMetricsSmoother queueMetrics;
public:
TLogQueuingMetricsReply lastReply;
bool valid;
UID id;
double getSmoothFreeSpace() const { return smoothFreeSpace.smoothTotal(); }
double getSmoothTotalSpace() const { return smoothTotalSpace.smoothTotal(); }
double getSmoothDurableBytes() const { return smoothDurableBytes.smoothTotal(); }
double getSmoothInputBytesRate() const { return smoothInputBytes.smoothRate(); }
double getVerySmoothDurableBytesRate() const { return verySmoothDurableBytes.smoothRate(); }
double getSmoothFreeSpace() const { return queueMetrics.getSmoothFreeSpace(); }
double getSmoothTotalSpace() const { return queueMetrics.getSmoothTotalSpace(); }
double getSmoothDurableBytes() const { return queueMetrics.getSmoothDurableBytes(); }
double getSmoothInputBytesRate() const { return queueMetrics.getSmoothInputBytesRate(); }
double getVerySmoothDurableBytesRate() const { return queueMetrics.getVerySmoothDurableBytesRate(); }
explicit TLogQueueInfo(UID id);
Version getLastCommittedVersion() const { return lastReply.v; }

17
fdbserver/storageserver/TransactionTagCounter.cpp
View File

@ -19,6 +19,7 @@
*/
#include "fdbclient/NativeAPI.actor.h"
#include "fdbserver/core/BusyTagCollector.h"
#include "fdbserver/core/Knobs.h"
#include "TransactionTagCounter.h"
#include "flow/Coroutines.h"
@ -36,24 +37,14 @@ class TransactionTagCounterImpl {
Reference<EventCacheHolder> busiestReadTagEventHolder;
std::vector<BusyTagInfo> getBusiestTagsFromLastInterval(double elapsed) const {
std::priority_queue<BusyTagInfo, std::vector<BusyTagInfo>, std::greater<BusyTagInfo>> topKTags;
BusyTagCollector busiestTags(maxTagsTracked, minRateTracked);
for (auto const& [tag, cost] : intervalCosts) {
auto const rate = cost / elapsed;
auto const fractionalBusyness = std::min(1.0, cost / intervalTotalCost);
if (rate < minRateTracked) {
continue;
} else if (topKTags.size() < maxTagsTracked) {
topKTags.emplace(tag, rate, fractionalBusyness);
} else if (topKTags.top().rate < rate) {
topKTags.pop();
topKTags.emplace(tag, rate, fractionalBusyness);
}
busiestTags.add(tag, rate, fractionalBusyness);
}
std::vector<BusyTagInfo> result;
while (!topKTags.empty()) {
result.push_back(topKTags.top());
topKTags.pop();
}
busiestTags.drainInto(result);
return result;
}

36
fdbserver/storageserver/storageserver.cpp
View File

@ -11589,28 +11589,30 @@ Future<Void> serveGetValueRequests(StorageServer* self, FutureStream<GetValueReq
}
}
Future<Void> serveGetKeyValuesRequests(StorageServer* self, FutureStream<GetKeyValuesRequest> getKeyValues) {
getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKeyValues;
template <class Request, class Handler>
Future<Void> serveGuardedReadRequests(StorageServer* self,
FutureStream<Request> requests,
TransactionLineage::Operation operation,
Handler handler) {
getCurrentLineage()->modify(&TransactionLineage::operation) = operation;
while (true) {
GetKeyValuesRequest req = co_await getKeyValues;
Request req = co_await requests;
// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
// downgrade before doing real work
self->actors.add(self->readGuard(req, getKeyValuesQ));
self->actors.add(self->readGuard(req, handler));
}
}
Future<Void> serveGetKeyValuesRequests(StorageServer* self, FutureStream<GetKeyValuesRequest> getKeyValues) {
return serveGuardedReadRequests(
self, std::move(getKeyValues), TransactionLineage::Operation::GetKeyValues, getKeyValuesQ);
}
Future<Void> serveGetMappedKeyValuesRequests(StorageServer* self,
FutureStream<GetMappedKeyValuesRequest> getMappedKeyValues) {
// TODO: Is it fine to keep TransactionLineage::Operation::GetKeyValues here?
getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKeyValues;
while (true) {
GetMappedKeyValuesRequest req = co_await getMappedKeyValues;
// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
// downgrade before doing real work
self->actors.add(self->readGuard(req, getMappedKeyValuesQ));
}
return serveGuardedReadRequests(
self, std::move(getMappedKeyValues), TransactionLineage::Operation::GetKeyValues, getMappedKeyValuesQ);
}
Future<Void> serveGetKeyValuesStreamRequests(StorageServer* self,
@ -11625,13 +11627,7 @@ Future<Void> serveGetKeyValuesStreamRequests(StorageServer* self,
}
Future<Void> serveGetKeyRequests(StorageServer* self, FutureStream<GetKeyRequest> getKey) {
getCurrentLineage()->modify(&TransactionLineage::operation) = TransactionLineage::Operation::GetKey;
while (true) {
GetKeyRequest req = co_await getKey;
// Warning: This code is executed at extremely high priority (TaskPriority::LoadBalancedEndpoint), so
// downgrade before doing real work
self->actors.add(self->readGuard(req, getKeyQ));
}
return serveGuardedReadRequests(self, std::move(getKey), TransactionLineage::Operation::GetKey, getKeyQ);
}
Future<Void> watchValueWaitForVersion(StorageServer* self,

18
flow/FastAlloc.cpp
View File

@ -110,9 +110,6 @@ bool valgrindPrecise() {
}
#endif
template <int Size>
void* FastAllocator<Size>::freelist = nullptr;
std::atomic<int64_t> g_hugeArenaMemory(0);
double hugeArenaLastLogged = 0;
@ -398,7 +395,6 @@ void* FastAllocator<Size>::allocate() {
}
#endif
#if FASTALLOC_THREAD_SAFE
ThreadData& thr = threadData();
if (!thr.freelist) {
ASSERT(thr.count == 0);
@ -418,15 +414,6 @@ void* FastAllocator<Size>::allocate() {
thr.freelist = *(void**)p;
ASSERT(!thr.freelist == (thr.count == 0)); // freelist is empty if and only if count is 0
// check( p, true );
#else
void* p = freelist;
if (!p)
getMagazine();
#if VALGRIND
VALGRIND_MAKE_MEM_DEFINED(p, sizeof(void*));
#endif
freelist = *(void**)p;
#endif
#if VALGRIND
VALGRIND_MALLOCLIKE_BLOCK(p, Size, 0, 0);
#endif
@ -481,7 +468,6 @@ void FastAllocator<Size>::release(void* ptr) {
}
#endif
#if FASTALLOC_THREAD_SAFE
ThreadData& thr = threadData();
if (thr.count == magazine_size) {
if (thr.alternate) // Two full magazines, return one
@ -500,10 +486,6 @@ void FastAllocator<Size>::release(void* ptr) {
*(void**)ptr = thr.freelist;
// check(ptr, false);
thr.freelist = ptr;
#else
*(void**)ptr = freelist;
freelist = ptr;
#endif
#if VALGRIND
VALGRIND_FREELIKE_BLOCK(ptr, 0);

51
flow/Net2.cpp
View File

@ -922,7 +922,8 @@ public:
static Future<Reference<IConnection>> connect(boost::asio::io_service* ios,
Reference<ReferencedObject<boost::asio::ssl::context>> context,
NetworkAddress addr,
tcp::socket* existingSocket = nullptr) {
tcp::socket* existingSocket = nullptr,
std::string hostname = {}) {
std::pair<IPAddress, uint16_t> peerIP = std::make_pair(addr.ip, addr.port);
auto iter(g_network->networkInfo.serverTLSConnectionThrottler.find(peerIP));
if (iter != g_network->networkInfo.serverTLSConnectionThrottler.end()) {
@ -949,6 +950,7 @@ public:
Reference<SSLConnection> self(new SSLConnection(*ios, context));
self->peer_address = addr;
self->sni_hostname = std::move(hostname);
try {
auto to = tcpEndpoint(self->peer_address);
BindPromise p("N2_ConnectError", self->id, self->peer_address);
@ -965,51 +967,6 @@ public:
}
}
// Connect with hostname for SNI (Server Name Indication) support
static Future<Reference<IConnection>> connectWithHostname(
boost::asio::io_service* ios,
Reference<ReferencedObject<boost::asio::ssl::context>> context,
NetworkAddress addr,
std::string hostname) {
std::pair<IPAddress, uint16_t> peerIP = std::make_pair(addr.ip, addr.port);
auto iter(g_network->networkInfo.serverTLSConnectionThrottler.find(peerIP));
if (iter != g_network->networkInfo.serverTLSConnectionThrottler.end()) {
if (now() < iter->second.second) {
if (iter->second.first >= FLOW_KNOBS->TLS_CLIENT_CONNECTION_THROTTLE_ATTEMPTS) {
TraceEvent("TLSOutgoingConnectionThrottlingWarning").suppressFor(1.0).detail("PeerIP", addr);
co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
throw connection_failed();
}
} else {
g_network->networkInfo.serverTLSConnectionThrottler.erase(peerIP);
}
}
Reference<SSLConnection> self(new SSLConnection(*ios, context));
self->peer_address = addr;
self->sni_hostname = hostname; // Store hostname for SNI during handshake
// Store hostname for SNI use during handshake
try {
auto to = tcpEndpoint(self->peer_address);
BindPromise p("N2_ConnectError", self->id, self->peer_address);
Future<Void> onConnected = p.getFuture();
self->socket.async_connect(to, std::move(p));
co_await onConnected;
// SNI will be set later in doConnectHandshake before SSL handshake
self->init();
co_return self;
} catch (Error&) {
// Either the connection failed, or was cancelled by the caller
self->closeSocket();
throw;
}
}
// This is not part of the IConnection interface, because it is wrapped by IListener::accept()
void accept(NetworkAddress peerAddr) {
this->peer_address = peerAddr;
@ -2020,7 +1977,7 @@ Future<Reference<IConnection>> Net2::connectExternal(NetworkAddress toAddr) {
Future<Reference<IConnection>> Net2::connectExternalWithHostname(NetworkAddress toAddr, const std::string& hostname) {
if (toAddr.isTLS()) {
initTLS(ETLSInitState::CONNECT);
return SSLConnection::connectWithHostname(&this->reactor.ios, this->sslContextVar.get(), toAddr, hostname);
return SSLConnection::connect(&this->reactor.ios, this->sslContextVar.get(), toAddr, nullptr, hostname);
}
return connect(toAddr);
}

180
flow/TLSConfig.cpp
View File

@ -26,6 +26,7 @@
TLSPolicy::~TLSPolicy() {}
#include <algorithm>
#include <array>
#include <cstring>
#include <exception>
#include <map>
@ -241,60 +242,6 @@ bool TLSConfig::getDisablePlainTextConnection() const {
return tlsDisablePlainTextConnection;
}
LoadedTLSConfig TLSConfig::loadSync() const {
LoadedTLSConfig loaded;
const std::string certPath = getCertificatePathSync();
if (!certPath.empty()) {
try {
loaded.tlsCertBytes = readFileBytes(certPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
} catch (Error& e) {
fprintf(stderr, "Warning: Error reading TLS Certificate [%s]: %s\n", certPath.c_str(), e.what());
throw;
}
} else {
loaded.tlsCertBytes = tlsCertBytes;
}
const std::string keyPath = getKeyPathSync();
if (!keyPath.empty()) {
try {
loaded.tlsKeyBytes = readFileBytes(keyPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
} catch (Error& e) {
fprintf(stderr, "Warning: Error reading TLS Key [%s]: %s\n", keyPath.c_str(), e.what());
throw;
}
} else {
loaded.tlsKeyBytes = tlsKeyBytes;
}
const std::string CAPath = getCAPathSync();
if (!CAPath.empty()) {
try {
loaded.tlsCABytes = readFileBytes(CAPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
} catch (Error& e) {
fprintf(stderr, "Warning: Error reading TLS CA [%s]: %s\n", CAPath.c_str(), e.what());
throw;
}
} else {
loaded.tlsCABytes = tlsCABytes;
}
loaded.tlsPassword = tlsPassword;
loaded.tlsVerifyPeers = tlsVerifyPeers;
loaded.endpointType = endpointType;
loaded.tlsDisablePlainTextConnection = tlsDisablePlainTextConnection;
return loaded;
}
TLSPolicy::TLSPolicy(const LoadedTLSConfig& loaded, std::function<void()> on_failure)
: rules(), on_failure(std::move(on_failure)), is_client(loaded.getEndpointType() == TLSEndpointType::CLIENT) {
set_verify_peers(loaded.getVerifyPeers());
}
// And now do the same thing, but async...
static Future<Void> readEntireFile(std::string filename, std::string* destination) {
Reference<IAsyncFile> file = co_await IAsyncFileSystem::filesystem()->open(
filename, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0);
@ -306,59 +253,102 @@ static Future<Void> readEntireFile(std::string filename, std::string* destinatio
co_await file->read(&((*destination)[0]), filesize, 0);
}
class TLSMaterialSource {
public:
TLSMaterialSource(const char* name, std::string path, const std::string& configuredBytes, std::string& loadedBytes)
: name(name), path(std::move(path)), configuredBytes(configuredBytes), loadedBytes(loadedBytes) {}
bool hasFile() const { return !path.empty(); }
void copyConfiguredBytes() const { loadedBytes = configuredBytes; }
void loadSync() const {
if (!hasFile()) {
copyConfiguredBytes();
return;
}
try {
loadedBytes = readFileBytes(path, FLOW_KNOBS->CERT_FILE_MAX_SIZE);
} catch (Error& error) {
reportError(error);
throw;
}
}
Future<Void> loadAsync() const { return readEntireFile(path, &loadedBytes); }
void reportError(const Error& error) const {
fprintf(stderr, "Warning: Error reading TLS %s [%s]: %s\n", name, path.c_str(), error.what());
}
private:
const char* name;
std::string path;
const std::string& configuredBytes;
std::string& loadedBytes;
};
static std::array<TLSMaterialSource, 3> getTLSMaterialSources(const TLSConfig& config, LoadedTLSConfig& loaded) {
return { TLSMaterialSource(
"Certificate", config.getCertificatePathSync(), config.tlsCertBytes, loaded.tlsCertBytes),
TLSMaterialSource("Key", config.getKeyPathSync(), config.tlsKeyBytes, loaded.tlsKeyBytes),
TLSMaterialSource("CA", config.getCAPathSync(), config.tlsCABytes, loaded.tlsCABytes) };
}
static void copyTLSSettings(const TLSConfig& config, LoadedTLSConfig& loaded) {
loaded.tlsPassword = config.tlsPassword;
loaded.tlsVerifyPeers = config.tlsVerifyPeers;
loaded.endpointType = config.endpointType;
loaded.tlsDisablePlainTextConnection = config.tlsDisablePlainTextConnection;
}
LoadedTLSConfig TLSConfig::loadSync() const {
LoadedTLSConfig loaded;
for (const auto& material : getTLSMaterialSources(*this, loaded)) {
material.loadSync();
}
copyTLSSettings(*this, loaded);
return loaded;
}
TLSPolicy::TLSPolicy(const LoadedTLSConfig& loaded, std::function<void()> on_failure)
: rules(), on_failure(std::move(on_failure)), is_client(loaded.getEndpointType() == TLSEndpointType::CLIENT) {
set_verify_peers(loaded.getVerifyPeers());
}
Future<LoadedTLSConfig> TLSConfig::loadAsync(const TLSConfig* self) {
LoadedTLSConfig loaded;
auto materials = getTLSMaterialSources(*self, loaded);
std::vector<Future<Void>> reads;
int32_t certIdx = -1;
int32_t keyIdx = -1;
int32_t caIdx = -1;
std::string certPath = self->getCertificatePathSync();
if (!certPath.empty()) {
reads.push_back(readEntireFile(certPath, &loaded.tlsCertBytes));
certIdx = reads.size() - 1;
} else {
loaded.tlsCertBytes = self->tlsCertBytes;
}
std::string keyPath = self->getKeyPathSync();
if (!keyPath.empty()) {
reads.push_back(readEntireFile(keyPath, &loaded.tlsKeyBytes));
keyIdx = reads.size() - 1;
} else {
loaded.tlsKeyBytes = self->tlsKeyBytes;
}
std::string CAPath = self->getCAPathSync();
if (!CAPath.empty()) {
reads.push_back(readEntireFile(CAPath, &loaded.tlsCABytes));
caIdx = reads.size() - 1;
} else {
loaded.tlsCABytes = self->tlsCABytes;
for (const auto& material : materials) {
if (material.hasFile()) {
reads.push_back(material.loadAsync());
} else {
material.copyConfiguredBytes();
}
}
try {
co_await waitForAll(reads);
} catch (Error& e) {
if (certIdx != -1 && reads[certIdx].isError()) {
fprintf(stderr, "Warning: Error reading TLS Certificate [%s]: %s\n", certPath.c_str(), e.what());
} else if (keyIdx != -1 && reads[keyIdx].isError()) {
fprintf(stderr, "Warning: Error reading TLS Key [%s]: %s\n", keyPath.c_str(), e.what());
} else if (caIdx != -1 && reads[caIdx].isError()) {
fprintf(stderr, "Warning: Error reading TLS Key [%s]: %s\n", CAPath.c_str(), e.what());
} else {
fprintf(stderr, "Warning: Error reading TLS needed file: %s\n", e.what());
} catch (Error& error) {
bool reported = false;
size_t readIndex = 0;
for (const auto& material : materials) {
if (material.hasFile() && reads[readIndex++].isError()) {
material.reportError(error);
reported = true;
break;
}
}
if (!reported) {
fprintf(stderr, "Warning: Error reading TLS needed file: %s\n", error.what());
}
throw;
}
loaded.tlsPassword = self->tlsPassword;
loaded.tlsVerifyPeers = self->tlsVerifyPeers;
loaded.endpointType = self->endpointType;
loaded.tlsDisablePlainTextConnection = self->tlsDisablePlainTextConnection;
copyTLSSettings(*self, loaded);
co_return loaded;
}

188
flow/include/flow/CoroutinesImpl.h
View File

@ -606,12 +606,58 @@ template <class Awaiter, class ValueType>
struct AwaitableResume<Awaiter, ValueType, /* IsStream = */ true, /* ReturnsExplicitVoid = */ true>
: AwaitableResume<Awaiter, ValueType, /* IsStream = */ true, /* ReturnsExplicitVoid = */ false> {};
template <class Awaiter, class PromiseType>
class AwaitableFutureState : public AwaitCancelHandler {
public:
[[maybe_unused]] [[nodiscard]] bool await_ready() const {
auto* self = static_cast<const Awaiter*>(this);
if (actorWaitStateIsCancelled(self->pt->waitState())) {
self->pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
return true;
}
return self->future.isReady();
}
[[maybe_unused]] void await_suspend(n_coroutine::coroutine_handle<> handle) {
auto* self = static_cast<Awaiter*>(this);
self->pt->setHandle(handle);
self->pt->waitState() = ACTOR_WAIT_STATE_WAITING;
auto callbackFuture = self->getCallbackFuture();
callbackFuture.addCallbackAndClear(self);
self->pt->setCancelHandler(this);
}
// NoThrowOnCancel destroys the coroutine frame instead of resuming this
// awaiter, so detach its callback before destroying the frame.
void cancelWait() override { static_cast<Awaiter*>(this)->remove(); }
bool resumeImpl() {
auto* self = static_cast<Awaiter*>(this);
self->pt->clearCancelHandler(this);
switch (self->pt->waitState()) {
case ACTOR_WAIT_STATE_CANCELLED:
self->remove();
case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
// await_ready() observed cancellation before callback registration.
throw actor_cancelled();
}
bool wasReady = self->pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
if (actorWaitStateIsWaiting(self->pt->waitState())) {
self->remove();
self->pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
}
return wasReady;
}
};
// Awaiter for `Future<T>` and `FutureStream<T>` values transformed through a
// coroutine promise.
template <class PromiseType, class ValueType, bool IsStream, bool ReturnsExplicitVoid = false>
struct AwaitableFuture
: std::conditional_t<IsStream, SingleCallback<ToFutureVal<ValueType>>, Callback<ToFutureVal<ValueType>>>,
AwaitCancelHandler,
AwaitableFutureState<AwaitableFuture<PromiseType, ValueType, IsStream, ReturnsExplicitVoid>, PromiseType>,
AwaitableResume<AwaitableFuture<PromiseType, ValueType, IsStream, ReturnsExplicitVoid>,
ValueType,
IsStream,
@ -647,60 +693,18 @@ struct AwaitableFuture
pt->resume();
}
[[maybe_unused]] [[nodiscard]] bool await_ready() const {
if (actorWaitStateIsCancelled(pt->waitState())) {
pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
return true;
}
return future.isReady();
}
[[maybe_unused]] void await_suspend(n_coroutine::coroutine_handle<> h) {
// Create a coroutine callback if it's the first time being suspended
pt->setHandle(h);
// Set wait_state and add callback
pt->waitState() = ACTOR_WAIT_STATE_WAITING;
auto getCallbackFuture() const {
if constexpr (IsStream) {
auto sf = future;
sf.addCallbackAndClear(this);
return future;
} else {
StrictFuture<FutureValue> sf = future;
sf.addCallbackAndClear(this);
return StrictFuture<FutureValue>(future);
}
pt->setCancelHandler(this);
}
// NoThrowOnCancel destroys the coroutine frame instead of resuming this
// awaiter, so detach the callback from the Future before that happens.
void cancelWait() override { this->remove(); }
bool resumeImpl() {
pt->clearCancelHandler(this);
// If actor is cancelled, then throw actor_cancelled()
switch (pt->waitState()) {
case ACTOR_WAIT_STATE_CANCELLED:
this->remove();
case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
// await_ready() observed cancellation before await_suspend() registered a callback, so there is nothing to
// remove here.
throw actor_cancelled();
}
bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
// Actor return from waiting, remove callback and reset wait_state.
if (actorWaitStateIsWaiting(pt->waitState())) {
this->remove();
pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
}
return wasReady;
}
};
template <class PromiseType, class ValueType>
struct AwaitableFutureOwning : Callback<ToFutureVal<ValueType>>, AwaitCancelHandler {
struct AwaitableFutureOwning : Callback<ToFutureVal<ValueType>>,
AwaitableFutureState<AwaitableFutureOwning<PromiseType, ValueType>, PromiseType> {
using FutureValue = ToFutureVal<ValueType>;
Future<FutureValue> future;
PromiseType* pt = nullptr;
@ -711,43 +715,7 @@ struct AwaitableFutureOwning : Callback<ToFutureVal<ValueType>>, AwaitCancelHand
void fire(FutureValue&&) override { pt->resume(); }
void error(Error) override { pt->resume(); }
[[maybe_unused]] [[nodiscard]] bool await_ready() const {
if (actorWaitStateIsCancelled(pt->waitState())) {
pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
return true;
}
return future.isReady();
}
[[maybe_unused]] void await_suspend(n_coroutine::coroutine_handle<> h) {
pt->setHandle(h);
pt->waitState() = ACTOR_WAIT_STATE_WAITING;
StrictFuture<FutureValue> sf = future;
sf.addCallbackAndClear(this);
pt->setCancelHandler(this);
}
// NoThrowOnCancel destroys the coroutine frame instead of resuming this
// awaiter, so detach the callback from the Future before that happens.
void cancelWait() override { this->remove(); }
bool resumeImpl() {
pt->clearCancelHandler(this);
switch (pt->waitState()) {
case ACTOR_WAIT_STATE_CANCELLED:
this->remove();
case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
throw actor_cancelled();
}
bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
if (actorWaitStateIsWaiting(pt->waitState())) {
this->remove();
pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
}
return wasReady;
}
StrictFuture<FutureValue> getCallbackFuture() const { return future; }
};
template <class PromiseType, class ValueType>
@ -799,7 +767,7 @@ struct AwaitableFutureErrorOr : AwaitableFutureOwning<PromiseType, SourceValue>
template <class PromiseType, class ValueType, bool ReturnsExplicitVoid = false>
struct ThreadAwaitableFutureStream
: SingleCallback<ToFutureVal<ValueType>>,
AwaitCancelHandler,
AwaitableFutureState<ThreadAwaitableFutureStream<PromiseType, ValueType, ReturnsExplicitVoid>, PromiseType>,
AwaitableResume<ThreadAwaitableFutureStream<PromiseType, ValueType, ReturnsExplicitVoid>,
ValueType,
true,
@ -826,53 +794,7 @@ struct ThreadAwaitableFutureStream
pt->resume();
}
[[maybe_unused]] [[nodiscard]] bool await_ready() const {
if (actorWaitStateIsCancelled(pt->waitState())) {
pt->waitState() = ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK;
// actor was cancelled
return true;
}
return future.isReady();
}
[[maybe_unused]] void await_suspend(n_coroutine::coroutine_handle<> h) {
// Create a coroutine callback if it's the first time being suspended
pt->setHandle(h);
// Set wait_state and add callback
pt->waitState() = ACTOR_WAIT_STATE_WAITING;
auto sf = future;
sf.addCallbackAndClear(this);
pt->setCancelHandler(this);
}
// NoThrowOnCancel destroys the coroutine frame instead of resuming this
// awaiter, so detach the callback from the FutureStream before that happens.
void cancelWait() override { this->remove(); }
bool resumeImpl() {
pt->clearCancelHandler(this);
// If actor is cancelled, then throw actor_cancelled()
switch (pt->waitState()) {
case ACTOR_WAIT_STATE_CANCELLED:
this->remove();
case ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK:
// await_ready() observed cancellation before await_suspend() registered a callback, so there is nothing to
// remove here.
// if the wait_state is -1 we still have to throw, so we fall through to the -2 case
throw actor_cancelled();
}
bool wasReady = pt->waitState() == ACTOR_WAIT_STATE_NOT_WAITING;
// Actor return from waiting, remove callback and reset wait_state.
if (actorWaitStateIsWaiting(pt->waitState())) {
this->remove();
pt->waitState() = ACTOR_WAIT_STATE_NOT_WAITING;
}
return wasReady;
}
FutureType getCallbackFuture() const { return future; }
};
// Promise for fire-and-forget coroutines. It deliberately has no result SAV:

9
flow/include/flow/FastAlloc.h
View File

@ -32,13 +32,6 @@
// #define ALLOC_INSTRUMENTATION_STDOUT ENABLED(NOT_IN_CLEAN)
// #define ALLOC_INSTRUMENTATION ENABLED(NOT_IN_CLEAN)
// The form "(1==1)" in this context is used to satisfy both clang and vc++ with a single syntax. Clang rejects "1"
// and vc++ rejects "true".
// FIXME: this has been set to true for 4+ years. We probably do not need the "not thread safe"
// version of the code. Consider removing this and just making it thread safe.
// Also, explain why thread safety is required here and not elsewhere (e.g. Arena and ArenaBlock).
#define FASTALLOC_THREAD_SAFE (FLOW_THREAD_SAFE || (1 == 1))
#if VALGRIND
#include <drd.h>
#include <memcheck.h>
@ -174,8 +167,6 @@ private:
return data;
}
static void* freelist;
static void getMagazine();
static void releaseMagazine(void*);
};

42
tests/TestRunner/fdb_test_runner/local_cluster.py
View File

@ -531,17 +531,27 @@ knob_min_trace_severity=5
)
self.active_servers.remove(server_id)
def _wait_for_status_update(self, expected, read_status, description, timeout):
deadline = time.monotonic() + timeout
while True:
actual = read_status()
if actual == expected:
return
if time.monotonic() >= deadline:
raise AssertionError(
"Failed to apply {} changes after {}sec. Expected: {}, Actual: {}".format(
description, timeout, expected, actual
)
)
time.sleep(RETRY_INTERVAL_SEC)
# Wait until changes to the set of servers (additions & removals) are applied
def wait_for_server_update(self, timeout=CLUSTER_UPDATE_TIMEOUT_SEC):
time_limit = time.time() + timeout
servers_found = set()
while time.time() <= time_limit:
servers_found = self.get_all_servers_from_status()
if servers_found != self.active_servers:
break
time.sleep(RETRY_INTERVAL_SEC)
assert "Failed to apply server changes after {}sec. Expected: {}, Actual: {}".format(
timeout, self.active_servers, servers_found
self._wait_for_status_update(
self.active_servers,
self.get_all_servers_from_status,
"server",
timeout,
)
# Apply changes to the set of the coordinators, based on the current value of self.coordinators
@ -554,15 +564,11 @@ knob_min_trace_severity=5
# Wait until the changes to the set of the coordinators are applied
def wait_for_coordinator_update(self, timeout=CLUSTER_UPDATE_TIMEOUT_SEC):
time_limit = time.time() + timeout
coord_found = set()
while time.time() <= time_limit:
coord_found = self.get_coordinators_from_status()
if coord_found != self.coordinators:
break
time.sleep(RETRY_INTERVAL_SEC)
assert "Failed to apply coordinator changes after {}sec. Expected: {}, Actual: {}".format(
timeout, self.coordinators, coord_found
self._wait_for_status_update(
self.coordinators,
self.get_coordinators_from_status,
"coordinator",
timeout,
)
# Check if the cluster file was successfully updated too
connection_string = open(self.cluster_file, "r").read()