wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into trace-log-refactor 

# Conflicts:
#	fdbserver/QuietDatabase.actor.cpp
#	fdbserver/Status.actor.cpp
#	flow/Trace.cpp
This commit is contained in:
A.J. Beamon 2018-06-08 13:24:30 -07:00
parent 5848055620 943dfa278b
commit 0ca51989bb
118 changed files with 1590 additions and 1542 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/flow/tester/Tester.actor.cpp
View File

@ -1816,7 +1816,7 @@ int main( int argc, char** argv ) {
} }
catch (std::exception& e) { catch (std::exception& e) {
fprintf(stderr, "std::exception: %s\n", e.what()); fprintf(stderr, "std::exception: %s\n", e.what());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("std::exception", e.what()); TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
flushAndExit(FDB_EXIT_MAIN_EXCEPTION); flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
} }
} }

24
documentation/sphinx/source/downloads.rst
View File

@ -10,38 +10,38 @@ macOS
The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server.
* `FoundationDB-5.2.1.pkg <https://www.foundationdb.org/downloads/5.2.1/macOS/installers/FoundationDB-5.2.1.pkg>`_ * `FoundationDB-5.2.3.pkg <https://www.foundationdb.org/downloads/5.2.3/macOS/installers/FoundationDB-5.2.3.pkg>`_
Ubuntu Ubuntu
------ ------
The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
* `foundationdb-clients-5.2.1-1_amd64.deb <https://www.foundationdb.org/downloads/5.2.1/ubuntu/installers/foundationdb-clients_5.2.1-1_amd64.deb>`_ * `foundationdb-clients-5.2.3-1_amd64.deb <https://www.foundationdb.org/downloads/5.2.3/ubuntu/installers/foundationdb-clients_5.2.3-1_amd64.deb>`_
* `foundationdb-server-5.2.1-1_amd64.deb <https://www.foundationdb.org/downloads/5.2.1/ubuntu/installers/foundationdb-server_5.2.1-1_amd64.deb>`_ (depends on the clients package) * `foundationdb-server-5.2.3-1_amd64.deb <https://www.foundationdb.org/downloads/5.2.3/ubuntu/installers/foundationdb-server_5.2.3-1_amd64.deb>`_ (depends on the clients package)
RHEL/CentOS EL6 RHEL/CentOS EL6
--------------- ---------------
The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x.
* `foundationdb-clients-5.2.1-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.1/rhel6/installers/foundationdb-clients-5.2.1-1.el6.x86_64.rpm>`_ * `foundationdb-clients-5.2.3-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.3/rhel6/installers/foundationdb-clients-5.2.3-1.el6.x86_64.rpm>`_
* `foundationdb-server-5.2.1-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.1/rhel6/installers/foundationdb-server-5.2.1-1.el6.x86_64.rpm>`_ (depends on the clients package) * `foundationdb-server-5.2.3-1.el6.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.3/rhel6/installers/foundationdb-server-5.2.3-1.el6.x86_64.rpm>`_ (depends on the clients package)
RHEL/CentOS EL7 RHEL/CentOS EL7
--------------- ---------------
The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x.
* `foundationdb-clients-5.2.1-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.1/rhel7/installers/foundationdb-clients-5.2.1-1.el7.x86_64.rpm>`_ * `foundationdb-clients-5.2.3-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.3/rhel7/installers/foundationdb-clients-5.2.3-1.el7.x86_64.rpm>`_
* `foundationdb-server-5.2.1-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.1/rhel7/installers/foundationdb-server-5.2.1-1.el7.x86_64.rpm>`_ (depends on the clients package) * `foundationdb-server-5.2.3-1.el7.x86_64.rpm <https://www.foundationdb.org/downloads/5.2.3/rhel7/installers/foundationdb-server-5.2.3-1.el7.x86_64.rpm>`_ (depends on the clients package)
Windows Windows
------- -------
The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server.
* `foundationdb-5.2.1-x64.msi <https://www.foundationdb.org/downloads/5.2.1/windows/installers/foundationdb-5.2.1-x64.msi>`_ * `foundationdb-5.2.3-x64.msi <https://www.foundationdb.org/downloads/5.2.3/windows/installers/foundationdb-5.2.3-x64.msi>`_
API Language Bindings API Language Bindings
===================== =====================
@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part
If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package: If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package:
* `foundationdb-5.2.1.tar.gz <https://www.foundationdb.org/downloads/5.2.1/bindings/python/foundationdb-5.2.1.tar.gz>`_ * `foundationdb-5.2.3.tar.gz <https://www.foundationdb.org/downloads/5.2.3/bindings/python/foundationdb-5.2.3.tar.gz>`_
Ruby 1.9.3/2.0.0+ Ruby 1.9.3/2.0.0+
----------------- -----------------
* `fdb-5.2.1.gem <https://www.foundationdb.org/downloads/5.2.1/bindings/ruby/fdb-5.2.1.gem>`_ * `fdb-5.2.3.gem <https://www.foundationdb.org/downloads/5.2.3/bindings/ruby/fdb-5.2.3.gem>`_
Java 8+ Java 8+
------- -------
* `fdb-java-5.2.1.jar <https://www.foundationdb.org/downloads/5.2.1/bindings/java/fdb-java-5.2.1.jar>`_ * `fdb-java-5.2.3.jar <https://www.foundationdb.org/downloads/5.2.3/bindings/java/fdb-java-5.2.3.jar>`_
* `fdb-java-5.2.1-javadoc.jar <https://www.foundationdb.org/downloads/5.2.1/bindings/java/fdb-java-5.2.1-javadoc.jar>`_ * `fdb-java-5.2.3-javadoc.jar <https://www.foundationdb.org/downloads/5.2.3/bindings/java/fdb-java-5.2.3-javadoc.jar>`_
Go 1.1+ Go 1.1+
------- -------

2
documentation/sphinx/source/old-release-notes/release-notes-450.rst
View File

@ -83,7 +83,7 @@ Fixes
* A multi-version API external client that was a duplicate could cause a crash at client shutdown. <rdar://problem/25838039> * A multi-version API external client that was a duplicate could cause a crash at client shutdown. <rdar://problem/25838039>
* A multi-version API external client that failed to load would cause a crash at client shutdown. <rdar://problem/26611919> * A multi-version API external client that failed to load would cause a crash at client shutdown. <rdar://problem/26611919>
* Status could report an incorrect database size in rare scenarios. <rdar://problem/25295683> * Status could report an incorrect database size in rare scenarios. <rdar://problem/25295683>
* Data distribution would stopped working (both exclude and new recruitment) if a call to commit on the storage server could hang indefinitely. <rdar://problem/26276166> * Data distribution would stop working (both exclude and new recruitment) if a call to commit on the storage server could hang indefinitely. <rdar://problem/26276166>
* ``fdbcli`` would quit on some commands that were not errors or continue after some real errors when run with ``--exec``. <rdar://problem/25755317> * ``fdbcli`` would quit on some commands that were not errors or continue after some real errors when run with ``--exec``. <rdar://problem/25755317>
* Trace files generated by clients using the multi-version API could have name collisions. <rdar://problem/25884486> * Trace files generated by clients using the multi-version API could have name collisions. <rdar://problem/25884486>

1
documentation/sphinx/source/release-notes.rst
View File

@ -29,6 +29,7 @@ Other Changes
------------- -------------
* Does not support upgrades from any version older than 5.0. * Does not support upgrades from any version older than 5.0.
* Renamed the trace log attribute ``logGroup`` to ``LogGroup``.
Earlier release notes Earlier release notes
--------------------- ---------------------

4
fdbbackup/backup.actor.cpp
View File

@ -1159,7 +1159,7 @@ ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr
if (backupVersion[i].get().present()) { if (backupVersion[i].get().present()) {
double seconds_behind = ((double)readVer - BinaryReader::fromStringRef<Version>(backupVersion[i].get().get(), Unversioned())) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; double seconds_behind = ((double)readVer - BinaryReader::fromStringRef<Version>(backupVersion[i].get().get(), Unversioned())) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
tagRoot.create("seconds_behind") = seconds_behind; tagRoot.create("seconds_behind") = seconds_behind;
//TraceEvent("BackupMetrics").detail("secondsBehind", seconds_behind); //TraceEvent("BackupMetrics").detail("SecondsBehind", seconds_behind);
} }
tagRoot.create("backup_state") = BackupAgentBase::getStateText(status); tagRoot.create("backup_state") = BackupAgentBase::getStateText(status);
@ -2999,7 +2999,7 @@ int main(int argc, char* argv[]) {
TraceEvent(SevError, "MainError").error(e); TraceEvent(SevError, "MainError").error(e);
status = FDB_EXIT_MAIN_ERROR; status = FDB_EXIT_MAIN_ERROR;
} catch (std::exception& e) { } catch (std::exception& e) {
TraceEvent(SevError, "MainError").error(unknown_error()).detail("std::exception", e.what()); TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
status = FDB_EXIT_MAIN_EXCEPTION; status = FDB_EXIT_MAIN_EXCEPTION;
} }

6
fdbcli/fdbcli.actor.cpp
View File

@ -2060,7 +2060,7 @@ void fdbcli_comp_cmd(std::string const& text, std::vector<std::string>& lc) {
void LogCommand(std::string line, UID randomID, std::string errMsg) { void LogCommand(std::string line, UID randomID, std::string errMsg) {
printf("%s\n", errMsg.c_str()); printf("%s\n", errMsg.c_str());
TraceEvent(SevInfo, "CLICommandLog", randomID).detail("command", printable(StringRef(line))).detail("error", printable(StringRef(errMsg))); TraceEvent(SevInfo, "CLICommandLog", randomID).detail("Command", printable(StringRef(line))).detail("Error", printable(StringRef(errMsg)));
} }
struct CLIOptions { struct CLIOptions {
@ -2305,7 +2305,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
try { try {
state UID randomID = g_random->randomUniqueID(); state UID randomID = g_random->randomUniqueID();
TraceEvent(SevInfo, "CLICommandLog", randomID).detail("command", printable(StringRef(line))); TraceEvent(SevInfo, "CLICommandLog", randomID).detail("Command", printable(StringRef(line)));
bool malformed, partial; bool malformed, partial;
state std::vector<std::vector<StringRef>> parsed = parseLine(line, malformed, partial); state std::vector<std::vector<StringRef>> parsed = parseLine(line, malformed, partial);
@ -3079,7 +3079,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
is_error = true; is_error = true;
} }
TraceEvent(SevInfo, "CLICommandLog", randomID).detail("command", printable(StringRef(line))).detail("is_error", is_error); TraceEvent(SevInfo, "CLICommandLog", randomID).detail("Command", printable(StringRef(line))).detail("IsError", is_error);
} catch (Error& e) { } catch (Error& e) {
if(e.code() != error_code_actor_cancelled) if(e.code() != error_code_actor_cancelled)

2
fdbclient/BackupAgent.h
View File

@ -423,7 +423,7 @@ bool copyParameter(Reference<Task> source, Reference<Task> dest, Key key);
Version getVersionFromString(std::string const& value); Version getVersionFromString(std::string const& value);
Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE); Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version endVersion, Key destUidValue, int blockSize = CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE);
Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid); Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid);
Future<Void> eraseLogData(Database cx, Key logUidValue, Key destUidValue, Optional<Version> beginVersion = Optional<Version>(), Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0); Future<Void> eraseLogData(Database cx, Key logUidValue, Key destUidValue, Optional<Version> endVersion = Optional<Version>(), bool checkBackupUid = false, Version backupUid = 0);
Key getApplyKey( Version version, Key backupUid ); Key getApplyKey( Version version, Key backupUid );
std::pair<uint64_t, uint32_t> decodeBKMutationLogKey(Key key); std::pair<uint64_t, uint32_t> decodeBKMutationLogKey(Key key);
Standalone<VectorRef<MutationRef>> decodeBackupLogValue(StringRef value); Standalone<VectorRef<MutationRef>> decodeBackupLogValue(StringRef value);

256
fdbclient/BackupAgentBase.actor.cpp
View File

@ -77,7 +77,7 @@ Standalone<VectorRef<KeyRangeRef>> getLogRanges(Version beginVersion, Version en
Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin); Key baLogRangePrefix = destUidValue.withPrefix(backupLogKeys.begin);
//TraceEvent("getLogRanges").detail("destUidValue", destUidValue).detail("prefix", printable(StringRef(baLogRangePrefix))); //TraceEvent("GetLogRanges").detail("DestUidValue", destUidValue).detail("Prefix", printable(StringRef(baLogRangePrefix)));
for (int64_t vblock = beginVersion / blockSize; vblock < (endVersion + blockSize - 1) / blockSize; ++vblock) { for (int64_t vblock = beginVersion / blockSize; vblock < (endVersion + blockSize - 1) / blockSize; ++vblock) {
int64_t tb = vblock * blockSize / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; int64_t tb = vblock * blockSize / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
@ -100,7 +100,7 @@ Standalone<VectorRef<KeyRangeRef>> getApplyRanges(Version beginVersion, Version
Key baLogRangePrefix = backupUid.withPrefix(applyLogKeys.begin); Key baLogRangePrefix = backupUid.withPrefix(applyLogKeys.begin);
//TraceEvent("getLogRanges").detail("backupUid", backupUid).detail("prefix", printable(StringRef(baLogRangePrefix))); //TraceEvent("GetLogRanges").detail("BackupUid", backupUid).detail("Prefix", printable(StringRef(baLogRangePrefix)));
for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) { for (int64_t vblock = beginVersion / CLIENT_KNOBS->APPLY_BLOCK_SIZE; vblock < (endVersion + CLIENT_KNOBS->APPLY_BLOCK_SIZE - 1) / CLIENT_KNOBS->APPLY_BLOCK_SIZE; ++vblock) {
int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; int64_t tb = vblock * CLIENT_KNOBS->APPLY_BLOCK_SIZE / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
@ -148,8 +148,8 @@ Standalone<VectorRef<MutationRef>> decodeBackupLogValue(StringRef value) {
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t)); memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t); offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001){ if (protocolVersion <= 0x0FDB00A200090001){
TraceEvent(SevError, "decodeBackupLogValue").detail("incompatible_protocol_version", protocolVersion) TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion)
.detail("valueSize", value.size()).detail("value", printable(value)); .detail("ValueSize", value.size()).detail("Value", printable(value));
throw incompatible_protocol_version(); throw incompatible_protocol_version();
} }
@ -188,14 +188,14 @@ Standalone<VectorRef<MutationRef>> decodeBackupLogValue(StringRef value) {
ASSERT(consumed == totalBytes); ASSERT(consumed == totalBytes);
if (value.size() != offset) { if (value.size() != offset) {
TraceEvent(SevError, "BA_decodeBackupLogValue").detail("unexpected_extra_data_size", value.size()).detail("offset", offset).detail("totalBytes", totalBytes).detail("consumed", consumed).detail("originalOffset", originalOffset); TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset);
throw restore_corrupted_data(); throw restore_corrupted_data();
} }
return result; return result;
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_decodeBackupLogValue").error(e).GetLastError().detail("valueSize", value.size()).detail("value", printable(value)); TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", printable(value));
throw; throw;
} }
} }
@ -207,8 +207,8 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
memcpy(&protocolVersion, value.begin(), sizeof(uint64_t)); memcpy(&protocolVersion, value.begin(), sizeof(uint64_t));
offset += sizeof(uint64_t); offset += sizeof(uint64_t);
if (protocolVersion <= 0x0FDB00A200090001){ if (protocolVersion <= 0x0FDB00A200090001){
TraceEvent(SevError, "decodeBackupLogValue").detail("incompatible_protocol_version", protocolVersion) TraceEvent(SevError, "DecodeBackupLogValue").detail("IncompatibleProtocolVersion", protocolVersion)
.detail("valueSize", value.size()).detail("value", printable(value)); .detail("ValueSize", value.size()).detail("Value", printable(value));
throw incompatible_protocol_version(); throw incompatible_protocol_version();
} }
@ -280,7 +280,7 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
} }
else { else {
Version ver = key_version->rangeContaining(logValue.param1).value(); Version ver = key_version->rangeContaining(logValue.param1).value();
//TraceEvent("ApplyMutation").detail("logValue", logValue.toString()).detail("version", version).detail("ver", ver).detail("apply", version > ver && ver != invalidVersion); //TraceEvent("ApplyMutation").detail("LogValue", logValue.toString()).detail("Version", version).detail("Ver", ver).detail("Apply", version > ver && ver != invalidVersion);
if (version > ver && ver != invalidVersion) { if (version > ver && ver != invalidVersion) {
if(removePrefix.size()) { if(removePrefix.size()) {
logValue.param1 = logValue.param1.removePrefix(removePrefix); logValue.param1 = logValue.param1.removePrefix(removePrefix);
@ -298,12 +298,12 @@ void decodeBackupLogValue(Arena& arena, VectorRef<MutationRef>& result, int& mut
ASSERT(consumed == totalBytes); ASSERT(consumed == totalBytes);
if (value.size() != offset) { if (value.size() != offset) {
TraceEvent(SevError, "BA_decodeBackupLogValue").detail("unexpected_extra_data_size", value.size()).detail("offset", offset).detail("totalBytes", totalBytes).detail("consumed", consumed).detail("originalOffset", originalOffset); TraceEvent(SevError, "BA_DecodeBackupLogValue").detail("UnexpectedExtraDataSize", value.size()).detail("Offset", offset).detail("TotalBytes", totalBytes).detail("Consumed", consumed).detail("OriginalOffset", originalOffset);
throw restore_corrupted_data(); throw restore_corrupted_data();
} }
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_decodeBackupLogValue").error(e).GetLastError().detail("valueSize", value.size()).detail("value", printable(value)); TraceEvent(e.code() == error_code_restore_missing_data ? SevWarn : SevError, "BA_DecodeBackupLogValue").error(e).GetLastError().detail("ValueSize", value.size()).detail("Value", printable(value));
throw; throw;
} }
} }
@ -312,7 +312,7 @@ ACTOR Future<Void> logErrorWorker(Reference<ReadYourWritesTransaction> tr, Key k
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if(now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) { if(now() - lastErrorTime > CLIENT_KNOBS->BACKUP_ERROR_DELAY) {
TraceEvent("BA_logError").detail("key", printable(keyErrors)).detail("message", message); TraceEvent("BA_LogError").detail("Key", printable(keyErrors)).detail("Message", message);
lastErrorTime = now(); lastErrorTime = now();
} }
tr->set(keyErrors, message); tr->set(keyErrors, message);
@ -423,19 +423,19 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
int index(0); int index(0);
for (auto & s : rangevalue){ for (auto & s : rangevalue){
uint64_t groupKey = groupBy(s.key).first; uint64_t groupKey = groupBy(s.key).first;
//TraceEvent("log_readCommitted").detail("groupKey", groupKey).detail("skipGroup", skipGroup).detail("nextKey", printable(nextKey.key)).detail("end", printable(end.key)).detail("valuesize", value.size()).detail("index",index++).detail("size",s.value.size()); //TraceEvent("Log_ReadCommitted").detail("GroupKey", groupKey).detail("SkipGroup", skipGroup).detail("NextKey", printable(nextKey.key)).detail("End", printable(end.key)).detail("Valuesize", value.size()).detail("Index",index++).detail("Size",s.value.size());
if (groupKey != skipGroup){ if (groupKey != skipGroup){
if (rcGroup.version == -1){ if (rcGroup.version == -1){
rcGroup.version = tr.getReadVersion().get(); rcGroup.version = tr.getReadVersion().get();
rcGroup.groupKey = groupKey; rcGroup.groupKey = groupKey;
} }
else if (rcGroup.groupKey != groupKey) { else if (rcGroup.groupKey != groupKey) {
//TraceEvent("log_readCommitted").detail("sendGroup0", rcGroup.groupKey).detail("itemSize", rcGroup.items.size()).detail("data_length",rcGroup.items[0].value.size()); //TraceEvent("Log_ReadCommitted").detail("SendGroup0", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("Data_length",rcGroup.items[0].value.size());
//state uint32_t len(0); //state uint32_t len(0);
//for (size_t j = 0; j < rcGroup.items.size(); ++j) { //for (size_t j = 0; j < rcGroup.items.size(); ++j) {
// len += rcGroup.items[j].value.size(); // len += rcGroup.items[j].value.size();
//} //}
//TraceEvent("SendGroup").detail("groupKey", rcGroup.groupKey).detail("version", rcGroup.version).detail("length", len).detail("releaser.remaining", releaser.remaining); //TraceEvent("SendGroup").detail("GroupKey", rcGroup.groupKey).detail("Version", rcGroup.version).detail("Length", len).detail("Releaser.remaining", releaser.remaining);
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
results.send(rcGroup); results.send(rcGroup);
@ -454,7 +454,7 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
if (rcGroup.version != -1){ if (rcGroup.version != -1){
releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point releaser.remaining -= rcGroup.items.expectedSize(); //its the responsibility of the caller to release after this point
ASSERT(releaser.remaining >= 0); ASSERT(releaser.remaining >= 0);
//TraceEvent("log_readCommitted").detail("sendGroup1", rcGroup.groupKey).detail("itemSize", rcGroup.items.size()).detail("data_length", rcGroup.items[0].value.size()); //TraceEvent("Log_ReadCommitted").detail("SendGroup1", rcGroup.groupKey).detail("ItemSize", rcGroup.items.size()).detail("DataLength", rcGroup.items[0].value.size());
results.send(rcGroup); results.send(rcGroup);
} }
@ -619,151 +619,117 @@ ACTOR Future<Void> applyMutations(Database cx, Key uid, Key addPrefix, Key remov
beginVersion = newEndVersion; beginVersion = newEndVersion;
} }
} catch( Error &e ) { } catch( Error &e ) {
TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "AM_error").error(e); TraceEvent(e.code() == error_code_restore_missing_data ? SevWarnAlways : SevError, "ApplyMutationsError").error(e);
throw; throw;
} }
} }
ACTOR Future<Void> _clearLogRanges(Reference<ReadYourWritesTransaction> tr, bool clearVersionHistory, Key logUidValue, Key destUidValue, Version beginVersion, Version endVersion) { ACTOR static Future<Void> _eraseLogData(Database cx, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix); state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix);
state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath); state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY)); if (!destUidValue.size()) {
// Make sure version history key does exist and lower the beginVersion if needed
bool foundSelf = false;
for (auto backupVersion : backupVersions) {
Key currLogUidValue = backupVersion.key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currLogUidValue == logUidValue) {
foundSelf = true;
beginVersion = std::min(beginVersion, BinaryReader::fromStringRef<Version>(backupVersion.value, Unversioned()));
}
}
// Do not clear anything if version history key cannot be found
if (!foundSelf) {
return Void(); return Void();
} }
Version nextSmallestVersion = endVersion; state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
bool clearLogRangesRequired = true; loop{
try {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
// More than one backup/DR with the same range if (checkBackupUid) {
if (backupVersions.size() > 1) { Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
for (auto backupVersion : backupVersions) { Optional<Value> v = wait( tr->get( sourceStates.pack(DatabaseBackupAgent::keyFolderId) ) );
Key currLogUidValue = backupVersion.key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue); if(v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
Version currVersion = BinaryReader::fromStringRef<Version>(backupVersion.value, Unversioned());
if (currLogUidValue == logUidValue) {
continue;
} else if (currVersion > beginVersion) {
nextSmallestVersion = std::min(currVersion, nextSmallestVersion);
} else {
// If we can find a version less than or equal to beginVersion, clearing log ranges is not required
clearLogRangesRequired = false;
break;
}
}
}
if (clearVersionHistory && backupVersions.size() == 1) {
// Clear version history
tr->clear(prefixRange(backupLatestVersionsPath));
// Clear everything under blog/[destUid]
tr->clear(prefixRange(destUidValue.withPrefix(backupLogKeys.begin)));
// Disable committing mutations into blog
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
} else {
if (clearVersionHistory) {
// Clear current backup version history
tr->clear(backupLatestVersionsKey);
} else {
// Update current backup latest version
tr->set(backupLatestVersionsKey, BinaryWriter::toValue<Version>(endVersion, Unversioned()));
}
// Clear log ranges if needed
if (clearLogRangesRequired) {
Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(beginVersion, nextSmallestVersion, destUidValue);
for (auto& range : ranges) {
tr->clear(range);
}
}
}
return Void();
}
// The difference between beginVersion and endVersion should not be too large
Future<Void> clearLogRanges(Reference<ReadYourWritesTransaction> tr, bool clearVersionHistory, Key logUidValue, Key destUidValue, Version beginVersion, Version endVersion) {
return _clearLogRanges(tr, clearVersionHistory, logUidValue, destUidValue, beginVersion, endVersion);
}
ACTOR static Future<Void> _eraseLogData(Database cx, Key logUidValue, Key destUidValue, Optional<Version> beginVersion, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
if ((beginVersion.present() && endVersion.present() && endVersion.get() <= beginVersion.get()) || !destUidValue.size())
return Void();
state Version currBeginVersion;
state Version endVersionValue;
state Version currEndVersion;
state bool clearVersionHistory;
ASSERT(beginVersion.present() == endVersion.present());
if (beginVersion.present()) {
currBeginVersion = beginVersion.get();
endVersionValue = endVersion.get();
clearVersionHistory = false;
} else {
// If beginVersion and endVersion are not presented, it means backup is done and we need to clear version history.
// Set currBeginVersion to INTMAX_MAX and it will be set to the correct version in clearLogRanges().
// Set endVersionValue to INTMAX_MAX since we need to clear log ranges up to next smallest version.
currBeginVersion = endVersionValue = currEndVersion = INTMAX_MAX;
clearVersionHistory = true;
}
while (currBeginVersion < endVersionValue || clearVersionHistory) {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
loop{
try {
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (checkBackupUid) {
Subspace sourceStates = Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceStates).get(logUidValue);
Optional<Value> v = wait( tr->get( sourceStates.pack(DatabaseBackupAgent::keyFolderId) ) );
if(v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) > backupUid)
return Void();
}
if (!clearVersionHistory) {
currEndVersion = std::min(currBeginVersion + CLIENT_KNOBS->CLEAR_LOG_RANGE_COUNT * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE, endVersionValue);
}
Void _ = wait(clearLogRanges(tr, clearVersionHistory, logUidValue, destUidValue, currBeginVersion, currEndVersion));
Void _ = wait(tr->commit());
if (clearVersionHistory) {
return Void(); return Void();
}
state Standalone<RangeResultRef> backupVersions = wait(tr->getRange(KeyRangeRef(backupLatestVersionsPath, strinc(backupLatestVersionsPath)), CLIENT_KNOBS->TOO_MANY));
// Make sure version history key does exist and lower the beginVersion if needed
state Version currBeginVersion = invalidVersion;
for (auto backupVersion : backupVersions) {
Key currLogUidValue = backupVersion.key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
if (currLogUidValue == logUidValue) {
currBeginVersion = BinaryReader::fromStringRef<Version>(backupVersion.value, Unversioned());
break;
}
}
// Do not clear anything if version history key cannot be found
if (currBeginVersion == invalidVersion) {
return Void();
}
state Version currEndVersion = currBeginVersion + CLIENT_KNOBS->CLEAR_LOG_RANGE_COUNT * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE;
if(endVersion.present()) {
currEndVersion = std::min(currEndVersion, endVersion.get());
}
state Version nextSmallestVersion = currEndVersion;
bool clearLogRangesRequired = true;
// More than one backup/DR with the same range
if (backupVersions.size() > 1) {
for (auto backupVersion : backupVersions) {
Key currLogUidValue = backupVersion.key.removePrefix(backupLatestVersionsPrefix).removePrefix(destUidValue);
Version currVersion = BinaryReader::fromStringRef<Version>(backupVersion.value, Unversioned());
if (currLogUidValue == logUidValue) {
continue;
} else if (currVersion > currBeginVersion) {
nextSmallestVersion = std::min(currVersion, nextSmallestVersion);
} else {
// If we can find a version less than or equal to beginVersion, clearing log ranges is not required
clearLogRangesRequired = false;
break;
}
}
}
if (!endVersion.present() && backupVersions.size() == 1) {
// Clear version history
tr->clear(prefixRange(backupLatestVersionsPath));
// Clear everything under blog/[destUid]
tr->clear(prefixRange(destUidValue.withPrefix(backupLogKeys.begin)));
// Disable committing mutations into blog
tr->clear(prefixRange(destUidValue.withPrefix(logRangesRange.begin)));
} else {
if (!endVersion.present() && currEndVersion >= nextSmallestVersion) {
// Clear current backup version history
tr->clear(backupLatestVersionsKey);
} else {
// Update current backup latest version
tr->set(backupLatestVersionsKey, BinaryWriter::toValue<Version>(currEndVersion, Unversioned()));
} }
currBeginVersion = currEndVersion; // Clear log ranges if needed
break; if (clearLogRangesRequired) {
} catch (Error &e) { Standalone<VectorRef<KeyRangeRef>> ranges = getLogRanges(currBeginVersion, nextSmallestVersion, destUidValue);
Void _ = wait(tr->onError(e)); for (auto& range : ranges) {
tr->clear(range);
}
}
} }
Void _ = wait(tr->commit());
if (!endVersion.present() && (backupVersions.size() == 1 || currEndVersion >= nextSmallestVersion)) {
return Void();
}
if(endVersion.present() && currEndVersion == endVersion.get()) {
return Void();
}
tr->reset();
} catch (Error &e) {
Void _ = wait(tr->onError(e));
} }
} }
return Void();
} }
Future<Void> eraseLogData(Database cx, Key logUidValue, Key destUidValue, Optional<Version> beginVersion, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) { Future<Void> eraseLogData(Database cx, Key logUidValue, Key destUidValue, Optional<Version> endVersion, bool checkBackupUid, Version backupUid) {
return _eraseLogData(cx, logUidValue, destUidValue, beginVersion, endVersion, checkBackupUid, backupUid); return _eraseLogData(cx, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid);
} }

4
fdbclient/BackupContainer.actor.cpp
View File

@ -1184,7 +1184,7 @@ Reference<IBackupContainer> IBackupContainer::openContainer(std::string url)
m.detail("Description", "Invalid container specification. See help.").detail("URL", url); m.detail("Description", "Invalid container specification. See help.").detail("URL", url);
if(e.code() == error_code_backup_invalid_url) if(e.code() == error_code_backup_invalid_url)
m.detail("lastOpenError", lastOpenError); m.detail("LastOpenError", lastOpenError);
throw; throw;
} }
} }
@ -1222,7 +1222,7 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
m.detail("Description", "Invalid backup container URL prefix. See help.").detail("URL", baseURL); m.detail("Description", "Invalid backup container URL prefix. See help.").detail("URL", baseURL);
if(e.code() == error_code_backup_invalid_url) if(e.code() == error_code_backup_invalid_url)
m.detail("lastOpenError", IBackupContainer::lastOpenError); m.detail("LastOpenError", IBackupContainer::lastOpenError);
throw; throw;
} }
} }

2
fdbclient/CoordinationInterface.h
View File

@ -93,7 +93,7 @@ private:
struct LeaderInfo { struct LeaderInfo {
UID changeID; UID changeID;
uint64_t mask = ~(127ll << 57); static const uint64_t mask = ~(127ll << 57);
Value serializedInfo; Value serializedInfo;
bool forward; // If true, serializedInfo is a connection string instead! bool forward; // If true, serializedInfo is a connection string instead!

71
fdbclient/DatabaseBackupAgent.actor.cpp
View File

@ -109,7 +109,7 @@ namespace dbBackup {
Future<Void> checkTaskVersion(Tr tr, Reference<Task> task, StringRef name, uint32_t version) { Future<Void> checkTaskVersion(Tr tr, Reference<Task> task, StringRef name, uint32_t version) {
uint32_t taskVersion = task->getVersion(); uint32_t taskVersion = task->getVersion();
if (taskVersion > version) { if (taskVersion > version) {
TraceEvent(SevError, "BA_BackupRangeTaskFunc_execute").detail("taskVersion", taskVersion).detail("Name", printable(name)).detail("Version", version); TraceEvent(SevError, "BA_BackupRangeTaskFunc_execute").detail("TaskVersion", taskVersion).detail("Name", printable(name)).detail("Version", version);
Void _ = wait(logError(tr, Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyErrors).pack(task->params[BackupAgentBase::keyConfigLogUid]), Void _ = wait(logError(tr, Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keyErrors).pack(task->params[BackupAgentBase::keyConfigLogUid]),
format("ERROR: %s task version `%lu' is greater than supported version `%lu'", task->params[Task::reservedTaskParamKeyType].toString().c_str(), (unsigned long)taskVersion, (unsigned long)version))); format("ERROR: %s task version `%lu' is greater than supported version `%lu'", task->params[Task::reservedTaskParamKeyType].toString().c_str(), (unsigned long)taskVersion, (unsigned long)version)));
@ -299,7 +299,7 @@ namespace dbBackup {
return Void(); return Void();
} }
//TraceEvent("DBA_Range").detail("range", printable(KeyRangeRef(rangeBegin, rangeEnd))).detail("version", values.second).detail("size", values.first.size()).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid])).detail("addPrefix", printable(addPrefix)).detail("removePrefix", printable(removePrefix)); //TraceEvent("DBA_Range").detail("Range", printable(KeyRangeRef(rangeBegin, rangeEnd))).detail("Version", values.second).detail("Size", values.first.size()).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid])).detail("AddPrefix", printable(addPrefix)).detail("RemovePrefix", printable(removePrefix));
Subspace krv(conf.get(DatabaseBackupAgent::keyRangeVersions)); Subspace krv(conf.get(DatabaseBackupAgent::keyRangeVersions));
state KeyRange versionRange = singleKeyRange(krv.pack(values.second)); state KeyRange versionRange = singleKeyRange(krv.pack(values.second));
@ -334,7 +334,7 @@ namespace dbBackup {
break; break;
if( values.first[valueLoc].key >= backupVersions.get()[versionLoc].key ) { if( values.first[valueLoc].key >= backupVersions.get()[versionLoc].key ) {
//TraceEvent("DBA_set", debugID).detail("key", printable(values.first[valueLoc].key)).detail("value", printable(values.first[valueLoc].value)); //TraceEvent("DBA_Set", debugID).detail("Key", printable(values.first[valueLoc].key)).detail("Value", printable(values.first[valueLoc].value));
tr->set(values.first[valueLoc].key.removePrefix(removePrefix).withPrefix(addPrefix), values.first[valueLoc].value); tr->set(values.first[valueLoc].key.removePrefix(removePrefix).withPrefix(addPrefix), values.first[valueLoc].value);
bytesSet += values.first[valueLoc].expectedSize() - removePrefix.expectedSize() + addPrefix.expectedSize(); bytesSet += values.first[valueLoc].expectedSize() - removePrefix.expectedSize() + addPrefix.expectedSize();
} }
@ -347,7 +347,7 @@ namespace dbBackup {
Void _ = wait(tr->commit()); Void _ = wait(tr->commit());
Params.bytesWritten().set(task, Params.bytesWritten().getOrDefault(task) + bytesSet); Params.bytesWritten().set(task, Params.bytesWritten().getOrDefault(task) + bytesSet);
//TraceEvent("DBA_setComplete", debugID).detail("ver", values.second).detail("logVersion", logVersion).detail("readVersion", readVer).detail("commitVer", tr.getCommittedVersion()).detail("range", printable(versionRange)); //TraceEvent("DBA_SetComplete", debugID).detail("Ver", values.second).detail("LogVersion", logVersion).detail("ReadVersion", readVer).detail("CommitVer", tr.getCommittedVersion()).detail("Range", printable(versionRange));
if(backupVersions.get().more) { if(backupVersions.get().more) {
tr->reset(); tr->reset();
@ -431,7 +431,7 @@ namespace dbBackup {
srcTr.setOption(FDBTransactionOptions::LOCK_AWARE); srcTr.setOption(FDBTransactionOptions::LOCK_AWARE);
Version readVersion = wait(srcTr.getReadVersion()); Version readVersion = wait(srcTr.getReadVersion());
tr->set(states.pack(DatabaseBackupAgent::keyCopyStop), BinaryWriter::toValue(readVersion, Unversioned())); tr->set(states.pack(DatabaseBackupAgent::keyCopyStop), BinaryWriter::toValue(readVersion, Unversioned()));
TraceEvent("DBA_FinishFullBackup").detail("copyStop", readVersion); TraceEvent("DBA_FinishFullBackup").detail("CopyStop", readVersion);
Void _ = wait(taskBucket->finish(tr, task)); Void _ = wait(taskBucket->finish(tr, task));
return Void(); return Void();
@ -476,21 +476,20 @@ namespace dbBackup {
Void _ = wait(checkTaskVersion(cx, task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version)); Void _ = wait(checkTaskVersion(cx, task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version));
Version beginVersion = BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyBeginVersion], Unversioned());
Version endVersion = BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyEndVersion], Unversioned()); Version endVersion = BinaryReader::fromStringRef<Version>(task->params[DatabaseBackupAgent::keyEndVersion], Unversioned());
Void _ = wait(eraseLogData(taskBucket->src, task->params[BackupAgentBase::keyConfigLogUid], task->params[BackupAgentBase::destUid], Optional<Version>(beginVersion), Optional<Version>(endVersion), true, BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned()))); Void _ = wait(eraseLogData(taskBucket->src, task->params[BackupAgentBase::keyConfigLogUid], task->params[BackupAgentBase::destUid], Optional<Version>(endVersion), true, BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned())));
return Void(); return Void();
} }
ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<Task> parentTask, Version beginVersion, Version endVersion, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>()) { ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<Task> parentTask, Version endVersion, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key doneKey = wait(completionKey.get(tr, taskBucket)); Key doneKey = wait(completionKey.get(tr, taskBucket));
Reference<Task> task(new Task(EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version, doneKey, 1)); Reference<Task> task(new Task(EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version, doneKey, 1));
copyDefaultParameters(parentTask, task); copyDefaultParameters(parentTask, task);
task->params[DatabaseBackupAgent::keyBeginVersion] = BinaryWriter::toValue(beginVersion, Unversioned()); task->params[DatabaseBackupAgent::keyBeginVersion] = BinaryWriter::toValue(1, Unversioned()); //FIXME: remove in 6.X, only needed for 5.2 backward compatibility
task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned()); task->params[DatabaseBackupAgent::keyEndVersion] = BinaryWriter::toValue(endVersion, Unversioned());
if (!waitFor) { if (!waitFor) {
@ -732,7 +731,7 @@ namespace dbBackup {
state Version stopVersionData = stopValue.present() ? BinaryReader::fromStringRef<Version>(stopValue.get(), Unversioned()) : -1; state Version stopVersionData = stopValue.present() ? BinaryReader::fromStringRef<Version>(stopValue.get(), Unversioned()) : -1;
if(endVersion - beginVersion > g_random->randomInt64(0, CLIENT_KNOBS->BACKUP_VERSION_DELAY)) { if(endVersion - beginVersion > g_random->randomInt64(0, CLIENT_KNOBS->BACKUP_VERSION_DELAY)) {
TraceEvent("DBA_CopyLogs").detail("beginVersion", beginVersion).detail("applyVersion", applyVersion).detail("endVersion", endVersion).detail("stopVersionData", stopVersionData).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid])); TraceEvent("DBA_CopyLogs").detail("BeginVersion", beginVersion).detail("ApplyVersion", applyVersion).detail("EndVersion", endVersion).detail("StopVersionData", stopVersionData).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid]));
} }
if ((stopVersionData == -1) || (stopVersionData >= applyVersion)) { if ((stopVersionData == -1) || (stopVersionData >= applyVersion)) {
@ -749,7 +748,7 @@ namespace dbBackup {
// Do not erase at the first time // Do not erase at the first time
if (prevBeginVersion > 0) { if (prevBeginVersion > 0) {
addTaskVector.push_back(EraseLogRangeTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::joinWith(allPartsDone))); addTaskVector.push_back(EraseLogRangeTaskFunc::addTask(tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(allPartsDone)));
} }
Void _ = wait(waitForAll(addTaskVector) && taskBucket->finish(tr, task)); Void _ = wait(waitForAll(addTaskVector) && taskBucket->finish(tr, task));
@ -814,7 +813,7 @@ namespace dbBackup {
Optional<Value> endValue = wait(tr2.get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin))); Optional<Value> endValue = wait(tr2.get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin)));
Version endVersion = endValue.present() ? BinaryReader::fromStringRef<Version>(endValue.get(), Unversioned()) : -1; Version endVersion = endValue.present() ? BinaryReader::fromStringRef<Version>(endValue.get(), Unversioned()) : -1;
//TraceEvent("DBA_FinishedFullBackup").detail("applied", appliedVersion).detail("endVer", endVersion); //TraceEvent("DBA_FinishedFullBackup").detail("Applied", appliedVersion).detail("EndVer", endVersion);
if(appliedVersion < endVersion) { if(appliedVersion < endVersion) {
Void _ = wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); Void _ = wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY));
task->params[FinishedFullBackupTaskFunc::keyInsertTask] = StringRef(); task->params[FinishedFullBackupTaskFunc::keyInsertTask] = StringRef();
@ -856,7 +855,7 @@ namespace dbBackup {
} }
Version backupUid = BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned()); Version backupUid = BinaryReader::fromStringRef<Version>(task->params[BackupAgentBase::keyFolderId], Unversioned());
Void _ = wait(eraseLogData(taskBucket->src, logUidValue, destUidValue, Optional<Version>(), Optional<Version>(), true, backupUid)); Void _ = wait(eraseLogData(taskBucket->src, logUidValue, destUidValue, Optional<Version>(), true, backupUid));
return Void(); return Void();
} }
@ -936,7 +935,7 @@ namespace dbBackup {
Optional<Value> stopWhenDone = wait(fStopWhenDone); Optional<Value> stopWhenDone = wait(fStopWhenDone);
if(endVersion - beginVersion > g_random->randomInt64(0, CLIENT_KNOBS->BACKUP_VERSION_DELAY)) { if(endVersion - beginVersion > g_random->randomInt64(0, CLIENT_KNOBS->BACKUP_VERSION_DELAY)) {
TraceEvent("DBA_CopyDiffLogs").detail("beginVersion", beginVersion).detail("endVersion", endVersion).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid])); TraceEvent("DBA_CopyDiffLogs").detail("BeginVersion", beginVersion).detail("EndVersion", endVersion).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid]));
} }
if (!stopWhenDone.present()) { if (!stopWhenDone.present()) {
@ -952,7 +951,7 @@ namespace dbBackup {
} }
if (prevBeginVersion > 0) { if (prevBeginVersion > 0) {
addTaskVector.push_back(EraseLogRangeTaskFunc::addTask(tr, taskBucket, task, prevBeginVersion, beginVersion, TaskCompletionKey::joinWith(allPartsDone))); addTaskVector.push_back(EraseLogRangeTaskFunc::addTask(tr, taskBucket, task, beginVersion, TaskCompletionKey::joinWith(allPartsDone)));
} }
Void _ = wait(waitForAll(addTaskVector) && taskBucket->finish(tr, task)); Void _ = wait(waitForAll(addTaskVector) && taskBucket->finish(tr, task));
@ -1212,7 +1211,7 @@ namespace dbBackup {
} }
} }
TraceEvent("DBA_abort_old_backup").detail("tagName", tagNameKey.printable()); TraceEvent("DBA_AbortOldBackup").detail("TagName", tagNameKey.printable());
Void _ = wait(srcDrAgent.abortBackup(cx, tagNameKey, false, true)); Void _ = wait(srcDrAgent.abortBackup(cx, tagNameKey, false, true));
return Void(); return Void();
@ -1405,7 +1404,7 @@ namespace dbBackup {
state UID logUid = BinaryReader::fromStringRef<UID>(task->params[DatabaseBackupAgent::keyConfigLogUid], Unversioned()); state UID logUid = BinaryReader::fromStringRef<UID>(task->params[DatabaseBackupAgent::keyConfigLogUid], Unversioned());
TraceEvent("DBA_Complete").detail("restoreVersion", restoreVersion).detail("differential", stopWhenDone.present()).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid])); TraceEvent("DBA_Complete").detail("RestoreVersion", restoreVersion).detail("Differential", stopWhenDone.present()).detail("LogUID", printable(task->params[BackupAgentBase::keyConfigLogUid]));
// Start the complete task, if differential is not enabled // Start the complete task, if differential is not enabled
if (stopWhenDone.present()) { if (stopWhenDone.present()) {
@ -1634,7 +1633,7 @@ public:
state UID logUid = wait(backupAgent->getLogUid(cx, tagName)); state UID logUid = wait(backupAgent->getLogUid(cx, tagName));
state Key drVersionKey = backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyDrVersion); state Key drVersionKey = backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyDrVersion);
TraceEvent("DRU_watchLatestDrVersion").detail("drVersionKey", drVersionKey.printable()).detail("logUid", BinaryWriter::toValue(logUid, Unversioned()).printable()); TraceEvent("DRU_WatchLatestDrVersion").detail("DrVersionKey", drVersionKey.printable()).detail("LogUid", BinaryWriter::toValue(logUid, Unversioned()).printable());
loop { loop {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx)); state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
@ -1645,7 +1644,7 @@ public:
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> drVersion = wait(tr->get(drVersionKey)); Optional<Value> drVersion = wait(tr->get(drVersionKey));
TraceEvent("DRU_versionCheck").detail("current", drVersion.present() ? BinaryReader::fromStringRef<int>(drVersion.get(), Unversioned()) : -1).detail("expected", DatabaseBackupAgent::LATEST_DR_VERSION).detail("logUid", BinaryWriter::toValue(logUid, Unversioned()).printable()); TraceEvent("DRU_VersionCheck").detail("Current", drVersion.present() ? BinaryReader::fromStringRef<int>(drVersion.get(), Unversioned()) : -1).detail("Expected", DatabaseBackupAgent::LATEST_DR_VERSION).detail("LogUid", BinaryWriter::toValue(logUid, Unversioned()).printable());
if (drVersion.present() && BinaryReader::fromStringRef<int>(drVersion.get(), Unversioned()) == DatabaseBackupAgent::LATEST_DR_VERSION) { if (drVersion.present() && BinaryReader::fromStringRef<int>(drVersion.get(), Unversioned()) == DatabaseBackupAgent::LATEST_DR_VERSION) {
return Void(); return Void();
} }
@ -1825,8 +1824,8 @@ public:
else else
Void _ = wait(checkDatabaseLock(tr, logUid)); Void _ = wait(checkDatabaseLock(tr, logUid));
TraceEvent("DBA_Submit").detail("logUid", logUid).detail("lock", lockDB).detail("LogUID", printable(logUidValue)).detail("tag", printable(tagName)) TraceEvent("DBA_Submit").detail("LogUid", logUid).detail("Lock", lockDB).detail("LogUID", printable(logUidValue)).detail("Tag", printable(tagName))
.detail("key", printable(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId))).detail("mapPrefix", printable(mapPrefix)); .detail("Key", printable(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyFolderId))).detail("MapPrefix", printable(mapPrefix));
return Void(); return Void();
} }
@ -1834,7 +1833,7 @@ public:
ACTOR static Future<Void> unlockBackup(DatabaseBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, Key tagName) { ACTOR static Future<Void> unlockBackup(DatabaseBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, Key tagName) {
UID logUid = wait(backupAgent->getLogUid(tr, tagName)); UID logUid = wait(backupAgent->getLogUid(tr, tagName));
Void _ = wait(unlockDatabase(tr, logUid)); Void _ = wait(unlockDatabase(tr, logUid));
TraceEvent("DBA_unlock").detail("tag", printable(tagName)); TraceEvent("DBA_Unlock").detail("Tag", printable(tagName));
return Void(); return Void();
} }
@ -1843,7 +1842,7 @@ public:
state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName)); state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName));
state int status = wait(backupAgent->getStateValue(dest, destlogUid)); state int status = wait(backupAgent->getStateValue(dest, destlogUid));
TraceEvent("DBA_switchover_start").detail("status", status); TraceEvent("DBA_SwitchoverStart").detail("Status", status);
if (status != BackupAgentBase::STATE_DIFFERENTIAL && status != BackupAgentBase::STATE_COMPLETED) { if (status != BackupAgentBase::STATE_DIFFERENTIAL && status != BackupAgentBase::STATE_COMPLETED) {
throw backup_duplicate(); throw backup_duplicate();
} }
@ -1872,7 +1871,7 @@ public:
} }
} }
TraceEvent("DBA_switchover_locked").detail("version", commitVersion); TraceEvent("DBA_SwitchoverLocked").detail("Version", commitVersion);
// Wait for the destination to apply mutations up to the lock commit before switching over. // Wait for the destination to apply mutations up to the lock commit before switching over.
state ReadYourWritesTransaction tr2(dest); state ReadYourWritesTransaction tr2(dest);
@ -1881,11 +1880,11 @@ public:
tr2.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr2.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr2.setOption(FDBTransactionOptions::LOCK_AWARE); tr2.setOption(FDBTransactionOptions::LOCK_AWARE);
state Optional<Value> backupUid = wait(tr2.get(backupAgent->states.get(BinaryWriter::toValue(destlogUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId))); state Optional<Value> backupUid = wait(tr2.get(backupAgent->states.get(BinaryWriter::toValue(destlogUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId)));
TraceEvent("DBA_switchover_backupUID").detail("uid", printable(backupUid)).detail("key", printable(backupAgent->states.get(BinaryWriter::toValue(destlogUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId))); TraceEvent("DBA_SwitchoverBackupUID").detail("Uid", printable(backupUid)).detail("Key", printable(backupAgent->states.get(BinaryWriter::toValue(destlogUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId)));
if(!backupUid.present()) if(!backupUid.present())
throw backup_duplicate(); throw backup_duplicate();
Optional<Value> v = wait(tr2.get(BinaryWriter::toValue(destlogUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin))); Optional<Value> v = wait(tr2.get(BinaryWriter::toValue(destlogUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin)));
TraceEvent("DBA_switchover_version").detail("version", v.present() ? BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) : 0); TraceEvent("DBA_SwitchoverVersion").detail("Version", v.present() ? BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) : 0);
if( v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >= commitVersion) if( v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >= commitVersion)
break; break;
@ -1898,7 +1897,7 @@ public:
} }
} }
TraceEvent("DBA_switchover_ready"); TraceEvent("DBA_SwitchoverReady");
try { try {
Void _ = wait( backupAgent->discontinueBackup(dest, tagName) ); Void _ = wait( backupAgent->discontinueBackup(dest, tagName) );
@ -1909,7 +1908,7 @@ public:
int _ = wait( backupAgent->waitBackup(dest, tagName, true) ); int _ = wait( backupAgent->waitBackup(dest, tagName, true) );
TraceEvent("DBA_switchover_stopped"); TraceEvent("DBA_SwitchoverStopped");
state ReadYourWritesTransaction tr3(dest); state ReadYourWritesTransaction tr3(dest);
loop { loop {
@ -1917,7 +1916,7 @@ public:
tr3.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr3.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr3.setOption(FDBTransactionOptions::LOCK_AWARE); tr3.setOption(FDBTransactionOptions::LOCK_AWARE);
Version destVersion = wait(tr3.getReadVersion()); Version destVersion = wait(tr3.getReadVersion());
TraceEvent("DBA_switchover_version_upgrade").detail("src", commitVersion).detail("dest", destVersion); TraceEvent("DBA_SwitchoverVersionUpgrade").detail("Src", commitVersion).detail("Dest", destVersion);
if (destVersion <= commitVersion) { if (destVersion <= commitVersion) {
TEST(true); // Forcing dest backup cluster to higher version TEST(true); // Forcing dest backup cluster to higher version
tr3.set(minRequiredCommitVersionKey, BinaryWriter::toValue(commitVersion+1, Unversioned())); tr3.set(minRequiredCommitVersionKey, BinaryWriter::toValue(commitVersion+1, Unversioned()));
@ -1930,7 +1929,7 @@ public:
} }
} }
TraceEvent("DBA_switchover_version_upgraded"); TraceEvent("DBA_SwitchoverVersionUpgraded");
try { try {
Void _ = wait( drAgent.submitBackup(backupAgent->taskBucket->src, tagName, backupRanges, false, addPrefix, removePrefix, true, true) ); Void _ = wait( drAgent.submitBackup(backupAgent->taskBucket->src, tagName, backupRanges, false, addPrefix, removePrefix, true, true) );
@ -1939,15 +1938,15 @@ public:
throw; throw;
} }
TraceEvent("DBA_switchover_submitted"); TraceEvent("DBA_SwitchoverSubmitted");
int _ = wait( drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName) ); int _ = wait( drAgent.waitSubmitted(backupAgent->taskBucket->src, tagName) );
TraceEvent("DBA_switchover_started"); TraceEvent("DBA_SwitchoverStarted");
Void _ = wait( backupAgent->unlockBackup(dest, tagName) ); Void _ = wait( backupAgent->unlockBackup(dest, tagName) );
TraceEvent("DBA_switchover_unlocked"); TraceEvent("DBA_SwitchoverUnlocked");
return Void(); return Void();
} }
@ -1957,7 +1956,7 @@ public:
state UID logUid = wait(backupAgent->getLogUid(tr, tagName)); state UID logUid = wait(backupAgent->getLogUid(tr, tagName));
state int status = wait(backupAgent->getStateValue(tr, logUid)); state int status = wait(backupAgent->getStateValue(tr, logUid));
TraceEvent("DBA_discontinue").detail("status", status); TraceEvent("DBA_Discontinue").detail("Status", status);
if (!DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status)) { if (!DatabaseBackupAgent::isRunnable((BackupAgentBase::enumState)status)) {
throw backup_unneeded(); throw backup_unneeded();
} }
@ -2018,7 +2017,7 @@ public:
tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_PARTIALLY_ABORTED))); tr->set(StringRef(backupAgent->states.get(logUidValue).pack(DatabaseBackupAgent::keyStateStatus)), StringRef(DatabaseBackupAgent::getStateText(BackupAgentBase::STATE_PARTIALLY_ABORTED)));
Void _ = wait(tr->commit()); Void _ = wait(tr->commit());
TraceEvent("DBA_Abort").detail("commitVersion", tr->getCommittedVersion()); TraceEvent("DBA_Abort").detail("CommitVersion", tr->getCommittedVersion());
break; break;
} }
catch (Error &e) { catch (Error &e) {
@ -2044,7 +2043,7 @@ public:
if (lastApplied.present()) { if (lastApplied.present()) {
Version current = tr->getReadVersion().get(); Version current = tr->getReadVersion().get();
Version applied = BinaryReader::fromStringRef<Version>(lastApplied.get(), Unversioned()); Version applied = BinaryReader::fromStringRef<Version>(lastApplied.get(), Unversioned());
TraceEvent("DBA_abort_version_upgrade").detail("src", applied).detail("dest", current); TraceEvent("DBA_AbortVersionUpgrade").detail("Src", applied).detail("Dest", current);
if (current <= applied) { if (current <= applied) {
TEST(true); // Upgrading version of local database. TEST(true); // Upgrading version of local database.
// The +1 is because we want to make sure that a versionstamped operation can't reuse // The +1 is because we want to make sure that a versionstamped operation can't reuse

3
fdbclient/DatabaseContext.h
View File

@ -141,7 +141,8 @@ public:
int64_t transactionsTooOld; int64_t transactionsTooOld;
int64_t transactionsFutureVersions; int64_t transactionsFutureVersions;
int64_t transactionsNotCommitted; int64_t transactionsNotCommitted;
int64_t transactionsMaybeCommitted; int64_t transactionsMaybeCommitted;
int64_t transactionsResourceConstrained;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit; ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit;
int outstandingWatches; int outstandingWatches;

75
fdbclient/FileBackupAgent.actor.cpp
View File

@ -705,7 +705,7 @@ namespace fileBackup {
if (taskVersion > version) { if (taskVersion > version) {
state Error err = task_invalid_version(); state Error err = task_invalid_version();
TraceEvent(SevWarn, "BA_BackupRangeTaskFunc_execute").detail("taskVersion", taskVersion).detail("Name", printable(name)).detail("Version", version); TraceEvent(SevWarn, "BA_BackupRangeTaskFunc_Execute").detail("TaskVersion", taskVersion).detail("Name", printable(name)).detail("Version", version);
if (KeyBackedConfig::TaskParams.uid().exists(task)) { if (KeyBackedConfig::TaskParams.uid().exists(task)) {
std::string msg = format("%s task version `%lu' is greater than supported version `%lu'", task->params[Task::reservedTaskParamKeyType].toString().c_str(), (unsigned long)taskVersion, (unsigned long)version); std::string msg = format("%s task version `%lu' is greater than supported version `%lu'", task->params[Task::reservedTaskParamKeyType].toString().c_str(), (unsigned long)taskVersion, (unsigned long)version);
Void _ = wait(BackupConfig(task).logError(cx, err, msg)); Void _ = wait(BackupConfig(task).logError(cx, err, msg));
@ -724,7 +724,7 @@ namespace fileBackup {
state Subspace tagNames = backupAgent->subspace.get(BackupAgentBase::keyTagName); state Subspace tagNames = backupAgent->subspace.get(BackupAgentBase::keyTagName);
Optional<Value> uidStr = wait(tr->get(tagNames.pack(Key(tagName)))); Optional<Value> uidStr = wait(tr->get(tagNames.pack(Key(tagName))));
if (!uidStr.present()) { if (!uidStr.present()) {
TraceEvent(SevWarn, "FileBackupAbortIncompatibleBackup_TagNotFound").detail("tagName", tagName.c_str()); TraceEvent(SevWarn, "FileBackupAbortIncompatibleBackup_TagNotFound").detail("TagName", tagName.c_str());
return Void(); return Void();
} }
state UID uid = BinaryReader::fromStringRef<UID>(uidStr.get(), Unversioned()); state UID uid = BinaryReader::fromStringRef<UID>(uidStr.get(), Unversioned());
@ -737,8 +737,8 @@ namespace fileBackup {
state EBackupState status = !statusStr.present() ? FileBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString()); state EBackupState status = !statusStr.present() ? FileBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString());
TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup") TraceEvent(SevInfo, "FileBackupAbortIncompatibleBackup")
.detail("tagName", tagName.c_str()) .detail("TagName", tagName.c_str())
.detail("status", BackupAgentBase::getStateText(status)); .detail("Status", BackupAgentBase::getStateText(status));
// Clear the folder id to prevent future tasks from executing at all // Clear the folder id to prevent future tasks from executing at all
tr->clear(singleKeyRange(StringRef(globalConfig.pack(FileBackupAgent::keyFolderId)))); tr->clear(singleKeyRange(StringRef(globalConfig.pack(FileBackupAgent::keyFolderId))));
@ -770,8 +770,8 @@ namespace fileBackup {
TEST(true); // Canceling old backup task TEST(true); // Canceling old backup task
TraceEvent(SevInfo, "FileBackupCancelOldTask") TraceEvent(SevInfo, "FileBackupCancelOldTask")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("tagName", tagName); .detail("TagName", tagName);
Void _ = wait(abortFiveZeroBackup(&backupAgent, tr, tagName)); Void _ = wait(abortFiveZeroBackup(&backupAgent, tr, tagName));
Void _ = wait(taskBucket->finish(tr, task)); Void _ = wait(taskBucket->finish(tr, task));
@ -779,7 +779,7 @@ namespace fileBackup {
} }
virtual StringRef getName() const { virtual StringRef getName() const {
TraceEvent(SevError, "FileBackupError").detail("cause", "AbortFiveZeroBackupTaskFunc::name() should never be called"); TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveZeroBackupTaskFunc::name() should never be called");
ASSERT(false); ASSERT(false);
return StringRef(); return StringRef();
} }
@ -812,9 +812,9 @@ namespace fileBackup {
throw backup_unneeded(); throw backup_unneeded();
} }
TraceEvent(SevInfo, "FBA_abortFileOneBackup") TraceEvent(SevInfo, "FBA_AbortFileOneBackup")
.detail("tagName", tagName.c_str()) .detail("TagName", tagName.c_str())
.detail("status", BackupAgentBase::getStateText(status)); .detail("Status", BackupAgentBase::getStateText(status));
// Cancel backup task through tag // Cancel backup task through tag
Void _ = wait(tag.cancel(tr)); Void _ = wait(tag.cancel(tr));
@ -840,8 +840,8 @@ namespace fileBackup {
TEST(true); // Canceling 5.1 backup task TEST(true); // Canceling 5.1 backup task
TraceEvent(SevInfo, "FileBackupCancelFiveOneTask") TraceEvent(SevInfo, "FileBackupCancelFiveOneTask")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("tagName", tagName); .detail("TagName", tagName);
Void _ = wait(abortFiveOneBackup(&backupAgent, tr, tagName)); Void _ = wait(abortFiveOneBackup(&backupAgent, tr, tagName));
Void _ = wait(taskBucket->finish(tr, task)); Void _ = wait(taskBucket->finish(tr, task));
@ -849,7 +849,7 @@ namespace fileBackup {
} }
virtual StringRef getName() const { virtual StringRef getName() const {
TraceEvent(SevError, "FileBackupError").detail("cause", "AbortFiveOneBackupTaskFunc::name() should never be called"); TraceEvent(SevError, "FileBackupError").detail("Cause", "AbortFiveOneBackupTaskFunc::name() should never be called");
ASSERT(false); ASSERT(false);
return StringRef(); return StringRef();
} }
@ -1911,30 +1911,25 @@ namespace fileBackup {
state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES)); state Reference<FlowLock> lock(new FlowLock(CLIENT_KNOBS->BACKUP_LOCK_BYTES));
Void _ = wait(checkTaskVersion(cx, task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version)); Void _ = wait(checkTaskVersion(cx, task, EraseLogRangeTaskFunc::name, EraseLogRangeTaskFunc::version));
state Version beginVersion = Params.beginVersion().get(task);
state Version endVersion = Params.endVersion().get(task); state Version endVersion = Params.endVersion().get(task);
state Key destUidValue = Params.destUidValue().get(task); state Key destUidValue = Params.destUidValue().get(task);
state BackupConfig config(task); state BackupConfig config(task);
state Key logUidValue = config.getUidAsKey(); state Key logUidValue = config.getUidAsKey();
if (beginVersion == 0) { Void _ = wait(eraseLogData(cx, logUidValue, destUidValue, endVersion != 0 ? Optional<Version>(endVersion) : Optional<Version>()));
Void _ = wait(eraseLogData(cx, logUidValue, destUidValue));
} else {
Void _ = wait(eraseLogData(cx, logUidValue, destUidValue, Optional<Version>(beginVersion), Optional<Version>(endVersion)));
}
return Void(); return Void();
} }
ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, UID logUid, TaskCompletionKey completionKey, Key destUidValue, Version beginVersion = 0, Version endVersion = 0, Reference<TaskFuture> waitFor = Reference<TaskFuture>()) { ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, UID logUid, TaskCompletionKey completionKey, Key destUidValue, Version endVersion = 0, Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
Key key = wait(addBackupTask(EraseLogRangeTaskFunc::name, Key key = wait(addBackupTask(EraseLogRangeTaskFunc::name,
EraseLogRangeTaskFunc::version, EraseLogRangeTaskFunc::version,
tr, taskBucket, completionKey, tr, taskBucket, completionKey,
BackupConfig(logUid), BackupConfig(logUid),
waitFor, waitFor,
[=](Reference<Task> task) { [=](Reference<Task> task) {
Params.beginVersion().set(task, beginVersion); Params.beginVersion().set(task, 1); //FIXME: remove in 6.X, only needed for 5.2 backward compatibility
Params.endVersion().set(task, endVersion); Params.endVersion().set(task, endVersion);
Params.destUidValue().set(task, destUidValue); Params.destUidValue().set(task, destUidValue);
}, },
@ -2039,7 +2034,7 @@ namespace fileBackup {
// Do not erase at the first time // Do not erase at the first time
if (prevBeginVersion > 0) { if (prevBeginVersion > 0) {
state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); state Key destUidValue = wait(config.destUidValue().getOrThrow(tr));
Key _ = wait(EraseLogRangeTaskFunc::addTask(tr, taskBucket, config.getUid(), TaskCompletionKey::joinWith(logDispatchBatchFuture), destUidValue, prevBeginVersion, beginVersion)); Key _ = wait(EraseLogRangeTaskFunc::addTask(tr, taskBucket, config.getUid(), TaskCompletionKey::joinWith(logDispatchBatchFuture), destUidValue, beginVersion));
} }
Void _ = wait(taskBucket->finish(tr, task)); Void _ = wait(taskBucket->finish(tr, task));
@ -3406,10 +3401,10 @@ public:
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
TraceEvent(SevInfo, "FBA_submitBackup") TraceEvent(SevInfo, "FBA_SubmitBackup")
.detail("tagName", tagName.c_str()) .detail("TagName", tagName.c_str())
.detail("stopWhenDone", stopWhenDone) .detail("StopWhenDone", stopWhenDone)
.detail("outContainer", outContainer.toString()); .detail("OutContainer", outContainer.toString());
state KeyBackedTag tag = makeBackupTag(tagName); state KeyBackedTag tag = makeBackupTag(tagName);
Optional<UidAndAbortedFlagT> uidAndAbortedFlag = wait(tag.get(tr)); Optional<UidAndAbortedFlagT> uidAndAbortedFlag = wait(tag.get(tr));
@ -3481,8 +3476,8 @@ public:
tr->set(destUidLookupPath, destUidValue); tr->set(destUidLookupPath, destUidValue);
} }
} }
Version initVersion = 1;
tr->set(config.getUidAsKey().withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix), BinaryWriter::toValue<Version>(initVersion, Unversioned())); tr->set(config.getUidAsKey().withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix), BinaryWriter::toValue<Version>(tr->getReadVersion().get(), Unversioned()));
config.destUidValue().set(tr, destUidValue); config.destUidValue().set(tr, destUidValue);
// Point the tag to this new uid // Point the tag to this new uid
@ -3632,10 +3627,10 @@ public:
// and clear the mutation logging config and data - but set its state as COMPLETED instead of ABORTED. // and clear the mutation logging config and data - but set its state as COMPLETED instead of ABORTED.
state Optional<Version> latestRestorableVersion = wait(config.getLatestRestorableVersion(tr)); state Optional<Version> latestRestorableVersion = wait(config.getLatestRestorableVersion(tr));
TraceEvent(SevInfo, "FBA_discontinueBackup") TraceEvent(SevInfo, "FBA_DiscontinueBackup")
.detail("AlreadyRestorable", latestRestorableVersion.present() ? "Yes" : "No") .detail("AlreadyRestorable", latestRestorableVersion.present() ? "Yes" : "No")
.detail("tagName", tag.tagName.c_str()) .detail("TagName", tag.tagName.c_str())
.detail("status", BackupAgentBase::getStateText(status)); .detail("Status", BackupAgentBase::getStateText(status));
if(latestRestorableVersion.present()) { if(latestRestorableVersion.present()) {
// Cancel all backup tasks through tag // Cancel all backup tasks through tag
@ -3679,9 +3674,9 @@ public:
throw backup_unneeded(); throw backup_unneeded();
} }
TraceEvent(SevInfo, "FBA_abortBackup") TraceEvent(SevInfo, "FBA_AbortBackup")
.detail("tagName", tagName.c_str()) .detail("TagName", tagName.c_str())
.detail("status", BackupAgentBase::getStateText(status)); .detail("Status", BackupAgentBase::getStateText(status));
// Cancel backup task through tag // Cancel backup task through tag
Void _ = wait(tag.cancel(tr)); Void _ = wait(tag.cancel(tr));
@ -3946,7 +3941,7 @@ public:
Void _ = wait( lockDatabase(&tr, randomUid) ); Void _ = wait( lockDatabase(&tr, randomUid) );
Void _ = wait(tr.commit()); Void _ = wait(tr.commit());
commitVersion = tr.getCommittedVersion(); commitVersion = tr.getCommittedVersion();
TraceEvent("AS_locked").detail("commitVer", commitVersion); TraceEvent("AS_Locked").detail("CommitVer", commitVersion);
break; break;
} catch( Error &e ) { } catch( Error &e ) {
Void _ = wait(tr.onError(e)); Void _ = wait(tr.onError(e));
@ -3958,7 +3953,7 @@ public:
try { try {
Optional<Version> restoreVersion = wait( backupConfig.getLatestRestorableVersion(ryw_tr) ); Optional<Version> restoreVersion = wait( backupConfig.getLatestRestorableVersion(ryw_tr) );
if(restoreVersion.present() && restoreVersion.get() >= commitVersion) { if(restoreVersion.present() && restoreVersion.get() >= commitVersion) {
TraceEvent("AS_restoreVersion").detail("restoreVer", restoreVersion.get()); TraceEvent("AS_RestoreVersion").detail("RestoreVer", restoreVersion.get());
break; break;
} else { } else {
ryw_tr->reset(); ryw_tr->reset();
@ -3974,7 +3969,7 @@ public:
try { try {
Void _ = wait( discontinueBackup(backupAgent, ryw_tr, tagName) ); Void _ = wait( discontinueBackup(backupAgent, ryw_tr, tagName) );
Void _ = wait( ryw_tr->commit() ); Void _ = wait( ryw_tr->commit() );
TraceEvent("AS_discontinuedBackup"); TraceEvent("AS_DiscontinuedBackup");
break; break;
} catch( Error &e ) { } catch( Error &e ) {
if(e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate){ if(e.code() == error_code_backup_unneeded || e.code() == error_code_backup_duplicate){
@ -3985,7 +3980,7 @@ public:
} }
int _ = wait( waitBackup(backupAgent, cx, tagName.toString(), true) ); int _ = wait( waitBackup(backupAgent, cx, tagName.toString(), true) );
TraceEvent("AS_backupStopped"); TraceEvent("AS_BackupStopped");
ryw_tr->reset(); ryw_tr->reset();
loop { loop {
@ -3995,7 +3990,7 @@ public:
ryw_tr->addReadConflictRange(range); ryw_tr->addReadConflictRange(range);
ryw_tr->clear(range); ryw_tr->clear(range);
Void _ = wait( ryw_tr->commit() ); Void _ = wait( ryw_tr->commit() );
TraceEvent("AS_clearedRange"); TraceEvent("AS_ClearedRange");
break; break;
} catch( Error &e ) { } catch( Error &e ) {
Void _ = wait( ryw_tr->onError(e) ); Void _ = wait( ryw_tr->onError(e) );
@ -4004,7 +3999,7 @@ public:
Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx)); Reference<IBackupContainer> bc = wait(backupConfig.backupContainer().getOrThrow(cx));
TraceEvent("AS_startRestore"); TraceEvent("AS_StartRestore");
Version ver = wait( restore(backupAgent, cx, tagName, KeyRef(bc->getURL()), true, -1, true, range, addPrefix, removePrefix, true, randomUid) ); Version ver = wait( restore(backupAgent, cx, tagName, KeyRef(bc->getURL()), true, -1, true, range, addPrefix, removePrefix, true, randomUid) );
return ver; return ver;
} }

8
fdbclient/KeyBackedTypes.h
View File

@ -123,10 +123,10 @@ public:
auto backtrace = platform::get_backtrace(); auto backtrace = platform::get_backtrace();
return map(get(tr, snapshot), [=](Optional<T> val) -> T { return map(get(tr, snapshot), [=](Optional<T> val) -> T {
if (!val.present()) { if (!val.present()) {
TraceEvent(SevInfo, "KeyBackedProperty_keyNotFound") TraceEvent(SevInfo, "KeyBackedProperty_KeyNotFound")
.detail("key", printable(keyCopy)) .detail("Key", printable(keyCopy))
.detail("err", err.code()) .detail("Err", err.code())
.detail("parentTrace", backtrace.c_str()); .detail("ParentTrace", backtrace.c_str());
throw err; throw err;
} }

1
fdbclient/Knobs.cpp
View File

@ -48,6 +48,7 @@ ClientKnobs::ClientKnobs(bool randomize) {
init( DEFAULT_BACKOFF, .01 ); if( randomize && BUGGIFY ) DEFAULT_BACKOFF = g_random->random01(); init( DEFAULT_BACKOFF, .01 ); if( randomize && BUGGIFY ) DEFAULT_BACKOFF = g_random->random01();
init( DEFAULT_MAX_BACKOFF, 1.0 ); init( DEFAULT_MAX_BACKOFF, 1.0 );
init( BACKOFF_GROWTH_RATE, 2.0 ); init( BACKOFF_GROWTH_RATE, 2.0 );
init( RESOURCE_CONSTRAINED_MAX_BACKOFF, 30.0 );
init( TRANSACTION_SIZE_LIMIT, 1e7 ); init( TRANSACTION_SIZE_LIMIT, 1e7 );
init( KEY_SIZE_LIMIT, 1e4 ); init( KEY_SIZE_LIMIT, 1e4 );

1
fdbclient/Knobs.h
View File

@ -46,6 +46,7 @@ public:
double DEFAULT_BACKOFF; double DEFAULT_BACKOFF;
double DEFAULT_MAX_BACKOFF; double DEFAULT_MAX_BACKOFF;
double BACKOFF_GROWTH_RATE; double BACKOFF_GROWTH_RATE;
double RESOURCE_CONSTRAINED_MAX_BACKOFF;
int64_t TRANSACTION_SIZE_LIMIT; int64_t TRANSACTION_SIZE_LIMIT;
int64_t KEY_SIZE_LIMIT; int64_t KEY_SIZE_LIMIT;

18
fdbclient/ManagementAPI.actor.cpp
View File

@ -192,13 +192,13 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
std::string mode = it.toString(); std::string mode = it.toString();
auto m = configForToken( mode ); auto m = configForToken( mode );
if( !m.size() ) { if( !m.size() ) {
TraceEvent(SevWarnAlways, "UnknownOption").detail("option", mode); TraceEvent(SevWarnAlways, "UnknownOption").detail("Option", mode);
return ConfigurationResult::UNKNOWN_OPTION; return ConfigurationResult::UNKNOWN_OPTION;
} }
for( auto t = m.begin(); t != m.end(); ++t ) { for( auto t = m.begin(); t != m.end(); ++t ) {
if( outConf.count( t->first ) ) { if( outConf.count( t->first ) ) {
TraceEvent(SevWarnAlways, "ConflictingOption").detail("option", printable(StringRef(t->first))); TraceEvent(SevWarnAlways, "ConflictingOption").detail("Option", printable(StringRef(t->first)));
return ConfigurationResult::CONFLICTING_OPTIONS; return ConfigurationResult::CONFLICTING_OPTIONS;
} }
outConf[t->first] = t->second; outConf[t->first] = t->second;
@ -1107,7 +1107,7 @@ ACTOR Future<int> setDDMode( Database cx, int mode ) {
Void _ = wait( tr.commit() ); Void _ = wait( tr.commit() );
return oldMode; return oldMode;
} catch (Error& e) { } catch (Error& e) {
TraceEvent("setDDModeRetrying").error(e); TraceEvent("SetDDModeRetrying").error(e);
Void _ = wait (tr.onError(e)); Void _ = wait (tr.onError(e));
} }
} }
@ -1249,7 +1249,7 @@ ACTOR Future<Void> lockDatabase( Transaction* tr, UID id ) {
if(BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) == id) { if(BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) == id) {
return Void(); return Void();
} else { } else {
//TraceEvent("DBA_lock_locked").detail("expecting", id).detail("lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())); //TraceEvent("DBA_LockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
throw database_locked(); throw database_locked();
} }
} }
@ -1268,7 +1268,7 @@ ACTOR Future<Void> lockDatabase( Reference<ReadYourWritesTransaction> tr, UID id
if(BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) == id) { if(BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) == id) {
return Void(); return Void();
} else { } else {
//TraceEvent("DBA_lock_locked").detail("expecting", id).detail("lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())); //TraceEvent("DBA_LockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
throw database_locked(); throw database_locked();
} }
} }
@ -1302,7 +1302,7 @@ ACTOR Future<Void> unlockDatabase( Transaction* tr, UID id ) {
return Void(); return Void();
if(val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) { if(val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
//TraceEvent("DBA_unlock_locked").detail("expecting", id).detail("lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())); //TraceEvent("DBA_UnlockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
throw database_locked(); throw database_locked();
} }
@ -1319,7 +1319,7 @@ ACTOR Future<Void> unlockDatabase( Reference<ReadYourWritesTransaction> tr, UID
return Void(); return Void();
if(val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) { if(val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
//TraceEvent("DBA_unlock_locked").detail("expecting", id).detail("lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())); //TraceEvent("DBA_UnlockLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()));
throw database_locked(); throw database_locked();
} }
@ -1348,7 +1348,7 @@ ACTOR Future<Void> checkDatabaseLock( Transaction* tr, UID id ) {
Optional<Value> val = wait( tr->get(databaseLockedKey) ); Optional<Value> val = wait( tr->get(databaseLockedKey) );
if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) { if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
//TraceEvent("DBA_check_locked").detail("expecting", id).detail("lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())).backtrace(); //TraceEvent("DBA_CheckLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())).backtrace();
throw database_locked(); throw database_locked();
} }
@ -1361,7 +1361,7 @@ ACTOR Future<Void> checkDatabaseLock( Reference<ReadYourWritesTransaction> tr, U
Optional<Value> val = wait( tr->get(databaseLockedKey) ); Optional<Value> val = wait( tr->get(databaseLockedKey) );
if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) { if (val.present() && BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned()) != id) {
//TraceEvent("DBA_check_locked").detail("expecting", id).detail("lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())).backtrace(); //TraceEvent("DBA_CheckLocked").detail("Expecting", id).detail("Lock", BinaryReader::fromStringRef<UID>(val.get().substr(10), Unversioned())).backtrace();
throw database_locked(); throw database_locked();
} }

2
fdbclient/MasterProxyInterface.h
View File

@ -55,7 +55,7 @@ struct MasterProxyInterface {
void initEndpoints() { void initEndpoints() {
getConsistentReadVersion.getEndpoint(TaskProxyGetConsistentReadVersion); getConsistentReadVersion.getEndpoint(TaskProxyGetConsistentReadVersion);
getRawCommittedVersion.getEndpoint(TaskProxyGetRawCommittedVersion); getRawCommittedVersion.getEndpoint(TaskProxyGetRawCommittedVersion);
commit.getEndpoint(TaskProxyCommit); commit.getEndpoint(TaskProxyCommitDispatcher);
getKeyServersLocations.getEndpoint(TaskProxyGetKeyServersLocations); getKeyServersLocations.getEndpoint(TaskProxyGetKeyServersLocations);
} }
}; };

44
fdbclient/MetricLogger.actor.cpp
View File

@ -446,12 +446,12 @@ TEST_CASE("fdbserver/metrics/TraceEvents") {
stringMetric = Standalone<StringRef>(StringRef((uint8_t *)s, strlen(s)), arena); stringMetric = Standalone<StringRef>(StringRef((uint8_t *)s, strlen(s)), arena);
TraceEvent("Dummy") TraceEvent("Dummy")
.detail("a", x) .detail("A", x)
.detail("x", 1.5 * x) .detail("X", 1.5 * x)
.detail("d", s) .detail("D", s)
.detail("j", sin(2.0 * x)) .detail("J", sin(2.0 * x))
.detail("k", sin(3.0 * x)) .detail("K", sin(3.0 * x))
.detail("s", sstart + (double)chunk * sin(10.0 * i / chunk)); .detail("S", sstart + (double)chunk * sin(10.0 * i / chunk));
} }
Void _ = wait(delay(w)); Void _ = wait(delay(w));
@ -460,14 +460,14 @@ TEST_CASE("fdbserver/metrics/TraceEvents") {
intMetric = x; intMetric = x;
boolMetric = x % 2 > 0; boolMetric = x % 2 > 0;
TraceEvent("Dummy") TraceEvent("Dummy")
.detail("a", x) .detail("A", x)
.detail("x", 1.5 * x) .detail("X", 1.5 * x)
.detail("b", x*2) .detail("B", x*2)
.detail("y", 3.0 * x) .detail("Y", 3.0 * x)
.detail("d", d[x % 3]) .detail("D", d[x % 3])
.detail("j", sin(2.0 * x)) .detail("J", sin(2.0 * x))
.detail("k", sin(3.0 * x)) .detail("K", sin(3.0 * x))
.detail("s", sstart + (double)chunk * sin(40.0 * i / chunk)); .detail("S", sstart + (double)chunk * sin(40.0 * i / chunk));
} }
Void _ = wait(delay(w)); Void _ = wait(delay(w));
@ -476,14 +476,14 @@ TEST_CASE("fdbserver/metrics/TraceEvents") {
intMetric = x; intMetric = x;
boolMetric = x % 2 > 0; boolMetric = x % 2 > 0;
TraceEvent("Dummy") TraceEvent("Dummy")
.detail("a", x) .detail("A", x)
.detail("x", 1.5 * x) .detail("X", 1.5 * x)
.detail("c", x*3) .detail("C", x*3)
.detail("z", 4.5 * x) .detail("Z", 4.5 * x)
.detail("d", d[x % 3]) .detail("D", d[x % 3])
.detail("j", sin(2.0 * x)) .detail("J", sin(2.0 * x))
.detail("k", sin(3.0 * x)) .detail("K", sin(3.0 * x))
.detail("s", sstart + (double)chunk * sin(160.0 * i / chunk)); .detail("S", sstart + (double)chunk * sin(160.0 * i / chunk));
} }
Void _ = wait(delay(w)); Void _ = wait(delay(w));

82
fdbclient/MonitorLeader.actor.cpp
View File

@ -325,45 +325,53 @@ ACTOR Future<Void> monitorNominee( Key key, ClientLeaderRegInterface coord, Asyn
} }
// Also used in fdbserver/LeaderElection.actor.cpp! // Also used in fdbserver/LeaderElection.actor.cpp!
Optional<LeaderInfo> getLeader( vector<Optional<LeaderInfo>> nominees ) { // bool represents if the LeaderInfo is a majority answer or not.
// This function also masks the first 7 bits of changeId of the nominees and returns the Leader with masked changeId
Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInfo>>& nominees ) {
vector<LeaderInfo> maskedNominees;
maskedNominees.reserve(nominees.size());
for (auto &nominee : nominees) {
if (nominee.present()) {
maskedNominees.push_back(nominee.get());
maskedNominees.back().changeID = UID(maskedNominees.back().changeID.first() & LeaderInfo::mask, maskedNominees.back().changeID.second());
}
}
// If any coordinator says that the quorum is forwarded, then it is // If any coordinator says that the quorum is forwarded, then it is
for(int i=0; i<nominees.size(); i++) for(int i=0; i<maskedNominees.size(); i++)
if (nominees[i].present() && nominees[i].get().forward) if (maskedNominees[i].forward)
return nominees[i].get(); return std::pair<LeaderInfo, bool>(maskedNominees[i], true);
if(!maskedNominees.size())
return Optional<std::pair<LeaderInfo, bool>>();
std::sort(maskedNominees.begin(), maskedNominees.end(),
[](const LeaderInfo& l, const LeaderInfo& r) { return l.changeID < r.changeID; });
if(!nominees.size())
return Optional<LeaderInfo>();
// There is a leader if a majority of the nominees are the same.
// If there is a majority, the median item is in it.
int bestCount = 0; int bestCount = 0;
Optional<LeaderInfo> currentNominee; LeaderInfo bestNominee;
for(int i=0; i<nominees.size(); i++) { LeaderInfo currentNominee;
if( (nominees[i].present() != currentNominee.present()) || (currentNominee.present() && !currentNominee.get().equalInternalId(nominees[i].get()) ) ) { int curCount = 0;
if(bestCount > 0) { for (int i = 0; i < maskedNominees.size(); i++) {
bestCount--; if (currentNominee == maskedNominees[i]) {
} else { curCount++;
bestCount = 1; }
currentNominee = nominees[i]; else {
if (curCount > bestCount) {
bestNominee = currentNominee;
bestCount = curCount;
} }
} else { currentNominee = maskedNominees[i];
bestCount++; curCount = 1;
} }
} }
if (curCount > bestCount) {
if(!currentNominee.present()) bestNominee = currentNominee;
return Optional<LeaderInfo>(); bestCount = curCount;
int amountBest = 0;
for(int i=0; i<nominees.size(); i++) {
if( nominees[i].present() && currentNominee.get().equalInternalId(nominees[i].get()) ) {
amountBest++;
}
} }
if(amountBest >= nominees.size()/2 + 1) { bool majority = bestCount >= nominees.size() / 2 + 1;
return currentNominee; return std::pair<LeaderInfo, bool>(bestNominee, majority);
}
return Optional<LeaderInfo>();
} }
struct MonitorLeaderInfo { struct MonitorLeaderInfo {
@ -389,12 +397,12 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
allActors = waitForAll(actors); allActors = waitForAll(actors);
loop { loop {
Optional<LeaderInfo> leader = getLeader(nominees); Optional<std::pair<LeaderInfo, bool>> leader = getLeader(nominees);
TraceEvent("MonitorLeaderChange").detail("NewLeader", leader.present() ? leader.get().changeID : UID(1,1)); TraceEvent("MonitorLeaderChange").detail("NewLeader", leader.present() ? leader.get().first.changeID : UID(1,1));
if (leader.present()) { if (leader.present()) {
if( leader.get().forward ) { if( leader.get().first.forward ) {
TraceEvent("MonitorLeaderForwarding").detail("NewConnStr", leader.get().serializedInfo.toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString()); TraceEvent("MonitorLeaderForwarding").detail("NewConnStr", leader.get().first.serializedInfo.toString()).detail("OldConnStr", info.intermediateConnFile->getConnectionString().toString());
info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(leader.get().serializedInfo.toString()))); info.intermediateConnFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile->getFilename(), ClusterConnectionString(leader.get().first.serializedInfo.toString())));
return info; return info;
} }
if(connFile != info.intermediateConnFile) { if(connFile != info.intermediateConnFile) {
@ -410,7 +418,7 @@ ACTOR Future<MonitorLeaderInfo> monitorLeaderOneGeneration( Reference<ClusterCon
info.hasConnected = true; info.hasConnected = true;
connFile->notifyConnected(); connFile->notifyConnected();
outSerializedLeaderInfo->set( leader.get().serializedInfo ); outSerializedLeaderInfo->set( leader.get().first.serializedInfo );
} }
Void _ = wait( nomineeChange.onTrigger() || allActors ); Void _ = wait( nomineeChange.onTrigger() || allActors );
} }

2
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -1240,7 +1240,7 @@ ThreadFuture<Reference<ICluster>> MultiVersionApi::createCluster(const char *clu
} }
else { else {
for( auto it : externalClients ) { for( auto it : externalClients ) {
TraceEvent("CreatingClusterOnExternalClient").detail("LibraryPath", it.second->libPath).detail("failed", it.second->failed); TraceEvent("CreatingClusterOnExternalClient").detail("LibraryPath", it.second->libPath).detail("Failed", it.second->failed);
} }
return mapThreadFuture<Reference<ICluster>, Reference<ICluster>>(clusterFuture, [this, clusterFile](ErrorOr<Reference<ICluster>> cluster) { return mapThreadFuture<Reference<ICluster>, Reference<ICluster>>(clusterFuture, [this, clusterFile](ErrorOr<Reference<ICluster>> cluster) {
if(cluster.isError()) { if(cluster.isError()) {

91
fdbclient/NativeAPI.actor.cpp
View File

@ -221,20 +221,21 @@ ACTOR Future<Void> databaseLogger( DatabaseContext *cx ) {
.detail("FutureVersions", cx->transactionsFutureVersions) .detail("FutureVersions", cx->transactionsFutureVersions)
.detail("NotCommitted", cx->transactionsNotCommitted) .detail("NotCommitted", cx->transactionsNotCommitted)
.detail("MaybeCommitted", cx->transactionsMaybeCommitted) .detail("MaybeCommitted", cx->transactionsMaybeCommitted)
.detail("MeanLatency", 1000 * cx->latencies.mean()) .detail("ResourceConstrained", cx->transactionsResourceConstrained)
.detail("MedianLatency", 1000 * cx->latencies.median()) .detail("MeanLatency", cx->latencies.mean())
.detail("Latency90", 1000 * cx->latencies.percentile(0.90)) .detail("MedianLatency", cx->latencies.median())
.detail("Latency98", 1000 * cx->latencies.percentile(0.98)) .detail("Latency90", cx->latencies.percentile(0.90))
.detail("MaxLatency", 1000 * cx->latencies.max()) .detail("Latency98", cx->latencies.percentile(0.98))
.detail("MeanRowReadLatency", 1000 * cx->readLatencies.mean()) .detail("MaxLatency", cx->latencies.max())
.detail("MedianRowReadLatency", 1000 * cx->readLatencies.median()) .detail("MeanRowReadLatency", cx->readLatencies.mean())
.detail("MaxRowReadLatency", 1000 * cx->readLatencies.max()) .detail("MedianRowReadLatency", cx->readLatencies.median())
.detail("MeanGRVLatency", 1000 * cx->GRVLatencies.mean()) .detail("MaxRowReadLatency", cx->readLatencies.max())
.detail("MedianGRVLatency", 1000 * cx->GRVLatencies.median()) .detail("MeanGRVLatency", cx->GRVLatencies.mean())
.detail("MaxGRVLatency", 1000 * cx->GRVLatencies.max()) .detail("MedianGRVLatency", cx->GRVLatencies.median())
.detail("MeanCommitLatency", 1000 * cx->commitLatencies.mean()) .detail("MaxGRVLatency", cx->GRVLatencies.max())
.detail("MedianCommitLatency", 1000 * cx->commitLatencies.median()) .detail("MeanCommitLatency", cx->commitLatencies.mean())
.detail("MaxCommitLatency", 1000 * cx->commitLatencies.max()) .detail("MedianCommitLatency", cx->commitLatencies.median())
.detail("MaxCommitLatency", cx->commitLatencies.max())
.detail("MeanMutationsPerCommit", cx->mutationsPerCommit.mean()) .detail("MeanMutationsPerCommit", cx->mutationsPerCommit.mean())
.detail("MedianMutationsPerCommit", cx->mutationsPerCommit.median()) .detail("MedianMutationsPerCommit", cx->mutationsPerCommit.median())
.detail("MaxMutationsPerCommit", cx->mutationsPerCommit.max()) .detail("MaxMutationsPerCommit", cx->mutationsPerCommit.max())
@ -470,7 +471,7 @@ DatabaseContext::DatabaseContext(
int taskID, LocalityData clientLocality, bool enableLocalityLoadBalance, bool lockAware ) int taskID, LocalityData clientLocality, bool enableLocalityLoadBalance, bool lockAware )
: clientInfo(clientInfo), masterProxiesChangeTrigger(), cluster(cluster), clientInfoMonitor(clientInfoMonitor), dbName(dbName), dbId(dbId), : clientInfo(clientInfo), masterProxiesChangeTrigger(), cluster(cluster), clientInfoMonitor(clientInfoMonitor), dbName(dbName), dbId(dbId),
transactionReadVersions(0), transactionLogicalReads(0), transactionPhysicalReads(0), transactionCommittedMutations(0), transactionCommittedMutationBytes(0), transactionsCommitStarted(0), transactionReadVersions(0), transactionLogicalReads(0), transactionPhysicalReads(0), transactionCommittedMutations(0), transactionCommittedMutationBytes(0), transactionsCommitStarted(0),
transactionsCommitCompleted(0), transactionsTooOld(0), transactionsFutureVersions(0), transactionsNotCommitted(0), transactionsMaybeCommitted(0), taskID(taskID), transactionsCommitCompleted(0), transactionsTooOld(0), transactionsFutureVersions(0), transactionsNotCommitted(0), transactionsMaybeCommitted(0), transactionsResourceConstrained(0), taskID(taskID),
outstandingWatches(0), maxOutstandingWatches(CLIENT_KNOBS->DEFAULT_MAX_OUTSTANDING_WATCHES), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware), outstandingWatches(0), maxOutstandingWatches(CLIENT_KNOBS->DEFAULT_MAX_OUTSTANDING_WATCHES), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware),
latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000) latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000)
{ {
@ -969,7 +970,7 @@ AddressExclusion AddressExclusion::parse( StringRef const& key ) {
std::string s = key.toString(); std::string s = key.toString();
int a,b,c,d,port,count=-1; int a,b,c,d,port,count=-1;
if (sscanf(s.c_str(), "%d.%d.%d.%d%n", &a,&b,&c,&d, &count)<4) { if (sscanf(s.c_str(), "%d.%d.%d.%d%n", &a,&b,&c,&d, &count)<4) {
TraceEvent(SevWarnAlways, "AddressExclusionParseError").detail("s", printable(key)); TraceEvent(SevWarnAlways, "AddressExclusionParseError").detail("String", printable(key));
return AddressExclusion(); return AddressExclusion();
} }
s = s.substr(count); s = s.substr(count);
@ -977,7 +978,7 @@ AddressExclusion AddressExclusion::parse( StringRef const& key ) {
if (!s.size()) if (!s.size())
return AddressExclusion( ip ); return AddressExclusion( ip );
if (sscanf( s.c_str(), ":%d%n", &port, &count ) < 1 || count != s.size()) { if (sscanf( s.c_str(), ":%d%n", &port, &count ) < 1 || count != s.size()) {
TraceEvent(SevWarnAlways, "AddressExclusionParseError").detail("s", printable(key)); TraceEvent(SevWarnAlways, "AddressExclusionParseError").detail("String", printable(key));
return AddressExclusion(); return AddressExclusion();
} }
return AddressExclusion( ip, port ); return AddressExclusion( ip, port );
@ -1249,11 +1250,11 @@ ACTOR Future<Key> getKey( Database cx, KeySelector k, Future<Version> version, T
try { try {
if( info.debugID.present() ) if( info.debugID.present() )
g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.Before"); //.detail("StartKey", printable(k.getKey())).detail("offset",k.offset).detail("orEqual",k.orEqual); g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.Before"); //.detail("StartKey", printable(k.getKey())).detail("Offset",k.offset).detail("OrEqual",k.orEqual);
++cx->transactionPhysicalReads; ++cx->transactionPhysicalReads;
GetKeyReply reply = wait( loadBalance( ssi.second, &StorageServerInterface::getKey, GetKeyRequest(k, version.get()), TaskDefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL ) ); GetKeyReply reply = wait( loadBalance( ssi.second, &StorageServerInterface::getKey, GetKeyRequest(k, version.get()), TaskDefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL ) );
if( info.debugID.present() ) if( info.debugID.present() )
g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.After"); //.detail("NextKey",printable(reply.sel.key)).detail("offset", reply.sel.offset).detail("orEqual", k.orEqual); g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.After"); //.detail("NextKey",printable(reply.sel.key)).detail("Offset", reply.sel.offset).detail("OrEqual", k.orEqual);
k = reply.sel; k = reply.sel;
if (!k.offset && k.orEqual) { if (!k.offset && k.orEqual) {
return k.getKey(); return k.getKey();
@ -1265,7 +1266,7 @@ ACTOR Future<Key> getKey( Database cx, KeySelector k, Future<Version> version, T
Void _ = wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID)); Void _ = wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID));
} else { } else {
if(e.code() != error_code_actor_cancelled) { if(e.code() != error_code_actor_cancelled) {
TraceEvent(SevInfo, "getKeyError") TraceEvent(SevInfo, "GetKeyError")
.error(e) .error(e)
.detail("AtKey", printable(k.getKey())) .detail("AtKey", printable(k.getKey()))
.detail("Offset", k.offset); .detail("Offset", k.offset);
@ -1290,7 +1291,7 @@ ACTOR Future<Version> waitForCommittedVersion( Database cx, Version version ) {
} }
} }
} catch (Error& e) { } catch (Error& e) {
TraceEvent(SevError, "waitForCommittedVersionError").error(e); TraceEvent(SevError, "WaitForCommittedVersionError").error(e);
throw; throw;
} }
} }
@ -1323,7 +1324,7 @@ ACTOR Future< Void > watchValue( Future<Version> version, Key key, Optional<Valu
//cannot do this until the storage server is notified on knownCommittedVersion changes from tlog (faster than the current update loop) //cannot do this until the storage server is notified on knownCommittedVersion changes from tlog (faster than the current update loop)
Version v = wait( waitForCommittedVersion( cx, resp ) ); Version v = wait( waitForCommittedVersion( cx, resp ) );
//TraceEvent("watcherCommitted").detail("committedVersion", v).detail("watchVersion", resp).detail("key", printable( key )).detail("value", printable(value)); //TraceEvent("WatcherCommitted").detail("CommittedVersion", v).detail("WatchVersion", resp).detail("Key", printable( key )).detail("Value", printable(value));
if( v - resp < 50000000 ) // False if there is a master failure between getting the response and getting the committed version, Dependent on SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT if( v - resp < 50000000 ) // False if there is a master failure between getting the response and getting the committed version, Dependent on SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT
return Void(); return Void();
@ -1484,7 +1485,7 @@ ACTOR Future<Standalone<RangeResultRef>> getExactRange( Database cx, Version ver
Void _ = wait( delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID )); Void _ = wait( delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID ));
break; break;
} else { } else {
TraceEvent(SevInfo, "getExactRangeError") TraceEvent(SevInfo, "GetExactRangeError")
.error(e) .error(e)
.detail("ShardBegin", printable(locations[shard].first.begin)) .detail("ShardBegin", printable(locations[shard].first.begin))
.detail("ShardEnd", printable(locations[shard].first.end)); .detail("ShardEnd", printable(locations[shard].first.end));
@ -1663,11 +1664,11 @@ ACTOR Future<Standalone<RangeResultRef>> getRange( Database cx, Reference<Transa
/*TraceEvent("TransactionDebugGetRangeInfo", info.debugID.get()) /*TraceEvent("TransactionDebugGetRangeInfo", info.debugID.get())
.detail("ReqBeginKey", printable(req.begin.getKey())) .detail("ReqBeginKey", printable(req.begin.getKey()))
.detail("ReqEndKey", printable(req.end.getKey())) .detail("ReqEndKey", printable(req.end.getKey()))
.detail("originalBegin", originalBegin.toString()) .detail("OriginalBegin", originalBegin.toString())
.detail("originalEnd", originalEnd.toString()) .detail("OriginalEnd", originalEnd.toString())
.detail("Begin", begin.toString()) .detail("Begin", begin.toString())
.detail("End", end.toString()) .detail("End", end.toString())
.detail("shard", printable(shard)) .detail("Shard", printable(shard))
.detail("ReqLimit", req.limit) .detail("ReqLimit", req.limit)
.detail("ReqLimitBytes", req.limitBytes) .detail("ReqLimitBytes", req.limitBytes)
.detail("ReqVersion", req.version) .detail("ReqVersion", req.version)
@ -2165,9 +2166,10 @@ void Transaction::addWriteConflictRange( const KeyRangeRef& keys ) {
t.write_conflict_ranges.push_back_deep( req.arena, r ); t.write_conflict_ranges.push_back_deep( req.arena, r );
} }
double Transaction::getBackoff() { double Transaction::getBackoff(int errCode) {
double b = backoff * g_random->random01(); double b = backoff * g_random->random01();
backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, options.maxBackoff); backoff = errCode == error_code_proxy_memory_limit_exceeded ? std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF) :
std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, options.maxBackoff);
return b; return b;
} }
@ -2446,8 +2448,8 @@ ACTOR static Future<Void> tryCommit( Database cx, Reference<TransactionLogInfo>
// The user needs to be informed that we aren't sure whether the commit happened. Standard retry loops retry it anyway (relying on transaction idempotence) but a client might do something else. // The user needs to be informed that we aren't sure whether the commit happened. Standard retry loops retry it anyway (relying on transaction idempotence) but a client might do something else.
throw commit_unknown_result(); throw commit_unknown_result();
} else { } else {
if (e.code() != error_code_transaction_too_old && e.code() != error_code_not_committed && e.code() != error_code_database_locked) if (e.code() != error_code_transaction_too_old && e.code() != error_code_not_committed && e.code() != error_code_database_locked && e.code() != error_code_proxy_memory_limit_exceeded)
TraceEvent(SevError, "tryCommitError").error(e); TraceEvent(SevError, "TryCommitError").error(e);
if (trLogInfo) if (trLogInfo)
trLogInfo->addLog(FdbClientLogEvents::EventCommitError(startTime, static_cast<int>(e.code()), req)); trLogInfo->addLog(FdbClientLogEvents::EventCommitError(startTime, static_cast<int>(e.code()), req));
throw; throw;
@ -2456,11 +2458,6 @@ ACTOR static Future<Void> tryCommit( Database cx, Reference<TransactionLogInfo>
} }
Future<Void> Transaction::commitMutations() { Future<Void> Transaction::commitMutations() {
cx->transactionsCommitStarted++;
if(options.readOnly)
return transaction_read_only();
try { try {
//if this is a read-only transaction return immediately //if this is a read-only transaction return immediately
if( !tr.transaction.write_conflict_ranges.size() && !tr.transaction.mutations.size() ) { if( !tr.transaction.write_conflict_ranges.size() && !tr.transaction.mutations.size() ) {
@ -2471,6 +2468,11 @@ Future<Void> Transaction::commitMutations() {
return Void(); return Void();
} }
cx->transactionsCommitStarted++;
if(options.readOnly)
return transaction_read_only();
cx->mutationsPerCommit.addSample(tr.transaction.mutations.size()); cx->mutationsPerCommit.addSample(tr.transaction.mutations.size());
cx->bytesPerCommit.addSample(tr.transaction.mutations.expectedSize()); cx->bytesPerCommit.addSample(tr.transaction.mutations.expectedSize());
@ -2479,8 +2481,8 @@ Future<Void> Transaction::commitMutations() {
TraceEvent(!g_network->isSimulated() ? SevWarnAlways : SevWarn, "LargeTransaction") TraceEvent(!g_network->isSimulated() ? SevWarnAlways : SevWarn, "LargeTransaction")
.detail("Size", transactionSize) .detail("Size", transactionSize)
.detail("NumMutations", tr.transaction.mutations.size()) .detail("NumMutations", tr.transaction.mutations.size())
.detail("readConflictSize", tr.transaction.read_conflict_ranges.expectedSize()) .detail("ReadConflictSize", tr.transaction.read_conflict_ranges.expectedSize())
.detail("writeConflictSize", tr.transaction.write_conflict_ranges.expectedSize()) .detail("WriteConflictSize", tr.transaction.write_conflict_ranges.expectedSize())
.suppressFor(1.0); .suppressFor(1.0);
} }
@ -2796,14 +2798,17 @@ Future<Void> Transaction::onError( Error const& e ) {
} }
if (e.code() == error_code_not_committed || if (e.code() == error_code_not_committed ||
e.code() == error_code_commit_unknown_result || e.code() == error_code_commit_unknown_result ||
e.code() == error_code_database_locked) e.code() == error_code_database_locked ||
e.code() == error_code_proxy_memory_limit_exceeded)
{ {
if(e.code() == error_code_not_committed) if(e.code() == error_code_not_committed)
cx->transactionsNotCommitted++; cx->transactionsNotCommitted++;
if(e.code() == error_code_commit_unknown_result) if(e.code() == error_code_commit_unknown_result)
cx->transactionsMaybeCommitted++; cx->transactionsMaybeCommitted++;
if (e.code() == error_code_proxy_memory_limit_exceeded)
cx->transactionsResourceConstrained++;
double backoff = getBackoff(); double backoff = getBackoff(e.code());
reset(); reset();
return delay( backoff, info.taskID ); return delay( backoff, info.taskID );
} }
@ -2922,7 +2927,7 @@ ACTOR Future< StorageMetrics > waitStorageMetrics(
} }
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) {
TraceEvent(SevError, "waitStorageMetricsError").error(e); TraceEvent(SevError, "WaitStorageMetricsError").error(e);
throw; throw;
} }
cx->invalidateCache(keys); cx->invalidateCache(keys);
@ -2963,7 +2968,7 @@ ACTOR Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( Database cx,
else { else {
results.push_back_deep( results.arena(), keys.begin ); results.push_back_deep( results.arena(), keys.begin );
try { try {
//TraceEvent("SplitStorageMetrics").detail("locations", locations.size()); //TraceEvent("SplitStorageMetrics").detail("Locations", locations.size());
state int i = 0; state int i = 0;
for(; i<locations.size(); i++) { for(; i<locations.size(); i++) {
@ -2979,7 +2984,7 @@ ACTOR Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( Database cx,
} }
used = res.used; used = res.used;
//TraceEvent("SplitStorageMetricsResult").detail("used", used.bytes).detail("location", i).detail("size", res.splits.size()); //TraceEvent("SplitStorageMetricsResult").detail("Used", used.bytes).detail("Location", i).detail("Size", res.splits.size());
} }
if( used.allLessOrEqual( limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT ) ) { if( used.allLessOrEqual( limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT ) ) {
@ -2990,7 +2995,7 @@ ACTOR Future< Standalone<VectorRef<KeyRef>> > splitStorageMetrics( Database cx,
return results; return results;
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) {
TraceEvent(SevError, "splitStorageMetricsError").error(e); TraceEvent(SevError, "SplitStorageMetricsError").error(e);
throw; throw;
} }
cx->invalidateCache( keys ); cx->invalidateCache( keys );

2
fdbclient/NativeAPI.h
View File

@ -284,7 +284,7 @@ public:
void reset(); void reset();
void fullReset(); void fullReset();
double getBackoff(); double getBackoff(int errCode);
void debugTransaction(UID dID) { info.debugID = dID; } void debugTransaction(UID dID) { info.debugID = dID; }
Future<Void> commitMutations(); Future<Void> commitMutations();

48
fdbclient/RYWIterator.cpp
View File

@ -558,7 +558,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
KeyRangeRef range = RandomTestImpl::getRandomRange(arena); KeyRangeRef range = RandomTestImpl::getRandomRange(arena);
writes.addConflictRange(range); writes.addConflictRange(range);
conflictMap.insert(range, true); conflictMap.insert(range, true);
TraceEvent("RWMT_addConflictRange").detail("range", printable(range)); TraceEvent("RWMT_AddConflictRange").detail("Range", printable(range));
} }
else if(r == 1) { else if(r == 1) {
KeyRangeRef range = RandomTestImpl::getRandomRange(arena); KeyRangeRef range = RandomTestImpl::getRandomRange(arena);
@ -567,7 +567,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
conflictMap.insert(range, false); conflictMap.insert(range, false);
clearMap.insert(range, false); clearMap.insert(range, false);
unreadableMap.insert(range, true); unreadableMap.insert(range, true);
TraceEvent("RWMT_addUnmodifiedAndUnreadableRange").detail("range", printable(range)); TraceEvent("RWMT_AddUnmodifiedAndUnreadableRange").detail("Range", printable(range));
} }
else if (r == 2) { else if (r == 2) {
bool addConflict = g_random->random01() < 0.5; bool addConflict = g_random->random01() < 0.5;
@ -578,7 +578,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
conflictMap.insert(range, true); conflictMap.insert(range, true);
clearMap.insert(range, true); clearMap.insert(range, true);
unreadableMap.insert(range, false); unreadableMap.insert(range, false);
TraceEvent("RWMT_clear").detail("range", printable(range)).detail("addConflict", addConflict); TraceEvent("RWMT_Clear").detail("Range", printable(range)).detail("AddConflict", addConflict);
} }
else if (r == 3) { else if (r == 3) {
bool addConflict = g_random->random01() < 0.5; bool addConflict = g_random->random01() < 0.5;
@ -590,7 +590,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
conflictMap.insert(key, true); conflictMap.insert(key, true);
clearMap.insert(key, false); clearMap.insert(key, false);
unreadableMap.insert(key, true); unreadableMap.insert(key, true);
TraceEvent("RWMT_setVersionstampedValue").detail("key", printable(key)).detail("value", value.size()).detail("addConflict", addConflict); TraceEvent("RWMT_SetVersionstampedValue").detail("Key", printable(key)).detail("Value", value.size()).detail("AddConflict", addConflict);
} }
else if (r == 4) { else if (r == 4) {
bool addConflict = g_random->random01() < 0.5; bool addConflict = g_random->random01() < 0.5;
@ -602,7 +602,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
conflictMap.insert(key, true); conflictMap.insert(key, true);
clearMap.insert(key, false); clearMap.insert(key, false);
unreadableMap.insert(key, true); unreadableMap.insert(key, true);
TraceEvent("RWMT_setVersionstampedKey").detail("key", printable(key)).detail("value", value.size()).detail("addConflict", addConflict); TraceEvent("RWMT_SetVersionstampedKey").detail("Key", printable(key)).detail("Value", value.size()).detail("AddConflict", addConflict);
} }
else if (r == 5) { else if (r == 5) {
bool addConflict = g_random->random01() < 0.5; bool addConflict = g_random->random01() < 0.5;
@ -622,7 +622,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
if (addConflict) if (addConflict)
conflictMap.insert(key, true); conflictMap.insert(key, true);
clearMap.insert(key, false); clearMap.insert(key, false);
TraceEvent("RWMT_and").detail("key", printable(key)).detail("value", value.size()).detail("addConflict", addConflict); TraceEvent("RWMT_And").detail("Key", printable(key)).detail("Value", value.size()).detail("AddConflict", addConflict);
} }
else { else {
bool addConflict = g_random->random01() < 0.5; bool addConflict = g_random->random01() < 0.5;
@ -636,7 +636,7 @@ TEST_CASE("fdbclient/WriteMap/random") {
if (addConflict) if (addConflict)
conflictMap.insert(key, true); conflictMap.insert(key, true);
clearMap.insert(key, false); clearMap.insert(key, false);
TraceEvent("RWMT_set").detail("key", printable(key)).detail("value", value.size()).detail("addConflict", addConflict); TraceEvent("RWMT_Set").detail("Key", printable(key)).detail("Value", value.size()).detail("AddConflict", addConflict);
} }
} }
@ -648,23 +648,23 @@ TEST_CASE("fdbclient/WriteMap/random") {
for (; it.beginKey() < allKeys.end; ++it) { for (; it.beginKey() < allKeys.end; ++it) {
if (it.is_operation()) { if (it.is_operation()) {
ASSERT(setIter != setEnd); ASSERT(setIter != setEnd);
TraceEvent("RWMT_checkOperation") TraceEvent("RWMT_CheckOperation")
.detail("wm_key", printable(it.beginKey().toStandaloneStringRef())) .detail("WmKey", printable(it.beginKey().toStandaloneStringRef()))
.detail("wm_size", it.op().size()) .detail("WmSize", it.op().size())
.detail("wm_value", it.op().top().value.present() ? std::to_string(it.op().top().value.get().size()) : "Not Found") .detail("WmValue", it.op().top().value.present() ? std::to_string(it.op().top().value.get().size()) : "Not Found")
.detail("wm_type", (int)it.op().top().type) .detail("WmType", (int)it.op().top().type)
.detail("sm_key", printable(setIter->first)) .detail("SmKey", printable(setIter->first))
.detail("sm_size", setIter->second.size()) .detail("SmSize", setIter->second.size())
.detail("sm_value", setIter->second.top().value.present() ? std::to_string(setIter->second.top().value.get().size()) : "Not Found") .detail("SmValue", setIter->second.top().value.present() ? std::to_string(setIter->second.top().value.get().size()) : "Not Found")
.detail("sm_type", (int)setIter->second.top().type); .detail("SmType", (int)setIter->second.top().type);
ASSERT(it.beginKey() == setIter->first && it.op() == setIter->second); ASSERT(it.beginKey() == setIter->first && it.op() == setIter->second);
++setIter; ++setIter;
} }
} }
TraceEvent("RWMT_checkOperationFinal") TraceEvent("RWMT_CheckOperationFinal")
.detail("wm_key", printable(it.beginKey().toStandaloneStringRef())) .detail("WmKey", printable(it.beginKey().toStandaloneStringRef()))
.detail("sm_iter", setIter == setEnd); .detail("SmIter", setIter == setEnd);
ASSERT(it.beginKey() >= allKeys.end && setIter == setEnd); ASSERT(it.beginKey() >= allKeys.end && setIter == setEnd);
@ -711,11 +711,11 @@ TEST_CASE("fdbclient/WriteMap/random") {
auto unreadableEnd = unreadableRanges.end(); auto unreadableEnd = unreadableRanges.end();
while (it.beginKey() < allKeys.end && unreadableIter != unreadableEnd) { while (it.beginKey() < allKeys.end && unreadableIter != unreadableEnd) {
TraceEvent("RWMT_checkUnreadable") TraceEvent("RWMT_CheckUnreadable")
.detail("writeMapRange", printable(KeyRangeRef(it.beginKey().toStandaloneStringRef(), it.endKey().toStandaloneStringRef()))) .detail("WriteMapRange", printable(KeyRangeRef(it.beginKey().toStandaloneStringRef(), it.endKey().toStandaloneStringRef())))
.detail("unreadableMapRange", printable(unreadableIter.range())) .detail("UnreadableMapRange", printable(unreadableIter.range()))
.detail("writeMapValue", it.is_unreadable()) .detail("WriteMapValue", it.is_unreadable())
.detail("unreadableMapValue", unreadableIter.value()); .detail("UnreadableMapValue", unreadableIter.value());
ASSERT(unreadableIter.value() == it.is_unreadable()); ASSERT(unreadableIter.value() == it.is_unreadable());
if (unreadableIter.range().end < it.endKey()) { if (unreadableIter.range().end < it.endKey()) {
++unreadableIter; ++unreadableIter;

52
fdbclient/ReadYourWrites.actor.cpp
View File

@ -490,15 +490,15 @@ public:
//TraceEvent("RYWSelectorsStartForward", randomID).detail("ByteLimit", limits.bytes).detail("RowLimit", limits.rows); //TraceEvent("RYWSelectorsStartForward", randomID).detail("ByteLimit", limits.bytes).detail("RowLimit", limits.rows);
loop { loop {
/*TraceEvent("RYWSelectors", randomID).detail("begin", begin.toString()) /*TraceEvent("RYWSelectors", randomID).detail("Begin", begin.toString())
.detail("end", end.toString()) .detail("End", end.toString())
.detail("reached", limits.isReached()) .detail("Reached", limits.isReached())
.detail("itemsPastEnd", itemsPastEnd) .detail("ItemsPastEnd", itemsPastEnd)
.detail("endOffset", -end.offset) .detail("EndOffset", -end.offset)
.detail("itBegin", printable(it.beginKey().toStandaloneStringRef())) .detail("ItBegin", printable(it.beginKey().toStandaloneStringRef()))
.detail("itEnd", printable(itEnd.beginKey().toStandaloneStringRef())) .detail("ItEnd", printable(itEnd.beginKey().toStandaloneStringRef()))
.detail("unknown", it.is_unknown_range()) .detail("Unknown", it.is_unknown_range())
.detail("requests", requestCount);*/ .detail("Requests", requestCount);*/
if( !result.size() && actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey() ) { if( !result.size() && actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey() ) {
return RangeResultRef(false, false); return RangeResultRef(false, false);
@ -589,13 +589,13 @@ public:
ASSERT( !requestLimit.hasRowLimit() || requestLimit.rows > 0 ); ASSERT( !requestLimit.hasRowLimit() || requestLimit.rows > 0 );
ASSERT( requestLimit.hasRowLimit() || requestLimit.hasByteLimit() ); ASSERT( requestLimit.hasRowLimit() || requestLimit.hasByteLimit() );
//TraceEvent("RYWIssuing", randomID).detail("begin", read_begin.toString()).detail("end", read_end.toString()).detail("bytes", requestLimit.bytes).detail("rows", requestLimit.rows).detail("limits", limits.bytes).detail("reached", limits.isReached()).detail("requestCount", requestCount).detail("singleClears", singleClears).detail("ucEnd", printable(ucEnd.beginKey().toStandaloneStringRef())).detail("minRows", requestLimit.minRows); //TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", printable(ucEnd.beginKey().toStandaloneStringRef())).detail("MinRows", requestLimit.minRows);
additionalRows = 0; additionalRows = 0;
Standalone<RangeResultRef> snapshot_read = wait( ryw->tr.getRange( read_begin, read_end, requestLimit, true, false ) ); Standalone<RangeResultRef> snapshot_read = wait( ryw->tr.getRange( read_begin, read_end, requestLimit, true, false ) );
KeyRangeRef range = getKnownKeyRange( snapshot_read, read_begin, read_end, ryw->arena ); KeyRangeRef range = getKnownKeyRange( snapshot_read, read_begin, read_end, ryw->arena );
//TraceEvent("RYWCacheInsert", randomID).detail("Range", printable(range)).detail("expectedSize", snapshot_read.expectedSize()).detail("rows", snapshot_read.size()).detail("results", printable(snapshot_read)).detail("more", snapshot_read.more).detail("readToBegin", snapshot_read.readToBegin).detail("readThroughEnd", snapshot_read.readThroughEnd).detail("readThrough", printable(snapshot_read.readThrough)); //TraceEvent("RYWCacheInsert", randomID).detail("Range", printable(range)).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", printable(snapshot_read)).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", printable(snapshot_read.readThrough));
if( ryw->cache.insert( range, snapshot_read ) ) if( ryw->cache.insert( range, snapshot_read ) )
ryw->arena.dependsOn(snapshot_read.arena()); ryw->arena.dependsOn(snapshot_read.arena());
@ -615,7 +615,7 @@ public:
itemsPastEnd += maxCount - count; itemsPastEnd += maxCount - count;
//TraceEvent("RYWaddKV", randomID).detail("key", printable(it.beginKey().toStandaloneStringRef())).detail("count", count).detail("maxCount", maxCount).detail("itemsPastEnd", itemsPastEnd); //TraceEvent("RYWaddKV", randomID).detail("Key", printable(it.beginKey().toStandaloneStringRef())).detail("Count", count).detail("MaxCount", maxCount).detail("ItemsPastEnd", itemsPastEnd);
if( count ) result.append( result.arena(), start, count ); if( count ) result.append( result.arena(), start, count );
++it; ++it;
} else } else
@ -756,19 +756,19 @@ public:
resolveKeySelectorFromCache( begin, itEnd, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualBeginOffset ); resolveKeySelectorFromCache( begin, itEnd, ryw->getMaxReadKey(), &readToBegin, &readThroughEnd, &actualBeginOffset );
} }
//TraceEvent("RYWSelectorsStartReverse", randomID).detail("byteLimit", limits.bytes).detail("rowLimit", limits.rows); //TraceEvent("RYWSelectorsStartReverse", randomID).detail("ByteLimit", limits.bytes).detail("RowLimit", limits.rows);
loop { loop {
/*TraceEvent("RYWSelectors", randomID).detail("begin", begin.toString()) /*TraceEvent("RYWSelectors", randomID).detail("Begin", begin.toString())
.detail("end", end.toString()) .detail("End", end.toString())
.detail("reached", limits.isReached()) .detail("Reached", limits.isReached())
.detail("itemsPastBegin", itemsPastBegin) .detail("ItemsPastBegin", itemsPastBegin)
.detail("endOffset", end.offset) .detail("EndOffset", end.offset)
.detail("itBegin", printable(it.beginKey().toStandaloneStringRef())) .detail("ItBegin", printable(it.beginKey().toStandaloneStringRef()))
.detail("itEnd", printable(itEnd.beginKey().toStandaloneStringRef())) .detail("ItEnd", printable(itEnd.beginKey().toStandaloneStringRef()))
.detail("unknown", it.is_unknown_range()) .detail("Unknown", it.is_unknown_range())
.detail("kv", it.is_kv()) .detail("Kv", it.is_kv())
.detail("requests", requestCount);*/ .detail("Requests", requestCount);*/
if(!result.size() && actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) { if(!result.size() && actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) {
return RangeResultRef(false, false); return RangeResultRef(false, false);
@ -862,13 +862,13 @@ public:
ASSERT( !requestLimit.hasRowLimit() || requestLimit.rows > 0 ); ASSERT( !requestLimit.hasRowLimit() || requestLimit.rows > 0 );
ASSERT( requestLimit.hasRowLimit() || requestLimit.hasByteLimit() ); ASSERT( requestLimit.hasRowLimit() || requestLimit.hasByteLimit() );
//TraceEvent("RYWIssuing", randomID).detail("begin", read_begin.toString()).detail("end", read_end.toString()).detail("bytes", requestLimit.bytes).detail("rows", requestLimit.rows).detail("limits", limits.bytes).detail("reached", limits.isReached()).detail("requestCount", requestCount).detail("singleClears", singleClears).detail("ucEnd", printable(ucEnd.beginKey().toStandaloneStringRef())).detail("minRows", requestLimit.minRows); //TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", printable(ucEnd.beginKey().toStandaloneStringRef())).detail("MinRows", requestLimit.minRows);
additionalRows = 0; additionalRows = 0;
Standalone<RangeResultRef> snapshot_read = wait( ryw->tr.getRange( read_begin, read_end, requestLimit, true, true ) ); Standalone<RangeResultRef> snapshot_read = wait( ryw->tr.getRange( read_begin, read_end, requestLimit, true, true ) );
KeyRangeRef range = getKnownKeyRangeBack( snapshot_read, read_begin, read_end, ryw->arena ); KeyRangeRef range = getKnownKeyRangeBack( snapshot_read, read_begin, read_end, ryw->arena );
//TraceEvent("RYWCacheInsert", randomID).detail("Range", printable(range)).detail("expectedSize", snapshot_read.expectedSize()).detail("rows", snapshot_read.size()).detail("results", printable(snapshot_read)).detail("more", snapshot_read.more).detail("readToBegin", snapshot_read.readToBegin).detail("readThroughEnd", snapshot_read.readThroughEnd).detail("readThrough", printable(snapshot_read.readThrough)); //TraceEvent("RYWCacheInsert", randomID).detail("Range", printable(range)).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", printable(snapshot_read)).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", printable(snapshot_read.readThrough));
RangeResultRef reversed; RangeResultRef reversed;
reversed.resize(ryw->arena, snapshot_read.size()); reversed.resize(ryw->arena, snapshot_read.size());
@ -895,7 +895,7 @@ public:
} }
itemsPastBegin += maxCount - count; itemsPastBegin += maxCount - count;
//TraceEvent("RYWaddKV", randomID).detail("key", printable(it.beginKey().toStandaloneStringRef())).detail("count", count).detail("maxCount", maxCount).detail("itemsPastBegin", itemsPastBegin); //TraceEvent("RYWaddKV", randomID).detail("Key", printable(it.beginKey().toStandaloneStringRef())).detail("Count", count).detail("MaxCount", maxCount).detail("ItemsPastBegin", itemsPastBegin);
if( count ) { if( count ) {
int size = result.size(); int size = result.size();
result.resize(result.arena(),size+count); result.resize(result.arena(),size+count);

1
fdbclient/ReadYourWrites.h
View File

@ -119,7 +119,6 @@ public:
void cancel(); void cancel();
void reset(); void reset();
double getBackoff() { return tr.getBackoff(); }
void debugTransaction(UID dID) { tr.debugTransaction(dID); } void debugTransaction(UID dID) { tr.debugTransaction(dID); }
Future<Void> debug_onIdle() { return reading; } Future<Void> debug_onIdle() { return reading; }

2
fdbclient/SnapshotCache.h
View File

@ -331,7 +331,7 @@ public:
void dump() { void dump() {
for( auto it = entries.begin(); it != entries.end(); ++it ) { for( auto it = entries.begin(); it != entries.end(); ++it ) {
TraceEvent("CacheDump").detail("begin", printable(it->beginKey)).detail("end", printable(it->endKey.toStandaloneStringRef())).detail("values", printable(it->values)); TraceEvent("CacheDump").detail("Begin", printable(it->beginKey)).detail("End", printable(it->endKey.toStandaloneStringRef())).detail("Values", printable(it->values));
} }
} }

2
fdbclient/SystemData.cpp
View File

@ -453,7 +453,7 @@ Key logRangesEncodeKey(KeyRef keyBegin, UID logUid) {
// Returns the start key and optionally the logRange Uid // Returns the start key and optionally the logRange Uid
KeyRef logRangesDecodeKey(KeyRef key, UID* logUid) { KeyRef logRangesDecodeKey(KeyRef key, UID* logUid) {
if (key.size() < logRangesRange.begin.size() + sizeof(UID)) { if (key.size() < logRangesRange.begin.size() + sizeof(UID)) {
TraceEvent(SevError, "InvalidDecodeKey").detail("key", printable(key)); TraceEvent(SevError, "InvalidDecodeKey").detail("Key", printable(key));
ASSERT(false); ASSERT(false);
} }

54
fdbclient/TaskBucket.actor.cpp
View File

@ -229,17 +229,17 @@ public:
ACTOR static Future<bool> taskVerify(Reference<TaskBucket> tb, Reference<ReadYourWritesTransaction> tr, Reference<Task> task) { ACTOR static Future<bool> taskVerify(Reference<TaskBucket> tb, Reference<ReadYourWritesTransaction> tr, Reference<Task> task) {
if (task->params.find(Task::reservedTaskParamValidKey) == task->params.end()) { if (task->params.find(Task::reservedTaskParamValidKey) == task->params.end()) {
TraceEvent("TB_taskVerify_invalidTask") TraceEvent("TB_TaskVerifyInvalidTask")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("reservedTaskParamValidKey", "missing"); .detail("ReservedTaskParamValidKey", "missing");
return false; return false;
} }
if (task->params.find(Task::reservedTaskParamValidValue) == task->params.end()) { if (task->params.find(Task::reservedTaskParamValidValue) == task->params.end()) {
TraceEvent("TB_taskVerify_invalidTask") TraceEvent("TB_TaskVerifyInvalidTask")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("reservedTaskParamValidKey", printable(task->params[Task::reservedTaskParamValidKey])) .detail("ReservedTaskParamValidKey", printable(task->params[Task::reservedTaskParamValidKey]))
.detail("reservedTaskParamValidValue", "missing"); .detail("ReservedTaskParamValidValue", "missing");
return false; return false;
} }
@ -248,20 +248,20 @@ public:
Optional<Value> keyValue = wait(tr->get(task->params[Task::reservedTaskParamValidKey])); Optional<Value> keyValue = wait(tr->get(task->params[Task::reservedTaskParamValidKey]));
if (!keyValue.present()) { if (!keyValue.present()) {
TraceEvent("TB_taskVerify_invalidTask") TraceEvent("TB_TaskVerifyInvalidTask")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("reservedTaskParamValidKey", printable(task->params[Task::reservedTaskParamValidKey])) .detail("ReservedTaskParamValidKey", printable(task->params[Task::reservedTaskParamValidKey]))
.detail("reservedTaskParamValidValue", printable(task->params[Task::reservedTaskParamValidValue])) .detail("ReservedTaskParamValidValue", printable(task->params[Task::reservedTaskParamValidValue]))
.detail("keyValue", "missing"); .detail("KeyValue", "missing");
return false; return false;
} }
if (keyValue.get().compare(StringRef(task->params[Task::reservedTaskParamValidValue]))) { if (keyValue.get().compare(StringRef(task->params[Task::reservedTaskParamValidValue]))) {
TraceEvent("TB_taskVerify_abortedTask") TraceEvent("TB_TaskVerifyAbortedTask")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("reservedTaskParamValidKey", printable(task->params[Task::reservedTaskParamValidKey])) .detail("ReservedTaskParamValidKey", printable(task->params[Task::reservedTaskParamValidKey]))
.detail("reservedTaskParamValidValue", printable(task->params[Task::reservedTaskParamValidValue])) .detail("ReservedTaskParamValidValue", printable(task->params[Task::reservedTaskParamValidValue]))
.detail("keyValue", printable(keyValue.get())); .detail("KeyValue", printable(keyValue.get()));
return false; return false;
} }
@ -704,14 +704,14 @@ public:
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
Standalone<RangeResultRef> values = wait(tr->getRange(subspace.range(), CLIENT_KNOBS->TOO_MANY)); Standalone<RangeResultRef> values = wait(tr->getRange(subspace.range(), CLIENT_KNOBS->TOO_MANY));
TraceEvent("TaskBucket").detail("debugPrintRange", "Print DB Range").detail("key", printable(subspace.key())).detail("count", values.size()).detail("msg", printable(msg)); TraceEvent("TaskBucket").detail("DebugPrintRange", "Print DB Range").detail("Key", printable(subspace.key())).detail("Count", values.size()).detail("Msg", printable(msg));
/* /*
printf("debugPrintRange key: (%d) %s\n", values.size(), printable(subspace.key()).c_str()); printf("debugPrintRange key: (%d) %s\n", values.size(), printable(subspace.key()).c_str());
for (auto & s : values) { for (auto & s : values) {
printf(" key: %-40s value: %s\n", printable(s.key).c_str(), printable(s.value).c_str()); printf(" key: %-40s value: %s\n", printable(s.key).c_str(), printable(s.value).c_str());
TraceEvent("TaskBucket").detail("debugPrintRange", printable(msg)) TraceEvent("TaskBucket").detail("DebugPrintRange", printable(msg))
.detail("key", printable(s.key)) .detail("Key", printable(s.key))
.detail("value", printable(s.value)); .detail("Value", printable(s.value));
}*/ }*/
return Void(); return Void();
@ -835,9 +835,9 @@ ACTOR static Future<Key> actorAddTask(TaskBucket* tb, Reference<ReadYourWritesTr
Optional<Value> validationValue = wait(tr->get(validationKey)); Optional<Value> validationValue = wait(tr->get(validationKey));
if (!validationValue.present()) { if (!validationValue.present()) {
TraceEvent(SevError, "TB_addTask_invalidKey") TraceEvent(SevError, "TB_AddTaskInvalidKey")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("validationKey", printable(validationKey)); .detail("ValidationKey", printable(validationKey));
throw invalid_option_value(); throw invalid_option_value();
} }
@ -1103,9 +1103,9 @@ public:
Optional<Value> validationValue = wait(tr->get(validationKey)); Optional<Value> validationValue = wait(tr->get(validationKey));
if (!validationValue.present()) { if (!validationValue.present()) {
TraceEvent(SevError, "TB_onSetAddTask_invalidKey") TraceEvent(SevError, "TB_OnSetAddTaskInvalidKey")
.detail("task", printable(task->params[Task::reservedTaskParamKeyType])) .detail("Task", printable(task->params[Task::reservedTaskParamKeyType]))
.detail("validationKey", printable(validationKey)); .detail("ValidationKey", printable(validationKey));
throw invalid_option_value(); throw invalid_option_value();
} }

2
fdbclient/VersionedMap.h
View File

@ -635,7 +635,7 @@ public:
int count=0, height=0; int count=0, height=0;
PTreeImpl::validate<MapPair<K,std::pair<T,Version>>>( root, at, NULL, NULL, count, height ); PTreeImpl::validate<MapPair<K,std::pair<T,Version>>>( root, at, NULL, NULL, count, height );
if ( height > 100 ) if ( height > 100 )
TraceEvent(SevWarnAlways, "DiabolicalPTreeSize").detail("size", count).detail("height", height); TraceEvent(SevWarnAlways, "DiabolicalPTreeSize").detail("Size", count).detail("Height", height);
} }
private: private:
Tree root; Tree root;

4
fdbclient/vexillographer/fdb.options
View File

@ -46,8 +46,8 @@ description is not currently required but encouraged.
paramType="Int" paramDescription="max total size of trace files" paramType="Int" paramDescription="max total size of trace files"
description="Sets the maximum size of all the trace output files put together. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time."/> description="Sets the maximum size of all the trace output files put together. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time."/>
<Option name="trace_log_group" code="33" <Option name="trace_log_group" code="33"
paramType="String" paramDescription="value of the logGroup attribute" paramType="String" paramDescription="value of the LogGroup attribute"
description="Sets the 'logGroup' attribute with the specified value for all events in the trace output files. The default log group is 'default'."/> description="Sets the 'LogGroup' attribute with the specified value for all events in the trace output files. The default log group is 'default'."/>
<Option name="knob" code="40" <Option name="knob" code="40"
paramType="String" paramDescription="knob_name=knob_value" paramType="String" paramDescription="knob_name=knob_value"
description="Set internal tuning or debugging knobs"/> description="Set internal tuning or debugging knobs"/>

6
fdbrpc/AsyncFileCached.actor.h
View File

@ -229,15 +229,15 @@ private:
ACTOR static Future<Reference<IAsyncFile>> open_impl( std::string filename, int flags, int mode, Reference<EvictablePageCache> pageCache ) { ACTOR static Future<Reference<IAsyncFile>> open_impl( std::string filename, int flags, int mode, Reference<EvictablePageCache> pageCache ) {
try { try {
TraceEvent("AFCUnderlyingOpenBegin").detail("filename", filename); TraceEvent("AFCUnderlyingOpenBegin").detail("Filename", filename);
if(flags & IAsyncFile::OPEN_CACHED_READ_ONLY) if(flags & IAsyncFile::OPEN_CACHED_READ_ONLY)
flags = flags & ~IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_READONLY; flags = flags & ~IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_READONLY;
else else
flags = flags & ~IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_READWRITE; flags = flags & ~IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_READWRITE;
state Reference<IAsyncFile> f = wait( IAsyncFileSystem::filesystem()->open(filename, flags | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, mode) ); state Reference<IAsyncFile> f = wait( IAsyncFileSystem::filesystem()->open(filename, flags | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_UNBUFFERED, mode) );
TraceEvent("AFCUnderlyingOpenEnd").detail("filename", filename); TraceEvent("AFCUnderlyingOpenEnd").detail("Filename", filename);
int64_t l = wait( f->size() ); int64_t l = wait( f->size() );
TraceEvent("AFCUnderlyingSize").detail("filename", filename).detail("size", l); TraceEvent("AFCUnderlyingSize").detail("Filename", filename).detail("Size", l);
auto& of = openFiles[filename]; auto& of = openFiles[filename];
of.f = new AsyncFileCached(f, filename, l, pageCache); of.f = new AsyncFileCached(f, filename, l, pageCache);
of.opened = Future<Reference<IAsyncFile>>(); of.opened = Future<Reference<IAsyncFile>>();

12
fdbrpc/AsyncFileEIO.actor.h
View File

@ -83,10 +83,10 @@ public:
TraceEvent(notFound ? SevWarn : SevWarnAlways, "FileOpenError").error(e).GetLastError().detail("File", filename).detail("Flags", flags).detail("Mode", mode); TraceEvent(notFound ? SevWarn : SevWarnAlways, "FileOpenError").error(e).GetLastError().detail("File", filename).detail("Flags", flags).detail("Mode", mode);
throw e; throw e;
} }
TraceEvent("AsyncFileOpened").detail("Filename", filename).detail("fd", r->result).detail("Flags", flags).suppressFor(1.0); TraceEvent("AsyncFileOpened").detail("Filename", filename).detail("Fd", r->result).detail("Flags", flags).suppressFor(1.0);
if ((flags & OPEN_LOCK) && !lock_fd(r->result)) { if ((flags & OPEN_LOCK) && !lock_fd(r->result)) {
TraceEvent(SevError, "UnableToLockFile").detail("filename", filename).GetLastError(); TraceEvent(SevError, "UnableToLockFile").detail("Filename", filename).GetLastError();
throw io_error(); throw io_error();
} }
@ -142,7 +142,7 @@ public:
struct stat buf; struct stat buf;
if (fstat( fd, &buf )) { if (fstat( fd, &buf )) {
TraceEvent("AsyncFileEIOFStatError").detail("fd",fd).GetLastError(); TraceEvent("AsyncFileEIOFStatError").detail("Fd",fd).GetLastError();
return io_error(); return io_error();
} }
return buf.st_size; return buf.st_size;
@ -183,7 +183,7 @@ public:
// rename() is atomic // rename() is atomic
if (rename( part_filename.c_str(), final_filename.c_str() )) { if (rename( part_filename.c_str(), final_filename.c_str() )) {
TraceEvent("AsyncFileEIORenameError").detail("filename", final_filename).GetLastError(); TraceEvent("AsyncFileEIORenameError").detail("Filename", final_filename).GetLastError();
throw io_error(); throw io_error();
} }
@ -254,7 +254,7 @@ private:
static void error( const char* context, int fd, eio_req* r, Reference<ErrorInfo> const& err = Reference<ErrorInfo>() ) { static void error( const char* context, int fd, eio_req* r, Reference<ErrorInfo> const& err = Reference<ErrorInfo>() ) {
Error e = io_error(); Error e = io_error();
errno = r->errorno; errno = r->errorno;
TraceEvent(context).detail("fd", fd).detail("Result", r->result).GetLastError().error(e); TraceEvent(context).detail("Fd", fd).detail("Result", r->result).GetLastError().error(e);
if (err) err->set(e); if (err) err->set(e);
else throw e; else throw e;
} }
@ -264,7 +264,7 @@ private:
state eio_req* r = eio_close(fd, 0, eio_callback, &p); state eio_req* r = eio_close(fd, 0, eio_callback, &p);
Void _ = wait( p.getFuture() ); Void _ = wait( p.getFuture() );
if (r->result) error( "CloseError", fd, r ); if (r->result) error( "CloseError", fd, r );
TraceEvent("AsyncFileClosed").detail("fd", fd).suppressFor(1.0); TraceEvent("AsyncFileClosed").detail("Fd", fd).suppressFor(1.0);
} }
ACTOR static Future<int> read_impl( int fd, void* data, int length, int64_t offset ) { ACTOR static Future<int> read_impl( int fd, void* data, int length, int64_t offset ) {

24
fdbrpc/AsyncFileKAIO.actor.h
View File

@ -113,7 +113,7 @@ public:
int ecode = errno; // Save errno in case it is modified before it is used below int ecode = errno; // Save errno in case it is modified before it is used below
TraceEvent ev("AsyncFileKAIOOpenFailed"); TraceEvent ev("AsyncFileKAIOOpenFailed");
ev.detail("Filename", filename).detailf("Flags", "%x", flags) ev.detail("Filename", filename).detailf("Flags", "%x", flags)
.detailf("OSFlags", "%x", openFlags(flags) | O_DIRECT).detailf("mode", "0%o", mode).error(e).GetLastError(); .detailf("OSFlags", "%x", openFlags(flags) | O_DIRECT).detailf("Mode", "0%o", mode).error(e).GetLastError();
if(ecode == EINVAL) if(ecode == EINVAL)
ev.detail("Description", "Invalid argument - Does the target filesystem support KAIO?"); ev.detail("Description", "Invalid argument - Does the target filesystem support KAIO?");
return e; return e;
@ -121,8 +121,8 @@ public:
TraceEvent("AsyncFileKAIOOpen") TraceEvent("AsyncFileKAIOOpen")
.detail("Filename", filename) .detail("Filename", filename)
.detail("Flags", flags) .detail("Flags", flags)
.detail("mode", mode) .detail("Mode", mode)
.detail("fd", fd); .detail("Fd", fd);
} }
Reference<AsyncFileKAIO> r(new AsyncFileKAIO( fd, flags, filename )); Reference<AsyncFileKAIO> r(new AsyncFileKAIO( fd, flags, filename ));
@ -136,14 +136,14 @@ public:
lockDesc.l_len = 0; // "Specifying 0 for l_len has the special meaning: lock all bytes starting at the location specified by l_whence and l_start through to the end of file, no matter how large the file grows." lockDesc.l_len = 0; // "Specifying 0 for l_len has the special meaning: lock all bytes starting at the location specified by l_whence and l_start through to the end of file, no matter how large the file grows."
lockDesc.l_pid = 0; lockDesc.l_pid = 0;
if (fcntl(fd, F_SETLK, &lockDesc) == -1) { if (fcntl(fd, F_SETLK, &lockDesc) == -1) {
TraceEvent(SevError, "UnableToLockFile").detail("filename", filename).GetLastError(); TraceEvent(SevError, "UnableToLockFile").detail("Filename", filename).GetLastError();
return io_error(); return io_error();
} }
} }
struct stat buf; struct stat buf;
if (fstat( fd, &buf )) { if (fstat( fd, &buf )) {
TraceEvent("AsyncFileKAIOFStatError").detail("fd",fd).detail("filename", filename).GetLastError(); TraceEvent("AsyncFileKAIOFStatError").detail("Fd",fd).detail("Filename", filename).GetLastError();
return io_error(); return io_error();
} }
@ -262,7 +262,7 @@ public:
result = fallocate( fd, 0, 0, size); result = fallocate( fd, 0, 0, size);
if (result != 0) { if (result != 0) {
int fallocateErrCode = errno; int fallocateErrCode = errno;
TraceEvent("AsyncFileKAIOAllocateError").detail("fd",fd).detail("filename", filename).GetLastError(); TraceEvent("AsyncFileKAIOAllocateError").detail("Fd",fd).detail("Filename", filename).GetLastError();
if ( fallocateErrCode == EOPNOTSUPP ) { if ( fallocateErrCode == EOPNOTSUPP ) {
// Mark fallocate as unsupported. Try again with truncate. // Mark fallocate as unsupported. Try again with truncate.
ctx.fallocateSupported = false; ctx.fallocateSupported = false;
@ -280,7 +280,7 @@ public:
KAIOLogEvent(logFile, id, OpLogEntry::TRUNCATE, OpLogEntry::COMPLETE, size / 4096, result); KAIOLogEvent(logFile, id, OpLogEntry::TRUNCATE, OpLogEntry::COMPLETE, size / 4096, result);
if(result != 0) { if(result != 0) {
TraceEvent("AsyncFileKAIOTruncateError").detail("fd",fd).detail("filename", filename).GetLastError(); TraceEvent("AsyncFileKAIOTruncateError").detail("Fd",fd).detail("Filename", filename).GetLastError();
return io_error(); return io_error();
} }
@ -409,7 +409,7 @@ public:
double elapsed = timer_monotonic() - begin; double elapsed = timer_monotonic() - begin;
g_network->networkMetrics.secSquaredSubmit += elapsed*elapsed/2; g_network->networkMetrics.secSquaredSubmit += elapsed*elapsed/2;
//TraceEvent("Launched").detail("n", rc).detail("queued", ctx.queue.size()).detail("ms", elapsed*1e3).detail("oustanding", ctx.outstanding+rc); //TraceEvent("Launched").detail("N", rc).detail("Queued", ctx.queue.size()).detail("Elapsed", elapsed).detail("Outstanding", ctx.outstanding+rc);
//printf("launched: %d/%d in %f us (%d outstanding; lowest prio %d)\n", rc, ctx.queue.size(), elapsed*1e6, ctx.outstanding + rc, toStart[n-1]->getTask()); //printf("launched: %d/%d in %f us (%d outstanding; lowest prio %d)\n", rc, ctx.queue.size(), elapsed*1e6, ctx.outstanding + rc, toStart[n-1]->getTask());
if (rc<0) { if (rc<0) {
if (errno == EAGAIN) { if (errno == EAGAIN) {
@ -478,16 +478,16 @@ private:
fstat( aio_fildes, &fst ); fstat( aio_fildes, &fst );
errno = -r; errno = -r;
TraceEvent("AsyncFileKAIOIOError").GetLastError().detail("fd", aio_fildes).detail("op", aio_lio_opcode).detail("nbytes", nbytes).detail("offset", offset).detail("ptr", int64_t(buf)) TraceEvent("AsyncFileKAIOIOError").GetLastError().detail("Fd", aio_fildes).detail("Op", aio_lio_opcode).detail("Nbytes", nbytes).detail("Offset", offset).detail("Ptr", int64_t(buf))
.detail("Size", fst.st_size).detail("filename", owner->filename); .detail("Size", fst.st_size).detail("Filename", owner->filename);
} }
deliver( result, owner->failed, r, getTask() ); deliver( result, owner->failed, r, getTask() );
delete this; delete this;
} }
void timeout(bool warnOnly) { void timeout(bool warnOnly) {
TraceEvent(SevWarnAlways, "AsyncFileKAIOTimeout").detail("fd", aio_fildes).detail("op", aio_lio_opcode).detail("nbytes", nbytes).detail("offset", offset).detail("ptr", int64_t(buf)) TraceEvent(SevWarnAlways, "AsyncFileKAIOTimeout").detail("Fd", aio_fildes).detail("Op", aio_lio_opcode).detail("Nbytes", nbytes).detail("Offset", offset).detail("Ptr", int64_t(buf))
.detail("filename", owner->filename); .detail("Filename", owner->filename);
g_network->setGlobal(INetwork::enASIOTimedOut, (flowGlobalType)true); g_network->setGlobal(INetwork::enASIOTimedOut, (flowGlobalType)true);
if(!warnOnly) if(!warnOnly)

4
fdbrpc/AsyncFileNonDurable.actor.h
View File

@ -200,7 +200,7 @@ public:
state int currentTaskID = g_network->getCurrentTask(); state int currentTaskID = g_network->getCurrentTask();
state Future<Void> shutdown = success(currentProcess->shutdownSignal.getFuture()); state Future<Void> shutdown = success(currentProcess->shutdownSignal.getFuture());
//TraceEvent("AsyncFileNonDurableOpenBegin").detail("Filename", filename).detail("addr", g_simulator.getCurrentProcess()->address); //TraceEvent("AsyncFileNonDurableOpenBegin").detail("Filename", filename).detail("Addr", g_simulator.getCurrentProcess()->address);
Void _ = wait( g_simulator.onMachine( currentProcess ) ); Void _ = wait( g_simulator.onMachine( currentProcess ) );
try { try {
Void _ = wait(success(wrappedFile) || shutdown); Void _ = wait(success(wrappedFile) || shutdown);
@ -238,7 +238,7 @@ public:
state Error err = e; state Error err = e;
std::string currentFilename = ( wrappedFile.isReady() && !wrappedFile.isError() ) ? wrappedFile.get()->getFilename() : actualFilename; std::string currentFilename = ( wrappedFile.isReady() && !wrappedFile.isError() ) ? wrappedFile.get()->getFilename() : actualFilename;
currentProcess->machine->openFiles.erase( currentFilename ); currentProcess->machine->openFiles.erase( currentFilename );
//TraceEvent("AsyncFileNonDurableOpenError").detail("Filename", filename).detail("address", currentProcess->address).error(e, true).detail("addr", g_simulator.getCurrentProcess()->address); //TraceEvent("AsyncFileNonDurableOpenError").detail("Filename", filename).detail("Address", currentProcess->address).error(e, true).detail("Addr", g_simulator.getCurrentProcess()->address);
Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) ); Void _ = wait( g_simulator.onProcess( currentProcess, currentTaskID ) );
throw err; throw err;
} }

10
fdbrpc/FlowTests.actor.cpp
View File

@ -1098,11 +1098,11 @@ TEST_CASE("flow/flow/YieldedAsyncMap/basic")
//yam.triggerRange(0, 4); //yam.triggerRange(0, 4);
state Future<Void> y2 = yam.onChange(1); state Future<Void> y2 = yam.onChange(1);
Void _ = wait(reportErrors(y0, "y0")); Void _ = wait(reportErrors(y0, "Y0"));
Void _ = wait(reportErrors(y1, "y1")); Void _ = wait(reportErrors(y1, "Y1"));
Void _ = wait(reportErrors(y1a, "y1a")); Void _ = wait(reportErrors(y1a, "Y1a"));
Void _ = wait(reportErrors(y1b, "y1b")); Void _ = wait(reportErrors(y1b, "Y1b"));
Void _ = wait(reportErrors(timeout(y2, 5, Void()), "y2")); Void _ = wait(reportErrors(timeout(y2, 5, Void()), "Y2"));
return Void(); return Void();
} }

6
fdbrpc/FlowTransport.actor.cpp
View File

@ -516,14 +516,14 @@ static void scanPackets( TransportData* transport, uint8_t*& unprocessed_begin,
uint32_t calculatedChecksum = crc32c_append(0, p, packetLen); uint32_t calculatedChecksum = crc32c_append(0, p, packetLen);
if (calculatedChecksum != packetChecksum) { if (calculatedChecksum != packetChecksum) {
if (isBuggifyEnabled) { if (isBuggifyEnabled) {
TraceEvent(SevInfo, "ChecksumMismatchExp").detail("packetChecksum", (int)packetChecksum).detail("calculatedChecksum", (int)calculatedChecksum); TraceEvent(SevInfo, "ChecksumMismatchExp").detail("PacketChecksum", (int)packetChecksum).detail("CalculatedChecksum", (int)calculatedChecksum);
} else { } else {
TraceEvent(SevWarnAlways, "ChecksumMismatchUnexp").detail("packetChecksum", (int)packetChecksum).detail("calculatedChecksum", (int)calculatedChecksum); TraceEvent(SevWarnAlways, "ChecksumMismatchUnexp").detail("PacketChecksum", (int)packetChecksum).detail("CalculatedChecksum", (int)calculatedChecksum);
} }
throw checksum_failed(); throw checksum_failed();
} else { } else {
if (isBuggifyEnabled) { if (isBuggifyEnabled) {
TraceEvent(SevError, "ChecksumMatchUnexp").detail("packetChecksum", (int)packetChecksum).detail("calculatedChecksum", (int)calculatedChecksum); TraceEvent(SevError, "ChecksumMatchUnexp").detail("PacketChecksum", (int)packetChecksum).detail("CalculatedChecksum", (int)calculatedChecksum);
} }
} }
} }

20
fdbrpc/batcher.actor.h
View File

@ -47,7 +47,7 @@ bool firstInBatch(CommitTransactionRequest x) {
} }
ACTOR template <class X> ACTOR template <class X>
Future<Void> batcher(PromiseStream<std::vector<X>> out, FutureStream<X> in, double avgMinDelay, double* avgMaxDelay, double emptyBatchTimeout, int maxCount, int desiredBytes, int maxBytes, Optional<PromiseStream<Void>> batchStartedStream, int taskID = TaskDefaultDelay, Counter* counter = 0) Future<Void> batcher(PromiseStream<std::pair<std::vector<X>, int> > out, FutureStream<X> in, double avgMinDelay, double* avgMaxDelay, double emptyBatchTimeout, int maxCount, int desiredBytes, int maxBytes, Optional<PromiseStream<Void>> batchStartedStream, int64_t *commitBatchesMemBytesCount, int64_t commitBatchesMemBytesLimit, int taskID = TaskDefaultDelay, Counter* counter = 0)
{ {
Void _ = wait( delayJittered(*avgMaxDelay, taskID) ); // smooth out Void _ = wait( delayJittered(*avgMaxDelay, taskID) ); // smooth out
// This is set up to deliver even zero-size batches if emptyBatchTimeout elapses, because that's what master proxy wants. The source control history // This is set up to deliver even zero-size batches if emptyBatchTimeout elapses, because that's what master proxy wants. The source control history
@ -68,6 +68,15 @@ Future<Void> batcher(PromiseStream<std::vector<X>> out, FutureStream<X> in, doub
while (!timeout.isReady() && !(batch.size() == maxCount || batchBytes >= desiredBytes)) { while (!timeout.isReady() && !(batch.size() == maxCount || batchBytes >= desiredBytes)) {
choose { choose {
when ( X x = waitNext(in) ) { when ( X x = waitNext(in) ) {
int bytes = getBytes(x);
// Drop requests if memory is under severe pressure
if (*commitBatchesMemBytesCount + bytes > commitBatchesMemBytesLimit) {
x.reply.sendError(proxy_memory_limit_exceeded());
TraceEvent(SevWarnAlways, "ProxyCommitBatchMemoryThresholdExceeded").detail("CommitBatchesMemBytesCount", *commitBatchesMemBytesCount).detail("CommitBatchesMemLimit", commitBatchesMemBytesLimit).suppressFor(60, true);
continue;
}
// Process requests in the normal case
if (counter) ++*counter; if (counter) ++*counter;
logOnReceive(x); logOnReceive(x);
if (!batch.size()) { if (!batch.size()) {
@ -79,10 +88,9 @@ Future<Void> batcher(PromiseStream<std::vector<X>> out, FutureStream<X> in, doub
timeout = delayJittered(*avgMaxDelay - (now() - lastBatch), taskID); timeout = delayJittered(*avgMaxDelay - (now() - lastBatch), taskID);
} }
int bytes = getBytes( x );
bool first = firstInBatch( x ); bool first = firstInBatch( x );
if((batchBytes + bytes > maxBytes || first) && batch.size()) { if((batchBytes + bytes > maxBytes || first) && batch.size()) {
out.send(batch); out.send({ batch, batchBytes });
lastBatch = now(); lastBatch = now();
if(batchStartedStream.present()) if(batchStartedStream.present())
batchStartedStream.get().send(Void()); batchStartedStream.get().send(Void());
@ -93,14 +101,14 @@ Future<Void> batcher(PromiseStream<std::vector<X>> out, FutureStream<X> in, doub
batch.push_back(x); batch.push_back(x);
batchBytes += bytes; batchBytes += bytes;
*commitBatchesMemBytesCount += bytes;
} }
when ( Void _ = wait( timeout ) ) {} when ( Void _ = wait( timeout ) ) {}
} }
} }
out.send({std::move(batch), batchBytes});
out.send(batch);
lastBatch = now(); lastBatch = now();
} }
} }
#endif #endif

56
fdbrpc/sim2.actor.cpp
View File

@ -633,7 +633,7 @@ private:
std::string sourceFilename = self->filename + ".part"; std::string sourceFilename = self->filename + ".part";
if(machineCache.count(sourceFilename)) { if(machineCache.count(sourceFilename)) {
TraceEvent("SimpleFileRename").detail("From", sourceFilename).detail("To", self->filename).detail("sourceCount", machineCache.count(sourceFilename)).detail("fileCount", machineCache.count(self->filename)); TraceEvent("SimpleFileRename").detail("From", sourceFilename).detail("To", self->filename).detail("SourceCount", machineCache.count(sourceFilename)).detail("FileCount", machineCache.count(self->filename));
renameFile( sourceFilename.c_str(), self->filename.c_str() ); renameFile( sourceFilename.c_str(), self->filename.c_str() );
ASSERT(!machineCache.count(self->filename)); ASSERT(!machineCache.count(self->filename));
@ -866,7 +866,7 @@ public:
diskSpace.totalSpace = 5e9 + g_random->random01() * 100e9; //Total space between 5GB and 105GB diskSpace.totalSpace = 5e9 + g_random->random01() * 100e9; //Total space between 5GB and 105GB
diskSpace.baseFreeSpace = std::min<int64_t>(diskSpace.totalSpace, std::max(5e9, (g_random->random01() * (1 - .075) + .075) * diskSpace.totalSpace) + totalFileSize); //Minimum 5GB or 7.5% total disk space, whichever is higher diskSpace.baseFreeSpace = std::min<int64_t>(diskSpace.totalSpace, std::max(5e9, (g_random->random01() * (1 - .075) + .075) * diskSpace.totalSpace) + totalFileSize); //Minimum 5GB or 7.5% total disk space, whichever is higher
TraceEvent("Sim2DiskSpaceInitialization").detail("TotalSpace", diskSpace.totalSpace).detail("BaseFreeSpace", diskSpace.baseFreeSpace).detail("totalFileSize", totalFileSize).detail("NumFiles", numFiles); TraceEvent("Sim2DiskSpaceInitialization").detail("TotalSpace", diskSpace.totalSpace).detail("BaseFreeSpace", diskSpace.baseFreeSpace).detail("TotalFileSize", totalFileSize).detail("NumFiles", numFiles);
} }
else { else {
int64_t maxDelta = std::min(5.0, (now() - diskSpace.lastUpdate)) * (BUGGIFY ? 10e6 : 1e6); //External processes modifying the disk int64_t maxDelta = std::min(5.0, (now() - diskSpace.lastUpdate)) * (BUGGIFY ? 10e6 : 1e6); //External processes modifying the disk
@ -880,7 +880,7 @@ public:
free = std::max<int64_t>(0, diskSpace.baseFreeSpace - totalFileSize); free = std::max<int64_t>(0, diskSpace.baseFreeSpace - totalFileSize);
if(free == 0) if(free == 0)
TraceEvent(SevWarnAlways, "Sim2NoFreeSpace").detail("TotalSpace", diskSpace.totalSpace).detail("BaseFreeSpace", diskSpace.baseFreeSpace).detail("totalFileSize", totalFileSize).detail("NumFiles", numFiles); TraceEvent(SevWarnAlways, "Sim2NoFreeSpace").detail("TotalSpace", diskSpace.totalSpace).detail("BaseFreeSpace", diskSpace.baseFreeSpace).detail("TotalFileSize", totalFileSize).detail("NumFiles", numFiles);
} }
virtual bool isAddressOnThisHost( NetworkAddress const& addr ) { virtual bool isAddressOnThisHost( NetworkAddress const& addr ) {
return addr.ip == getCurrentProcess()->address.ip; return addr.ip == getCurrentProcess()->address.ip;
@ -964,8 +964,8 @@ public:
for( int i = 0; i < machine.processes.size(); i++ ) { for( int i = 0; i < machine.processes.size(); i++ ) {
if( machine.processes[i]->locality.zoneId() != locality.zoneId() ) { // SOMEDAY: compute ip from locality to avoid this check if( machine.processes[i]->locality.zoneId() != locality.zoneId() ) { // SOMEDAY: compute ip from locality to avoid this check
TraceEvent("Sim2Mismatch").detail("IP", format("%x", ip)) TraceEvent("Sim2Mismatch").detail("IP", format("%x", ip))
.detailext("zoneId", locality.zoneId()).detail("NewName", name) .detailext("ZoneId", locality.zoneId()).detail("NewName", name)
.detailext("ExistingmachineId", machine.processes[i]->locality.zoneId()).detail("ExistingName", machine.processes[i]->name); .detailext("ExistingMachineId", machine.processes[i]->locality.zoneId()).detail("ExistingName", machine.processes[i]->name);
ASSERT( false ); ASSERT( false );
} }
ASSERT( machine.processes[i]->address.port != port ); ASSERT( machine.processes[i]->address.port != port );
@ -993,7 +993,7 @@ public:
m->setGlobal(enNetworkConnections, (flowGlobalType) m->network); m->setGlobal(enNetworkConnections, (flowGlobalType) m->network);
m->setGlobal(enASIOTimedOut, (flowGlobalType) false); m->setGlobal(enASIOTimedOut, (flowGlobalType) false);
TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId()).detail("Excluded", m->excluded).detail("Cleared", m->cleared); TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("ZoneId", m->locality.zoneId()).detail("Excluded", m->excluded).detail("Cleared", m->cleared);
// FIXME: Sometimes, connections to/from this process will explicitly close // FIXME: Sometimes, connections to/from this process will explicitly close
@ -1123,21 +1123,21 @@ public:
if (tooManyDead) { if (tooManyDead) {
newKt = Reboot; newKt = Reboot;
canSurvive = false; canSurvive = false;
TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("tLogPolicy", tLogPolicy->info()).detail("Reason", "tLogPolicy validates against dead processes."); TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("TLogPolicy", tLogPolicy->info()).detail("Reason", "tLogPolicy validates against dead processes.");
} }
// Reboot and Delete if remaining machines do NOT fulfill policies // Reboot and Delete if remaining machines do NOT fulfill policies
else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && notEnoughLeft) { else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && notEnoughLeft) {
newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot; newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
canSurvive = false; canSurvive = false;
TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("tLogPolicy", tLogPolicy->info()).detail("Reason", "tLogPolicy does not validates against remaining processes."); TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("TLogPolicy", tLogPolicy->info()).detail("Reason", "tLogPolicy does not validates against remaining processes.");
} }
else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (nQuorum > uniqueMachines.size())) { else if ((kt != RebootAndDelete) && (kt != RebootProcessAndDelete) && (nQuorum > uniqueMachines.size())) {
newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot; newKt = (g_random->random01() < 0.33) ? RebootAndDelete : Reboot;
canSurvive = false; canSurvive = false;
TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("storagePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("Reason", "Not enough unique machines to perform auto configuration of coordinators."); TraceEvent("KillChanged").detail("KillType", kt).detail("NewKillType", newKt).detail("StoragePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()).detail("Reason", "Not enough unique machines to perform auto configuration of coordinators.");
} }
else { else {
TraceEvent("CanSurviveKills").detail("KillType", kt).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size()); TraceEvent("CanSurviveKills").detail("KillType", kt).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info()).detail("Quorum", nQuorum).detail("Machines", uniqueMachines.size());
} }
} }
if (newKillType) *newKillType = newKt; if (newKillType) *newKillType = newKt;
@ -1145,7 +1145,7 @@ public:
} }
virtual void destroyProcess( ISimulator::ProcessInfo *p ) { virtual void destroyProcess( ISimulator::ProcessInfo *p ) {
TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("zoneId", p->locality.zoneId()); TraceEvent("ProcessDestroyed").detail("Name", p->name).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId());
currentlyRebootingProcesses.insert(std::pair<NetworkAddress, ProcessInfo*>(p->address, p)); currentlyRebootingProcesses.insert(std::pair<NetworkAddress, ProcessInfo*>(p->address, p));
std::vector<ProcessInfo*>& processes = machines[ p->locality.zoneId().get() ].processes; std::vector<ProcessInfo*>& processes = machines[ p->locality.zoneId().get() ].processes;
if( p != processes.back() ) { if( p != processes.back() ) {
@ -1202,7 +1202,7 @@ public:
} }
} }
virtual void killProcess( ProcessInfo* machine, KillType kt ) { virtual void killProcess( ProcessInfo* machine, KillType kt ) {
TraceEvent("attemptingKillProcess"); TraceEvent("AttemptingKillProcess");
if (kt < RebootAndDelete ) { if (kt < RebootAndDelete ) {
killProcess_internal( machine, kt ); killProcess_internal( machine, kt );
} }
@ -1241,7 +1241,7 @@ public:
// Do nothing, if no processes to kill // Do nothing, if no processes to kill
if (processesOnMachine == 0) { if (processesOnMachine == 0) {
TraceEvent(SevWarn, "AbortedKill", zoneId).detailext("ZoneId", zoneId).detail("Reason", "The target had no processes running.").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace(); TraceEvent(SevWarn, "AbortedKill", zoneId).detailext("ZoneId", zoneId).detail("Reason", "The target had no processes running.").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
if (ktFinal) *ktFinal = None; if (ktFinal) *ktFinal = None;
return false; return false;
} }
@ -1285,10 +1285,10 @@ public:
if ((kt != Reboot) && (!killIsSafe)) { if ((kt != Reboot) && (!killIsSafe)) {
kt = Reboot; kt = Reboot;
} }
TraceEvent("ChangedKillMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("Protected", protectedWorker).detail("Unavailable", unavailable).detail("Excluded", excluded).detail("Cleared", cleared).detail("ProtectedTotal", protectedAddresses.size()).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()); TraceEvent("ChangedKillMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("OrigKillType", ktOrig).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("Protected", protectedWorker).detail("Unavailable", unavailable).detail("Excluded", excluded).detail("Cleared", cleared).detail("ProtectedTotal", protectedAddresses.size()).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
} }
else if ((kt == KillInstantly) || (kt == InjectFaults)) { else if ((kt == KillInstantly) || (kt == InjectFaults)) {
TraceEvent("DeadMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()); TraceEvent("DeadMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
for (auto process : processesLeft) { for (auto process : processesLeft) {
TraceEvent("DeadMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process)); TraceEvent("DeadMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
} }
@ -1297,7 +1297,7 @@ public:
} }
} }
else { else {
TraceEvent("ClearMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("processesPerMachine", processesPerMachine).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()); TraceEvent("ClearMachine", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("TotalProcesses", machines.size()).detail("ProcessesPerMachine", processesPerMachine).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
for (auto process : processesLeft) { for (auto process : processesLeft) {
TraceEvent("ClearMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process)); TraceEvent("ClearMachineSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
} }
@ -1312,7 +1312,7 @@ public:
// Check if any processes on machine are rebooting // Check if any processes on machine are rebooting
if( processesOnMachine != processesPerMachine && kt >= RebootAndDelete ) { if( processesOnMachine != processesPerMachine && kt >= RebootAndDelete ) {
TEST(true); //Attempted reboot, but the target did not have all of its processes running TEST(true); //Attempted reboot, but the target did not have all of its processes running
TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace(); TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
if (ktFinal) *ktFinal = None; if (ktFinal) *ktFinal = None;
return false; return false;
} }
@ -1320,24 +1320,24 @@ public:
// Check if any processes on machine are rebooting // Check if any processes on machine are rebooting
if ( processesOnMachine != processesPerMachine) { if ( processesOnMachine != processesPerMachine) {
TEST(true); //Attempted reboot, but the target did not have all of its processes running TEST(true); //Attempted reboot, but the target did not have all of its processes running
TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("processes", processesOnMachine).detail("processesPerMachine", processesPerMachine).backtrace(); TraceEvent(SevWarn, "AbortedKill", zoneId).detail("KillType", kt).detailext("ZoneId", zoneId).detail("Reason", "Machine processes does not match number of processes per machine").detail("Processes", processesOnMachine).detail("ProcessesPerMachine", processesPerMachine).backtrace();
if (ktFinal) *ktFinal = None; if (ktFinal) *ktFinal = None;
return false; return false;
} }
TraceEvent("KillMachine", zoneId).detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig).detail("killIsSafe", killIsSafe); TraceEvent("KillMachine", zoneId).detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig).detail("KillIsSafe", killIsSafe);
if (kt < RebootAndDelete ) { if (kt < RebootAndDelete ) {
if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr) if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr)
killProcess_internal( machines[zoneId].machineProcess, kt ); killProcess_internal( machines[zoneId].machineProcess, kt );
for (auto& process : machines[zoneId].processes) { for (auto& process : machines[zoneId].processes) {
TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()).detail("failed", process->failed).detail("excluded", process->excluded).detail("cleared", process->cleared).detail("rebooting", process->rebooting); TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("StartingClass", process->startingClass.toString()).detail("Failed", process->failed).detail("Excluded", process->excluded).detail("Cleared", process->cleared).detail("Rebooting", process->rebooting);
if (process->startingClass != ProcessClass::TesterClass) if (process->startingClass != ProcessClass::TesterClass)
killProcess_internal( process, kt ); killProcess_internal( process, kt );
} }
} }
else if ( kt == Reboot || killIsSafe) { else if ( kt == Reboot || killIsSafe) {
for (auto& process : machines[zoneId].processes) { for (auto& process : machines[zoneId].processes) {
TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()).detail("failed", process->failed).detail("excluded", process->excluded).detail("cleared", process->cleared).detail("rebooting", process->rebooting); TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("StartingClass", process->startingClass.toString()).detail("Failed", process->failed).detail("Excluded", process->excluded).detail("Cleared", process->cleared).detail("Rebooting", process->rebooting);
if (process->startingClass != ProcessClass::TesterClass) if (process->startingClass != ProcessClass::TesterClass)
doReboot(process, kt ); doReboot(process, kt );
} }
@ -1367,7 +1367,7 @@ public:
if ((kt != Reboot) && (protectedAddresses.count(procRecord->address))) { if ((kt != Reboot) && (protectedAddresses.count(procRecord->address))) {
kt = Reboot; kt = Reboot;
TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig) TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig)
.detail("Reason", "Datacenter has protected process").detail("ProcessAddress", procRecord->address).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("excluded", procRecord->excluded).detail("cleared", procRecord->cleared).detail("Process", describe(*procRecord)); .detail("Reason", "Datacenter has protected process").detail("ProcessAddress", procRecord->address).detail("Failed", procRecord->failed).detail("Rebooting", procRecord->rebooting).detail("Excluded", procRecord->excluded).detail("Cleared", procRecord->cleared).detail("Process", describe(*procRecord));
} }
datacenterZones[processZoneId.get()] ++; datacenterZones[processZoneId.get()] ++;
dcProcesses ++; dcProcesses ++;
@ -1404,7 +1404,7 @@ public:
TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig); TraceEvent(SevWarn, "DcKillChanged").detailext("DataCenter", dcId).detail("KillType", kt).detail("OrigKillType", ktOrig);
} }
else { else {
TraceEvent("DeadDataCenter").detailext("DataCenter", dcId).detail("KillType", kt).detail("DcZones", datacenterZones.size()).detail("DcProcesses", dcProcesses).detail("ProcessesDead", processesDead.size()).detail("ProcessesLeft", processesLeft.size()).detail("tLogPolicy", tLogPolicy->info()).detail("storagePolicy", storagePolicy->info()); TraceEvent("DeadDataCenter").detailext("DataCenter", dcId).detail("KillType", kt).detail("DcZones", datacenterZones.size()).detail("DcProcesses", dcProcesses).detail("ProcessesDead", processesDead.size()).detail("ProcessesLeft", processesLeft.size()).detail("TLogPolicy", tLogPolicy->info()).detail("StoragePolicy", storagePolicy->info());
for (auto process : processesLeft) { for (auto process : processesLeft) {
auto zoneId = process->locality.zoneId(); auto zoneId = process->locality.zoneId();
TraceEvent("DeadDcSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process)); TraceEvent("DeadDcSurvivors", zoneId).detailext("ZoneId", zoneId).detail("KillType", kt).detail("ProcessesLeft", processesLeft.size()).detail("ProcessesDead", processesDead.size()).detail("SurvivingProcess", describe(*process));
@ -1420,7 +1420,7 @@ public:
for (auto& datacenterZone : datacenterZones) { for (auto& datacenterZone : datacenterZones) {
killMachine( datacenterZone.first, kt, (kt == RebootAndDelete), true, &ktResult); killMachine( datacenterZone.first, kt, (kt == RebootAndDelete), true, &ktResult);
if (ktResult != kt) { if (ktResult != kt) {
TraceEvent(SevWarn, "killDCFail") TraceEvent(SevWarn, "KillDCFail")
.detailext("Zone", datacenterZone.first) .detailext("Zone", datacenterZone.first)
.detail("KillType", kt) .detail("KillType", kt)
.detail("KillTypeResult", ktResult) .detail("KillTypeResult", ktResult)
@ -1430,7 +1430,7 @@ public:
ktMin = std::min<KillType>( ktResult, ktMin ); ktMin = std::min<KillType>( ktResult, ktMin );
} }
TraceEvent("killDataCenter") TraceEvent("KillDataCenter")
.detail("DcZones", datacenterZones.size()) .detail("DcZones", datacenterZones.size())
.detail("DcProcesses", dcProcesses) .detail("DcProcesses", dcProcesses)
.detailext("DCID", dcId) .detailext("DCID", dcId)
@ -1546,7 +1546,7 @@ public:
/*auto elapsed = getCPUTicks() - before; /*auto elapsed = getCPUTicks() - before;
currentProcess->cpuTicks += elapsed; currentProcess->cpuTicks += elapsed;
if (g_random->random01() < 0.01){ if (g_random->random01() < 0.01){
TraceEvent("st").detail("cpu", currentProcess->cpuTicks); TraceEvent("TaskDuration").detail("CpuTicks", currentProcess->cpuTicks);
currentProcess->cpuTicks = 0; currentProcess->cpuTicks = 0;
}*/ }*/
} catch (Error& e) { } catch (Error& e) {
@ -1627,7 +1627,7 @@ static double networkLatency() {
} }
ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) { ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) {
TraceEvent("RebootingProcessAttempt").detailext("ZoneId", p->locality.zoneId()).detail("KillType", kt).detail("Process", p->toString()).detail("startingClass", p->startingClass.toString()).detail("failed", p->failed).detail("excluded", p->excluded).detail("cleared", p->cleared).detail("rebooting", p->rebooting).detail("TaskDefaultDelay", TaskDefaultDelay); TraceEvent("RebootingProcessAttempt").detailext("ZoneId", p->locality.zoneId()).detail("KillType", kt).detail("Process", p->toString()).detail("StartingClass", p->startingClass.toString()).detail("Failed", p->failed).detail("Excluded", p->excluded).detail("Cleared", p->cleared).detail("Rebooting", p->rebooting).detail("TaskDefaultDelay", TaskDefaultDelay);
Void _ = wait( g_sim2.delay( 0, TaskDefaultDelay, p ) ); // Switch to the machine in question Void _ = wait( g_sim2.delay( 0, TaskDefaultDelay, p ) ); // Switch to the machine in question
@ -1641,7 +1641,7 @@ ACTOR void doReboot( ISimulator::ProcessInfo *p, ISimulator::KillType kt ) {
if( p->rebooting ) if( p->rebooting )
return; return;
TraceEvent("RebootingProcess").detail("KillType", kt).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId()).detailext("DataHall", p->locality.dataHallId()).detail("Locality", p->locality.toString()).detail("failed", p->failed).detail("excluded", p->excluded).detail("cleared", p->cleared).backtrace(); TraceEvent("RebootingProcess").detail("KillType", kt).detail("Address", p->address).detailext("ZoneId", p->locality.zoneId()).detailext("DataHall", p->locality.dataHallId()).detail("Locality", p->locality.toString()).detail("Failed", p->failed).detail("Excluded", p->excluded).detail("Cleared", p->cleared).backtrace();
p->rebooting = true; p->rebooting = true;
if ((kt == ISimulator::RebootAndDelete) || (kt == ISimulator::RebootProcessAndDelete)) { if ((kt == ISimulator::RebootAndDelete) || (kt == ISimulator::RebootProcessAndDelete)) {
p->cleared = true; p->cleared = true;

28
fdbserver/ApplyMetadataMutation.h
View File

@ -106,7 +106,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
MutationRef privatized = m; MutationRef privatized = m;
privatized.param1 = m.param1.withPrefix(systemKeys.begin, arena); privatized.param1 = m.param1.withPrefix(systemKeys.begin, arena);
TraceEvent(SevDebug, "SendingPrivateMutation", dbgid).detail("Original", m.toString()).detail("Privatized", privatized.toString()).detail("Server", serverKeysDecodeServer(m.param1)) TraceEvent(SevDebug, "SendingPrivateMutation", dbgid).detail("Original", m.toString()).detail("Privatized", privatized.toString()).detail("Server", serverKeysDecodeServer(m.param1))
.detail("tagKey", printable(serverTagKeyFor( serverKeysDecodeServer(m.param1) ))).detail("tag", decodeServerTagValue( txnStateStore->readValue( serverTagKeyFor( serverKeysDecodeServer(m.param1) ) ).get().get() ).toString()); .detail("TagKey", printable(serverTagKeyFor( serverKeysDecodeServer(m.param1) ))).detail("Tag", decodeServerTagValue( txnStateStore->readValue( serverTagKeyFor( serverKeysDecodeServer(m.param1) ) ).get().get() ).toString());
toCommit->addTag( decodeServerTagValue( txnStateStore->readValue( serverTagKeyFor( serverKeysDecodeServer(m.param1) ) ).get().get() ) ); toCommit->addTag( decodeServerTagValue( txnStateStore->readValue( serverTagKeyFor( serverKeysDecodeServer(m.param1) ) ).get().get() ) );
toCommit->addTypedMessage(privatized); toCommit->addTypedMessage(privatized);
@ -118,7 +118,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
if(toCommit) { if(toCommit) {
MutationRef privatized = m; MutationRef privatized = m;
privatized.param1 = m.param1.withPrefix(systemKeys.begin, arena); privatized.param1 = m.param1.withPrefix(systemKeys.begin, arena);
TraceEvent("ServerTag", dbgid).detail("server", id).detail("tag", tag.toString()); TraceEvent("ServerTag", dbgid).detail("Server", id).detail("Tag", tag.toString());
toCommit->addTag(tag); toCommit->addTag(tag);
toCommit->addTypedMessage(LogProtocolMessage()); toCommit->addTypedMessage(LogProtocolMessage());
@ -151,7 +151,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
if(Optional<StringRef>(m.param2) != txnStateStore->readValue(m.param1).get().cast_to<StringRef>()) { // FIXME: Make this check more specific, here or by reading configuration whenever there is a change if(Optional<StringRef>(m.param2) != txnStateStore->readValue(m.param1).get().cast_to<StringRef>()) { // FIXME: Make this check more specific, here or by reading configuration whenever there is a change
if(!m.param1.startsWith( excludedServersPrefix ) && m.param1 != excludedServersVersionKey) { if(!m.param1.startsWith( excludedServersPrefix ) && m.param1 != excludedServersVersionKey) {
auto t = txnStateStore->readValue(m.param1).get(); auto t = txnStateStore->readValue(m.param1).get();
TraceEvent("MutationRequiresRestart", dbgid).detail("M", m.toString()).detail("PrevValue", t.present() ? printable(t.get()) : "(none)").detail("toCommit", toCommit!=NULL); TraceEvent("MutationRequiresRestart", dbgid).detail("M", m.toString()).detail("PrevValue", t.present() ? printable(t.get()) : "(none)").detail("ToCommit", toCommit!=NULL);
if(confChange) *confChange = true; if(confChange) *confChange = true;
} }
} }
@ -227,8 +227,8 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
} }
// Log the modification // Log the modification
TraceEvent("LogRangeAdd").detail("logRanges", vecBackupKeys->size()).detail("mutationKey", printable(m.param1)) TraceEvent("LogRangeAdd").detail("LogRanges", vecBackupKeys->size()).detail("MutationKey", printable(m.param1))
.detail("logRangeBegin", printable(logRangeBegin)).detail("logRangeEnd", printable(logRangeEnd)); .detail("LogRangeBegin", printable(logRangeBegin)).detail("LogRangeEnd", printable(logRangeEnd));
} }
} }
else if (m.param1.startsWith(globalKeysPrefix)) { else if (m.param1.startsWith(globalKeysPrefix)) {
@ -254,7 +254,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
} }
else if (m.param1 == minRequiredCommitVersionKey) { else if (m.param1 == minRequiredCommitVersionKey) {
Version requested = BinaryReader::fromStringRef<Version>(m.param2, Unversioned()); Version requested = BinaryReader::fromStringRef<Version>(m.param2, Unversioned());
TraceEvent("MinRequiredCommitVersion", dbgid).detail("min", requested).detail("current", popVersion).detail("hasConf", !!confChange); TraceEvent("MinRequiredCommitVersion", dbgid).detail("Min", requested).detail("Current", popVersion).detail("HasConf", !!confChange);
if(!initialCommit) txnStateStore->set(KeyValueRef(m.param1, m.param2)); if(!initialCommit) txnStateStore->set(KeyValueRef(m.param1, m.param2));
if (confChange) *confChange = true; if (confChange) *confChange = true;
TEST(true); // Recovering at a higher version. TEST(true); // Recovering at a higher version.
@ -292,7 +292,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
auto serverKeysCleared = txnStateStore->readRange( range & serverTagKeys ).get(); // read is expected to be immediately available auto serverKeysCleared = txnStateStore->readRange( range & serverTagKeys ).get(); // read is expected to be immediately available
for(auto &kv : serverKeysCleared) { for(auto &kv : serverKeysCleared) {
Tag tag = decodeServerTagValue(kv.value); Tag tag = decodeServerTagValue(kv.value);
TraceEvent("ServerTagRemove").detail("popVersion", popVersion).detail("tag", tag.toString()).detail("server", decodeServerTagKey(kv.key)); TraceEvent("ServerTagRemove").detail("PopVersion", popVersion).detail("Tag", tag.toString()).detail("Server", decodeServerTagKey(kv.key));
logSystem->pop( popVersion, decodeServerTagValue(kv.value) ); logSystem->pop( popVersion, decodeServerTagValue(kv.value) );
(*tag_popped)[tag] = popVersion; (*tag_popped)[tag] = popVersion;
@ -320,7 +320,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
auto serverKeysCleared = txnStateStore->readRange( range & serverTagHistoryKeys ).get(); // read is expected to be immediately available auto serverKeysCleared = txnStateStore->readRange( range & serverTagHistoryKeys ).get(); // read is expected to be immediately available
for(auto &kv : serverKeysCleared) { for(auto &kv : serverKeysCleared) {
Tag tag = decodeServerTagValue(kv.value); Tag tag = decodeServerTagValue(kv.value);
TraceEvent("ServerTagHistoryRemove").detail("popVersion", popVersion).detail("tag", tag.toString()).detail("version", decodeServerTagHistoryKey(kv.key)); TraceEvent("ServerTagHistoryRemove").detail("PopVersion", popVersion).detail("Tag", tag.toString()).detail("Version", decodeServerTagHistoryKey(kv.key));
logSystem->pop( popVersion, tag ); logSystem->pop( popVersion, tag );
(*tag_popped)[tag] = popVersion; (*tag_popped)[tag] = popVersion;
} }
@ -362,8 +362,8 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
KeyRangeRef commonLogRange(range & logRangesRange); KeyRangeRef commonLogRange(range & logRangesRange);
TraceEvent("LogRangeClear") TraceEvent("LogRangeClear")
.detail("rangeBegin", printable(range.begin)).detail("rangeEnd", printable(range.end)) .detail("RangeBegin", printable(range.begin)).detail("RangeEnd", printable(range.end))
.detail("intersectBegin", printable(commonLogRange.begin)).detail("intersectEnd", printable(commonLogRange.end)); .detail("IntersectBegin", printable(commonLogRange.begin)).detail("IntersectEnd", printable(commonLogRange.end));
// Remove the key range from the vector, if defined // Remove the key range from the vector, if defined
if (vecBackupKeys) { if (vecBackupKeys) {
@ -374,7 +374,7 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
// read is expected to be immediately available // read is expected to be immediately available
auto logRangesAffected = txnStateStore->readRange(commonLogRange).get(); auto logRangesAffected = txnStateStore->readRange(commonLogRange).get();
TraceEvent("LogRangeClearBegin").detail("affectedLogRanges", logRangesAffected.size()); TraceEvent("LogRangeClearBegin").detail("AffectedLogRanges", logRangesAffected.size());
// Add the backup name to the backup locations that do not have it // Add the backup name to the backup locations that do not have it
for (auto logRangeAffected : logRangesAffected) for (auto logRangeAffected : logRangesAffected)
@ -385,9 +385,9 @@ static void applyMetadataMutations(UID const& dbgid, Arena &arena, VectorRef<Mut
// Decode the log destination and key value // Decode the log destination and key value
logKeyEnd = logRangesDecodeValue(logRangeAffected.value, &logDestination); logKeyEnd = logRangesDecodeValue(logRangeAffected.value, &logDestination);
TraceEvent("LogRangeErase").detail("affectedKey", printable(logRangeAffected.key)).detail("affectedValue", printable(logRangeAffected.value)) TraceEvent("LogRangeErase").detail("AffectedKey", printable(logRangeAffected.key)).detail("AffectedValue", printable(logRangeAffected.value))
.detail("logKeyBegin", printable(logKeyBegin)).detail("logKeyEnd", printable(logKeyEnd)) .detail("LogKeyBegin", printable(logKeyBegin)).detail("LogKeyEnd", printable(logKeyEnd))
.detail("logDestination", printable(logDestination)); .detail("LogDestination", printable(logDestination));
// Identify the locations to place the backup key // Identify the locations to place the backup key
auto logRanges = vecBackupKeys->modify(KeyRangeRef(logKeyBegin, logKeyEnd)); auto logRanges = vecBackupKeys->modify(KeyRangeRef(logKeyBegin, logKeyEnd));

42
fdbserver/ClusterController.actor.cpp
View File

@ -273,7 +273,7 @@ public:
logServerMap->add(worker.first.locality, &worker); logServerMap->add(worker.first.locality, &worker);
} }
if (logServerSet->size() < required) { if (logServerSet->size() < required) {
TraceEvent(SevWarn,"GWFTADTooFew", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("Required", required).detail("tLogPolicy", policy->info()).detail("DesiredLogs", desired); TraceEvent(SevWarn,"GWFTADTooFew", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("Required", required).detail("TLogPolicy", policy->info()).detail("DesiredLogs", desired);
} }
else if (logServerSet->size() <= desired) { else if (logServerSet->size() <= desired) {
if (logServerSet->validate(policy)) { if (logServerSet->validate(policy)) {
@ -283,7 +283,7 @@ public:
bCompleted = true; bCompleted = true;
break; break;
} }
TraceEvent(SevWarn,"GWFTADNotAcceptable", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("Required", required).detail("tLogPolicy",policy->info()).detail("DesiredLogs", desired); TraceEvent(SevWarn,"GWFTADNotAcceptable", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("Required", required).detail("TLogPolicy",policy->info()).detail("DesiredLogs", desired);
} }
// Try to select the desired size, if larger // Try to select the desired size, if larger
else { else {
@ -300,11 +300,11 @@ public:
tLocalities.push_back(object->first.locality); tLocalities.push_back(object->first.locality);
} }
TraceEvent("GWFTADBestResults", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("BestCount", bestSet.size()).detail("BestZones", ::describeZones(tLocalities)) TraceEvent("GWFTADBestResults", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("BestCount", bestSet.size()).detail("BestZones", ::describeZones(tLocalities))
.detail("BestDataHalls", ::describeDataHalls(tLocalities)).detail("tLogPolicy", policy->info()).detail("TotalResults", results.size()).detail("DesiredLogs", desired); .detail("BestDataHalls", ::describeDataHalls(tLocalities)).detail("TLogPolicy", policy->info()).detail("TotalResults", results.size()).detail("DesiredLogs", desired);
bCompleted = true; bCompleted = true;
break; break;
} }
TraceEvent(SevWarn,"GWFTADNoBest", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("Required", required).detail("tLogPolicy", policy->info()).detail("DesiredLogs", desired); TraceEvent(SevWarn,"GWFTADNoBest", id).detail("Fitness", fitness).detail("Processes", logServerSet->size()).detail("Required", required).detail("TLogPolicy", policy->info()).detail("DesiredLogs", desired);
} }
} }
@ -318,7 +318,7 @@ public:
TraceEvent(SevWarn, "GetTLogTeamFailed").detail("Policy", policy->info()).detail("Processes", logServerSet->size()).detail("Workers", id_worker.size()).detail("FitnessGroups", fitness_workers.size()) TraceEvent(SevWarn, "GetTLogTeamFailed").detail("Policy", policy->info()).detail("Processes", logServerSet->size()).detail("Workers", id_worker.size()).detail("FitnessGroups", fitness_workers.size())
.detail("TLogZones", ::describeZones(tLocalities)).detail("TLogDataHalls", ::describeDataHalls(tLocalities)).detail("MissingZones", ::describeZones(unavailableLocals)) .detail("TLogZones", ::describeZones(tLocalities)).detail("TLogDataHalls", ::describeDataHalls(tLocalities)).detail("MissingZones", ::describeZones(unavailableLocals))
.detail("MissingDataHalls", ::describeDataHalls(unavailableLocals)).detail("Required", required).detail("DesiredLogs", desired).detail("RatingTests",SERVER_KNOBS->POLICY_RATING_TESTS) .detail("MissingDataHalls", ::describeDataHalls(unavailableLocals)).detail("Required", required).detail("DesiredLogs", desired).detail("RatingTests",SERVER_KNOBS->POLICY_RATING_TESTS)
.detail("checkStable", checkStable).detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS).backtrace(); .detail("CheckStable", checkStable).detail("PolicyGenerations",SERVER_KNOBS->POLICY_GENERATIONS).backtrace();
logServerSet->clear(); logServerSet->clear();
logServerSet.clear(); logServerSet.clear();
@ -675,10 +675,10 @@ public:
} }
//If this cluster controller dies, do not prioritize recruiting the next one in the same DC //If this cluster controller dies, do not prioritize recruiting the next one in the same DC
desiredDcIds.set(vector<Optional<Key>>()); desiredDcIds.set(vector<Optional<Key>>());
TraceEvent("findWorkersForConfig").detail("replication", req.configuration.tLogReplicationFactor) TraceEvent("FindWorkersForConfig").detail("Replication", req.configuration.tLogReplicationFactor)
.detail("desiredLogs", req.configuration.getDesiredLogs()).detail("actualLogs", result.tLogs.size()) .detail("DesiredLogs", req.configuration.getDesiredLogs()).detail("ActualLogs", result.tLogs.size())
.detail("desiredProxies", req.configuration.getDesiredProxies()).detail("actualProxies", result.proxies.size()) .detail("DesiredProxies", req.configuration.getDesiredProxies()).detail("ActualProxies", result.proxies.size())
.detail("desiredResolvers", req.configuration.getDesiredResolvers()).detail("actualResolvers", result.resolvers.size()); .detail("DesiredResolvers", req.configuration.getDesiredResolvers()).detail("ActualResolvers", result.resolvers.size());
if( now() - startTime < SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY && if( now() - startTime < SERVER_KNOBS->WAIT_FOR_GOOD_RECRUITMENT_DELAY &&
( RoleFitness(tlogs, ProcessClass::TLog) > RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs()) || ( RoleFitness(tlogs, ProcessClass::TLog) > RoleFitness(SERVER_KNOBS->EXPECTED_TLOG_FITNESS, req.configuration.getDesiredLogs()) ||
@ -882,13 +882,13 @@ public:
if(oldInFit.betterFitness(newInFit)) return false; if(oldInFit.betterFitness(newInFit)) return false;
if(oldTLogFit > newTLogFit || oldInFit > newInFit || oldSatelliteTLogFit > newSatelliteTLogFit || oldRemoteTLogFit > newRemoteTLogFit || oldLogRoutersFit > newLogRoutersFit) { if(oldTLogFit > newTLogFit || oldInFit > newInFit || oldSatelliteTLogFit > newSatelliteTLogFit || oldRemoteTLogFit > newRemoteTLogFit || oldLogRoutersFit > newLogRoutersFit) {
TraceEvent("BetterMasterExists", id).detail("oldMasterFit", oldMasterFit).detail("newMasterFit", mworker.fitness) TraceEvent("BetterMasterExists", id).detail("OldMasterFit", oldMasterFit).detail("NewMasterFit", mworker.fitness)
.detail("oldTLogFitC", oldTLogFit.count).detail("newTLogFitC", newTLogFit.count) .detail("OldTLogFitC", oldTLogFit.count).detail("NewTLogFitC", newTLogFit.count)
.detail("oldTLogWorstFitT", oldTLogFit.worstFit).detail("newTLogWorstFitT", newTLogFit.worstFit) .detail("OldTLogWorstFitT", oldTLogFit.worstFit).detail("NewTLogWorstFitT", newTLogFit.worstFit)
.detail("oldTLogBestFitT", oldTLogFit.bestFit).detail("newTLogBestFitT", newTLogFit.bestFit) .detail("OldTLogBestFitT", oldTLogFit.bestFit).detail("NewTLogBestFitT", newTLogFit.bestFit)
.detail("oldInFitW", oldInFit.worstFit).detail("newInFitW", newInFit.worstFit) .detail("OldInFitW", oldInFit.worstFit).detail("NewInFitW", newInFit.worstFit)
.detail("oldInFitB", oldInFit.bestFit).detail("newInFitB", newInFit.bestFit) .detail("OldInFitB", oldInFit.bestFit).detail("NewInFitB", newInFit.bestFit)
.detail("oldInFitC", oldInFit.count).detail("newInFitC", newInFit.count); .detail("OldInFitC", oldInFit.count).detail("NewInFitC", newInFit.count);
return true; return true;
} }
@ -1022,7 +1022,7 @@ ACTOR Future<Void> clusterWatchDatabase( ClusterControllerData* cluster, Cluster
if (e.code() == error_code_actor_cancelled) throw; if (e.code() == error_code_actor_cancelled) throw;
bool ok = e.code() == error_code_no_more_servers; bool ok = e.code() == error_code_no_more_servers;
TraceEvent(ok ? SevWarn : SevError,"clusterWatchDatabaseRetrying", cluster->id).error(e); TraceEvent(ok ? SevWarn : SevError,"ClusterWatchDatabaseRetrying", cluster->id).error(e);
if (!ok) if (!ok)
throw e; throw e;
Void _ = wait( delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY) ); Void _ = wait( delay(SERVER_KNOBS->ATTEMPT_RECRUITMENT_DELAY) );
@ -1350,7 +1350,7 @@ ACTOR Future<Void> failureDetectionServer( UID uniqueID, FutureStream< FailureMo
pivotDelay = std::max(0.0, pivotDelay - FLOW_KNOBS->SERVER_REQUEST_INTERVAL); pivotDelay = std::max(0.0, pivotDelay - FLOW_KNOBS->SERVER_REQUEST_INTERVAL);
TraceEvent("FailureDetectionPoll", uniqueID).detail("PivotDelay", pivotDelay).detail("Clients", currentStatus.size()); TraceEvent("FailureDetectionPoll", uniqueID).detail("PivotDelay", pivotDelay).detail("Clients", currentStatus.size());
//TraceEvent("FailureDetectionAcceptableDelay").detail("ms", acceptableDelay*1000); //TraceEvent("FailureDetectionAcceptableDelay").detail("Delay", acceptableDelay1000);
for(auto it = currentStatus.begin(); it != currentStatus.end(); ) { for(auto it = currentStatus.begin(); it != currentStatus.end(); ) {
double delay = t - it->second.lastRequestTime; double delay = t - it->second.lastRequestTime;
@ -1454,13 +1454,13 @@ ACTOR Future<Void> clusterRecruitRemoteFromConfiguration( ClusterControllerData*
void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest const& req ) { void clusterRegisterMaster( ClusterControllerData* self, RegisterMasterRequest const& req ) {
req.reply.send( Void() ); req.reply.send( Void() );
TraceEvent("MasterRegistrationReceived", self->id).detail("dbName", printable(req.dbName)).detail("MasterId", req.id).detail("Master", req.mi.toString()).detail("Tlogs", describe(req.logSystemConfig.tLogs)).detail("Resolvers", req.resolvers.size()) TraceEvent("MasterRegistrationReceived", self->id).detail("DbName", printable(req.dbName)).detail("MasterId", req.id).detail("Master", req.mi.toString()).detail("Tlogs", describe(req.logSystemConfig.tLogs)).detail("Resolvers", req.resolvers.size())
.detail("RecoveryState", req.recoveryState).detail("RegistrationCount", req.registrationCount).detail("Proxies", req.proxies.size()).detail("RecoveryCount", req.recoveryCount); .detail("RecoveryState", req.recoveryState).detail("RegistrationCount", req.registrationCount).detail("Proxies", req.proxies.size()).detail("RecoveryCount", req.recoveryCount);
//make sure the request comes from an active database //make sure the request comes from an active database
auto db = &self->db; auto db = &self->db;
if ( db->serverInfo->get().master.id() != req.id || req.registrationCount <= db->masterRegistrationCount ) { if ( db->serverInfo->get().master.id() != req.id || req.registrationCount <= db->masterRegistrationCount ) {
TraceEvent("MasterRegistrationNotFound", self->id).detail("dbName", printable(req.dbName)).detail("MasterId", req.id).detail("existingId", db->serverInfo->get().master.id()).detail("RegCount", req.registrationCount).detail("ExistingRegCount", db->masterRegistrationCount); TraceEvent("MasterRegistrationNotFound", self->id).detail("DbName", printable(req.dbName)).detail("MasterId", req.id).detail("ExistingId", db->serverInfo->get().master.id()).detail("RegCount", req.registrationCount).detail("ExistingRegCount", db->masterRegistrationCount);
return; return;
} }
@ -1537,7 +1537,7 @@ void registerWorker( RegisterWorkerRequest req, ClusterControllerData *self ) {
auto info = self->id_worker.find( w.locality.processId() ); auto info = self->id_worker.find( w.locality.processId() );
ClusterControllerPriorityInfo newPriorityInfo = req.priorityInfo; ClusterControllerPriorityInfo newPriorityInfo = req.priorityInfo;
TraceEvent("ClusterControllerActualWorkers", self->id).detail("WorkerID",w.id()).detailext("ProcessID", w.locality.processId()).detailext("ZoneId", w.locality.zoneId()).detailext("DataHall", w.locality.dataHallId()).detail("pClass", req.processClass.toString()).detail("Workers", self->id_worker.size()).detail("Registered", (info == self->id_worker.end() ? "False" : "True")).backtrace(); TraceEvent("ClusterControllerActualWorkers", self->id).detail("WorkerId",w.id()).detailext("ProcessId", w.locality.processId()).detailext("ZoneId", w.locality.zoneId()).detailext("DataHall", w.locality.dataHallId()).detail("PClass", req.processClass.toString()).detail("Workers", self->id_worker.size()).detail("Registered", (info == self->id_worker.end() ? "False" : "True")).backtrace();
if ( w.address() == g_network->getLocalAddress() ) { if ( w.address() == g_network->getLocalAddress() ) {
self->clusterControllerProcessId = w.locality.processId(); self->clusterControllerProcessId = w.locality.processId();

6
fdbserver/CoordinatedState.actor.cpp
View File

@ -114,9 +114,9 @@ struct CoordinatedStateImpl {
UniqueGeneration wgen = wait( self->replicatedWrite( self, GenerationRegWriteRequest( KeyValueRef(self->coordinators.clusterKey, v), self->gen ) ) ); UniqueGeneration wgen = wait( self->replicatedWrite( self, GenerationRegWriteRequest( KeyValueRef(self->coordinators.clusterKey, v), self->gen ) ) );
self->stage = 6; self->stage = 6;
TraceEvent("CoordinatedStateSet").detail("gen", self->gen.generation).detail("wgen", wgen.generation) TraceEvent("CoordinatedStateSet").detail("Gen", self->gen.generation).detail("Wgen", wgen.generation)
.detail("genu", self->gen.uid).detail("wgenu", wgen.uid) .detail("Genu", self->gen.uid).detail("Wgenu", wgen.uid)
.detail("cgen", self->conflictGen); .detail("Cgen", self->conflictGen);
if (wgen == self->gen) if (wgen == self->gen)
return Void(); return Void();

6
fdbserver/Coordination.actor.cpp
View File

@ -150,11 +150,11 @@ ACTOR Future<Void> localGenerationReg( GenerationRegInterface interf, OnDemandSt
store->set( KeyValueRef( wrq.kv.key, BinaryWriter::toValue(v, IncludeVersion()) ) ); store->set( KeyValueRef( wrq.kv.key, BinaryWriter::toValue(v, IncludeVersion()) ) );
Void _ = wait(store->commit()); Void _ = wait(store->commit());
TraceEvent("GenerationRegWrote").detail("From", wrq.reply.getEndpoint().address).detail("Key", printable(wrq.kv.key)) TraceEvent("GenerationRegWrote").detail("From", wrq.reply.getEndpoint().address).detail("Key", printable(wrq.kv.key))
.detail("reqGen", wrq.gen.generation).detail("Returning", v.writeGen.generation); .detail("ReqGen", wrq.gen.generation).detail("Returning", v.writeGen.generation);
wrq.reply.send( v.writeGen ); wrq.reply.send( v.writeGen );
} else { } else {
TraceEvent("GenerationRegWriteFail").detail("From", wrq.reply.getEndpoint().address).detail("Key", printable(wrq.kv.key)) TraceEvent("GenerationRegWriteFail").detail("From", wrq.reply.getEndpoint().address).detail("Key", printable(wrq.kv.key))
.detail("reqGen", wrq.gen.generation).detail("readGen", v.readGen.generation).detail("writeGen", v.writeGen.generation); .detail("ReqGen", wrq.gen.generation).detail("ReadGen", v.readGen.generation).detail("WriteGen", v.writeGen.generation);
wrq.reply.send( std::max( v.readGen, v.writeGen ) ); wrq.reply.send( std::max( v.readGen, v.writeGen ) );
} }
} }
@ -416,7 +416,7 @@ ACTOR Future<Void> coordinationServer(std::string dataFolder) {
state GenerationRegInterface myInterface( g_network ); state GenerationRegInterface myInterface( g_network );
state OnDemandStore store( dataFolder, myID ); state OnDemandStore store( dataFolder, myID );
TraceEvent("CoordinationServer", myID).detail("myInterfaceAddr", myInterface.read.getEndpoint().address).detail("Folder", dataFolder); TraceEvent("CoordinationServer", myID).detail("MyInterfaceAddr", myInterface.read.getEndpoint().address).detail("Folder", dataFolder);
try { try {
Void _ = wait( localGenerationReg(myInterface, &store) || leaderServer(myLeaderInterface, &store) || store.getError() ); Void _ = wait( localGenerationReg(myInterface, &store) || leaderServer(myLeaderInterface, &store) || store.getError() );

42
fdbserver/DataDistribution.actor.cpp
View File

@ -323,8 +323,8 @@ ACTOR Future<Void> storageServerFailureTracker(
: waitFailureClient(server.waitFailure, SERVER_KNOBS->DATA_DISTRIBUTION_FAILURE_REACTION_TIME, 0, TaskDataDistribution) ) ) : waitFailureClient(server.waitFailure, SERVER_KNOBS->DATA_DISTRIBUTION_FAILURE_REACTION_TIME, 0, TaskDataDistribution) ) )
{ {
status->isFailed = !status->isFailed; status->isFailed = !status->isFailed;
TraceEvent("StatusMapChange", masterId).detail("ServerID", server.id()).detail("Status", status->toString()). TraceEvent("StatusMapChange", masterId).detail("ServerID", server.id()).detail("Status", status->toString())
detail("Available", IFailureMonitor::failureMonitor().getState(server.waitFailure.getEndpoint()).isAvailable()); .detail("Available", IFailureMonitor::failureMonitor().getState(server.waitFailure.getEndpoint()).isAvailable());
} }
when ( Void _ = wait( status->isUnhealthy() ? waitForAllDataRemoved(cx, server.id(), addedVersion) : Never() ) ) { break; } when ( Void _ = wait( status->isUnhealthy() ? waitForAllDataRemoved(cx, server.id(), addedVersion) : Never() ) ) { break; }
} }
@ -384,7 +384,7 @@ ACTOR Future<Reference<InitialDataDistribution>> getInitialDataDistribution( Dat
Void _ = wait( tr.onError(e) ); Void _ = wait( tr.onError(e) );
ASSERT(!succeeded); //We shouldn't be retrying if we have already started modifying result in this loop ASSERT(!succeeded); //We shouldn't be retrying if we have already started modifying result in this loop
TraceEvent("getInitialTeamsRetry", masterId); TraceEvent("GetInitialTeamsRetry", masterId);
} }
} }
@ -463,7 +463,7 @@ ACTOR Future<Reference<InitialDataDistribution>> getInitialDataDistribution( Dat
Void _ = wait( tr.onError(e) ); Void _ = wait( tr.onError(e) );
ASSERT(!succeeded); //We shouldn't be retrying if we have already started modifying result in this loop ASSERT(!succeeded); //We shouldn't be retrying if we have already started modifying result in this loop
TraceEvent("getInitialTeamsKeyServersRetry", masterId); TraceEvent("GetInitialTeamsKeyServersRetry", masterId);
} }
} }
@ -962,7 +962,7 @@ struct DDTeamCollection {
} }
if(totalServers.size() < configuration.storageTeamSize ) { if(totalServers.size() < configuration.storageTeamSize ) {
TraceEvent(SevWarn, "DataDistributionBuildTeams", masterId).detail("Reason","Not enough servers for a team").detail("Servers",totalServers.size()).detail("teamSize", configuration.storageTeamSize); TraceEvent(SevWarn, "DataDistributionBuildTeams", masterId).detail("Reason","Not enough servers for a team").detail("Servers",totalServers.size()).detail("TeamSize", configuration.storageTeamSize);
return addedTeams; return addedTeams;
} }
@ -1170,7 +1170,7 @@ struct DDTeamCollection {
} }
allServers.push_back( newServer.id() ); allServers.push_back( newServer.id() );
TraceEvent("AddedStorageServer", masterId).detail("ServerID", newServer.id()).detail("ProcessClass", processClass.toString()).detail("WaitFailureToken", newServer.waitFailure.getEndpoint().token).detail("address", newServer.waitFailure.getEndpoint().address); TraceEvent("AddedStorageServer", masterId).detail("ServerID", newServer.id()).detail("ProcessClass", processClass.toString()).detail("WaitFailureToken", newServer.waitFailure.getEndpoint().token).detail("Address", newServer.waitFailure.getEndpoint().address);
auto &r = server_info[newServer.id()] = Reference<TCServerInfo>( new TCServerInfo( newServer, processClass ) ); auto &r = server_info[newServer.id()] = Reference<TCServerInfo>( new TCServerInfo( newServer, processClass ) );
r->tracker = storageServerTracker( this, cx, r.getPtr(), &server_status, lock, masterId, &server_info, serverChanges, serverFailures, errorOut, addedVersion ); r->tracker = storageServerTracker( this, cx, r.getPtr(), &server_status, lock, masterId, &server_info, serverChanges, serverFailures, errorOut, addedVersion );
restartTeamBuilder.trigger(); restartTeamBuilder.trigger();
@ -1254,7 +1254,7 @@ ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<IDataDistribut
try { try {
loop { loop {
TraceEvent("TeamHealthChangeDetected", self->masterId).detail("isReady", self->initialFailureReactionDelay.isReady() ); TraceEvent("TeamHealthChangeDetected", self->masterId).detail("IsReady", self->initialFailureReactionDelay.isReady() );
// Check if the number of degraded machines has changed // Check if the number of degraded machines has changed
state vector<Future<Void>> change; state vector<Future<Void>> change;
auto servers = team->getServerIDs(); auto servers = team->getServerIDs();
@ -1287,8 +1287,8 @@ ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<IDataDistribut
if( serversLeft != lastServersLeft || anyUndesired != lastAnyUndesired || anyWrongConfiguration != lastWrongConfiguration || wrongSize || recheck ) { if( serversLeft != lastServersLeft || anyUndesired != lastAnyUndesired || anyWrongConfiguration != lastWrongConfiguration || wrongSize || recheck ) {
TraceEvent("TeamHealthChanged", self->masterId) TraceEvent("TeamHealthChanged", self->masterId)
.detail("Team", team->getDesc()).detail("serversLeft", serversLeft) .detail("Team", team->getDesc()).detail("ServersLeft", serversLeft)
.detail("lastServersLeft", lastServersLeft).detail("ContainsUndesiredServer", anyUndesired) .detail("LastServersLeft", lastServersLeft).detail("ContainsUndesiredServer", anyUndesired)
.detail("HealthyTeamsCount", self->healthyTeamCount).detail("IsWrongConfiguration", anyWrongConfiguration); .detail("HealthyTeamsCount", self->healthyTeamCount).detail("IsWrongConfiguration", anyWrongConfiguration);
bool healthy = matchesPolicy && !anyUndesired && team->getServerIDs().size() == self->configuration.storageTeamSize && team->getServerIDs().size() == serversLeft; bool healthy = matchesPolicy && !anyUndesired && team->getServerIDs().size() == self->configuration.storageTeamSize && team->getServerIDs().size() == serversLeft;
@ -1394,7 +1394,7 @@ ACTOR Future<Void> teamTracker( DDTeamCollection *self, Reference<IDataDistribut
} }
} }
} else { } else {
TraceEvent("TeamHealthNotReady", self->masterId).detail("healthyTeamCount", self->healthyTeamCount); TraceEvent("TeamHealthNotReady", self->masterId).detail("HealthyTeamCount", self->healthyTeamCount);
} }
} }
@ -1771,7 +1771,7 @@ ACTOR Future<Void> initializeStorage( DDTeamCollection *self, RecruitStorageRepl
isr.interfaceId = interfaceId; isr.interfaceId = interfaceId;
TraceEvent("DDRecruiting").detail("State", "Sending request to worker").detail("WorkerID", candidateWorker.worker.id()) TraceEvent("DDRecruiting").detail("State", "Sending request to worker").detail("WorkerID", candidateWorker.worker.id())
.detail("WorkerLocality", candidateWorker.worker.locality.toString()).detail("interf", interfaceId).detail("addr", candidateWorker.worker.address()); .detail("WorkerLocality", candidateWorker.worker.locality.toString()).detail("Interf", interfaceId).detail("Addr", candidateWorker.worker.address());
self->recruitingIds.insert(interfaceId); self->recruitingIds.insert(interfaceId);
self->recruitingLocalities.insert(candidateWorker.worker.address()); self->recruitingLocalities.insert(candidateWorker.worker.address());
@ -1782,7 +1782,7 @@ ACTOR Future<Void> initializeStorage( DDTeamCollection *self, RecruitStorageRepl
self->recruitingStream.set(self->recruitingStream.get()-1); self->recruitingStream.set(self->recruitingStream.get()-1);
TraceEvent("DDRecruiting").detail("State", "Finished request").detail("WorkerID", candidateWorker.worker.id()) TraceEvent("DDRecruiting").detail("State", "Finished request").detail("WorkerID", candidateWorker.worker.id())
.detail("WorkerLocality", candidateWorker.worker.locality.toString()).detail("interf", interfaceId).detail("addr", candidateWorker.worker.address()); .detail("WorkerLocality", candidateWorker.worker.locality.toString()).detail("Interf", interfaceId).detail("Addr", candidateWorker.worker.address());
if( newServer.isError() ) { if( newServer.isError() ) {
TraceEvent(SevWarn, "DDRecruitmentError").error(newServer.getError()); TraceEvent(SevWarn, "DDRecruitmentError").error(newServer.getError());
@ -1874,17 +1874,17 @@ ACTOR Future<Void> updateReplicasKey(DDTeamCollection* self, Optional<Key> dcId)
Void _ = wait(self->initialFailureReactionDelay); Void _ = wait(self->initialFailureReactionDelay);
Void _ = wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY, TaskLowPriority)); //After the team trackers wait on the initial failure reaction delay, they yield. We want to make sure every tracker has had the opportunity to send their relocations to the queue. Void _ = wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY, TaskLowPriority)); //After the team trackers wait on the initial failure reaction delay, they yield. We want to make sure every tracker has had the opportunity to send their relocations to the queue.
while(self->zeroHealthyTeams->get() || self->processingUnhealthy->get()) { while(self->zeroHealthyTeams->get() || self->processingUnhealthy->get()) {
TraceEvent("DDUpdatingStalled", self->masterId).detail("dcId", printable(dcId)).detail("zeroHealthy", self->zeroHealthyTeams->get()).detail("processingUnhealthy", self->processingUnhealthy->get()); TraceEvent("DDUpdatingStalled", self->masterId).detail("DcId", printable(dcId)).detail("ZeroHealthy", self->zeroHealthyTeams->get()).detail("ProcessingUnhealthy", self->processingUnhealthy->get());
Void _ = wait(self->zeroHealthyTeams->onChange() || self->processingUnhealthy->onChange()); Void _ = wait(self->zeroHealthyTeams->onChange() || self->processingUnhealthy->onChange());
} }
TraceEvent("DDUpdatingReplicas", self->masterId).detail("dcId", printable(dcId)).detail("replicas", self->configuration.storageTeamSize); TraceEvent("DDUpdatingReplicas", self->masterId).detail("DcId", printable(dcId)).detail("Replicas", self->configuration.storageTeamSize);
state Transaction tr(self->cx); state Transaction tr(self->cx);
loop { loop {
try { try {
tr.addReadConflictRange(singleKeyRange(datacenterReplicasKeyFor(dcId))); tr.addReadConflictRange(singleKeyRange(datacenterReplicasKeyFor(dcId)));
tr.set(datacenterReplicasKeyFor(dcId), datacenterReplicasValue(self->configuration.storageTeamSize)); tr.set(datacenterReplicasKeyFor(dcId), datacenterReplicasValue(self->configuration.storageTeamSize));
Void _ = wait( tr.commit() ); Void _ = wait( tr.commit() );
TraceEvent("DDUpdatedReplicas", self->masterId).detail("dcId", printable(dcId)).detail("replicas", self->configuration.storageTeamSize); TraceEvent("DDUpdatedReplicas", self->masterId).detail("DcId", printable(dcId)).detail("Replicas", self->configuration.storageTeamSize);
return Void(); return Void();
} catch( Error &e ) { } catch( Error &e ) {
Void _ = wait( tr.onError(e) ); Void _ = wait( tr.onError(e) );
@ -1916,9 +1916,9 @@ ACTOR Future<Void> dataDistributionTeamCollection(
state PromiseStream<Void> serverRemoved; state PromiseStream<Void> serverRemoved;
state Future<Void> error = actorCollection( self.addActor.getFuture() ); state Future<Void> error = actorCollection( self.addActor.getFuture() );
TraceEvent("DDTeamCollectionBegin", masterId).detail("primary", primary); TraceEvent("DDTeamCollectionBegin", masterId).detail("Primary", primary);
Void _ = wait( readyToStart ); Void _ = wait( readyToStart );
TraceEvent("DDTeamCollectionReadyToStart", masterId).detail("primary", primary); TraceEvent("DDTeamCollectionReadyToStart", masterId).detail("Primary", primary);
try { try {
self.init( *initData ); self.init( *initData );
initData = Reference<InitialDataDistribution>(); initData = Reference<InitialDataDistribution>();
@ -1964,7 +1964,7 @@ ACTOR Future<Void> dataDistributionTeamCollection(
} }
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_movekeys_conflict) if (e.code() != error_code_movekeys_conflict)
TraceEvent(SevError, "dataDistributionTeamCollectionError", masterId).error(e); TraceEvent(SevError, "DataDistributionTeamCollectionError", masterId).error(e);
throw e; throw e;
} }
} }
@ -2124,9 +2124,9 @@ ACTOR Future<Void> dataDistribution(
TraceEvent("DDInitTookMoveKeysLock", mi.id()); TraceEvent("DDInitTookMoveKeysLock", mi.id());
state Reference<InitialDataDistribution> initData = wait( getInitialDataDistribution(cx, mi.id(), lock, configuration.remoteTLogReplicationFactor > 0 ? remoteDcIds : std::vector<Optional<Key>>() ) ); state Reference<InitialDataDistribution> initData = wait( getInitialDataDistribution(cx, mi.id(), lock, configuration.remoteTLogReplicationFactor > 0 ? remoteDcIds : std::vector<Optional<Key>>() ) );
if(initData->shards.size() > 1) { if(initData->shards.size() > 1) {
TraceEvent("DDInitGotInitialDD", mi.id()).detail("b", printable(initData->shards.end()[-2].key)).detail("e", printable(initData->shards.end()[-1].key)).detail("src", describe(initData->shards.end()[-2].primarySrc)).detail("dest", describe(initData->shards.end()[-2].primaryDest)).trackLatest("InitialDD"); TraceEvent("DDInitGotInitialDD", mi.id()).detail("B", printable(initData->shards.end()[-2].key)).detail("E", printable(initData->shards.end()[-1].key)).detail("Src", describe(initData->shards.end()[-2].primarySrc)).detail("Dest", describe(initData->shards.end()[-2].primaryDest)).trackLatest("InitialDD");
} else { } else {
TraceEvent("DDInitGotInitialDD", mi.id()).detail("b","").detail("e", "").detail("src", "[no items]").detail("dest", "[no items]").trackLatest("InitialDD"); TraceEvent("DDInitGotInitialDD", mi.id()).detail("B","").detail("E", "").detail("Src", "[no items]").detail("Dest", "[no items]").trackLatest("InitialDD");
} }
if (initData->mode) break; if (initData->mode) break;
@ -2210,7 +2210,7 @@ ACTOR Future<Void> dataDistribution(
if( e.code() != error_code_movekeys_conflict ) if( e.code() != error_code_movekeys_conflict )
throw err; throw err;
bool ddEnabled = wait( isDataDistributionEnabled(cx) ); bool ddEnabled = wait( isDataDistributionEnabled(cx) );
TraceEvent("DataDistributionMoveKeysConflict").detail("ddEnabled", ddEnabled); TraceEvent("DataDistributionMoveKeysConflict").detail("DataDistributionEnabled", ddEnabled);
if( ddEnabled ) if( ddEnabled )
throw err; throw err;
} }

22
fdbserver/DataDistributionQueue.actor.cpp
View File

@ -486,10 +486,10 @@ struct DDQueueData {
for( int i = 0; i < it->second.ledger.size() - 1; i++ ) { for( int i = 0; i < it->second.ledger.size() - 1; i++ ) {
if( it->second.ledger[i] < it->second.ledger[i+1] ) if( it->second.ledger[i] < it->second.ledger[i+1] )
TraceEvent(SevError, "DDQueueValidateError12").detail("Problem", "ascending ledger problem") TraceEvent(SevError, "DDQueueValidateError12").detail("Problem", "ascending ledger problem")
.detail("ledgerLevel", i).detail("ledgerValueA", it->second.ledger[i]).detail("ledgerValueB", it->second.ledger[i+1]); .detail("LedgerLevel", i).detail("LedgerValueA", it->second.ledger[i]).detail("LedgerValueB", it->second.ledger[i+1]);
if( it->second.ledger[i] < 0.0 ) if( it->second.ledger[i] < 0.0 )
TraceEvent(SevError, "DDQueueValidateError13").detail("Problem", "negative ascending problem") TraceEvent(SevError, "DDQueueValidateError13").detail("Problem", "negative ascending problem")
.detail("ledgerLevel", i).detail("ledgerValue", it->second.ledger[i]); .detail("LedgerLevel", i).detail("LedgerValue", it->second.ledger[i]);
} }
} }
@ -833,7 +833,7 @@ struct DDQueueData {
//logRelocation( rd, "LaunchedRelocation" ); //logRelocation( rd, "LaunchedRelocation" );
} }
if( now() - startTime > .001 && g_random->random01()<0.001 ) if( now() - startTime > .001 && g_random->random01()<0.001 )
TraceEvent(SevWarnAlways, "LaunchingQueueSlowx1000").detail("elapsed", now() - startTime ); TraceEvent(SevWarnAlways, "LaunchingQueueSlowx1000").detail("Elapsed", now() - startTime );
/*if( startedHere > 0 ) { /*if( startedHere > 0 ) {
TraceEvent("StartedDDRelocators", mi.id()) TraceEvent("StartedDDRelocators", mi.id())
@ -984,7 +984,7 @@ ACTOR Future<Void> dataDistributionRelocator( DDQueueData *self, RelocateData rd
error = e; error = e;
} }
//TraceEvent("RelocateShardFinished", masterId).detail("relocateId", relocateShardInterval.pairID); //TraceEvent("RelocateShardFinished", masterId).detail("RelocateId", relocateShardInterval.pairID);
if( error.code() != error_code_move_to_removed_server ) { if( error.code() != error_code_move_to_removed_server ) {
if( !error.code() ) { if( !error.code() ) {
@ -1001,7 +1001,7 @@ ACTOR Future<Void> dataDistributionRelocator( DDQueueData *self, RelocateData rd
if( !error.code() ) { if( !error.code() ) {
TraceEvent(relocateShardInterval.end(), masterId).detail("Result","Success"); TraceEvent(relocateShardInterval.end(), masterId).detail("Result","Success");
if(rd.keys.begin == keyServersPrefix) { if(rd.keys.begin == keyServersPrefix) {
TraceEvent("MovedKeyServerKeys").detail("dest", destination.getDesc()).trackLatest("MovedKeyServers"); TraceEvent("MovedKeyServerKeys").detail("Dest", destination.getDesc()).trackLatest("MovedKeyServers");
} }
if( !signalledTransferComplete ) { if( !signalledTransferComplete ) {
@ -1057,11 +1057,11 @@ ACTOR Future<bool> rebalanceTeams( DDQueueData* self, int priority, Reference<ID
for( int i = 0; i < shards.size(); i++ ) { for( int i = 0; i < shards.size(); i++ ) {
if( moveShard == shards[i] ) { if( moveShard == shards[i] ) {
TraceEvent(priority == PRIORITY_REBALANCE_OVERUTILIZED_TEAM ? "BgDDMountainChopper" : "BgDDValleyFiller", self->mi.id()) TraceEvent(priority == PRIORITY_REBALANCE_OVERUTILIZED_TEAM ? "BgDDMountainChopper" : "BgDDValleyFiller", self->mi.id())
.detail("sourceBytes", sourceBytes) .detail("SourceBytes", sourceBytes)
.detail("destBytes", destBytes) .detail("DestBytes", destBytes)
.detail("shardBytes", metrics.bytes) .detail("ShardBytes", metrics.bytes)
.detail("sourceTeam", sourceTeam->getDesc()) .detail("SourceTeam", sourceTeam->getDesc())
.detail("destTeam", destTeam->getDesc()); .detail("DestTeam", destTeam->getDesc());
self->input.send( RelocateShard( moveShard, priority ) ); self->input.send( RelocateShard( moveShard, priority ) );
return true; return true;
@ -1258,7 +1258,7 @@ ACTOR Future<Void> dataDistributionQueue(
} catch (Error& e) { } catch (Error& e) {
if (e.code() != error_code_broken_promise && // FIXME: Get rid of these broken_promise errors every time we are killed by the master dying if (e.code() != error_code_broken_promise && // FIXME: Get rid of these broken_promise errors every time we are killed by the master dying
e.code() != error_code_movekeys_conflict) e.code() != error_code_movekeys_conflict)
TraceEvent(SevError, "dataDistributionQueueError", mi.id()).error(e); TraceEvent(SevError, "DataDistributionQueueError", mi.id()).error(e);
throw e; throw e;
} }
} }

4
fdbserver/DataDistributionTracker.actor.cpp
View File

@ -344,7 +344,7 @@ ACTOR Future<Void> shardSplitter(
.detail("MetricsBytes", metrics.bytes) .detail("MetricsBytes", metrics.bytes)
.detail("Bandwidth", bandwidthStatus == BandwidthStatusHigh ? "High" : bandwidthStatus == BandwidthStatusNormal ? "Normal" : "Low") .detail("Bandwidth", bandwidthStatus == BandwidthStatusHigh ? "High" : bandwidthStatus == BandwidthStatusNormal ? "Normal" : "Low")
.detail("BytesPerKSec", metrics.bytesPerKSecond) .detail("BytesPerKSec", metrics.bytesPerKSecond)
.detail("numShards", numShards); .detail("NumShards", numShards);
} }
if( numShards > 1 ) { if( numShards > 1 ) {
@ -695,7 +695,7 @@ ACTOR Future<Void> dataDistributionTracker(
when( Void _ = wait( self.sizeChanges.getResult() ) ) {} when( Void _ = wait( self.sizeChanges.getResult() ) ) {}
} }
} catch (Error& e) { } catch (Error& e) {
TraceEvent(SevError, "dataDistributionTrackerError", self.masterId).error(e); TraceEvent(SevError, "DataDistributionTrackerError", self.masterId).error(e);
throw e; throw e;
} }
} }

70
fdbserver/DiskQueue.actor.cpp
View File

@ -136,7 +136,7 @@ public:
Future<Standalone<StringRef>> readFirstAndLastPages( compare_pages compare ) { return readFirstAndLastPages(this,compare); } Future<Standalone<StringRef>> readFirstAndLastPages( compare_pages compare ) { return readFirstAndLastPages(this,compare); }
void setStartPage( int file, int64_t page ) { void setStartPage( int file, int64_t page ) {
TraceEvent("RDQSetStart", dbgid).detail("f",file).detail("p",page).detail("file0name", files[0].dbgFilename); TraceEvent("RDQSetStart", dbgid).detail("FileNum",file).detail("PageNum",page).detail("File0Name", files[0].dbgFilename);
readingFile = file; readingFile = file;
readingPage = page; readingPage = page;
} }
@ -235,7 +235,7 @@ public:
if(p > 0) { if(p > 0) {
toSync.push_back( files[1].syncQueue ); toSync.push_back( files[1].syncQueue );
/*TraceEvent("RDQWriteAndSwap", this->dbgid).detail("File1name", files[1].dbgFilename).detail("File1size", files[1].size) /*TraceEvent("RDQWriteAndSwap", this->dbgid).detail("File1name", files[1].dbgFilename).detail("File1size", files[1].size)
.detail("writingPos", writingPos).detail("writingBytes", p);*/ .detail("WritingPos", writingPos).detail("WritingBytes", p);*/
waitfor.push_back( files[1].f->write( pageData.begin(), p, writingPos ) ); waitfor.push_back( files[1].f->write( pageData.begin(), p, writingPos ) );
pageData = pageData.substr( p ); pageData = pageData.substr( p );
} }
@ -247,19 +247,19 @@ public:
} else { } else {
// Extend files[1] to accomodate the new write and about 10MB or 2x current size for future writes. // Extend files[1] to accomodate the new write and about 10MB or 2x current size for future writes.
/*TraceEvent("RDQExtend", this->dbgid).detail("File1name", files[1].dbgFilename).detail("File1size", files[1].size) /*TraceEvent("RDQExtend", this->dbgid).detail("File1name", files[1].dbgFilename).detail("File1size", files[1].size)
.detail("extensionBytes", fileExtensionBytes);*/ .detail("ExtensionBytes", fileExtensionBytes);*/
int64_t minExtension = pageData.size() + writingPos - files[1].size; int64_t minExtension = pageData.size() + writingPos - files[1].size;
files[1].size += std::min(std::max(fileExtensionBytes, minExtension), files[0].size+files[1].size+minExtension); files[1].size += std::min(std::max(fileExtensionBytes, minExtension), files[0].size+files[1].size+minExtension);
waitfor.push_back( files[1].f->truncate( files[1].size ) ); waitfor.push_back( files[1].f->truncate( files[1].size ) );
if(fileSizeWarningLimit > 0 && files[1].size > fileSizeWarningLimit) { if(fileSizeWarningLimit > 0 && files[1].size > fileSizeWarningLimit) {
TraceEvent(SevWarnAlways, "DiskQueueFileTooLarge", dbgid).detail("filename", filename(1)).detail("size", files[1].size).suppressFor(1.0); TraceEvent(SevWarnAlways, "DiskQueueFileTooLarge", dbgid).detail("Filename", filename(1)).detail("Size", files[1].size).suppressFor(1.0);
} }
} }
} }
/*TraceEvent("RDQWrite", this->dbgid).detail("File1name", files[1].dbgFilename).detail("File1size", files[1].size) /*TraceEvent("RDQWrite", this->dbgid).detail("File1name", files[1].dbgFilename).detail("File1size", files[1].size)
.detail("writingPos", writingPos).detail("writingBytes", pageData.size());*/ .detail("WritingPos", writingPos).detail("WritingBytes", pageData.size());*/
files[1].size = std::max( files[1].size, writingPos + pageData.size() ); files[1].size = std::max( files[1].size, writingPos + pageData.size() );
toSync.push_back( files[1].syncQueue ); toSync.push_back( files[1].syncQueue );
waitfor.push_back( files[1].f->write( pageData.begin(), pageData.size(), writingPos ) ); waitfor.push_back( files[1].f->write( pageData.begin(), pageData.size(), writingPos ) );
@ -322,7 +322,7 @@ public:
delete pageMem; delete pageMem;
TEST(true); // push error TEST(true); // push error
TEST(2==syncFiles.size()); // push spanning both files error TEST(2==syncFiles.size()); // push spanning both files error
TraceEvent(SevError, "RDQ_pushAndCommit_Error", dbgid).detail("InitialFilename0", filename).error(e, true); TraceEvent(SevError, "RDQPushAndCommitError", dbgid).detail("InitialFilename0", filename).error(e, true);
if (errorPromise.canBeSet()) errorPromise.sendError(e); if (errorPromise.canBeSet()) errorPromise.sendError(e);
if (pushing.canBeSet()) pushing.sendError(e); if (pushing.canBeSet()) pushing.sendError(e);
@ -512,7 +512,7 @@ public:
bool middleValid = compare( &firstPage[1], middlePage ); bool middleValid = compare( &firstPage[1], middlePage );
TraceEvent("RDQBS", self->dbgid).detail("b", begin).detail("e", end).detail("m", middle).detail("v", middleValid).detail("file0name", self->files[0].dbgFilename); TraceEvent("RDQBS", self->dbgid).detail("Begin", begin).detail("End", end).detail("Middle", middle).detail("Valid", middleValid).detail("File0Name", self->files[0].dbgFilename);
if (middleValid) if (middleValid)
begin = middle; begin = middle;
@ -533,7 +533,7 @@ public:
return result.str; return result.str;
} catch (Error& e) { } catch (Error& e) {
bool ok = e.code() == error_code_file_not_found; bool ok = e.code() == error_code_file_not_found;
TraceEvent(ok ? SevInfo : SevError, "RDQ_rfl_Error", self->dbgid).detail("file0name", self->files[0].dbgFilename).error(e, true); TraceEvent(ok ? SevInfo : SevError, "RDQReadFirstAndLastPagesError", self->dbgid).detail("File0Name", self->files[0].dbgFilename).error(e, true);
if (!self->error.isSet()) self->error.sendError(e); if (!self->error.isSet()) self->error.sendError(e);
throw; throw;
} }
@ -588,7 +588,7 @@ public:
return result; return result;
} catch (Error& e) { } catch (Error& e) {
TEST(true); // Read next page error TEST(true); // Read next page error
TraceEvent(SevError, "RDQ_rnp_Error", self->dbgid).detail("file0name", self->files[0].dbgFilename).error(e, true); TraceEvent(SevError, "RDQReadNextPageError", self->dbgid).detail("File0Name", self->files[0].dbgFilename).error(e, true);
if (!self->error.isSet()) self->error.sendError(e); if (!self->error.isSet()) self->error.sendError(e);
throw; throw;
} }
@ -633,7 +633,7 @@ public:
return Void(); return Void();
} catch (Error& e) { } catch (Error& e) {
TraceEvent(SevError, "RDQ_tblrp_Error", self->dbgid).detail("file0name", self->files[0].dbgFilename).error(e); TraceEvent(SevError, "RDQTruncateBeforeLastReadPageError", self->dbgid).detail("File0Name", self->files[0].dbgFilename).error(e);
if (!self->error.isSet()) self->error.sendError(e); if (!self->error.isSet()) self->error.sendError(e);
throw; throw;
} }
@ -676,7 +676,7 @@ public:
TraceEvent(SevError, "DQPopUncommittedData", dbgid) TraceEvent(SevError, "DQPopUncommittedData", dbgid)
.detail("UpTo", upTo) .detail("UpTo", upTo)
.detail("LastCommittedSeq", lastCommittedSeq) .detail("LastCommittedSeq", lastCommittedSeq)
.detail("file0name", rawQueue->files[0].dbgFilename); .detail("File0Name", rawQueue->files[0].dbgFilename);
} }
if (upTo.lo > poppedSeq) { if (upTo.lo > poppedSeq) {
poppedSeq = upTo.lo; poppedSeq = upTo.lo;
@ -712,17 +712,17 @@ public:
if( pushedPageCount() >= 8000 ) { if( pushedPageCount() >= 8000 ) {
TraceEvent( warnAlwaysForMemory ? SevWarnAlways : SevWarn, "DiskQueueMemoryWarning", dbgid) TraceEvent( warnAlwaysForMemory ? SevWarnAlways : SevWarn, "DiskQueueMemoryWarning", dbgid)
.detail("pushed_pages", pushedPageCount()) .detail("PushedPages", pushedPageCount())
.detail("nextPageSeq", nextPageSeq) .detail("NextPageSeq", nextPageSeq)
.detail("Details", format("%d pages", pushedPageCount())) .detail("Details", format("%d pages", pushedPageCount()))
.detail("file0name", rawQueue->files[0].dbgFilename) .detail("File0Name", rawQueue->files[0].dbgFilename)
.suppressFor(1.0); .suppressFor(1.0);
if(g_network->isSimulated()) if(g_network->isSimulated())
warnAlwaysForMemory = false; warnAlwaysForMemory = false;
} }
/*TraceEvent("DQCommit", dbgid).detail("Pages", pushedPageCount()).detail("lastPoppedSeq", lastPoppedSeq).detail("poppedSeq", poppedSeq).detail("nextPageSeq", nextPageSeq) /*TraceEvent("DQCommit", dbgid).detail("Pages", pushedPageCount()).detail("LastPoppedSeq", lastPoppedSeq).detail("PoppedSeq", poppedSeq).detail("NextPageSeq", nextPageSeq)
.detail("RawFile0Size", rawQueue->files[0].size).detail("RawFile1Size", rawQueue->files[1].size).detail("writingPos", rawQueue->writingPos) .detail("RawFile0Size", rawQueue->files[0].size).detail("RawFile1Size", rawQueue->files[1].size).detail("WritingPos", rawQueue->writingPos)
.detail("RawFile0Name", rawQueue->files[0].dbgFilename);*/ .detail("RawFile0Name", rawQueue->files[0].dbgFilename);*/
lastCommittedSeq = backPage().endSeq(); lastCommittedSeq = backPage().endSeq();
@ -742,17 +742,17 @@ public:
virtual Future<Void> getError() { return rawQueue->getError(); } virtual Future<Void> getError() { return rawQueue->getError(); }
virtual Future<Void> onClosed() { return rawQueue->onClosed(); } virtual Future<Void> onClosed() { return rawQueue->onClosed(); }
virtual void dispose() { virtual void dispose() {
TraceEvent("DQDestroy", dbgid).detail("lastPoppedSeq", lastPoppedSeq).detail("poppedSeq", poppedSeq).detail("nextPageSeq", nextPageSeq).detail("file0name", rawQueue->files[0].dbgFilename); TraceEvent("DQDestroy", dbgid).detail("LastPoppedSeq", lastPoppedSeq).detail("PoppedSeq", poppedSeq).detail("NextPageSeq", nextPageSeq).detail("File0Name", rawQueue->files[0].dbgFilename);
rawQueue->dispose(); rawQueue->dispose();
delete this; delete this;
} }
virtual void close() { virtual void close() {
TraceEvent("DQClose", dbgid) TraceEvent("DQClose", dbgid)
.detail("lastPoppedSeq", lastPoppedSeq) .detail("LastPoppedSeq", lastPoppedSeq)
.detail("poppedSeq", poppedSeq) .detail("PoppedSeq", poppedSeq)
.detail("nextPageSeq", nextPageSeq) .detail("NextPageSeq", nextPageSeq)
.detail("poppedCommitted", rawQueue->dbg_file0BeginSeq + rawQueue->files[0].popped + rawQueue->files[1].popped) .detail("PoppedCommitted", rawQueue->dbg_file0BeginSeq + rawQueue->files[0].popped + rawQueue->files[1].popped)
.detail("file0name", rawQueue->files[0].dbgFilename); .detail("File0Name", rawQueue->files[0].dbgFilename);
rawQueue->close(); rawQueue->close();
delete this; delete this;
} }
@ -818,9 +818,9 @@ private:
if (pushedPageCount() == 8000) { if (pushedPageCount() == 8000) {
TraceEvent("DiskQueueHighPageCount", dbgid) TraceEvent("DiskQueueHighPageCount", dbgid)
.detail("pushed_pages", pushedPageCount()) .detail("PushedPages", pushedPageCount())
.detail("nextPageSeq", nextPageSeq) .detail("NextPageSeq", nextPageSeq)
.detail("file0name", rawQueue->files[0].dbgFilename); .detail("File0Name", rawQueue->files[0].dbgFilename);
} }
} }
@ -857,7 +857,7 @@ private:
} }
self->readBufPos = self->nextReadLocation % sizeof(Page) - sizeof(PageHeader); self->readBufPos = self->nextReadLocation % sizeof(Page) - sizeof(PageHeader);
if (self->readBufPos < 0) { self->nextReadLocation -= self->readBufPos; self->readBufPos = 0; } if (self->readBufPos < 0) { self->nextReadLocation -= self->readBufPos; self->readBufPos = 0; }
TraceEvent("DQRecStart", self->dbgid).detail("readBufPos", self->readBufPos).detail("nextReadLoc", self->nextReadLocation).detail("file0name", self->rawQueue->files[0].dbgFilename); TraceEvent("DQRecStart", self->dbgid).detail("ReadBufPos", self->readBufPos).detail("NextReadLoc", self->nextReadLocation).detail("File0Name", self->rawQueue->files[0].dbgFilename);
} }
loop { loop {
@ -873,7 +873,7 @@ private:
Standalone<StringRef> page = wait( self->rawQueue->readNextPage() ); Standalone<StringRef> page = wait( self->rawQueue->readNextPage() );
if (!page.size()) { if (!page.size()) {
TraceEvent("DQRecEOF", self->dbgid).detail("nextReadLocation", self->nextReadLocation).detail("file0name", self->rawQueue->files[0].dbgFilename); TraceEvent("DQRecEOF", self->dbgid).detail("NextReadLocation", self->nextReadLocation).detail("File0Name", self->rawQueue->files[0].dbgFilename);
break; break;
} }
ASSERT( page.size() == sizeof(Page) ); ASSERT( page.size() == sizeof(Page) );
@ -881,12 +881,12 @@ private:
self->readBufArena = page.arena(); self->readBufArena = page.arena();
self->readBufPage = (Page*)page.begin(); self->readBufPage = (Page*)page.begin();
if (!self->readBufPage->checkHash() || self->readBufPage->seq < self->nextReadLocation/sizeof(Page)*sizeof(Page)) { if (!self->readBufPage->checkHash() || self->readBufPage->seq < self->nextReadLocation/sizeof(Page)*sizeof(Page)) {
TraceEvent("DQRecInvalidPage", self->dbgid).detail("nextReadLocation", self->nextReadLocation).detail("hashCheck", self->readBufPage->checkHash()) TraceEvent("DQRecInvalidPage", self->dbgid).detail("NextReadLocation", self->nextReadLocation).detail("HashCheck", self->readBufPage->checkHash())
.detail("seq", self->readBufPage->seq).detail("expect", self->nextReadLocation/sizeof(Page)*sizeof(Page)).detail("file0name", self->rawQueue->files[0].dbgFilename); .detail("Seq", self->readBufPage->seq).detail("Expect", self->nextReadLocation/sizeof(Page)*sizeof(Page)).detail("File0Name", self->rawQueue->files[0].dbgFilename);
Void _ = wait( self->rawQueue->truncateBeforeLastReadPage() ); Void _ = wait( self->rawQueue->truncateBeforeLastReadPage() );
break; break;
} }
//TraceEvent("DQRecPage", self->dbgid).detail("nextReadLoc", self->nextReadLocation).detail("Seq", self->readBufPage->seq).detail("Pop", self->readBufPage->popped).detail("Payload", self->readBufPage->payloadSize).detail("file0name", self->rawQueue->files[0].dbgFilename); //TraceEvent("DQRecPage", self->dbgid).detail("NextReadLoc", self->nextReadLocation).detail("Seq", self->readBufPage->seq).detail("Pop", self->readBufPage->popped).detail("Payload", self->readBufPage->payloadSize).detail("File0Name", self->rawQueue->files[0].dbgFilename);
ASSERT( self->readBufPage->seq == self->nextReadLocation/sizeof(Page)*sizeof(Page) ); ASSERT( self->readBufPage->seq == self->nextReadLocation/sizeof(Page)*sizeof(Page) );
self->lastPoppedSeq = self->readBufPage->popped; self->lastPoppedSeq = self->readBufPage->popped;
} }
@ -902,7 +902,7 @@ private:
self->nextPageSeq = self->nextReadLocation/sizeof(Page)*sizeof(Page); self->nextPageSeq = self->nextReadLocation/sizeof(Page)*sizeof(Page);
if (self->nextReadLocation % sizeof(Page) > 36) self->nextPageSeq += sizeof(Page); if (self->nextReadLocation % sizeof(Page) > 36) self->nextPageSeq += sizeof(Page);
TraceEvent("DQRecovered", self->dbgid).detail("lastPoppedSeq", self->lastPoppedSeq).detail("poppedSeq", self->poppedSeq).detail("nextPageSeq", self->nextPageSeq).detail("file0name", self->rawQueue->files[0].dbgFilename); TraceEvent("DQRecovered", self->dbgid).detail("LastPoppedSeq", self->lastPoppedSeq).detail("PoppedSeq", self->poppedSeq).detail("NextPageSeq", self->nextPageSeq).detail("File0Name", self->rawQueue->files[0].dbgFilename);
self->recovered = true; self->recovered = true;
ASSERT( self->poppedSeq <= self->endLocation() ); ASSERT( self->poppedSeq <= self->endLocation() );
self->recoveryFirstPages = Standalone<StringRef>(); self->recoveryFirstPages = Standalone<StringRef>();
@ -938,7 +938,7 @@ private:
state int sizeNum; state int sizeNum;
for( sizeNum=0; sizeNum < self->rawQueue->files[fileNum].size; sizeNum += sizeof(Page) ) { for( sizeNum=0; sizeNum < self->rawQueue->files[fileNum].size; sizeNum += sizeof(Page) ) {
int _ = wait( self->rawQueue->files[fileNum].f->read( testPage.get(), sizeof(Page), sizeNum ) ); int _ = wait( self->rawQueue->files[fileNum].f->read( testPage.get(), sizeof(Page), sizeNum ) );
TraceEvent("PageData").detail("file", self->rawQueue->files[fileNum].dbgFilename).detail("sizeNum", sizeNum).detail("seq", testPage->seq).detail("hash", testPage->checkHash()).detail("popped", testPage->popped); TraceEvent("PageData").detail("File", self->rawQueue->files[fileNum].dbgFilename).detail("SizeNum", sizeNum).detail("Seq", testPage->seq).detail("Hash", testPage->checkHash()).detail("Popped", testPage->popped);
} }
} }
*/ */
@ -962,7 +962,7 @@ private:
.detail("Page1Seq", p[1].seq) .detail("Page1Seq", p[1].seq)
.detail("Location", loc) .detail("Location", loc)
.detail("Context", context) .detail("Context", context)
.detail("file0name", rawQueue->files[0].dbgFilename); .detail("File0Name", rawQueue->files[0].dbgFilename);
for(int i=recoveryFirstPages.size() / sizeof(Page) - 2; i>=0; i--) for(int i=recoveryFirstPages.size() / sizeof(Page) - 2; i>=0; i--)
if ( p[i].checkHash() && p[i].seq <= (size_t)loc ) { if ( p[i].checkHash() && p[i].seq <= (size_t)loc ) {
@ -976,7 +976,7 @@ private:
.detail("PageSequence", p[i].seq) .detail("PageSequence", p[i].seq)
.detail("Location", loc) .detail("Location", loc)
.detail("Context", context) .detail("Context", context)
.detail("file0name", rawQueue->files[0].dbgFilename); .detail("File0Name", rawQueue->files[0].dbgFilename);
ok = true; ok = true;
break; break;
} }
@ -989,7 +989,7 @@ private:
.detail("Page1Seq", p[1].seq) .detail("Page1Seq", p[1].seq)
.detail("Location", loc) .detail("Location", loc)
.detail("Context", context) .detail("Context", context)
.detail("file0name", rawQueue->files[0].dbgFilename); .detail("File0Name", rawQueue->files[0].dbgFilename);
ASSERT( ok ); ASSERT( ok );
} }

28
fdbserver/KeyValueStoreMemory.actor.cpp
View File

@ -408,9 +408,9 @@ private:
bool ok = count < 1e6; bool ok = count < 1e6;
if( !ok ) { if( !ok ) {
TraceEvent(/*ok ? SevInfo : */SevWarnAlways, "KVSMemCommit_queue", id) TraceEvent(/*ok ? SevInfo : */SevWarnAlways, "KVSMemCommit_queue", id)
.detail("bytes", total) .detail("Bytes", total)
.detail("log", log) .detail("Log", log)
.detail("ops", count) .detail("Ops", count)
.detail("LastLoggedLocation", log_location) .detail("LastLoggedLocation", log_location)
.detail("Details", count); .detail("Details", count);
} }
@ -490,9 +490,9 @@ private:
if (h.op == OpSnapshotItem) { // snapshot data item if (h.op == OpSnapshotItem) { // snapshot data item
/*if (p1 < uncommittedNextKey) { /*if (p1 < uncommittedNextKey) {
TraceEvent(SevError, "RecSnapshotBack", self->id) TraceEvent(SevError, "RecSnapshotBack", self->id)
.detail("nextKey", printable(uncommittedNextKey)) .detail("NextKey", printable(uncommittedNextKey))
.detail("p1", printable(p1)) .detail("P1", printable(p1))
.detail("nextlocation", self->log->getNextReadLocation()); .detail("Nextlocation", self->log->getNextReadLocation());
} }
ASSERT( p1 >= uncommittedNextKey );*/ ASSERT( p1 >= uncommittedNextKey );*/
if( p1 >= uncommittedNextKey ) if( p1 >= uncommittedNextKey )
@ -502,9 +502,9 @@ private:
++dbgSnapshotItemCount; ++dbgSnapshotItemCount;
} else if (h.op == OpSnapshotEnd || h.op == OpSnapshotAbort) { // snapshot complete } else if (h.op == OpSnapshotEnd || h.op == OpSnapshotAbort) { // snapshot complete
TraceEvent("RecSnapshotEnd", self->id) TraceEvent("RecSnapshotEnd", self->id)
.detail("nextKey", printable(uncommittedNextKey)) .detail("NextKey", printable(uncommittedNextKey))
.detail("nextlocation", self->log->getNextReadLocation()) .detail("Nextlocation", self->log->getNextReadLocation())
.detail("isSnapshotEnd", h.op == OpSnapshotEnd); .detail("IsSnapshotEnd", h.op == OpSnapshotEnd);
if(h.op == OpSnapshotEnd) { if(h.op == OpSnapshotEnd) {
uncommittedPrevSnapshotEnd = uncommittedSnapshotEnd; uncommittedPrevSnapshotEnd = uncommittedSnapshotEnd;
@ -531,7 +531,7 @@ private:
} else if (h.op == OpRollback) { // rollback previous transaction } else if (h.op == OpRollback) { // rollback previous transaction
recoveryQueue.rollback(); recoveryQueue.rollback();
TraceEvent("KVSMemRecSnapshotRollback", self->id) TraceEvent("KVSMemRecSnapshotRollback", self->id)
.detail("nextKey", printable(uncommittedNextKey)); .detail("NextKey", printable(uncommittedNextKey));
uncommittedNextKey = self->recoveredSnapshotKey; uncommittedNextKey = self->recoveredSnapshotKey;
uncommittedPrevSnapshotEnd = self->previousSnapshotEnd; uncommittedPrevSnapshotEnd = self->previousSnapshotEnd;
uncommittedSnapshotEnd = self->currentSnapshotEnd; uncommittedSnapshotEnd = self->currentSnapshotEnd;
@ -646,10 +646,10 @@ private:
if (next == self->data.end()) { if (next == self->data.end()) {
auto thisSnapshotEnd = self->log_op( OpSnapshotEnd, StringRef(), StringRef() ); auto thisSnapshotEnd = self->log_op( OpSnapshotEnd, StringRef(), StringRef() );
//TraceEvent("SnapshotEnd", self->id) //TraceEvent("SnapshotEnd", self->id)
// .detail("lastKey", printable(lastKey.present() ? lastKey.get() : LiteralStringRef("<none>"))) // .detail("LastKey", printable(lastKey.present() ? lastKey.get() : LiteralStringRef("<none>")))
// .detail("currentSnapshotEndLoc", self->currentSnapshotEnd) // .detail("CurrentSnapshotEndLoc", self->currentSnapshotEnd)
// .detail("previousSnapshotEndLoc", self->previousSnapshotEnd) // .detail("PreviousSnapshotEndLoc", self->previousSnapshotEnd)
// .detail("thisSnapshotEnd", thisSnapshotEnd) // .detail("ThisSnapshotEnd", thisSnapshotEnd)
// .detail("Items", snapItems) // .detail("Items", snapItems)
// .detail("CommittedWrites", self->notifiedCommittedWriteBytes.get()) // .detail("CommittedWrites", self->notifiedCommittedWriteBytes.get())
// .detail("SnapshotSize", snapshotBytes); // .detail("SnapshotSize", snapshotBytes);

22
fdbserver/KeyValueStoreSQLite.actor.cpp
View File

@ -269,7 +269,7 @@ struct SQLiteDB : NonCopyable {
TraceEvent("BTreeIntegrityCheckBegin").detail("Filename", filename); TraceEvent("BTreeIntegrityCheckBegin").detail("Filename", filename);
char* e = sqlite3BtreeIntegrityCheck(btree, tables, 3, 1000, &errors, verbose); char* e = sqlite3BtreeIntegrityCheck(btree, tables, 3, 1000, &errors, verbose);
if (!(g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))) { if (!(g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting))) {
TraceEvent((errors||e) ? SevError : SevInfo, "BTreeIntegrityCheck").detail("filename", filename).detail("ErrorTotal", errors); TraceEvent((errors||e) ? SevError : SevInfo, "BTreeIntegrityCheck").detail("Filename", filename).detail("ErrorTotal", errors);
if(e != nullptr) { if(e != nullptr) {
// e is a string containing 1 or more lines. Create a separate trace event for each line. // e is a string containing 1 or more lines. Create a separate trace event for each line.
char *lineStart = e; char *lineStart = e;
@ -282,7 +282,7 @@ struct SQLiteDB : NonCopyable {
// If the line length found is not zero then print a trace event // If the line length found is not zero then print a trace event
if(*lineStart != '\0') if(*lineStart != '\0')
TraceEvent(SevError, "BTreeIntegrityCheck").detail("filename", filename).detail("ErrorDetail", lineStart); TraceEvent(SevError, "BTreeIntegrityCheck").detail("Filename", filename).detail("ErrorDetail", lineStart);
lineStart = lineEnd; lineStart = lineEnd;
} }
} }
@ -1262,7 +1262,7 @@ int SQLiteDB::checkAllPageChecksums() {
TraceEvent("SQLitePageChecksumScanEnd") TraceEvent("SQLitePageChecksumScanEnd")
.detail("Elapsed", DEBUG_DETERMINISM ? 0 : timer()-startT) .detail("Elapsed", DEBUG_DETERMINISM ? 0 : timer()-startT)
.detail("filename", filename) .detail("Filename", filename)
.detail("CorruptPages", corruptPages) .detail("CorruptPages", corruptPages)
.detail("ReadErrors", readErrors) .detail("ReadErrors", readErrors)
.detail("TotalErrors", totalErrors); .detail("TotalErrors", totalErrors);
@ -1275,7 +1275,7 @@ int SQLiteDB::checkAllPageChecksums() {
void SQLiteDB::open(bool writable) { void SQLiteDB::open(bool writable) {
ASSERT( !haveMutex ); ASSERT( !haveMutex );
double startT = timer(); double startT = timer();
//TraceEvent("KVThreadInitStage").detail("Stage",1).detail("filename", filename).detail("writable", writable); //TraceEvent("KVThreadInitStage").detail("Stage",1).detail("Filename", filename).detail("Writable", writable);
// First try to open an existing file // First try to open an existing file
std::string apath = abspath(filename); std::string apath = abspath(filename);
@ -1283,7 +1283,7 @@ void SQLiteDB::open(bool writable) {
ErrorOr<Reference<IAsyncFile>> dbFile = waitForAndGet( errorOr( IAsyncFileSystem::filesystem()->open( apath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0 ) ) ); ErrorOr<Reference<IAsyncFile>> dbFile = waitForAndGet( errorOr( IAsyncFileSystem::filesystem()->open( apath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0 ) ) );
ErrorOr<Reference<IAsyncFile>> walFile = waitForAndGet( errorOr( IAsyncFileSystem::filesystem()->open( walpath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0 ) ) ); ErrorOr<Reference<IAsyncFile>> walFile = waitForAndGet( errorOr( IAsyncFileSystem::filesystem()->open( walpath, IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_LOCK, 0 ) ) );
//TraceEvent("KVThreadInitStage").detail("Stage",15).detail("filename", apath).detail("writable", writable).detail("isErr", dbFile.isError()); //TraceEvent("KVThreadInitStage").detail("Stage",15).detail("Filename", apath).detail("Writable", writable).detail("IsErr", dbFile.isError());
if (writable) { if (writable) {
if (dbFile.isError() && dbFile.getError().code() == error_code_file_not_found && !fileExists(apath) && // db file is missing if (dbFile.isError() && dbFile.getError().code() == error_code_file_not_found && !fileExists(apath) && // db file is missing
@ -1318,7 +1318,7 @@ void SQLiteDB::open(bool writable) {
if (dbFile.isError()) throw dbFile.getError(); // If we've failed to open the file, throw an exception if (dbFile.isError()) throw dbFile.getError(); // If we've failed to open the file, throw an exception
if (walFile.isError()) throw walFile.getError(); // If we've failed to open the file, throw an exception if (walFile.isError()) throw walFile.getError(); // If we've failed to open the file, throw an exception
//TraceEvent("KVThreadInitStage").detail("Stage",2).detail("filename", filename).detail("writable", writable); //TraceEvent("KVThreadInitStage").detail("Stage",2).detail("Filename", filename).detail("Writable", writable);
// Now that the file itself is open and locked, let sqlite open the database // Now that the file itself is open and locked, let sqlite open the database
// Note that VFSAsync will also call g_network->open (including for the WAL), so its flags are important, too // Note that VFSAsync will also call g_network->open (including for the WAL), so its flags are important, too
@ -1330,7 +1330,7 @@ void SQLiteDB::open(bool writable) {
sqlite3_extended_result_codes(db, 1); sqlite3_extended_result_codes(db, 1);
//TraceEvent("KVThreadInitStage").detail("Stage",3).detail("filename", filename).detail("writable", writable); //TraceEvent("KVThreadInitStage").detail("Stage",3).detail("Filename", filename).detail("Writable", writable);
//Statement(*this, "PRAGMA cache_size = 100").execute(); //Statement(*this, "PRAGMA cache_size = 100").execute();
@ -1346,7 +1346,7 @@ void SQLiteDB::open(bool writable) {
Statement(*this, "PRAGMA wal_autocheckpoint = -1").nextRow(); Statement(*this, "PRAGMA wal_autocheckpoint = -1").nextRow();
} }
//TraceEvent("KVThreadInitStage").detail("Stage",4).detail("filename", filename).detail("writable", writable); //TraceEvent("KVThreadInitStage").detail("Stage",4).detail("Filename", filename).detail("Writable", writable);
sqlite3_mutex_enter(db->mutex); sqlite3_mutex_enter(db->mutex);
haveMutex = true; haveMutex = true;
@ -1356,7 +1356,7 @@ void SQLiteDB::open(bool writable) {
this->dbFile = dbFile.get(); this->dbFile = dbFile.get();
this->walFile = walFile.get(); this->walFile = walFile.get();
TraceEvent("KVThreadInitTime").detail("Elapsed", DEBUG_DETERMINISM ? 0 : timer()-startT).detail("filename", filename).detail("writable", writable); TraceEvent("KVThreadInitTime").detail("Elapsed", DEBUG_DETERMINISM ? 0 : timer()-startT).detail("Filename", filename).detail("Writable", writable);
ASSERT(vfsAsyncIsOpen(filename)); ASSERT(vfsAsyncIsOpen(filename));
} }
@ -1392,7 +1392,7 @@ void SQLiteDB::createFromScratch() {
ASSERT( freetable == table+1 ); ASSERT( freetable == table+1 );
endTransaction(); endTransaction();
} else { } else {
TraceEvent("pgnoRoot").detail("value", pgnoRoot); TraceEvent("pgnoRoot").detail("Value", pgnoRoot);
checkError("CheckTables", SQLITE_CORRUPT); checkError("CheckTables", SQLITE_CORRUPT);
} }
} }
@ -1662,7 +1662,7 @@ private:
checkFreePages(); checkFreePages();
++writesComplete; ++writesComplete;
if (t3-a.issuedTime > 10.0*g_random->random01()) if (t3-a.issuedTime > 10.0*g_random->random01())
TraceEvent("KVCommit10s_sample", dbgid).detail("Queued", t1-a.issuedTime).detail("Commit", t2-t1).detail("Checkpoint", t3-t2); TraceEvent("KVCommit10sSample", dbgid).detail("Queued", t1-a.issuedTime).detail("Commit", t2-t1).detail("Checkpoint", t3-t2);
diskBytesUsed = waitForAndGet( conn.dbFile->size() ) + waitForAndGet( conn.walFile->size() ); diskBytesUsed = waitForAndGet( conn.dbFile->size() ) + waitForAndGet( conn.walFile->size() );

4
fdbserver/Knobs.cpp
View File

@ -227,6 +227,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( COMMIT_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION, 0.1 ); init( COMMIT_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION, 0.1 );
init( COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA, 0.1 ); init( COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA, 0.1 );
init( COMMIT_TRANSACTION_BATCH_COUNT_MAX, 32768 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_COUNT_MAX = 1000; // Do NOT increase this number beyond 32768, as CommitIds only budget 2 bytes for storing transaction id within each batch init( COMMIT_TRANSACTION_BATCH_COUNT_MAX, 32768 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_COUNT_MAX = 1000; // Do NOT increase this number beyond 32768, as CommitIds only budget 2 bytes for storing transaction id within each batch
init( COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT, 8LL << 30 ); if (randomize && BUGGIFY) COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT = g_random->randomInt64(100LL << 20, 8LL << 30);
// these settings disable batch bytes scaling. Try COMMIT_TRANSACTION_BATCH_BYTES_MAX=1e6, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE=50000, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER=0.5? // these settings disable batch bytes scaling. Try COMMIT_TRANSACTION_BATCH_BYTES_MAX=1e6, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE=50000, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER=0.5?
init( COMMIT_TRANSACTION_BATCH_BYTES_MIN, 100000 ); init( COMMIT_TRANSACTION_BATCH_BYTES_MIN, 100000 );
@ -238,6 +239,8 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( RESOLVER_COALESCE_TIME, 1.0 ); init( RESOLVER_COALESCE_TIME, 1.0 );
init( BUGGIFIED_ROW_LIMIT, APPLY_MUTATION_BYTES ); if( randomize && BUGGIFY ) BUGGIFIED_ROW_LIMIT = g_random->randomInt(3, 30); init( BUGGIFIED_ROW_LIMIT, APPLY_MUTATION_BYTES ); if( randomize && BUGGIFY ) BUGGIFIED_ROW_LIMIT = g_random->randomInt(3, 30);
init( PROXY_SPIN_DELAY, 0.01 ); init( PROXY_SPIN_DELAY, 0.01 );
init( COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL, 0.5 );
init( COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR, 10.0 );
// Master Server // Master Server
init( MASTER_LOGGING_DELAY, 1.0 ); init( MASTER_LOGGING_DELAY, 1.0 );
@ -293,6 +296,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( MIN_REBOOT_TIME, 4.0 ); if( longReboots ) MIN_REBOOT_TIME = 10.0; init( MIN_REBOOT_TIME, 4.0 ); if( longReboots ) MIN_REBOOT_TIME = 10.0;
init( MAX_REBOOT_TIME, 5.0 ); if( longReboots ) MAX_REBOOT_TIME = 20.0; init( MAX_REBOOT_TIME, 5.0 ); if( longReboots ) MAX_REBOOT_TIME = 20.0;
init( LOG_DIRECTORY, "."); // Will be set to the command line flag. init( LOG_DIRECTORY, "."); // Will be set to the command line flag.
init(SERVER_MEM_LIMIT, 8LL << 30);
//Ratekeeper //Ratekeeper
bool slowRateKeeper = randomize && BUGGIFY; bool slowRateKeeper = randomize && BUGGIFY;

4
fdbserver/Knobs.h
View File

@ -178,6 +178,9 @@ public:
int COMMIT_TRANSACTION_BATCH_BYTES_MAX; int COMMIT_TRANSACTION_BATCH_BYTES_MAX;
double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE; double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE;
double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER; double COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER;
int64_t COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT;
double COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL;
double COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR;
double TRANSACTION_BUDGET_TIME; double TRANSACTION_BUDGET_TIME;
double RESOLVER_COALESCE_TIME; double RESOLVER_COALESCE_TIME;
@ -236,6 +239,7 @@ public:
double MIN_REBOOT_TIME; double MIN_REBOOT_TIME;
double MAX_REBOOT_TIME; double MAX_REBOOT_TIME;
std::string LOG_DIRECTORY; std::string LOG_DIRECTORY;
int64_t SERVER_MEM_LIMIT;
//Ratekeeper //Ratekeeper
double SMOOTHING_AMOUNT; double SMOOTHING_AMOUNT;

30
fdbserver/LeaderElection.actor.cpp
View File

@ -25,7 +25,7 @@
#include "fdbserver/CoordinationInterface.h" #include "fdbserver/CoordinationInterface.h"
#include "fdbclient/MonitorLeader.h" #include "fdbclient/MonitorLeader.h"
extern Optional<LeaderInfo> getLeader( vector<Optional<LeaderInfo>> nominees ); Optional<std::pair<LeaderInfo, bool>> getLeader( const vector<Optional<LeaderInfo>>& nominees );
ACTOR Future<Void> submitCandidacy( Key key, LeaderElectionRegInterface coord, LeaderInfo myInfo, UID prevChangeID, Reference<AsyncVar<vector<Optional<LeaderInfo>>>> nominees, int index ) { ACTOR Future<Void> submitCandidacy( Key key, LeaderElectionRegInterface coord, LeaderInfo myInfo, UID prevChangeID, Reference<AsyncVar<vector<Optional<LeaderInfo>>>> nominees, int index ) {
loop { loop {
@ -106,44 +106,44 @@ ACTOR Future<Void> tryBecomeLeaderInternal( ServerCoordinators coordinators, Val
candidacies = waitForAll(cand); candidacies = waitForAll(cand);
loop { loop {
state Optional<LeaderInfo> leader = getLeader( nominees->get() ); state Optional<std::pair<LeaderInfo, bool>> leader = getLeader( nominees->get() );
if( leader.present() && leader.get().forward ) { if( leader.present() && leader.get().first.forward ) {
// These coordinators are forwarded to another set. But before we change our own cluster file, we need to make // These coordinators are forwarded to another set. But before we change our own cluster file, we need to make
// sure that a majority of coordinators know that. // sure that a majority of coordinators know that.
// SOMEDAY: Wait briefly to see if other coordinators will tell us they already know, to save communication? // SOMEDAY: Wait briefly to see if other coordinators will tell us they already know, to save communication?
Void _ = wait( changeLeaderCoordinators( coordinators, leader.get().serializedInfo ) ); Void _ = wait( changeLeaderCoordinators( coordinators, leader.get().first.serializedInfo ) );
if(!hasConnected) { if(!hasConnected) {
TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", coordinators.ccf->getFilename()) TraceEvent(SevWarnAlways, "IncorrectClusterFileContentsAtConnection").detail("Filename", coordinators.ccf->getFilename())
.detail("ConnectionStringFromFile", coordinators.ccf->getConnectionString().toString()) .detail("ConnectionStringFromFile", coordinators.ccf->getConnectionString().toString())
.detail("CurrentConnectionString", leader.get().serializedInfo.toString()); .detail("CurrentConnectionString", leader.get().first.serializedInfo.toString());
} }
coordinators.ccf->setConnectionString( ClusterConnectionString( leader.get().serializedInfo.toString() ) ); coordinators.ccf->setConnectionString( ClusterConnectionString( leader.get().first.serializedInfo.toString() ) );
TraceEvent("LeaderForwarding").detail("ConnStr", coordinators.ccf->getConnectionString().toString()); TraceEvent("LeaderForwarding").detail("ConnStr", coordinators.ccf->getConnectionString().toString());
throw coordinators_changed(); throw coordinators_changed();
} }
if (leader.present()) { if (leader.present() && leader.get().second) {
hasConnected = true; hasConnected = true;
coordinators.ccf->notifyConnected(); coordinators.ccf->notifyConnected();
} }
if (leader.present() && leader.get().changeID == myInfo.changeID) { if (leader.present() && leader.get().second && leader.get().first.equalInternalId(myInfo)) {
TraceEvent("BecomingLeader", myInfo.changeID); TraceEvent("BecomingLeader", myInfo.changeID);
ASSERT( leader.get().serializedInfo == proposedSerializedInterface ); ASSERT( leader.get().first.serializedInfo == proposedSerializedInterface );
outSerializedLeader->set( leader.get().serializedInfo ); outSerializedLeader->set( leader.get().first.serializedInfo );
iAmLeader = true; iAmLeader = true;
break; break;
} }
if (leader.present()) { if (leader.present()) {
TraceEvent("LeaderChanged", myInfo.changeID).detail("ToID", leader.get().changeID); TraceEvent("LeaderChanged", myInfo.changeID).detail("ToID", leader.get().first.changeID);
if (leader.get().serializedInfo != proposedSerializedInterface) // We never set outSerializedLeader to our own interface unless we are ready to become leader! if (leader.get().first.serializedInfo != proposedSerializedInterface) // We never set outSerializedLeader to our own interface unless we are ready to become leader!
outSerializedLeader->set( leader.get().serializedInfo ); outSerializedLeader->set( leader.get().first.serializedInfo );
} }
// If more than 2*SERVER_KNOBS->POLLING_FREQUENCY elapses while we are nominated by some coordinator but there is no leader, // If more than 2*SERVER_KNOBS->POLLING_FREQUENCY elapses while we are nominated by some coordinator but there is no leader,
// we might be breaking the leader election process for someone with better communications but lower ID, so change IDs. // we might be breaking the leader election process for someone with better communications but lower ID, so change IDs.
if (!leader.present() && std::count( nominees->get().begin(), nominees->get().end(), myInfo )) { if ((!leader.present() || !leader.get().second) && std::count( nominees->get().begin(), nominees->get().end(), myInfo )) {
if (!badCandidateTimeout.isValid()) if (!badCandidateTimeout.isValid())
badCandidateTimeout = delay( SERVER_KNOBS->POLLING_FREQUENCY*2, TaskCoordinationReply ); badCandidateTimeout = delay( SERVER_KNOBS->POLLING_FREQUENCY*2, TaskCoordinationReply );
} else } else
@ -211,4 +211,4 @@ ACTOR Future<Void> tryBecomeLeaderInternal( ServerCoordinators coordinators, Val
if (SERVER_KNOBS->BUGGIFY_ALL_COORDINATION || BUGGIFY) Void _ = wait( delay( SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * g_random->random01() ) ); if (SERVER_KNOBS->BUGGIFY_ALL_COORDINATION || BUGGIFY) Void _ = wait( delay( SERVER_KNOBS->BUGGIFIED_EVENTUAL_CONSISTENCY * g_random->random01() ) );
return Void(); // We are no longer leader return Void(); // We are no longer leader
} }

12
fdbserver/LogRouter.actor.cpp
View File

@ -204,7 +204,7 @@ ACTOR Future<Void> pullAsyncData( LogRouterData *self ) {
Void _ = wait(self->minPopped.whenAtLeast(std::min(self->version.get(), ver - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS))); Void _ = wait(self->minPopped.whenAtLeast(std::min(self->version.get(), ver - SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)));
commitMessages(self, ver, messages); commitMessages(self, ver, messages);
self->version.set( ver ); self->version.set( ver );
//TraceEvent("LogRouterVersion").detail("ver",ver); //TraceEvent("LogRouterVersion").detail("Ver",ver);
} }
lastVer = ver; lastVer = ver;
ver = r->version().version; ver = r->version().version;
@ -249,7 +249,7 @@ void peekMessagesFromMemory( LogRouterData* self, TLogPeekRequest const& req, Bi
ASSERT( !messages.getLength() ); ASSERT( !messages.getLength() );
auto& deque = get_version_messages(self, req.tag); auto& deque = get_version_messages(self, req.tag);
//TraceEvent("tLogPeekMem", self->dbgid).detail("Tag", printable(req.tag1)).detail("pDS", self->persistentDataSequence).detail("pDDS", self->persistentDataDurableSequence).detail("Oldest", map1.empty() ? 0 : map1.begin()->key ).detail("OldestMsgCount", map1.empty() ? 0 : map1.begin()->value.size()); //TraceEvent("TLogPeekMem", self->dbgid).detail("Tag", printable(req.tag1)).detail("PDS", self->persistentDataSequence).detail("PDDS", self->persistentDataDurableSequence).detail("Oldest", map1.empty() ? 0 : map1.begin()->key ).detail("OldestMsgCount", map1.empty() ? 0 : map1.begin()->value.size());
auto it = std::lower_bound(deque.begin(), deque.end(), std::make_pair(req.begin, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>()); auto it = std::lower_bound(deque.begin(), deque.end(), std::make_pair(req.begin, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>());
@ -258,7 +258,7 @@ void peekMessagesFromMemory( LogRouterData* self, TLogPeekRequest const& req, Bi
if(it->first != currentVersion) { if(it->first != currentVersion) {
if (messages.getLength() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) { if (messages.getLength() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) {
endVersion = currentVersion + 1; endVersion = currentVersion + 1;
//TraceEvent("tLogPeekMessagesReached2", self->dbgid); //TraceEvent("TLogPeekMessagesReached2", self->dbgid);
break; break;
} }
@ -280,7 +280,7 @@ Version poppedVersion( LogRouterData* self, Tag tag) {
ACTOR Future<Void> logRouterPeekMessages( LogRouterData* self, TLogPeekRequest req ) { ACTOR Future<Void> logRouterPeekMessages( LogRouterData* self, TLogPeekRequest req ) {
state BinaryWriter messages(Unversioned()); state BinaryWriter messages(Unversioned());
//TraceEvent("LogRouterPeek1", self->dbgid).detail("from", req.reply.getEndpoint().address).detail("ver", self->version.get()).detail("begin", req.begin); //TraceEvent("LogRouterPeek1", self->dbgid).detail("From", req.reply.getEndpoint().address).detail("Ver", self->version.get()).detail("Begin", req.begin);
if( req.returnIfBlocked && self->version.get() < req.begin ) { if( req.returnIfBlocked && self->version.get() < req.begin ) {
//TraceEvent("LogRouterPeek2", self->dbgid); //TraceEvent("LogRouterPeek2", self->dbgid);
req.reply.sendError(end_of_stream()); req.reply.sendError(end_of_stream());
@ -297,7 +297,7 @@ ACTOR Future<Void> logRouterPeekMessages( LogRouterData* self, TLogPeekRequest r
if(poppedVer > req.begin || req.begin < self->startVersion) { if(poppedVer > req.begin || req.begin < self->startVersion) {
//This should only happen if a packet is sent multiple times and the reply is not needed. //This should only happen if a packet is sent multiple times and the reply is not needed.
// Since we are using popped differently, do not send a reply. // Since we are using popped differently, do not send a reply.
TraceEvent(SevWarnAlways, "LogRouterPeekPopped", self->dbgid).detail("begin", req.begin).detail("popped", poppedVer).detail("start", self->startVersion); TraceEvent(SevWarnAlways, "LogRouterPeekPopped", self->dbgid).detail("Begin", req.begin).detail("Popped", poppedVer).detail("Start", self->startVersion);
req.reply.send( Never() ); req.reply.send( Never() );
return Void(); return Void();
} }
@ -410,7 +410,7 @@ ACTOR Future<Void> logRouter(
Reference<AsyncVar<ServerDBInfo>> db) Reference<AsyncVar<ServerDBInfo>> db)
{ {
try { try {
TraceEvent("LogRouterStart", interf.id()).detail("start", req.startVersion).detail("tag", req.routerTag.toString()).detail("localities", req.tLogLocalities.size()).detail("hasBestPolicy", req.hasBestPolicy).detail("locality", req.locality); TraceEvent("LogRouterStart", interf.id()).detail("Start", req.startVersion).detail("Tag", req.routerTag.toString()).detail("Localities", req.tLogLocalities.size()).detail("HasBestPolicy", req.hasBestPolicy).detail("Locality", req.locality);
state Future<Void> core = logRouterCore(interf, req, db); state Future<Void> core = logRouterCore(interf, req, db);
loop choose{ loop choose{
when(Void _ = wait(core)) { return Void(); } when(Void _ = wait(core)) { return Void(); }

8
fdbserver/LogSystemDiskQueueAdapter.actor.cpp
View File

@ -29,14 +29,14 @@ public:
while (self->recoveryQueueDataSize < bytes) { while (self->recoveryQueueDataSize < bytes) {
if (self->recoveryLoc == self->logSystem->getEnd()) { if (self->recoveryLoc == self->logSystem->getEnd()) {
// Recovery will be complete once the current recoveryQueue is consumed, so we no longer need self->logSystem // Recovery will be complete once the current recoveryQueue is consumed, so we no longer need self->logSystem
TraceEvent("PeekNextEnd").detail("queue", self->recoveryQueue.size()).detail("bytes", bytes).detail("loc", self->recoveryLoc).detail("end", self->logSystem->getEnd()); TraceEvent("PeekNextEnd").detail("Queue", self->recoveryQueue.size()).detail("Bytes", bytes).detail("Loc", self->recoveryLoc).detail("End", self->logSystem->getEnd());
self->logSystem.clear(); self->logSystem.clear();
break; break;
} }
if(!self->cursor->hasMessage()) { if(!self->cursor->hasMessage()) {
Void _ = wait( self->cursor->getMore() ); Void _ = wait( self->cursor->getMore() );
TraceEvent("PeekNextGetMore").detail("queue", self->recoveryQueue.size()).detail("bytes", bytes).detail("loc", self->recoveryLoc).detail("end", self->logSystem->getEnd()); TraceEvent("PeekNextGetMore").detail("Queue", self->recoveryQueue.size()).detail("Bytes", bytes).detail("Loc", self->recoveryLoc).detail("End", self->logSystem->getEnd());
if(self->recoveryQueueDataSize == 0) { if(self->recoveryQueueDataSize == 0) {
self->recoveryQueueLoc = self->recoveryLoc; self->recoveryQueueLoc = self->recoveryLoc;
} }
@ -51,7 +51,7 @@ public:
self->cursor->nextMessage(); self->cursor->nextMessage();
if(!self->cursor->hasMessage()) self->recoveryLoc = self->cursor->version().version; if(!self->cursor->hasMessage()) self->recoveryLoc = self->cursor->version().version;
//TraceEvent("PeekNextResults").detail("from", self->recoveryLoc).detail("queue", self->recoveryQueue.size()).detail("bytes", bytes).detail("has", self->cursor->hasMessage()).detail("end", self->logSystem->getEnd()); //TraceEvent("PeekNextResults").detail("From", self->recoveryLoc).detail("Queue", self->recoveryQueue.size()).detail("Bytes", bytes).detail("Has", self->cursor->hasMessage()).detail("End", self->logSystem->getEnd());
} }
if(self->recoveryQueue.size() > 1) { if(self->recoveryQueue.size() > 1) {
self->recoveryQueue[0] = concatenate(self->recoveryQueue.begin(), self->recoveryQueue.end()); self->recoveryQueue[0] = concatenate(self->recoveryQueue.begin(), self->recoveryQueue.end());
@ -63,7 +63,7 @@ public:
ASSERT(self->recoveryQueue[0].size() == self->recoveryQueueDataSize); ASSERT(self->recoveryQueue[0].size() == self->recoveryQueueDataSize);
//TraceEvent("PeekNextReturn").detail("bytes", bytes).detail("queueSize", self->recoveryQueue.size()); //TraceEvent("PeekNextReturn").detail("Bytes", bytes).detail("QueueSize", self->recoveryQueue.size());
bytes = std::min(bytes, self->recoveryQueue[0].size()); bytes = std::min(bytes, self->recoveryQueue[0].size());
Standalone<StringRef> result( self->recoveryQueue[0].substr(0,bytes), self->recoveryQueue[0].arena() ); Standalone<StringRef> result( self->recoveryQueue[0].substr(0,bytes), self->recoveryQueue[0].arena() );
self->recoveryQueue[0].contents() = self->recoveryQueue[0].substr(bytes); self->recoveryQueue[0].contents() = self->recoveryQueue[0].substr(bytes);

52
fdbserver/LogSystemPeekCursor.actor.cpp
View File

@ -26,13 +26,13 @@
ILogSystem::ServerPeekCursor::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf, Tag tag, Version begin, Version end, bool returnIfBlocked, bool parallelGetMore ) ILogSystem::ServerPeekCursor::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>> const& interf, Tag tag, Version begin, Version end, bool returnIfBlocked, bool parallelGetMore )
: interf(interf), tag(tag), messageVersion(begin), end(end), hasMsg(false), rd(results.arena, results.messages, Unversioned()), randomID(g_random->randomUniqueID()), poppedVersion(0), returnIfBlocked(returnIfBlocked), sequence(0), parallelGetMore(parallelGetMore) { : interf(interf), tag(tag), messageVersion(begin), end(end), hasMsg(false), rd(results.arena, results.messages, Unversioned()), randomID(g_random->randomUniqueID()), poppedVersion(0), returnIfBlocked(returnIfBlocked), sequence(0), parallelGetMore(parallelGetMore) {
this->results.maxKnownVersion = 0; this->results.maxKnownVersion = 0;
//TraceEvent("SPC_starting", randomID).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).backtrace(); //TraceEvent("SPC_Starting", randomID).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).backtrace();
} }
ILogSystem::ServerPeekCursor::ServerPeekCursor( TLogPeekReply const& results, LogMessageVersion const& messageVersion, LogMessageVersion const& end, int32_t messageLength, int32_t rawLength, bool hasMsg, Version poppedVersion, Tag tag ) ILogSystem::ServerPeekCursor::ServerPeekCursor( TLogPeekReply const& results, LogMessageVersion const& messageVersion, LogMessageVersion const& end, int32_t messageLength, int32_t rawLength, bool hasMsg, Version poppedVersion, Tag tag )
: results(results), tag(tag), rd(results.arena, results.messages, Unversioned()), messageVersion(messageVersion), end(end), messageLength(messageLength), rawLength(rawLength), hasMsg(hasMsg), randomID(g_random->randomUniqueID()), poppedVersion(poppedVersion), returnIfBlocked(false), sequence(0), parallelGetMore(false) : results(results), tag(tag), rd(results.arena, results.messages, Unversioned()), messageVersion(messageVersion), end(end), messageLength(messageLength), rawLength(rawLength), hasMsg(hasMsg), randomID(g_random->randomUniqueID()), poppedVersion(poppedVersion), returnIfBlocked(false), sequence(0), parallelGetMore(false)
{ {
//TraceEvent("SPC_clone", randomID); //TraceEvent("SPC_Clone", randomID);
this->results.maxKnownVersion = 0; this->results.maxKnownVersion = 0;
if(hasMsg) if(hasMsg)
nextMessage(); nextMessage();
@ -55,12 +55,12 @@ ArenaReader* ILogSystem::ServerPeekCursor::reader() {
} }
bool ILogSystem::ServerPeekCursor::hasMessage() { bool ILogSystem::ServerPeekCursor::hasMessage() {
//TraceEvent("SPC_hasMessage", randomID).detail("hasMsg", hasMsg); //TraceEvent("SPC_HasMessage", randomID).detail("HasMsg", hasMsg);
return hasMsg; return hasMsg;
} }
void ILogSystem::ServerPeekCursor::nextMessage() { void ILogSystem::ServerPeekCursor::nextMessage() {
//TraceEvent("SPC_nextMessage", randomID).detail("messageVersion", messageVersion.toString()); //TraceEvent("SPC_NextMessage", randomID).detail("MessageVersion", messageVersion.toString());
ASSERT(hasMsg); ASSERT(hasMsg);
if (rd.empty()) { if (rd.empty()) {
messageVersion.reset(std::min(results.end, end.version)); messageVersion.reset(std::min(results.end, end.version));
@ -73,7 +73,7 @@ void ILogSystem::ServerPeekCursor::nextMessage() {
Version ver; Version ver;
rd >> dummy >> ver; rd >> dummy >> ver;
//TraceEvent("SPC_process_Seq", randomID).detail("messageVersion", messageVersion.toString()).detail("ver", ver).detail("tag", tag.toString()); //TraceEvent("SPC_ProcessSeq", randomID).detail("MessageVersion", messageVersion.toString()).detail("Ver", ver).detail("Tag", tag.toString());
//ASSERT( ver >= messageVersion.version ); //ASSERT( ver >= messageVersion.version );
messageVersion.reset(ver); messageVersion.reset(ver);
@ -96,11 +96,11 @@ void ILogSystem::ServerPeekCursor::nextMessage() {
rawLength = messageLength + sizeof(messageLength); rawLength = messageLength + sizeof(messageLength);
messageLength -= (sizeof(messageVersion.sub) + sizeof(tagCount) + tagCount*sizeof(Tag)); messageLength -= (sizeof(messageVersion.sub) + sizeof(tagCount) + tagCount*sizeof(Tag));
hasMsg = true; hasMsg = true;
//TraceEvent("SPC_nextMessageB", randomID).detail("messageVersion", messageVersion.toString()); //TraceEvent("SPC_NextMessageB", randomID).detail("MessageVersion", messageVersion.toString());
} }
StringRef ILogSystem::ServerPeekCursor::getMessage() { StringRef ILogSystem::ServerPeekCursor::getMessage() {
//TraceEvent("SPC_getMessage", randomID); //TraceEvent("SPC_GetMessage", randomID);
return StringRef( (uint8_t const*)rd.readBytes(messageLength), messageLength); return StringRef( (uint8_t const*)rd.readBytes(messageLength), messageLength);
} }
@ -114,7 +114,7 @@ const std::vector<Tag>& ILogSystem::ServerPeekCursor::getTags() {
} }
void ILogSystem::ServerPeekCursor::advanceTo(LogMessageVersion n) { void ILogSystem::ServerPeekCursor::advanceTo(LogMessageVersion n) {
//TraceEvent("SPC_advanceTo", randomID).detail("n", n.toString()); //TraceEvent("SPC_AdvanceTo", randomID).detail("N", n.toString());
while( messageVersion < n && hasMessage() ) { while( messageVersion < n && hasMessage() ) {
getMessage(); getMessage();
nextMessage(); nextMessage();
@ -157,7 +157,7 @@ ACTOR Future<Void> serverPeekParallelGetMore( ILogSystem::ServerPeekCursor* self
self->hasMsg = true; self->hasMsg = true;
self->nextMessage(); self->nextMessage();
self->advanceTo(skipSeq); self->advanceTo(skipSeq);
//TraceEvent("SPC_getMoreB", self->randomID).detail("has", self->hasMessage()).detail("end", res.end).detail("popped", res.popped.present() ? res.popped.get() : 0); //TraceEvent("SPC_GetMoreB", self->randomID).detail("Has", self->hasMessage()).detail("End", res.end).detail("Popped", res.popped.present() ? res.popped.get() : 0);
return Void(); return Void();
} }
when( Void _ = wait( self->interfaceChanged ) ) { when( Void _ = wait( self->interfaceChanged ) ) {
@ -202,7 +202,7 @@ ACTOR Future<Void> serverPeekGetMore( ILogSystem::ServerPeekCursor* self, int ta
self->hasMsg = true; self->hasMsg = true;
self->nextMessage(); self->nextMessage();
self->advanceTo(skipSeq); self->advanceTo(skipSeq);
//TraceEvent("SPC_getMoreB", self->randomID).detail("has", self->hasMessage()).detail("end", res.end).detail("popped", res.popped.present() ? res.popped.get() : 0); //TraceEvent("SPC_GetMoreB", self->randomID).detail("Has", self->hasMessage()).detail("End", res.end).detail("Popped", res.popped.present() ? res.popped.get() : 0);
return Void(); return Void();
} }
when( Void _ = wait( self->interf->onChange() ) ) {} when( Void _ = wait( self->interf->onChange() ) ) {}
@ -218,7 +218,7 @@ ACTOR Future<Void> serverPeekGetMore( ILogSystem::ServerPeekCursor* self, int ta
} }
Future<Void> ILogSystem::ServerPeekCursor::getMore(int taskID) { Future<Void> ILogSystem::ServerPeekCursor::getMore(int taskID) {
//TraceEvent("SPC_getMore", randomID).detail("hasMessage", hasMessage()).detail("more", !more.isValid() || more.isReady()).detail("messageVersion", messageVersion.toString()).detail("end", end.toString()); //TraceEvent("SPC_GetMore", randomID).detail("HasMessage", hasMessage()).detail("More", !more.isValid() || more.isReady()).detail("MessageVersion", messageVersion.toString()).detail("End", end.toString());
if( hasMessage() ) if( hasMessage() )
return Void(); return Void();
if( !more.isValid() || more.isReady() ) { if( !more.isValid() || more.isReady() ) {
@ -269,7 +269,7 @@ ILogSystem::MergedPeekCursor::MergedPeekCursor( std::vector<Reference<AsyncVar<O
tLogPolicy(tLogPolicy), tLogReplicationFactor(tLogReplicationFactor), collectTags(false) { tLogPolicy(tLogPolicy), tLogReplicationFactor(tLogReplicationFactor), collectTags(false) {
for( int i = 0; i < logServers.size(); i++ ) { for( int i = 0; i < logServers.size(); i++ ) {
Reference<ILogSystem::ServerPeekCursor> cursor( new ILogSystem::ServerPeekCursor( logServers[i], tag, begin, end, bestServer >= 0, parallelGetMore ) ); Reference<ILogSystem::ServerPeekCursor> cursor( new ILogSystem::ServerPeekCursor( logServers[i], tag, begin, end, bestServer >= 0, parallelGetMore ) );
//TraceEvent("MPC_starting", randomID).detail("cursor", cursor->randomID).detail("end", end); //TraceEvent("MPC_Starting", randomID).detail("Cursor", cursor->randomID).detail("End", end);
serverCursors.push_back( cursor ); serverCursors.push_back( cursor );
} }
sortedVersions.resize(serverCursors.size()); sortedVersions.resize(serverCursors.size());
@ -432,7 +432,7 @@ void ILogSystem::MergedPeekCursor::advanceTo(LogMessageVersion n) {
ACTOR Future<Void> mergedPeekGetMore(ILogSystem::MergedPeekCursor* self, LogMessageVersion startVersion, int taskID) { ACTOR Future<Void> mergedPeekGetMore(ILogSystem::MergedPeekCursor* self, LogMessageVersion startVersion, int taskID) {
loop { loop {
//TraceEvent("MPC_getMoreA", self->randomID).detail("start", startVersion.toString()); //TraceEvent("MPC_GetMoreA", self->randomID).detail("Start", startVersion.toString());
if(self->bestServer >= 0 && self->serverCursors[self->bestServer]->isActive()) { if(self->bestServer >= 0 && self->serverCursors[self->bestServer]->isActive()) {
ASSERT(!self->serverCursors[self->bestServer]->hasMessage()); ASSERT(!self->serverCursors[self->bestServer]->hasMessage());
Void _ = wait( self->serverCursors[self->bestServer]->getMore(taskID) || self->serverCursors[self->bestServer]->onFailed() ); Void _ = wait( self->serverCursors[self->bestServer]->getMore(taskID) || self->serverCursors[self->bestServer]->onFailed() );
@ -444,7 +444,7 @@ ACTOR Future<Void> mergedPeekGetMore(ILogSystem::MergedPeekCursor* self, LogMess
Void _ = wait(quorum(q, 1)); Void _ = wait(quorum(q, 1));
} }
self->calcHasMessage(); self->calcHasMessage();
//TraceEvent("MPC_getMoreB", self->randomID).detail("hasMessage", self->hasMessage()).detail("start", startVersion.toString()).detail("seq", self->version().toString()); //TraceEvent("MPC_GetMoreB", self->randomID).detail("HasMessage", self->hasMessage()).detail("Start", startVersion.toString()).detail("Seq", self->version().toString());
if (self->hasMessage() || self->version() > startVersion) { if (self->hasMessage() || self->version() > startVersion) {
self->messageArena = Arena(); self->messageArena = Arena();
return Void(); return Void();
@ -546,7 +546,7 @@ ArenaReader* ILogSystem::SetPeekCursor::reader() { return serverCursors[currentS
void ILogSystem::SetPeekCursor::calcHasMessage() { void ILogSystem::SetPeekCursor::calcHasMessage() {
if(bestSet >= 0 && bestServer >= 0) { if(bestSet >= 0 && bestServer >= 0) {
if(nextVersion.present()) { if(nextVersion.present()) {
//TraceEvent("LPC_calcNext").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage).detail("nextVersion", nextVersion.get().toString()); //TraceEvent("LPC_CalcNext").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage).detail("NextVersion", nextVersion.get().toString());
serverCursors[bestSet][bestServer]->advanceTo( nextVersion.get() ); serverCursors[bestSet][bestServer]->advanceTo( nextVersion.get() );
} }
if( serverCursors[bestSet][bestServer]->hasMessage() ) { if( serverCursors[bestSet][bestServer]->hasMessage() ) {
@ -555,7 +555,7 @@ void ILogSystem::SetPeekCursor::calcHasMessage() {
currentCursor = bestServer; currentCursor = bestServer;
hasNextMessage = true; hasNextMessage = true;
//TraceEvent("LPC_calc1").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage); //TraceEvent("LPC_Calc1").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage);
for (auto& cursors : serverCursors) { for (auto& cursors : serverCursors) {
for(auto& c : cursors) { for(auto& c : cursors) {
@ -578,10 +578,10 @@ void ILogSystem::SetPeekCursor::calcHasMessage() {
if(useBestSet) { if(useBestSet) {
updateMessage(bestSet, false); // Use Quorum logic updateMessage(bestSet, false); // Use Quorum logic
//TraceEvent("LPC_calc2").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage); //TraceEvent("LPC_Calc2").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage);
if(!hasNextMessage) { if(!hasNextMessage) {
updateMessage(bestSet, true); updateMessage(bestSet, true);
//TraceEvent("LPC_calc3").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage); //TraceEvent("LPC_Calc3").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage);
} }
} else { } else {
for(int i = 0; i < logSets.size() && !hasNextMessage; i++) { for(int i = 0; i < logSets.size() && !hasNextMessage; i++) {
@ -589,13 +589,13 @@ void ILogSystem::SetPeekCursor::calcHasMessage() {
updateMessage(i, false); // Use Quorum logic updateMessage(i, false); // Use Quorum logic
} }
} }
//TraceEvent("LPC_calc4").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage); //TraceEvent("LPC_Calc4").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage);
for(int i = 0; i < logSets.size() && !hasNextMessage; i++) { for(int i = 0; i < logSets.size() && !hasNextMessage; i++) {
if(i != bestSet) { if(i != bestSet) {
updateMessage(i, true); updateMessage(i, true);
} }
} }
//TraceEvent("LPC_calc5").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage); //TraceEvent("LPC_Calc5").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage);
} }
} }
@ -607,7 +607,7 @@ void ILogSystem::SetPeekCursor::updateMessage(int logIdx, bool usePolicy) {
auto& serverCursor = serverCursors[logIdx][i]; auto& serverCursor = serverCursors[logIdx][i];
if (nextVersion.present()) serverCursor->advanceTo(nextVersion.get()); if (nextVersion.present()) serverCursor->advanceTo(nextVersion.get());
sortedVersions.push_back(std::pair<LogMessageVersion, int>(serverCursor->version(), i)); sortedVersions.push_back(std::pair<LogMessageVersion, int>(serverCursor->version(), i));
//TraceEvent("LPC_update1").detail("ver", messageVersion.toString()).detail("tag", tag.toString()).detail("hasNextMessage", hasNextMessage).detail("serverVer", serverCursor->version().toString()).detail("i", i); //TraceEvent("LPC_Update1").detail("Ver", messageVersion.toString()).detail("Tag", tag.toString()).detail("HasNextMessage", hasNextMessage).detail("ServerVer", serverCursor->version().toString()).detail("I", i);
} }
if(usePolicy) { if(usePolicy) {
@ -687,10 +687,10 @@ void ILogSystem::SetPeekCursor::advanceTo(LogMessageVersion n) {
ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self, LogMessageVersion startVersion, int taskID) { ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self, LogMessageVersion startVersion, int taskID) {
loop { loop {
//TraceEvent("LPC_getMore1", self->randomID).detail("start", startVersion.toString()).detail("t", self->tag); //TraceEvent("LPC_GetMore1", self->randomID).detail("Start", startVersion.toString()).detail("Tag", self->tag);
if(self->bestServer >= 0 && self->bestSet >= 0 && self->serverCursors[self->bestSet][self->bestServer]->isActive()) { if(self->bestServer >= 0 && self->bestSet >= 0 && self->serverCursors[self->bestSet][self->bestServer]->isActive()) {
ASSERT(!self->serverCursors[self->bestSet][self->bestServer]->hasMessage()); ASSERT(!self->serverCursors[self->bestSet][self->bestServer]->hasMessage());
//TraceEvent("LPC_getMore2", self->randomID).detail("start", startVersion.toString()).detail("t", self->tag); //TraceEvent("LPC_GetMore2", self->randomID).detail("Start", startVersion.toString()).detail("Tag", self->tag);
Void _ = wait( self->serverCursors[self->bestSet][self->bestServer]->getMore(taskID) || self->serverCursors[self->bestSet][self->bestServer]->onFailed() ); Void _ = wait( self->serverCursors[self->bestSet][self->bestServer]->getMore(taskID) || self->serverCursors[self->bestSet][self->bestServer]->onFailed() );
self->useBestSet = true; self->useBestSet = true;
} else { } else {
@ -713,7 +713,7 @@ ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self, LogMessageVer
return Void(); return Void();
} }
//TraceEvent("LPC_getMore3", self->randomID).detail("start", startVersion.toString()).detail("t", self->tag.toString()).detail("bestSetSize", self->serverCursors[self->bestSet].size()); //TraceEvent("LPC_GetMore3", self->randomID).detail("Start", startVersion.toString()).detail("Tag", self->tag.toString()).detail("BestSetSize", self->serverCursors[self->bestSet].size());
vector<Future<Void>> q; vector<Future<Void>> q;
for (auto& c : self->serverCursors[self->bestSet]) { for (auto& c : self->serverCursors[self->bestSet]) {
if (!c->hasMessage()) { if (!c->hasMessage()) {
@ -727,7 +727,7 @@ ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self, LogMessageVer
} else { } else {
//FIXME: this will peeking way too many cursors when satellites exist, and does not need to peek bestSet cursors since we cannot get anymore data from them //FIXME: this will peeking way too many cursors when satellites exist, and does not need to peek bestSet cursors since we cannot get anymore data from them
vector<Future<Void>> q; vector<Future<Void>> q;
//TraceEvent("LPC_getMore4", self->randomID).detail("start", startVersion.toString()).detail("t", self->tag); //TraceEvent("LPC_GetMore4", self->randomID).detail("Start", startVersion.toString()).detail("Tag", self->tag);
for(auto& cursors : self->serverCursors) { for(auto& cursors : self->serverCursors) {
for (auto& c :cursors) { for (auto& c :cursors) {
if (!c->hasMessage()) { if (!c->hasMessage()) {
@ -740,7 +740,7 @@ ACTOR Future<Void> setPeekGetMore(ILogSystem::SetPeekCursor* self, LogMessageVer
} }
} }
self->calcHasMessage(); self->calcHasMessage();
//TraceEvent("LPC_getMoreB", self->randomID).detail("hasMessage", self->hasMessage()).detail("start", startVersion.toString()).detail("seq", self->version().toString()); //TraceEvent("LPC_GetMoreB", self->randomID).detail("HasMessage", self->hasMessage()).detail("Start", startVersion.toString()).detail("Seq", self->version().toString());
if (self->hasMessage() || self->version() > startVersion) if (self->hasMessage() || self->version() > startVersion)
return Void(); return Void();
} }

74
fdbserver/MasterProxyServer.actor.cpp
View File

@ -58,15 +58,16 @@ struct ProxyStats {
Future<Void> logger; Future<Void> logger;
explicit ProxyStats(UID id, Version* pVersion, NotifiedVersion* pCommittedVersion) explicit ProxyStats(UID id, Version* pVersion, NotifiedVersion* pCommittedVersion, int64_t *commitBatchesMemBytesCountPtr)
: cc("ProxyStats", id.toString()), : cc("ProxyStats", id.toString()),
txnStartIn("txnStartIn", cc), txnStartOut("txnStartOut", cc), txnStartBatch("txnStartBatch", cc), txnSystemPriorityStartIn("txnSystemPriorityStartIn", cc), txnSystemPriorityStartOut("txnSystemPriorityStartOut", cc), txnBatchPriorityStartIn("txnBatchPriorityStartIn", cc), txnBatchPriorityStartOut("txnBatchPriorityStartOut", cc), txnStartIn("TxnStartIn", cc), txnStartOut("TxnStartOut", cc), txnStartBatch("TxnStartBatch", cc), txnSystemPriorityStartIn("TxnSystemPriorityStartIn", cc), txnSystemPriorityStartOut("TxnSystemPriorityStartOut", cc), txnBatchPriorityStartIn("TxnBatchPriorityStartIn", cc), txnBatchPriorityStartOut("TxnBatchPriorityStartOut", cc),
txnDefaultPriorityStartIn("txnDefaultPriorityStartIn", cc), txnDefaultPriorityStartOut("txnDefaultPriorityStartOut", cc), txnCommitIn("txnCommitIn", cc), txnCommitVersionAssigned("txnCommitVersionAssigned", cc), txnCommitResolving("txnCommitResolving", cc), txnCommitResolved("txnCommitResolved", cc), txnCommitOut("txnCommitOut", cc), txnDefaultPriorityStartIn("TxnDefaultPriorityStartIn", cc), txnDefaultPriorityStartOut("TxnDefaultPriorityStartOut", cc), txnCommitIn("TxnCommitIn", cc), txnCommitVersionAssigned("TxnCommitVersionAssigned", cc), txnCommitResolving("TxnCommitResolving", cc), txnCommitResolved("TxnCommitResolved", cc), txnCommitOut("TxnCommitOut", cc),
txnCommitOutSuccess("txnCommitOutSuccess", cc), txnConflicts("txnConflicts", cc), commitBatchIn("commitBatchIn", cc), commitBatchOut("commitBatchOut", cc), mutationBytes("mutationBytes", cc), mutations("mutations", cc), conflictRanges("conflictRanges", cc), lastCommitVersionAssigned(0) txnCommitOutSuccess("TxnCommitOutSuccess", cc), txnConflicts("TxnConflicts", cc), commitBatchIn("CommitBatchIn", cc), commitBatchOut("CommitBatchOut", cc), mutationBytes("MutationBytes", cc), mutations("Mutations", cc), conflictRanges("ConflictRanges", cc), lastCommitVersionAssigned(0)
{ {
specialCounter(cc, "lastAssignedCommitVersion", [this](){return this->lastCommitVersionAssigned;}); specialCounter(cc, "LastAssignedCommitVersion", [this](){return this->lastCommitVersionAssigned;});
specialCounter(cc, "version", [pVersion](){return *pVersion; }); specialCounter(cc, "Version", [pVersion](){return *pVersion; });
specialCounter(cc, "committedVersion", [pCommittedVersion](){ return pCommittedVersion->get(); }); specialCounter(cc, "CommittedVersion", [pCommittedVersion](){ return pCommittedVersion->get(); });
specialCounter(cc, "CommitBatchesMemBytesCount", [commitBatchesMemBytesCountPtr]() { return *commitBatchesMemBytesCountPtr; });
logger = traceCounters("ProxyMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ProxyMetrics"); logger = traceCounters("ProxyMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "ProxyMetrics");
} }
}; };
@ -152,7 +153,7 @@ ACTOR Future<Void> queueTransactionStartRequests(std::priority_queue< std::pair<
ACTOR void discardCommit(UID id, Future<LogSystemDiskQueueAdapter::CommitMessage> fcm, Future<Void> dummyCommitState) { ACTOR void discardCommit(UID id, Future<LogSystemDiskQueueAdapter::CommitMessage> fcm, Future<Void> dummyCommitState) {
ASSERT(!dummyCommitState.isReady()); ASSERT(!dummyCommitState.isReady());
LogSystemDiskQueueAdapter::CommitMessage cm = wait(fcm); LogSystemDiskQueueAdapter::CommitMessage cm = wait(fcm);
TraceEvent("Discarding", id).detail("count", cm.messages.size()); TraceEvent("Discarding", id).detail("Count", cm.messages.size());
cm.acknowledge.send(Void()); cm.acknowledge.send(Void());
ASSERT(dummyCommitState.isReady()); ASSERT(dummyCommitState.isReady());
} }
@ -167,6 +168,7 @@ DESCR struct SingleKeyMutation {
struct ProxyCommitData { struct ProxyCommitData {
UID dbgid; UID dbgid;
int64_t commitBatchesMemBytesCount;
ProxyStats stats; ProxyStats stats;
MasterInterface master; MasterInterface master;
vector<ResolverInterface> resolvers; vector<ResolverInterface> resolvers;
@ -221,13 +223,14 @@ struct ProxyCommitData {
} }
ProxyCommitData(UID dbgid, MasterInterface master, RequestStream<GetReadVersionRequest> getConsistentReadVersion, Version recoveryTransactionVersion, RequestStream<CommitTransactionRequest> commit, Reference<AsyncVar<ServerDBInfo>> db, bool firstProxy) ProxyCommitData(UID dbgid, MasterInterface master, RequestStream<GetReadVersionRequest> getConsistentReadVersion, Version recoveryTransactionVersion, RequestStream<CommitTransactionRequest> commit, Reference<AsyncVar<ServerDBInfo>> db, bool firstProxy)
: dbgid(dbgid), stats(dbgid, &version, &committedVersion), master(master), : dbgid(dbgid), stats(dbgid, &version, &committedVersion, &commitBatchesMemBytesCount), master(master),
logAdapter(NULL), txnStateStore(NULL), logAdapter(NULL), txnStateStore(NULL),
committedVersion(recoveryTransactionVersion), version(0), committedVersion(recoveryTransactionVersion), version(0),
lastVersionTime(0), commitVersionRequestNumber(1), mostRecentProcessedRequestNumber(0), lastVersionTime(0), commitVersionRequestNumber(1), mostRecentProcessedRequestNumber(0),
getConsistentReadVersion(getConsistentReadVersion), commit(commit), lastCoalesceTime(0), getConsistentReadVersion(getConsistentReadVersion), commit(commit), lastCoalesceTime(0),
localCommitBatchesStarted(0), locked(false), firstProxy(firstProxy), localCommitBatchesStarted(0), locked(false), firstProxy(firstProxy),
cx(openDBOnServer(db, TaskDefaultEndpoint, true, true)), singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")) cx(openDBOnServer(db, TaskDefaultEndpoint, true, true)), singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")),
commitBatchesMemBytesCount(0)
{} {}
}; };
@ -315,7 +318,8 @@ struct ResolutionRequestBuilder {
ACTOR Future<Void> commitBatch( ACTOR Future<Void> commitBatch(
ProxyCommitData* self, ProxyCommitData* self,
vector<CommitTransactionRequest> trs, vector<CommitTransactionRequest> trs,
double *commitBatchTime) double *commitBatchTime,
int currentBatchMemBytesCount)
{ {
state int64_t localBatchNumber = ++self->localCommitBatchesStarted; state int64_t localBatchNumber = ++self->localCommitBatchesStarted;
state LogPushData toCommit(self->logSystem); state LogPushData toCommit(self->logSystem);
@ -325,6 +329,9 @@ ACTOR Future<Void> commitBatch(
ASSERT(SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS <= SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT); // since we are using just the former to limit the number of versions actually in flight! ASSERT(SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS <= SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT); // since we are using just the former to limit the number of versions actually in flight!
// Active load balancing runs at a very high priority (to obtain accurate estimate of memory used by commit batches) so we need to downgrade here
Void _ = wait(delay(0, TaskProxyCommit));
self->lastVersionTime = t1; self->lastVersionTime = t1;
++self->stats.commitBatchIn; ++self->stats.commitBatchIn;
@ -339,7 +346,7 @@ ACTOR Future<Void> commitBatch(
if(localBatchNumber == 2 && !debugID.present() && self->firstProxy && !g_network->isSimulated()) { if(localBatchNumber == 2 && !debugID.present() && self->firstProxy && !g_network->isSimulated()) {
debugID = g_random->randomUniqueID(); debugID = g_random->randomUniqueID();
TraceEvent("SecondCommitBatch", self->dbgid).detail("debugID", debugID.get()); TraceEvent("SecondCommitBatch", self->dbgid).detail("DebugID", debugID.get());
} }
if (debugID.present()) if (debugID.present())
@ -374,7 +381,7 @@ ACTOR Future<Void> commitBatch(
r->value().push_back(std::make_pair(versionReply.resolverChangesVersion,it.dest)); r->value().push_back(std::make_pair(versionReply.resolverChangesVersion,it.dest));
} }
//TraceEvent("ProxyGotVer", self->dbgid).detail("commit", commitVersion).detail("prev", prevVersion); //TraceEvent("ProxyGotVer", self->dbgid).detail("Commit", commitVersion).detail("Prev", prevVersion);
if (debugID.present()) if (debugID.present())
g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "MasterProxyServer.commitBatch.GotCommitVersion"); g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "MasterProxyServer.commitBatch.GotCommitVersion");
@ -730,8 +737,8 @@ ACTOR Future<Void> commitBatch(
// if (debugMutation("BackupProxyCommit", commitVersion, backupMutation)) { // if (debugMutation("BackupProxyCommit", commitVersion, backupMutation)) {
// TraceEvent("BackupProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("BackupMutation", backupMutation.toString()) // TraceEvent("BackupProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("BackupMutation", backupMutation.toString())
// .detail("BackupMutationSize", val.size()).detail("Version", commitVersion).detail("destPath", printable(logRangeMutation.first)) // .detail("BackupMutationSize", val.size()).detail("Version", commitVersion).detail("DestPath", printable(logRangeMutation.first))
// .detail("partIndex", part).detail("partIndexEndian", bigEndian32(part)).detail("partData", printable(backupMutation.param1)); // .detail("PartIndex", part).detail("PartIndexEndian", bigEndian32(part)).detail("PartData", printable(backupMutation.param1));
// } // }
} }
} }
@ -857,7 +864,7 @@ ACTOR Future<Void> commitBatch(
} }
self->keyResolvers.coalesce(allKeys); self->keyResolvers.coalesce(allKeys);
if(self->keyResolvers.size() != lastSize) if(self->keyResolvers.size() != lastSize)
TraceEvent("KeyResolverSize", self->dbgid).detail("size", self->keyResolvers.size()); TraceEvent("KeyResolverSize", self->dbgid).detail("Size", self->keyResolvers.size());
} }
// Dynamic batching for commits // Dynamic batching for commits
@ -868,6 +875,8 @@ ACTOR Future<Void> commitBatch(
target_latency * SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA + *commitBatchTime * (1-SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA))); target_latency * SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA + *commitBatchTime * (1-SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA)));
self->commitBatchesMemBytesCount -= currentBatchMemBytesCount;
ASSERT_ABORT(self->commitBatchesMemBytesCount >= 0);
return Void(); return Void();
} }
@ -1004,15 +1013,15 @@ ACTOR static Future<Void> transactionStarter(
forwardPromise(GRVTimer, delayJittered(SERVER_KNOBS->START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL, TaskProxyGRVTimer)); forwardPromise(GRVTimer, delayJittered(SERVER_KNOBS->START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL, TaskProxyGRVTimer));
/*TraceEvent("GRVBatch", proxy.id()) /*TraceEvent("GRVBatch", proxy.id())
.detail("elapsed", elapsed) .detail("Elapsed", elapsed)
.detail("nTransactionToStart", nTransactionsToStart) .detail("NTransactionToStart", nTransactionsToStart)
.detail("transactionRate", transactionRate) .detail("TransactionRate", transactionRate)
.detail("transactionQueueSize", transactionQueue.size()) .detail("TransactionQueueSize", transactionQueue.size())
.detail("numTransactionsStarted", transactionsStarted[0] + transactionsStarted[1]) .detail("NumTransactionsStarted", transactionsStarted[0] + transactionsStarted[1])
.detail("numSystemTransactionsStarted", systemTransactionsStarted[0] + systemTransactionsStarted[1]) .detail("NumSystemTransactionsStarted", systemTransactionsStarted[0] + systemTransactionsStarted[1])
.detail("numNonSystemTransactionsStarted", transactionsStarted[0] + transactionsStarted[1] - systemTransactionsStarted[0] - systemTransactionsStarted[1]) .detail("NumNonSystemTransactionsStarted", transactionsStarted[0] + transactionsStarted[1] - systemTransactionsStarted[0] - systemTransactionsStarted[1])
.detail("transactionBudget", transactionBudget) .detail("TransactionBudget", transactionBudget)
.detail("lastLeftToStart", leftToStart);*/ .detail("LastLeftToStart", leftToStart);*/
// dynamic batching // dynamic batching
ReplyPromise<GetReadVersionReply> GRVReply; ReplyPromise<GetReadVersionReply> GRVReply;
@ -1156,9 +1165,8 @@ ACTOR Future<Void> masterProxyServerCore(
state ProxyCommitData commitData(proxy.id(), master, proxy.getConsistentReadVersion, recoveryTransactionVersion, proxy.commit, db, firstProxy); state ProxyCommitData commitData(proxy.id(), master, proxy.getConsistentReadVersion, recoveryTransactionVersion, proxy.commit, db, firstProxy);
state Future<Sequence> sequenceFuture = (Sequence)0; state Future<Sequence> sequenceFuture = (Sequence)0;
state PromiseStream< vector<CommitTransactionRequest> > batchedCommits; state PromiseStream< std::pair<vector<CommitTransactionRequest>, int> > batchedCommits;
state Future<Void> commitBatcher; state Future<Void> commitBatcher;
state Future<Void> lastCommitComplete = Void(); state Future<Void> lastCommitComplete = Void();
state PromiseStream<Future<Void>> addActor; state PromiseStream<Future<Void>> addActor;
@ -1192,6 +1200,10 @@ ACTOR Future<Void> masterProxyServerCore(
commitData.logAdapter = new LogSystemDiskQueueAdapter(commitData.logSystem, txsTag, false); commitData.logAdapter = new LogSystemDiskQueueAdapter(commitData.logSystem, txsTag, false);
commitData.txnStateStore = keyValueStoreLogSystem(commitData.logAdapter, proxy.id(), 2e9, true, true); commitData.txnStateStore = keyValueStoreLogSystem(commitData.logAdapter, proxy.id(), 2e9, true, true);
onError = onError || commitData.logSystem->onError(); onError = onError || commitData.logSystem->onError();
// ((SERVER_MEM_LIMIT * COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) / COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR) is only a approximate formula for limiting the memory used.
// COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR is an estimate based on experiments and not an accurate one.
state int64_t commitBatchesMemoryLimit = std::min(SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT, static_cast<int64_t>((SERVER_KNOBS->SERVER_MEM_LIMIT * SERVER_KNOBS->COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) / SERVER_KNOBS->COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR));
TraceEvent(SevInfo, "CommitBatchesMemoryLimit").detail("BytesLimit", commitBatchesMemoryLimit);
addActor.send(transactionStarter(proxy, master, db, addActor, &commitData)); addActor.send(transactionStarter(proxy, master, db, addActor, &commitData));
addActor.send(readRequestServer(proxy, &commitData)); addActor.send(readRequestServer(proxy, &commitData));
@ -1203,7 +1215,7 @@ ACTOR Future<Void> masterProxyServerCore(
(int)std::min<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MAX, (int)std::min<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MAX,
std::max<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MIN, std::max<double>(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_MIN,
SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE * pow(db->get().client.proxies.size(), SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER))); SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE * pow(db->get().client.proxies.size(), SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER)));
commitBatcher = batcher(batchedCommits, proxy.commit.getFuture(), SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, &commitBatchInterval, SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL, SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_COUNT_MAX, commitBatchByteLimit, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT, commitData.commitBatchStartNotifications, TaskProxyCommitBatcher, &commitData.stats.txnCommitIn); commitBatcher = batcher(batchedCommits, proxy.commit.getFuture(), SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, &commitBatchInterval, SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL, SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_COUNT_MAX, commitBatchByteLimit, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT, commitData.commitBatchStartNotifications, &commitData.commitBatchesMemBytesCount, commitBatchesMemoryLimit, TaskProxyCommitBatcher, &commitData.stats.txnCommitIn);
loop choose{ loop choose{
when( Void _ = wait( dbInfoChange ) ) { when( Void _ = wait( dbInfoChange ) ) {
dbInfoChange = db->onChange(); dbInfoChange = db->onChange();
@ -1215,13 +1227,15 @@ ACTOR Future<Void> masterProxyServerCore(
} }
} }
when(Void _ = wait(onError)) {} when(Void _ = wait(onError)) {}
when(vector<CommitTransactionRequest> trs = waitNext(batchedCommits.getFuture())) { when(std::pair<vector<CommitTransactionRequest>, int> batchedRequests = waitNext(batchedCommits.getFuture())) {
const vector<CommitTransactionRequest> &trs = batchedRequests.first;
int batchBytes = batchedRequests.second;
//TraceEvent("MasterProxyCTR", proxy.id()).detail("CommitTransactions", trs.size()).detail("TransactionRate", transactionRate).detail("TransactionQueue", transactionQueue.size()).detail("ReleasedTransactionCount", transactionCount); //TraceEvent("MasterProxyCTR", proxy.id()).detail("CommitTransactions", trs.size()).detail("TransactionRate", transactionRate).detail("TransactionQueue", transactionQueue.size()).detail("ReleasedTransactionCount", transactionCount);
if (trs.size() || (db->get().recoveryState >= RecoveryState::FULLY_RECOVERED && now() - lastCommit >= SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL)) { if (trs.size() || (db->get().recoveryState >= RecoveryState::FULLY_RECOVERED && now() - lastCommit >= SERVER_KNOBS->MAX_COMMIT_BATCH_INTERVAL)) {
lastCommit = now(); lastCommit = now();
if (trs.size() || lastCommitComplete.isReady()) { if (trs.size() || lastCommitComplete.isReady()) {
lastCommitComplete = commitBatch(&commitData, trs, &commitBatchInterval); lastCommitComplete = commitBatch(&commitData, trs, &commitBatchInterval, batchBytes);
addActor.send(lastCommitComplete); addActor.send(lastCommitComplete);
} }
} }

8
fdbserver/MoveKeys.actor.cpp
View File

@ -315,7 +315,7 @@ ACTOR Future<Void> startMoveKeys( Database occ, KeyRange keys, vector<UID> serve
Void _ = wait( tr.onError(e) ); Void _ = wait( tr.onError(e) );
if(retries%10 == 0) { if(retries%10 == 0) {
TraceEvent(retries == 50 ? SevWarnAlways : SevWarn, "startMoveKeysRetrying", relocationIntervalId) TraceEvent(retries == 50 ? SevWarnAlways : SevWarn, "StartMoveKeysRetrying", relocationIntervalId)
.detail("Keys", printable(keys)) .detail("Keys", printable(keys))
.detail("BeginKey", printable(begin)) .detail("BeginKey", printable(begin))
.detail("NumTries", retries) .detail("NumTries", retries)
@ -556,7 +556,7 @@ ACTOR Future<Void> finishMoveKeys( Database occ, KeyRange keys, vector<UID> dest
ASSERT(false); ASSERT(false);
} }
waitInterval = TraceInterval("RelocateShard_FinishMoveKeys_WaitDurable"); waitInterval = TraceInterval("RelocateShard_FinishMoveKeysWaitDurable");
TraceEvent(SevDebug, waitInterval.begin(), relocationIntervalId) TraceEvent(SevDebug, waitInterval.begin(), relocationIntervalId)
.detail("KeyBegin", printable(keys.begin)) .detail("KeyBegin", printable(keys.begin))
.detail("KeyEnd", printable(keys.end)); .detail("KeyEnd", printable(keys.end));
@ -627,7 +627,7 @@ ACTOR Future<Void> finishMoveKeys( Database occ, KeyRange keys, vector<UID> dest
Void _ = wait( tr.onError(error) ); Void _ = wait( tr.onError(error) );
retries++; retries++;
if(retries%10 == 0) { if(retries%10 == 0) {
TraceEvent(retries == 20 ? SevWarnAlways : SevWarn, "RelocateShard_finishMoveKeysRetrying", relocationIntervalId) TraceEvent(retries == 20 ? SevWarnAlways : SevWarn, "RelocateShard_FinishMoveKeysRetrying", relocationIntervalId)
.error(err) .error(err)
.detail("KeyBegin", printable(keys.begin)) .detail("KeyBegin", printable(keys.begin))
.detail("KeyEnd", printable(keys.end)) .detail("KeyEnd", printable(keys.end))
@ -771,7 +771,7 @@ ACTOR Future<Void> removeStorageServer( Database cx, UID serverID, MoveKeysLock
TraceEvent(SevWarn,"NoCanRemove").detail("Count", noCanRemoveCount++).detail("ServerID", serverID); TraceEvent(SevWarn,"NoCanRemove").detail("Count", noCanRemoveCount++).detail("ServerID", serverID);
Void _ = wait( delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskDataDistributionLaunch) ); Void _ = wait( delayJittered(SERVER_KNOBS->REMOVE_RETRY_DELAY, TaskDataDistributionLaunch) );
tr.reset(); tr.reset();
TraceEvent("RemoveStorageServerRetrying").detail("canRemove", canRemove); TraceEvent("RemoveStorageServerRetrying").detail("CanRemove", canRemove);
} else { } else {
state Future<Optional<Value>> fListKey = tr.get( serverListKeyFor(serverID) ); state Future<Optional<Value>> fListKey = tr.get( serverListKeyFor(serverID) );

66
fdbserver/OldTLogServer.actor.cpp
View File

@ -407,8 +407,8 @@ namespace oldTLog {
explicit LogData(TLogData* tLogData, TLogInterface interf) : tLogData(tLogData), knownCommittedVersion(0), tli(interf), logId(interf.id()), explicit LogData(TLogData* tLogData, TLogInterface interf) : tLogData(tLogData), knownCommittedVersion(0), tli(interf), logId(interf.id()),
cc("TLog", interf.id().toString()), cc("TLog", interf.id().toString()),
bytesInput("bytesInput", cc), bytesInput("BytesInput", cc),
bytesDurable("bytesDurable", cc), bytesDurable("BytesDurable", cc),
// These are initialized differently on init() or recovery // These are initialized differently on init() or recovery
recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), recovery(Void()) recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), recovery(Void())
{ {
@ -419,22 +419,22 @@ namespace oldTLog {
version.initMetric(LiteralStringRef("TLog.Version"), cc.id); version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id); queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
specialCounter(cc, "version", [this](){ return this->version.get(); }); specialCounter(cc, "Version", [this](){ return this->version.get(); });
specialCounter(cc, "sharedBytesInput", [tLogData](){ return tLogData->bytesInput; }); specialCounter(cc, "SharedBytesInput", [tLogData](){ return tLogData->bytesInput; });
specialCounter(cc, "sharedBytesDurable", [tLogData](){ return tLogData->bytesDurable; }); specialCounter(cc, "SharedBytesDurable", [tLogData](){ return tLogData->bytesDurable; });
specialCounter(cc, "kvstoreBytesUsed", [tLogData](){ return tLogData->persistentData->getStorageBytes().used; }); specialCounter(cc, "KvstoreBytesUsed", [tLogData](){ return tLogData->persistentData->getStorageBytes().used; });
specialCounter(cc, "kvstoreBytesFree", [tLogData](){ return tLogData->persistentData->getStorageBytes().free; }); specialCounter(cc, "KvstoreBytesFree", [tLogData](){ return tLogData->persistentData->getStorageBytes().free; });
specialCounter(cc, "kvstoreBytesAvailable", [tLogData](){ return tLogData->persistentData->getStorageBytes().available; }); specialCounter(cc, "KvstoreBytesAvailable", [tLogData](){ return tLogData->persistentData->getStorageBytes().available; });
specialCounter(cc, "kvstoreBytesTotal", [tLogData](){ return tLogData->persistentData->getStorageBytes().total; }); specialCounter(cc, "KvstoreBytesTotal", [tLogData](){ return tLogData->persistentData->getStorageBytes().total; });
specialCounter(cc, "queueDiskBytesUsed", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().used; }); specialCounter(cc, "QueueDiskBytesUsed", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().used; });
specialCounter(cc, "queueDiskBytesFree", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().free; }); specialCounter(cc, "QueueDiskBytesFree", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().free; });
specialCounter(cc, "queueDiskBytesAvailable", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().available; }); specialCounter(cc, "QueueDiskBytesAvailable", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().available; });
specialCounter(cc, "queueDiskBytesTotal", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().total; }); specialCounter(cc, "QueueDiskBytesTotal", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().total; });
} }
~LogData() { ~LogData() {
tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue(); tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue();
TraceEvent("TLogBytesWhenRemoved", tli.id()).detail("sharedBytesInput", tLogData->bytesInput).detail("sharedBytesDurable", tLogData->bytesDurable).detail("localBytesInput", bytesInput.getValue()).detail("localBytesDurable", bytesDurable.getValue()); TraceEvent("TLogBytesWhenRemoved", tli.id()).detail("SharedBytesInput", tLogData->bytesInput).detail("SharedBytesDurable", tLogData->bytesDurable).detail("LocalBytesInput", bytesInput.getValue()).detail("LocalBytesDurable", bytesDurable.getValue());
ASSERT_ABORT(tLogData->bytesDurable <= tLogData->bytesInput); ASSERT_ABORT(tLogData->bytesDurable <= tLogData->bytesInput);
endRole(tli.id(), "TLog", "Error", true); endRole(tli.id(), "TLog", "Error", true);
@ -460,7 +460,7 @@ namespace oldTLog {
TEST( logData->stopped ); TEST( logData->stopped );
TEST( !logData->stopped ); TEST( !logData->stopped );
TraceEvent("TLogStop", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get()); TraceEvent("TLogStop", logData->logId).detail("Ver", stopVersion).detail("IsStopped", logData->stopped).detail("QueueCommitted", logData->queueCommittedVersion.get());
logData->stopped = true; logData->stopped = true;
if(logData->recoverySuccessful.canBeSet()) { if(logData->recoverySuccessful.canBeSet()) {
@ -476,7 +476,7 @@ namespace oldTLog {
result.end = stopVersion; result.end = stopVersion;
result.knownCommittedVersion = logData->knownCommittedVersion; result.knownCommittedVersion = logData->knownCommittedVersion;
TraceEvent("TLogStop2", self->dbgid).detail("logId", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get()); TraceEvent("TLogStop2", self->dbgid).detail("LogId", logData->logId).detail("Ver", stopVersion).detail("IsStopped", logData->stopped).detail("QueueCommitted", logData->queueCommittedVersion.get());
reply.send( result ); reply.send( result );
@ -504,7 +504,7 @@ namespace oldTLog {
ASSERT( newPersistentDataVersion > logData->persistentDataVersion ); ASSERT( newPersistentDataVersion > logData->persistentDataVersion );
ASSERT( logData->persistentDataVersion == logData->persistentDataDurableVersion ); ASSERT( logData->persistentDataVersion == logData->persistentDataDurableVersion );
//TraceEvent("updatePersistentData", self->dbgid).detail("seq", newPersistentDataSeq); //TraceEvent("UpdatePersistentData", self->dbgid).detail("Seq", newPersistentDataSeq);
state bool anyData = false; state bool anyData = false;
state Map<OldTag, LogData::TagData>::iterator tag; state Map<OldTag, LogData::TagData>::iterator tag;
@ -565,7 +565,7 @@ namespace oldTLog {
} }
if(logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) { if(logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) {
TraceEvent(SevError, "BytesDurableTooLarge", logData->logId).detail("sharedBytesInput", self->bytesInput).detail("sharedBytesDurable", self->bytesDurable).detail("localBytesInput", logData->bytesInput.getValue()).detail("localBytesDurable", logData->bytesDurable.getValue()); TraceEvent(SevError, "BytesDurableTooLarge", logData->logId).detail("SharedBytesInput", self->bytesInput).detail("SharedBytesDurable", self->bytesDurable).detail("LocalBytesInput", logData->bytesInput.getValue()).detail("LocalBytesDurable", logData->bytesDurable.getValue());
} }
ASSERT(logData->bytesDurable.getValue() <= logData->bytesInput.getValue()); ASSERT(logData->bytesDurable.getValue() <= logData->bytesInput.getValue());
@ -623,7 +623,7 @@ namespace oldTLog {
Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) ); Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) );
Void _ = wait( delay(0, TaskUpdateStorage) ); Void _ = wait( delay(0, TaskUpdateStorage) );
//TraceEvent("TlogUpdatePersist", self->dbgid).detail("logId", logData->logId).detail("nextVersion", nextVersion).detail("version", logData->version.get()).detail("persistentDataDurableVer", logData->persistentDataDurableVersion).detail("queueCommitVer", logData->queueCommittedVersion.get()).detail("persistDataVer", logData->persistentDataVersion); //TraceEvent("TlogUpdatePersist", self->dbgid).detail("LogId", logData->logId).detail("NextVersion", nextVersion).detail("Version", logData->version.get()).detail("PersistentDataDurableVer", logData->persistentDataDurableVersion).detail("QueueCommitVer", logData->queueCommittedVersion.get()).detail("PersistDataVer", logData->persistentDataVersion);
if (nextVersion > logData->persistentDataVersion) { if (nextVersion > logData->persistentDataVersion) {
self->updatePersist = updatePersistentData(self, logData, nextVersion); self->updatePersist = updatePersistentData(self, logData, nextVersion);
Void _ = wait( self->updatePersist ); Void _ = wait( self->updatePersist );
@ -670,7 +670,7 @@ namespace oldTLog {
nextVersion = std::max<Version>(nextVersion, logData->persistentDataVersion); nextVersion = std::max<Version>(nextVersion, logData->persistentDataVersion);
//TraceEvent("UpdateStorageVer", logData->logId).detail("nextVersion", nextVersion).detail("persistentDataVersion", logData->persistentDataVersion).detail("totalSize", totalSize); //TraceEvent("UpdateStorageVer", logData->logId).detail("NextVersion", nextVersion).detail("PersistentDataVersion", logData->persistentDataVersion).detail("TotalSize", totalSize);
Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) ); Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) );
Void _ = wait( delay(0, TaskUpdateStorage) ); Void _ = wait( delay(0, TaskUpdateStorage) );
@ -794,7 +794,7 @@ namespace oldTLog {
self->bytesInput += addedBytes; self->bytesInput += addedBytes;
bytesInput += addedBytes; bytesInput += addedBytes;
//TraceEvent("TLogPushed", self->dbgid).detail("Bytes", addedBytes).detail("MessageBytes", messages.size()).detail("Tags", tags.size()).detail("expectedBytes", expectedBytes).detail("mCount", mCount).detail("tCount", tCount); //TraceEvent("TLogPushed", self->dbgid).detail("Bytes", addedBytes).detail("MessageBytes", messages.size()).detail("Tags", tags.size()).detail("ExpectedBytes", expectedBytes).detail("MCount", mCount).detail("TCount", tCount);
} }
Version poppedVersion( Reference<LogData> self, OldTag tag) { Version poppedVersion( Reference<LogData> self, OldTag tag) {
@ -897,7 +897,7 @@ namespace oldTLog {
return Void(); return Void();
} }
//TraceEvent("tLogPeekMessages0", self->dbgid).detail("reqBeginEpoch", req.begin.epoch).detail("reqBeginSeq", req.begin.sequence).detail("epoch", self->epoch()).detail("persistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2)); //TraceEvent("tLogPeekMessages0", self->dbgid).detail("ReqBeginEpoch", req.begin.epoch).detail("ReqBeginSeq", req.begin.sequence).detail("Epoch", self->epoch()).detail("PersistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2));
// Wait until we have something to return that the caller doesn't already have // Wait until we have something to return that the caller doesn't already have
if( logData->version.get() < req.begin ) { if( logData->version.get() < req.begin ) {
Void _ = wait( logData->version.whenAtLeast( req.begin ) ); Void _ = wait( logData->version.whenAtLeast( req.begin ) );
@ -907,7 +907,7 @@ namespace oldTLog {
state Version endVersion = logData->version.get() + 1; state Version endVersion = logData->version.get() + 1;
//grab messages from disk //grab messages from disk
//TraceEvent("tLogPeekMessages", self->dbgid).detail("reqBeginEpoch", req.begin.epoch).detail("reqBeginSeq", req.begin.sequence).detail("epoch", self->epoch()).detail("persistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2)); //TraceEvent("tLogPeekMessages", self->dbgid).detail("ReqBeginEpoch", req.begin.epoch).detail("ReqBeginSeq", req.begin.sequence).detail("Epoch", self->epoch()).detail("PersistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2));
if( req.begin <= logData->persistentDataDurableVersion ) { if( req.begin <= logData->persistentDataDurableVersion ) {
// Just in case the durable version changes while we are waiting for the read, we grab this data from memory. We may or may not actually send it depending on // Just in case the durable version changes while we are waiting for the read, we grab this data from memory. We may or may not actually send it depending on
// whether we get enough data from disk. // whether we get enough data from disk.
@ -959,7 +959,7 @@ namespace oldTLog {
reply.messages = messages.toStringRef(); reply.messages = messages.toStringRef();
reply.end = endVersion; reply.end = endVersion;
} }
//TraceEvent("TlogPeek", self->dbgid).detail("logId", logData->logId).detail("endVer", reply.end).detail("msgBytes", reply.messages.expectedSize()).detail("ForAddress", req.reply.getEndpoint().address); //TraceEvent("TlogPeek", self->dbgid).detail("LogId", logData->logId).detail("EndVer", reply.end).detail("MsgBytes", reply.messages.expectedSize()).detail("ForAddress", req.reply.getEndpoint().address);
if(req.sequence.present()) { if(req.sequence.present()) {
auto& trackerData = self->peekTracker[peekId]; auto& trackerData = self->peekTracker[peekId];
@ -1026,7 +1026,7 @@ namespace oldTLog {
continue; continue;
} }
TraceEvent("commitQueueNewLog", self->dbgid).detail("logId", logData->logId).detail("version", logData->version.get()).detail("committing", logData->queueCommittingVersion).detail("commmitted", logData->queueCommittedVersion.get()); TraceEvent("CommitQueueNewLog", self->dbgid).detail("LogId", logData->logId).detail("Version", logData->version.get()).detail("Committing", logData->queueCommittingVersion).detail("Commmitted", logData->queueCommittedVersion.get());
loop { loop {
if(logData->stopped && logData->version.get() == std::max(logData->queueCommittingVersion, logData->queueCommittedVersion.get())) { if(logData->stopped && logData->version.get() == std::max(logData->queueCommittingVersion, logData->queueCommittedVersion.get())) {
@ -1074,8 +1074,8 @@ namespace oldTLog {
} }
if ( isDisplaced ) if ( isDisplaced )
{ {
TraceEvent("TLogDisplaced", tli.id()).detail("Reason", "DBInfoDoesNotContain").detail("recoveryCount", recoveryCount).detail("infRecoveryCount", inf.recoveryCount).detail("recoveryState", inf.recoveryState) TraceEvent("TLogDisplaced", tli.id()).detail("Reason", "DBInfoDoesNotContain").detail("RecoveryCount", recoveryCount).detail("InfRecoveryCount", inf.recoveryCount).detail("RecoveryState", inf.recoveryState)
.detail("logSysConf", describe(inf.logSystemConfig.tLogs)).detail("priorLogs", describe(inf.priorCommittedLogServers)).detail("oldLogGens", inf.logSystemConfig.oldTLogs.size()); .detail("LogSysConf", describe(inf.logSystemConfig.tLogs)).detail("PriorLogs", describe(inf.priorCommittedLogServers)).detail("OldLogGens", inf.logSystemConfig.oldTLogs.size());
if (BUGGIFY) Void _ = wait( delay( SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * g_random->random01() ) ); if (BUGGIFY) Void _ = wait( delay( SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * g_random->random01() ) );
throw worker_removed(); throw worker_removed();
} }
@ -1167,7 +1167,7 @@ namespace oldTLog {
} }
void removeLog( TLogData* self, Reference<LogData> logData ) { void removeLog( TLogData* self, Reference<LogData> logData ) {
TraceEvent("TLogRemoved", logData->logId).detail("input", logData->bytesInput.getValue()).detail("durable", logData->bytesDurable.getValue()); TraceEvent("TLogRemoved", logData->logId).detail("Input", logData->bytesInput.getValue()).detail("Durable", logData->bytesDurable.getValue());
logData->stopped = true; logData->stopped = true;
if(logData->recoverySuccessful.canBeSet()) { if(logData->recoverySuccessful.canBeSet()) {
logData->recoverySuccessful.send(false); logData->recoverySuccessful.send(false);
@ -1196,7 +1196,7 @@ namespace oldTLog {
return Void(); return Void();
} }
TraceEvent("newLogData", self->dbgid).detail("logId", logData->logId); TraceEvent("NewLogData", self->dbgid).detail("LogId", logData->logId);
logData->initialized = true; logData->initialized = true;
self->newLogData.trigger(); self->newLogData.trigger();
@ -1293,7 +1293,7 @@ namespace oldTLog {
logData->removed = rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture()); logData->removed = rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture());
removed.push_back(errorOr(logData->removed)); removed.push_back(errorOr(logData->removed));
TraceEvent("TLogRestorePersistentStateVer", id1).detail("ver", ver); TraceEvent("TLogRestorePersistentStateVer", id1).detail("Ver", ver);
// Restore popped keys. Pop operations that took place after the last (committed) updatePersistentDataVersion might be lost, but // Restore popped keys. Pop operations that took place after the last (committed) updatePersistentDataVersion might be lost, but
// that is fine because we will get the corresponding data back, too. // that is fine because we will get the corresponding data back, too.
@ -1342,8 +1342,8 @@ namespace oldTLog {
lastVer = qe.version; lastVer = qe.version;
} }
//TraceEvent("TLogRecoveredQE", self->dbgid).detail("logId", qe.id).detail("ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size()) //TraceEvent("TLogRecoveredQE", self->dbgid).detail("LogId", qe.id).detail("Ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size())
// .detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("version", logData->version.get()); // .detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("Version", logData->version.get());
if(logData) { if(logData) {
logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion); logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
@ -1376,7 +1376,7 @@ namespace oldTLog {
for(auto it : self->id_data) { for(auto it : self->id_data) {
if(it.second->queueCommittedVersion.get() == 0) { if(it.second->queueCommittedVersion.get() == 0) {
TraceEvent("TLogZeroVersion", self->dbgid).detail("logId", it.first); TraceEvent("TLogZeroVersion", self->dbgid).detail("LogId", it.first);
it.second->queueCommittedVersion.set(it.second->version.get()); it.second->queueCommittedVersion.set(it.second->version.get());
} }
self->sharedActors.send( tLogCore( self, it.second ) ); self->sharedActors.send( tLogCore( self, it.second ) );

12
fdbserver/QuietDatabase.actor.cpp
View File

@ -50,7 +50,7 @@ ACTOR Future<WorkerInterface> getMasterWorker( Database cx, Reference<AsyncVar<S
for( int i = 0; i < workers.size(); i++ ) { for( int i = 0; i < workers.size(); i++ ) {
if( workers[i].first.address() == dbInfo->get().master.address() ) { if( workers[i].first.address() == dbInfo->get().master.address() ) {
TraceEvent("GetMasterWorker").detail("Database", printable(cx->dbName)).detail("Stage", "GotWorkers").detail("masterId", dbInfo->get().master.id()).detail("workerId", workers[i].first.id()); TraceEvent("GetMasterWorker").detail("Database", printable(cx->dbName)).detail("Stage", "GotWorkers").detail("MasterId", dbInfo->get().master.id()).detail("WorkerId", workers[i].first.id());
return workers[i].first; return workers[i].first;
} }
} }
@ -94,8 +94,8 @@ int64_t getQueueSize( TraceEventFields md ) {
double inputRoughness, durableRoughness; double inputRoughness, durableRoughness;
int64_t inputBytes, durableBytes; int64_t inputBytes, durableBytes;
sscanf(md.getValue("bytesInput").c_str(), "%lf %lf %lld", &inputRate, &inputRoughness, &inputBytes); sscanf(md.getValue("BytesInput").c_str(), "%lf %lf %lld", &inputRate, &inputRoughness, &inputBytes);
sscanf(md.getValue("bytesDurable").c_str(), "%lf %lf %lld", &durableRate, &durableRoughness, &durableBytes); sscanf(md.getValue("BytesDurable").c_str(), "%lf %lf %lld", &durableRate, &durableRoughness, &durableBytes);
return inputBytes - durableBytes; return inputBytes - durableBytes;
} }
@ -330,9 +330,9 @@ ACTOR Future<Void> waitForQuietDatabase( Database cx, Reference<AsyncVar<ServerD
Void _ = wait( success( dataInFlight ) && success( tLogQueueSize ) && success( dataDistributionQueueSize ) Void _ = wait( success( dataInFlight ) && success( tLogQueueSize ) && success( dataDistributionQueueSize )
&& success( storageQueueSize ) && success( dataDistributionActive ) && success( storageServersRecruiting ) ); && success( storageQueueSize ) && success( dataDistributionActive ) && success( storageServersRecruiting ) );
TraceEvent(("QuietDatabase" + phase).c_str()) TraceEvent(("QuietDatabase" + phase).c_str())
.detail("dataInFlight", dataInFlight.get()).detail("maxTLogQueueSize", tLogQueueSize.get()).detail("dataDistributionQueueSize", dataDistributionQueueSize.get()) .detail("DataInFlight", dataInFlight.get()).detail("MaxTLogQueueSize", tLogQueueSize.get()).detail("DataDistributionQueueSize", dataDistributionQueueSize.get())
.detail("maxStorageQueueSize", storageQueueSize.get()).detail("dataDistributionActive", dataDistributionActive.get()) .detail("MaxStorageQueueSize", storageQueueSize.get()).detail("DataDistributionActive", dataDistributionActive.get())
.detail("storageServersRecruiting", storageServersRecruiting.get()); .detail("StorageServersRecruiting", storageServersRecruiting.get());
if( dataInFlight.get() > dataInFlightGate || tLogQueueSize.get() > maxTLogQueueGate if( dataInFlight.get() > dataInFlightGate || tLogQueueSize.get() > maxTLogQueueGate
|| dataDistributionQueueSize.get() > maxDataDistributionQueueSize || storageQueueSize.get() > maxStorageServerQueueGate || dataDistributionQueueSize.get() > maxDataDistributionQueueSize || storageQueueSize.get() > maxStorageServerQueueGate

4
fdbserver/Ratekeeper.actor.cpp
View File

@ -312,7 +312,7 @@ void updateRate( Ratekeeper* self ) {
.detail("ActualTPS", actualTPS) .detail("ActualTPS", actualTPS)
.detail("InputRate", inputRate) .detail("InputRate", inputRate)
.detail("VerySmoothDurableBytesRate", ss.verySmoothDurableBytes.smoothRate()) .detail("VerySmoothDurableBytesRate", ss.verySmoothDurableBytes.smoothRate())
.detail("b", b); .detail("B", b);
}*/ }*/
// Don't let any storage server use up its target bytes faster than its MVCC window! // Don't let any storage server use up its target bytes faster than its MVCC window!
@ -423,7 +423,7 @@ void updateRate( Ratekeeper* self ) {
if( tl.lastReply.bytesInput - tl.lastReply.bytesDurable > tl.lastReply.storageBytes.free - minFreeSpace / 2 ) { if( tl.lastReply.bytesInput - tl.lastReply.bytesDurable > tl.lastReply.storageBytes.free - minFreeSpace / 2 ) {
if(now() - self->lastWarning > 5.0) { if(now() - self->lastWarning > 5.0) {
self->lastWarning = now(); self->lastWarning = now();
TraceEvent(SevWarnAlways, "RkTlogMinFreeSpaceZero").detail("reasonId", tl.id); TraceEvent(SevWarnAlways, "RkTlogMinFreeSpaceZero").detail("ReasonId", tl.id);
} }
reasonID = tl.id; reasonID = tl.id;
limitReason = limitReason_t::log_server_min_free_space; limitReason = limitReason_t::log_server_min_free_space;

6
fdbserver/Resolver.actor.cpp
View File

@ -90,9 +90,9 @@ ACTOR Future<Void> resolveBatch(
while( self->totalStateBytes.get() > SERVER_KNOBS->RESOLVER_STATE_MEMORY_LIMIT && self->recentStateTransactionSizes.size() && while( self->totalStateBytes.get() > SERVER_KNOBS->RESOLVER_STATE_MEMORY_LIMIT && self->recentStateTransactionSizes.size() &&
proxyInfo.lastVersion > self->recentStateTransactionSizes.front().first && req.version > self->neededVersion.get() ) { proxyInfo.lastVersion > self->recentStateTransactionSizes.front().first && req.version > self->neededVersion.get() ) {
/*TraceEvent("ResolveBatchDelay").detail("From", proxyAddress).detail("stateBytes", self->totalStateBytes.get()).detail("RecentStateTransactionSize", self->recentStateTransactionSizes.size()) /*TraceEvent("ResolveBatchDelay").detail("From", proxyAddress).detail("StateBytes", self->totalStateBytes.get()).detail("RecentStateTransactionSize", self->recentStateTransactionSizes.size())
.detail("lastVersion", proxyInfo.lastVersion).detail("RequestVersion", req.version).detail("NeededVersion", self->neededVersion.get()) .detail("LastVersion", proxyInfo.lastVersion).detail("RequestVersion", req.version).detail("NeededVersion", self->neededVersion.get())
.detail("recentStateVer", self->recentStateTransactions.begin()->key);*/ .detail("RecentStateVer", self->recentStateTransactions.begin()->key);*/
Void _ = wait( self->totalStateBytes.onChange() || self->neededVersion.onChange() ); Void _ = wait( self->totalStateBytes.onChange() || self->neededVersion.onChange() );
} }

22
fdbserver/SimulatedCluster.actor.cpp
View File

@ -165,7 +165,7 @@ ACTOR Future<Void> runDr( Reference<ClusterConnectionFile> connFile ) {
Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1); Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1);
state Database extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get(); state Database extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get();
TraceEvent("StartingDrAgents").detail("connFile", connFile->getConnectionString().toString()).detail("extraString", extraFile->getConnectionString().toString()); TraceEvent("StartingDrAgents").detail("ConnFile", connFile->getConnectionString().toString()).detail("ExtraString", extraFile->getConnectionString().toString());
state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx); state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx);
state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB); state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB);
@ -219,7 +219,7 @@ ACTOR Future<ISimulator::KillType> simulatedFDBDRebooter(
TraceEvent("SimulatedFDBDPreWait").detail("Cycles", cycles).detail("RandomId", randomId) TraceEvent("SimulatedFDBDPreWait").detail("Cycles", cycles).detail("RandomId", randomId)
.detail("Address", NetworkAddress(ip, port, true, false)) .detail("Address", NetworkAddress(ip, port, true, false))
.detailext("ZoneId", localities.zoneId()) .detailext("ZoneId", localities.zoneId())
.detail("waitTime", waitTime).detail("Port", port); .detail("WaitTime", waitTime).detail("Port", port);
Void _ = wait( delay( waitTime ) ); Void _ = wait( delay( waitTime ) );
@ -422,8 +422,8 @@ ACTOR Future<Void> simulatedMachine(
.detail("CFolder0", coordFolders[0]) .detail("CFolder0", coordFolders[0])
.detail("MachineIPs", toIPVectorString(ips)) .detail("MachineIPs", toIPVectorString(ips))
.detail("SSL", sslEnabled) .detail("SSL", sslEnabled)
.detail("processes", processes.size()) .detail("Processes", processes.size())
.detail("bootCount", bootCount) .detail("BootCount", bootCount)
.detail("ProcessClass", processClass.toString()) .detail("ProcessClass", processClass.toString())
.detail("Restarting", restarting) .detail("Restarting", restarting)
.detail("UseSeedFile", useSeedFile) .detail("UseSeedFile", useSeedFile)
@ -657,12 +657,12 @@ ACTOR Future<Void> restartSimulatedSystem(
g_simulator.processesPerMachine = processesPerMachine; g_simulator.processesPerMachine = processesPerMachine;
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(SevError, "restartSimulationError").error(e); TraceEvent(SevError, "RestartSimulationError").error(e);
} }
TraceEvent("RestartSimulatorSettings") TraceEvent("RestartSimulatorSettings")
.detail("desiredCoordinators", g_simulator.desiredCoordinators) .detail("DesiredCoordinators", g_simulator.desiredCoordinators)
.detail("processesPerMachine", g_simulator.processesPerMachine); .detail("ProcessesPerMachine", g_simulator.processesPerMachine);
Void _ = wait(delay(1.0)); Void _ = wait(delay(1.0));
@ -951,7 +951,7 @@ void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseF
ASSERT(g_simulator.storagePolicy && g_simulator.tLogPolicy); ASSERT(g_simulator.storagePolicy && g_simulator.tLogPolicy);
ASSERT(!g_simulator.hasRemoteReplication || g_simulator.remoteTLogPolicy); ASSERT(!g_simulator.hasRemoteReplication || g_simulator.remoteTLogPolicy);
ASSERT(!g_simulator.hasSatelliteReplication || g_simulator.satelliteTLogPolicy); ASSERT(!g_simulator.hasSatelliteReplication || g_simulator.satelliteTLogPolicy);
TraceEvent("simulatorConfig").detail("ConfigString", printable(StringRef(startingConfigString))); TraceEvent("SimulatorConfig").detail("ConfigString", printable(StringRef(startingConfigString)));
const int dataCenters = simconfig.datacenters; const int dataCenters = simconfig.datacenters;
const int machineCount = simconfig.machine_count; const int machineCount = simconfig.machine_count;
@ -1063,9 +1063,9 @@ void setupSimulatedSystem( vector<Future<Void>> *systemActors, std::string baseF
g_simulator.processesPerMachine = processesPerMachine; g_simulator.processesPerMachine = processesPerMachine;
TraceEvent("SetupSimulatorSettings") TraceEvent("SetupSimulatorSettings")
.detail("desiredCoordinators", g_simulator.desiredCoordinators) .detail("DesiredCoordinators", g_simulator.desiredCoordinators)
.detail("physicalDatacenters", g_simulator.physicalDatacenters) .detail("PhysicalDatacenters", g_simulator.physicalDatacenters)
.detail("processesPerMachine", g_simulator.processesPerMachine); .detail("ProcessesPerMachine", g_simulator.processesPerMachine);
// SOMEDAY: add locality for testers to simulate network topology // SOMEDAY: add locality for testers to simulate network topology
// FIXME: Start workers with tester class instead, at least sometimes run tests with the testers-only flag // FIXME: Start workers with tester class instead, at least sometimes run tests with the testers-only flag

54
fdbserver/Status.actor.cpp
View File

@ -408,15 +408,15 @@ struct RolesInfo {
obj["id"] = iface.id().shortString(); obj["id"] = iface.id().shortString();
obj["role"] = role; obj["role"] = role;
try { try {
obj["stored_bytes"] = parseInt64(metrics.getValue("bytesStored")); obj["stored_bytes"] = parseInt64(metrics.getValue("BytesStored"));
obj["kvstore_used_bytes"] = parseInt64(metrics.getValue("kvstoreBytesUsed")); obj["kvstore_used_bytes"] = parseInt64(metrics.getValue("KvstoreBytesUsed"));
obj["kvstore_free_bytes"] = parseInt64(metrics.getValue("kvstoreBytesFree")); obj["kvstore_free_bytes"] = parseInt64(metrics.getValue("KvstoreBytesFree"));
obj["kvstore_available_bytes"] = parseInt64(metrics.getValue("kvstoreBytesAvailable")); obj["kvstore_available_bytes"] = parseInt64(metrics.getValue("KvstoreBytesAvailable"));
obj["kvstore_total_bytes"] = parseInt64(metrics.getValue("kvstoreBytesTotal")); obj["kvstore_total_bytes"] = parseInt64(metrics.getValue("KvstoreBytesTotal"));
obj["input_bytes"] = parseCounter(metrics.getValue("bytesInput")); obj["input_bytes"] = parseCounter(metrics.getValue("BytesInput"));
obj["durable_bytes"] = parseCounter(metrics.getValue("bytesDurable")); obj["durable_bytes"] = parseCounter(metrics.getValue("BytesDurable"));
obj["query_queue_max"] = parseInt(metrics.getValue("QueryQueueMax")); obj["query_queue_max"] = parseInt(metrics.getValue("QueryQueueMax"));
obj["finished_queries"] = parseCounter(metrics.getValue("finishedQueries")); obj["finished_queries"] = parseCounter(metrics.getValue("FinishedQueries"));
Version version = parseInt64(metrics.getValue("version")); Version version = parseInt64(metrics.getValue("version"));
obj["data_version"] = version; obj["data_version"] = version;
@ -436,17 +436,17 @@ struct RolesInfo {
obj["id"] = iface.id().shortString(); obj["id"] = iface.id().shortString();
obj["role"] = role; obj["role"] = role;
try { try {
obj["kvstore_used_bytes"] = parseInt64(metrics.getValue("kvstoreBytesUsed")); obj["kvstore_used_bytes"] = parseInt64(metrics.getValue("KvstoreBytesUsed"));
obj["kvstore_free_bytes"] = parseInt64(metrics.getValue("kvstoreBytesFree")); obj["kvstore_free_bytes"] = parseInt64(metrics.getValue("KvstoreBytesFree"));
obj["kvstore_available_bytes"] = parseInt64(metrics.getValue("kvstoreBytesAvailable")); obj["kvstore_available_bytes"] = parseInt64(metrics.getValue("KvstoreBytesAvailable"));
obj["kvstore_total_bytes"] = parseInt64(metrics.getValue("kvstoreBytesTotal")); obj["kvstore_total_bytes"] = parseInt64(metrics.getValue("KvstoreBytesTotal"));
obj["queue_disk_used_bytes"] = parseInt64(metrics.getValue("queueDiskBytesUsed")); obj["queue_disk_used_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesUsed"));
obj["queue_disk_free_bytes"] = parseInt64(metrics.getValue("queueDiskBytesFree")); obj["queue_disk_free_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesFree"));
obj["queue_disk_available_bytes"] = parseInt64(metrics.getValue("queueDiskBytesAvailable")); obj["queue_disk_available_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesAvailable"));
obj["queue_disk_total_bytes"] = parseInt64(metrics.getValue("queueDiskBytesTotal")); obj["queue_disk_total_bytes"] = parseInt64(metrics.getValue("QueueDiskBytesTotal"));
obj["input_bytes"] = parseCounter(metrics.getValue("bytesInput")); obj["input_bytes"] = parseCounter(metrics.getValue("BytesInput"));
obj["durable_bytes"] = parseCounter(metrics.getValue("bytesDurable")); obj["durable_bytes"] = parseCounter(metrics.getValue("BytesDurable"));
obj["data_version"] = parseInt64(metrics.getValue("version")); obj["data_version"] = parseInt64(metrics.getValue("Version"));
} catch (Error& e) { } catch (Error& e) {
if(e.code() != error_code_attribute_not_found) if(e.code() != error_code_attribute_not_found)
throw e; throw e;
@ -1214,11 +1214,11 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
StatusObject mutations=makeCounter(), mutationBytes=makeCounter(), txnConflicts=makeCounter(), txnStartOut=makeCounter(), txnCommitOutSuccess=makeCounter(); StatusObject mutations=makeCounter(), mutationBytes=makeCounter(), txnConflicts=makeCounter(), txnStartOut=makeCounter(), txnCommitOutSuccess=makeCounter();
for (auto &ps : proxyStats) { for (auto &ps : proxyStats) {
mutations = addCounters( mutations, parseCounter(ps.getValue("mutations")) ); mutations = addCounters( mutations, parseCounter(ps.getValue("Mutations")) );
mutationBytes = addCounters( mutationBytes, parseCounter(ps.getValue("mutationBytes")) ); mutationBytes = addCounters( mutationBytes, parseCounter(ps.getValue("MutationBytes")) );
txnConflicts = addCounters( txnConflicts, parseCounter(ps.getValue("txnConflicts")) ); txnConflicts = addCounters( txnConflicts, parseCounter(ps.getValue("TxnConflicts")) );
txnStartOut = addCounters( txnStartOut, parseCounter(ps.getValue("txnStartOut")) ); txnStartOut = addCounters( txnStartOut, parseCounter(ps.getValue("TxnStartOut")) );
txnCommitOutSuccess = addCounters( txnCommitOutSuccess, parseCounter(ps.getValue("txnCommitOutSuccess")) ); txnCommitOutSuccess = addCounters( txnCommitOutSuccess, parseCounter(ps.getValue("TxnCommitOutSuccess")) );
} }
operationsObj["writes"] = mutations; operationsObj["writes"] = mutations;
@ -1307,9 +1307,9 @@ ACTOR static Future<StatusObject> workloadStatusFetcher(Reference<AsyncVar<struc
StatusObject readBytes = makeCounter(); StatusObject readBytes = makeCounter();
for(auto &ss : storageServers.get()) { for(auto &ss : storageServers.get()) {
reads = addCounters(reads, parseCounter(ss.second.getValue("finishedQueries"))); reads = addCounters(reads, parseCounter(ss.second.getValue("FinishedQueries")));
readKeys = addCounters(readKeys, parseCounter(ss.second.getValue("rowsQueried"))); readKeys = addCounters(readKeys, parseCounter(ss.second.getValue("RowsQueried")));
readBytes = addCounters(readBytes, parseCounter(ss.second.getValue("bytesQueried"))); readBytes = addCounters(readBytes, parseCounter(ss.second.getValue("BytesQueried")));
} }
operationsObj["reads"] = reads; operationsObj["reads"] = reads;

10
fdbserver/StorageMetrics.actor.h
View File

@ -72,7 +72,7 @@ struct StorageMetricSample {
} }
// If we didn't return above, we didn't find anything. // If we didn't return above, we didn't find anything.
TraceEvent(SevWarnAlways, "CannotSplitLastSampleKey").detail("range", printable(range)).detail("offset", offset); TraceEvent(SevWarnAlways, "CannotSplitLastSampleKey").detail("Range", printable(range)).detail("Offset", offset);
return front ? range.end : range.begin; return front ? range.end : range.begin;
} }
}; };
@ -349,10 +349,10 @@ struct StorageServerMetrics {
if (sb.free < 1e9 && g_random->random01() < 0.1) if (sb.free < 1e9 && g_random->random01() < 0.1)
TraceEvent(SevWarn, "PhysicalDiskMetrics") TraceEvent(SevWarn, "PhysicalDiskMetrics")
.detail("free", sb.free) .detail("Free", sb.free)
.detail("total", sb.total) .detail("Total", sb.total)
.detail("available", sb.available) .detail("Available", sb.available)
.detail("load", rep.load.bytes); .detail("Load", rep.load.bytes);
rep.free.bytes = sb.free; rep.free.bytes = sb.free;
rep.free.iosPerKSecond = 10e6; rep.free.iosPerKSecond = 10e6;

222
fdbserver/TLogServer.actor.cpp
View File

@ -283,43 +283,43 @@ struct TLogData : NonCopyable {
struct LogData : NonCopyable, public ReferenceCounted<LogData> { struct LogData : NonCopyable, public ReferenceCounted<LogData> {
struct TagData : NonCopyable, public ReferenceCounted<TagData> { struct TagData : NonCopyable, public ReferenceCounted<TagData> {
std::deque<std::pair<Version, LengthPrefixedStringRef>> version_messages; std::deque<std::pair<Version, LengthPrefixedStringRef>> versionMessages;
bool nothing_persistent; // true means tag is *known* to have no messages in persistentData. false means nothing. bool nothingPersistent; // true means tag is *known* to have no messages in persistentData. false means nothing.
bool popped_recently; // `popped` has changed since last updatePersistentData bool poppedRecently; // `popped` has changed since last updatePersistentData
Version popped; // see popped version tracking contract below Version popped; // see popped version tracking contract below
bool update_version_sizes; bool updateVersionSizes;
bool unpoppedRecovered; bool unpoppedRecovered;
Tag tag; Tag tag;
TagData( Tag tag, Version popped, bool nothing_persistent, bool popped_recently, bool unpoppedRecovered ) : tag(tag), nothing_persistent(nothing_persistent), popped(popped), popped_recently(popped_recently), unpoppedRecovered(unpoppedRecovered), update_version_sizes(tag != txsTag) {} TagData( Tag tag, Version popped, bool nothingPersistent, bool poppedRecently, bool unpoppedRecovered ) : tag(tag), nothingPersistent(nothingPersistent), popped(popped), poppedRecently(poppedRecently), unpoppedRecovered(unpoppedRecovered), updateVersionSizes(tag != txsTag) {}
TagData(TagData&& r) noexcept(true) : version_messages(std::move(r.version_messages)), nothing_persistent(r.nothing_persistent), popped_recently(r.popped_recently), popped(r.popped), update_version_sizes(r.update_version_sizes), tag(r.tag), unpoppedRecovered(r.unpoppedRecovered) {} TagData(TagData&& r) noexcept(true) : versionMessages(std::move(r.versionMessages)), nothingPersistent(r.nothingPersistent), poppedRecently(r.poppedRecently), popped(r.popped), updateVersionSizes(r.updateVersionSizes), tag(r.tag), unpoppedRecovered(r.unpoppedRecovered) {}
void operator= (TagData&& r) noexcept(true) { void operator= (TagData&& r) noexcept(true) {
version_messages = std::move(r.version_messages); versionMessages = std::move(r.versionMessages);
nothing_persistent = r.nothing_persistent; nothingPersistent = r.nothingPersistent;
popped_recently = r.popped_recently; poppedRecently = r.poppedRecently;
popped = r.popped; popped = r.popped;
update_version_sizes = r.update_version_sizes; updateVersionSizes = r.updateVersionSizes;
tag = r.tag; tag = r.tag;
unpoppedRecovered = r.unpoppedRecovered; unpoppedRecovered = r.unpoppedRecovered;
} }
// Erase messages not needed to update *from* versions >= before (thus, messages with toversion <= before) // Erase messages not needed to update *from* versions >= before (thus, messages with toversion <= before)
ACTOR Future<Void> eraseMessagesBefore( TagData *self, Version before, int64_t* gBytesErased, Reference<LogData> tlogData, int taskID ) { ACTOR Future<Void> eraseMessagesBefore( TagData *self, Version before, int64_t* gBytesErased, Reference<LogData> tlogData, int taskID ) {
while(!self->version_messages.empty() && self->version_messages.front().first < before) { while(!self->versionMessages.empty() && self->versionMessages.front().first < before) {
Version version = self->version_messages.front().first; Version version = self->versionMessages.front().first;
std::pair<int, int> &sizes = tlogData->version_sizes[version]; std::pair<int, int> &sizes = tlogData->version_sizes[version];
int64_t messagesErased = 0; int64_t messagesErased = 0;
while(!self->version_messages.empty() && self->version_messages.front().first == version) { while(!self->versionMessages.empty() && self->versionMessages.front().first == version) {
auto const& m = self->version_messages.front(); auto const& m = self->versionMessages.front();
++messagesErased; ++messagesErased;
if(self->update_version_sizes) { if(self->updateVersionSizes) {
sizes.first -= m.second.expectedSize(); sizes.first -= m.second.expectedSize();
} }
self->version_messages.pop_front(); self->versionMessages.pop_front();
} }
int64_t bytesErased = messagesErased * SERVER_KNOBS->VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD; int64_t bytesErased = messagesErased * SERVER_KNOBS->VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD;
@ -336,7 +336,7 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
} }
}; };
Map<Version, IDiskQueue::location> version_location; // For the version of each entry that was push()ed, the end location of the serialized bytes Map<Version, IDiskQueue::location> versionLocation; // For the version of each entry that was push()ed, the end location of the serialized bytes
/* /*
Popped version tracking contract needed by log system to implement ILogCursor::popped(): Popped version tracking contract needed by log system to implement ILogCursor::popped():
@ -378,8 +378,8 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
} }
//only callable after getTagData returns a null reference //only callable after getTagData returns a null reference
Reference<TagData> createTagData(Tag tag, Version popped, bool nothing_persistent, bool popped_recently, bool unpoppedRecovered) { Reference<TagData> createTagData(Tag tag, Version popped, bool nothingPersistent, bool poppedRecently, bool unpoppedRecovered) {
Reference<TagData> newTagData = Reference<TagData>( new TagData(tag, popped, nothing_persistent, popped_recently, unpoppedRecovered) ); Reference<TagData> newTagData = Reference<TagData>( new TagData(tag, popped, nothingPersistent, poppedRecently, unpoppedRecovered) );
int idx = tag.locality >= 0 ? 2*tag.locality : 1-(2*tag.locality); int idx = tag.locality >= 0 ? 2*tag.locality : 1-(2*tag.locality);
tag_data[idx][tag.id] = newTagData; tag_data[idx][tag.id] = newTagData;
return newTagData; return newTagData;
@ -408,7 +408,7 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
UID recruitmentID; UID recruitmentID;
explicit LogData(TLogData* tLogData, TLogInterface interf, Tag remoteTag, bool isPrimary, int logRouterTags, UID recruitmentID) : tLogData(tLogData), knownCommittedVersion(1), logId(interf.id()), explicit LogData(TLogData* tLogData, TLogInterface interf, Tag remoteTag, bool isPrimary, int logRouterTags, UID recruitmentID) : tLogData(tLogData), knownCommittedVersion(1), logId(interf.id()),
cc("TLog", interf.id().toString()), bytesInput("bytesInput", cc), bytesDurable("bytesDurable", cc), remoteTag(remoteTag), isPrimary(isPrimary), logRouterTags(logRouterTags), recruitmentID(recruitmentID), cc("TLog", interf.id().toString()), bytesInput("BytesInput", cc), bytesDurable("BytesDurable", cc), remoteTag(remoteTag), isPrimary(isPrimary), logRouterTags(logRouterTags), recruitmentID(recruitmentID),
logSystem(new AsyncVar<Reference<ILogSystem>>()), logRouterPoppedVersion(0), durableKnownCommittedVersion(0), logSystem(new AsyncVar<Reference<ILogSystem>>()), logRouterPoppedVersion(0), durableKnownCommittedVersion(0),
// These are initialized differently on init() or recovery // These are initialized differently on init() or recovery
recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), unrecoveredBefore(1), recoveredAt(1), unpoppedRecoveredTags(0), recoveryCount(), stopped(false), initialized(false), queueCommittingVersion(0), newPersistentDataVersion(invalidVersion), unrecoveredBefore(1), recoveredAt(1), unpoppedRecoveredTags(0),
@ -421,22 +421,22 @@ struct LogData : NonCopyable, public ReferenceCounted<LogData> {
version.initMetric(LiteralStringRef("TLog.Version"), cc.id); version.initMetric(LiteralStringRef("TLog.Version"), cc.id);
queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id); queueCommittedVersion.initMetric(LiteralStringRef("TLog.QueueCommittedVersion"), cc.id);
specialCounter(cc, "version", [this](){ return this->version.get(); }); specialCounter(cc, "Version", [this](){ return this->version.get(); });
specialCounter(cc, "sharedBytesInput", [tLogData](){ return tLogData->bytesInput; }); specialCounter(cc, "SharedBytesInput", [tLogData](){ return tLogData->bytesInput; });
specialCounter(cc, "sharedBytesDurable", [tLogData](){ return tLogData->bytesDurable; }); specialCounter(cc, "SharedBytesDurable", [tLogData](){ return tLogData->bytesDurable; });
specialCounter(cc, "kvstoreBytesUsed", [tLogData](){ return tLogData->persistentData->getStorageBytes().used; }); specialCounter(cc, "KvstoreBytesUsed", [tLogData](){ return tLogData->persistentData->getStorageBytes().used; });
specialCounter(cc, "kvstoreBytesFree", [tLogData](){ return tLogData->persistentData->getStorageBytes().free; }); specialCounter(cc, "KvstoreBytesFree", [tLogData](){ return tLogData->persistentData->getStorageBytes().free; });
specialCounter(cc, "kvstoreBytesAvailable", [tLogData](){ return tLogData->persistentData->getStorageBytes().available; }); specialCounter(cc, "KvstoreBytesAvailable", [tLogData](){ return tLogData->persistentData->getStorageBytes().available; });
specialCounter(cc, "kvstoreBytesTotal", [tLogData](){ return tLogData->persistentData->getStorageBytes().total; }); specialCounter(cc, "KvstoreBytesTotal", [tLogData](){ return tLogData->persistentData->getStorageBytes().total; });
specialCounter(cc, "queueDiskBytesUsed", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().used; }); specialCounter(cc, "QueueDiskBytesUsed", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().used; });
specialCounter(cc, "queueDiskBytesFree", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().free; }); specialCounter(cc, "QueueDiskBytesFree", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().free; });
specialCounter(cc, "queueDiskBytesAvailable", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().available; }); specialCounter(cc, "QueueDiskBytesAvailable", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().available; });
specialCounter(cc, "queueDiskBytesTotal", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().total; }); specialCounter(cc, "QueueDiskBytesTotal", [tLogData](){ return tLogData->rawPersistentQueue->getStorageBytes().total; });
} }
~LogData() { ~LogData() {
tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue(); tLogData->bytesDurable += bytesInput.getValue() - bytesDurable.getValue();
TraceEvent("TLogBytesWhenRemoved", logId).detail("sharedBytesInput", tLogData->bytesInput).detail("sharedBytesDurable", tLogData->bytesDurable).detail("localBytesInput", bytesInput.getValue()).detail("localBytesDurable", bytesDurable.getValue()); TraceEvent("TLogBytesWhenRemoved", logId).detail("SharedBytesInput", tLogData->bytesInput).detail("SharedBytesDurable", tLogData->bytesDurable).detail("LocalBytesInput", bytesInput.getValue()).detail("LocalBytesDurable", bytesDurable.getValue());
ASSERT_ABORT(tLogData->bytesDurable <= tLogData->bytesInput); ASSERT_ABORT(tLogData->bytesDurable <= tLogData->bytesInput);
endRole(logId, "TLog", "Error", true); endRole(logId, "TLog", "Error", true);
@ -468,28 +468,28 @@ void TLogQueue::push( T const& qe, Reference<LogData> logData ) {
*(uint32_t*)wr.getData() = wr.getLength() - sizeof(uint32_t) - sizeof(uint8_t); *(uint32_t*)wr.getData() = wr.getLength() - sizeof(uint32_t) - sizeof(uint8_t);
auto loc = queue->push( wr.toStringRef() ); auto loc = queue->push( wr.toStringRef() );
//TraceEvent("TLogQueueVersionWritten", dbgid).detail("Size", wr.getLength() - sizeof(uint32_t) - sizeof(uint8_t)).detail("Loc", loc); //TraceEvent("TLogQueueVersionWritten", dbgid).detail("Size", wr.getLength() - sizeof(uint32_t) - sizeof(uint8_t)).detail("Loc", loc);
logData->version_location[qe.version] = loc; logData->versionLocation[qe.version] = loc;
} }
void TLogQueue::pop( Version upTo, Reference<LogData> logData ) { void TLogQueue::pop( Version upTo, Reference<LogData> logData ) {
// Keep only the given and all subsequent version numbers // Keep only the given and all subsequent version numbers
// Find the first version >= upTo // Find the first version >= upTo
auto v = logData->version_location.lower_bound(upTo); auto v = logData->versionLocation.lower_bound(upTo);
if (v == logData->version_location.begin()) return; if (v == logData->versionLocation.begin()) return;
if(v == logData->version_location.end()) { if(v == logData->versionLocation.end()) {
v = logData->version_location.lastItem(); v = logData->versionLocation.lastItem();
} }
else { else {
v.decrementNonEnd(); v.decrementNonEnd();
} }
queue->pop( v->value ); queue->pop( v->value );
logData->version_location.erase( logData->version_location.begin(), v ); // ... and then we erase that previous version and all prior versions logData->versionLocation.erase( logData->versionLocation.begin(), v ); // ... and then we erase that previous version and all prior versions
} }
void TLogQueue::updateVersionSizes( const TLogQueueEntry& result, TLogData* tLog ) { void TLogQueue::updateVersionSizes( const TLogQueueEntry& result, TLogData* tLog ) {
auto it = tLog->id_data.find(result.id); auto it = tLog->id_data.find(result.id);
if(it != tLog->id_data.end()) { if(it != tLog->id_data.end()) {
it->second->version_location[result.version] = queue->getNextReadLocation(); it->second->versionLocation[result.version] = queue->getNextReadLocation();
} }
} }
@ -500,7 +500,7 @@ ACTOR Future<Void> tLogLock( TLogData* self, ReplyPromise< TLogLockResult > repl
TEST( logData->stopped ); TEST( logData->stopped );
TEST( !logData->stopped ); TEST( !logData->stopped );
TraceEvent("TLogStop", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get()); TraceEvent("TLogStop", logData->logId).detail("Ver", stopVersion).detail("IsStopped", logData->stopped).detail("QueueCommitted", logData->queueCommittedVersion.get());
logData->stopped = true; logData->stopped = true;
if(!logData->recoveryComplete.isSet()) { if(!logData->recoveryComplete.isSet()) {
@ -516,24 +516,24 @@ ACTOR Future<Void> tLogLock( TLogData* self, ReplyPromise< TLogLockResult > repl
result.end = stopVersion; result.end = stopVersion;
result.knownCommittedVersion = logData->knownCommittedVersion; result.knownCommittedVersion = logData->knownCommittedVersion;
TraceEvent("TLogStop2", self->dbgid).detail("logId", logData->logId).detail("Ver", stopVersion).detail("isStopped", logData->stopped).detail("queueCommitted", logData->queueCommittedVersion.get()).detail("knownCommitted", result.knownCommittedVersion); TraceEvent("TLogStop2", self->dbgid).detail("LogId", logData->logId).detail("Ver", stopVersion).detail("IsStopped", logData->stopped).detail("QueueCommitted", logData->queueCommittedVersion.get()).detail("KnownCommitted", result.knownCommittedVersion);
reply.send( result ); reply.send( result );
return Void(); return Void();
} }
void updatePersistentPopped( TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data ) { void updatePersistentPopped( TLogData* self, Reference<LogData> logData, Reference<LogData::TagData> data ) {
if (!data->popped_recently) return; if (!data->poppedRecently) return;
self->persistentData->set(KeyValueRef( persistTagPoppedKey(logData->logId, data->tag), persistTagPoppedValue(data->popped) )); self->persistentData->set(KeyValueRef( persistTagPoppedKey(logData->logId, data->tag), persistTagPoppedValue(data->popped) ));
data->popped_recently = false; data->poppedRecently = false;
if (data->nothing_persistent) return; if (data->nothingPersistent) return;
self->persistentData->clear( KeyRangeRef( self->persistentData->clear( KeyRangeRef(
persistTagMessagesKey( logData->logId, data->tag, Version(0) ), persistTagMessagesKey( logData->logId, data->tag, Version(0) ),
persistTagMessagesKey( logData->logId, data->tag, data->popped ) ) ); persistTagMessagesKey( logData->logId, data->tag, data->popped ) ) );
if (data->popped > logData->persistentDataVersion) if (data->popped > logData->persistentDataVersion)
data->nothing_persistent = true; data->nothingPersistent = true;
} }
ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logData, Version newPersistentDataVersion ) { ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logData, Version newPersistentDataVersion ) {
@ -543,30 +543,30 @@ ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logD
ASSERT( newPersistentDataVersion > logData->persistentDataVersion ); ASSERT( newPersistentDataVersion > logData->persistentDataVersion );
ASSERT( logData->persistentDataVersion == logData->persistentDataDurableVersion ); ASSERT( logData->persistentDataVersion == logData->persistentDataDurableVersion );
//TraceEvent("updatePersistentData", self->dbgid).detail("seq", newPersistentDataSeq); //TraceEvent("UpdatePersistentData", self->dbgid).detail("Seq", newPersistentDataSeq);
state bool anyData = false; state bool anyData = false;
// For all existing tags // For all existing tags
state int tag_locality = 0; state int tagLocality = 0;
state int tag_id = 0; state int tagId = 0;
for(tag_locality = 0; tag_locality < logData->tag_data.size(); tag_locality++) { for(tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for(tag_id = 0; tag_id < logData->tag_data[tag_locality].size(); tag_id++) { for(tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
state Reference<LogData::TagData> tagData = logData->tag_data[tag_locality][tag_id]; state Reference<LogData::TagData> tagData = logData->tag_data[tagLocality][tagId];
if(tagData) { if(tagData) {
state Version currentVersion = 0; state Version currentVersion = 0;
// Clear recently popped versions from persistentData if necessary // Clear recently popped versions from persistentData if necessary
updatePersistentPopped( self, logData, tagData ); updatePersistentPopped( self, logData, tagData );
// Transfer unpopped messages with version numbers less than newPersistentDataVersion to persistentData // Transfer unpopped messages with version numbers less than newPersistentDataVersion to persistentData
state std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator msg = tagData->version_messages.begin(); state std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator msg = tagData->versionMessages.begin();
while(msg != tagData->version_messages.end() && msg->first <= newPersistentDataVersion) { while(msg != tagData->versionMessages.end() && msg->first <= newPersistentDataVersion) {
currentVersion = msg->first; currentVersion = msg->first;
anyData = true; anyData = true;
tagData->nothing_persistent = false; tagData->nothingPersistent = false;
BinaryWriter wr( Unversioned() ); BinaryWriter wr( Unversioned() );
for(; msg != tagData->version_messages.end() && msg->first == currentVersion; ++msg) for(; msg != tagData->versionMessages.end() && msg->first == currentVersion; ++msg)
wr << msg->second.toStringRef(); wr << msg->second.toStringRef();
self->persistentData->set( KeyValueRef( persistTagMessagesKey( logData->logId, tagData->tag, currentVersion ), wr.toStringRef() ) ); self->persistentData->set( KeyValueRef( persistTagMessagesKey( logData->logId, tagData->tag, currentVersion ), wr.toStringRef() ) );
@ -574,7 +574,7 @@ ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logD
Future<Void> f = yield(TaskUpdateStorage); Future<Void> f = yield(TaskUpdateStorage);
if(!f.isReady()) { if(!f.isReady()) {
Void _ = wait(f); Void _ = wait(f);
msg = std::upper_bound(tagData->version_messages.begin(), tagData->version_messages.end(), std::make_pair(currentVersion, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>()); msg = std::upper_bound(tagData->versionMessages.begin(), tagData->versionMessages.end(), std::make_pair(currentVersion, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>());
} }
} }
@ -595,10 +595,10 @@ ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logD
TEST(anyData); // TLog moved data to persistentData TEST(anyData); // TLog moved data to persistentData
logData->persistentDataDurableVersion = newPersistentDataVersion; logData->persistentDataDurableVersion = newPersistentDataVersion;
for(tag_locality = 0; tag_locality < logData->tag_data.size(); tag_locality++) { for(tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for(tag_id = 0; tag_id < logData->tag_data[tag_locality].size(); tag_id++) { for(tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
if(logData->tag_data[tag_locality][tag_id]) { if(logData->tag_data[tagLocality][tagId]) {
Void _ = wait(logData->tag_data[tag_locality][tag_id]->eraseMessagesBefore( newPersistentDataVersion+1, &self->bytesDurable, logData, TaskUpdateStorage )); Void _ = wait(logData->tag_data[tagLocality][tagId]->eraseMessagesBefore( newPersistentDataVersion+1, &self->bytesDurable, logData, TaskUpdateStorage ));
Void _ = wait(yield(TaskUpdateStorage)); Void _ = wait(yield(TaskUpdateStorage));
} }
} }
@ -617,7 +617,7 @@ ACTOR Future<Void> updatePersistentData( TLogData* self, Reference<LogData> logD
} }
if(logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) { if(logData->bytesDurable.getValue() > logData->bytesInput.getValue() || self->bytesDurable > self->bytesInput) {
TraceEvent(SevError, "BytesDurableTooLarge", logData->logId).detail("sharedBytesInput", self->bytesInput).detail("sharedBytesDurable", self->bytesDurable).detail("localBytesInput", logData->bytesInput.getValue()).detail("localBytesDurable", logData->bytesDurable.getValue()); TraceEvent(SevError, "BytesDurableTooLarge", logData->logId).detail("SharedBytesInput", self->bytesInput).detail("SharedBytesDurable", self->bytesDurable).detail("LocalBytesInput", logData->bytesInput.getValue()).detail("LocalBytesDurable", logData->bytesDurable.getValue());
} }
ASSERT(logData->bytesDurable.getValue() <= logData->bytesInput.getValue()); ASSERT(logData->bytesDurable.getValue() <= logData->bytesInput.getValue());
@ -647,8 +647,8 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
state Version nextVersion = 0; state Version nextVersion = 0;
state int totalSize = 0; state int totalSize = 0;
state int tag_locality = 0; state int tagLocality = 0;
state int tag_id = 0; state int tagId = 0;
state Reference<LogData::TagData> tagData; state Reference<LogData::TagData> tagData;
if(logData->stopped) { if(logData->stopped) {
@ -656,11 +656,11 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
while(logData->persistentDataDurableVersion != logData->version.get()) { while(logData->persistentDataDurableVersion != logData->version.get()) {
std::vector<std::pair<std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator, std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator>> iters; std::vector<std::pair<std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator, std::deque<std::pair<Version, LengthPrefixedStringRef>>::iterator>> iters;
for(tag_locality = 0; tag_locality < logData->tag_data.size(); tag_locality++) { for(tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for(tag_id = 0; tag_id < logData->tag_data[tag_locality].size(); tag_id++) { for(tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
tagData = logData->tag_data[tag_locality][tag_id]; tagData = logData->tag_data[tagLocality][tagId];
if(tagData) { if(tagData) {
iters.push_back(std::make_pair(tagData->version_messages.begin(), tagData->version_messages.end())); iters.push_back(std::make_pair(tagData->versionMessages.begin(), tagData->versionMessages.end()));
} }
} }
} }
@ -686,7 +686,7 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) ); Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) );
Void _ = wait( delay(0, TaskUpdateStorage) ); Void _ = wait( delay(0, TaskUpdateStorage) );
//TraceEvent("TlogUpdatePersist", self->dbgid).detail("logId", logData->logId).detail("nextVersion", nextVersion).detail("version", logData->version.get()).detail("persistentDataDurableVer", logData->persistentDataDurableVersion).detail("queueCommitVer", logData->queueCommittedVersion.get()).detail("persistDataVer", logData->persistentDataVersion); //TraceEvent("TlogUpdatePersist", self->dbgid).detail("LogId", logData->logId).detail("NextVersion", nextVersion).detail("Version", logData->version.get()).detail("PersistentDataDurableVer", logData->persistentDataDurableVersion).detail("QueueCommitVer", logData->queueCommittedVersion.get()).detail("PersistDataVer", logData->persistentDataVersion);
if (nextVersion > logData->persistentDataVersion) { if (nextVersion > logData->persistentDataVersion) {
self->updatePersist = updatePersistentData(self, logData, nextVersion); self->updatePersist = updatePersistentData(self, logData, nextVersion);
Void _ = wait( self->updatePersist ); Void _ = wait( self->updatePersist );
@ -718,12 +718,12 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
++sizeItr; ++sizeItr;
nextVersion = sizeItr == logData->version_sizes.end() ? logData->version.get() : sizeItr->key; nextVersion = sizeItr == logData->version_sizes.end() ? logData->version.get() : sizeItr->key;
for(tag_locality = 0; tag_locality < logData->tag_data.size(); tag_locality++) { for(tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for(tag_id = 0; tag_id < logData->tag_data[tag_locality].size(); tag_id++) { for(tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
tagData = logData->tag_data[tag_locality][tag_id]; tagData = logData->tag_data[tagLocality][tagId];
if(tagData) { if(tagData) {
auto it = std::lower_bound(tagData->version_messages.begin(), tagData->version_messages.end(), std::make_pair(prevVersion, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>()); auto it = std::lower_bound(tagData->versionMessages.begin(), tagData->versionMessages.end(), std::make_pair(prevVersion, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>());
for(; it != tagData->version_messages.end() && it->first < nextVersion; ++it) { for(; it != tagData->versionMessages.end() && it->first < nextVersion; ++it) {
totalSize += it->second.expectedSize(); totalSize += it->second.expectedSize();
} }
@ -737,7 +737,7 @@ ACTOR Future<Void> updateStorage( TLogData* self ) {
nextVersion = std::max<Version>(nextVersion, logData->persistentDataVersion); nextVersion = std::max<Version>(nextVersion, logData->persistentDataVersion);
//TraceEvent("UpdateStorageVer", logData->logId).detail("nextVersion", nextVersion).detail("persistentDataVersion", logData->persistentDataVersion).detail("totalSize", totalSize); //TraceEvent("UpdateStorageVer", logData->logId).detail("NextVersion", nextVersion).detail("PersistentDataVersion", logData->persistentDataVersion).detail("TotalSize", totalSize);
Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) ); Void _ = wait( logData->queueCommittedVersion.whenAtLeast( nextVersion ) );
Void _ = wait( delay(0, TaskUpdateStorage) ); Void _ = wait( delay(0, TaskUpdateStorage) );
@ -820,12 +820,12 @@ void commitMessages( Reference<LogData> self, Version version, const std::vector
} }
if (version >= tagData->popped) { if (version >= tagData->popped) {
tagData->version_messages.push_back(std::make_pair(version, LengthPrefixedStringRef((uint32_t*)(block.end() - msg.message.size())))); tagData->versionMessages.push_back(std::make_pair(version, LengthPrefixedStringRef((uint32_t*)(block.end() - msg.message.size()))));
if(tagData->version_messages.back().second.expectedSize() > SERVER_KNOBS->MAX_MESSAGE_SIZE) { if(tagData->versionMessages.back().second.expectedSize() > SERVER_KNOBS->MAX_MESSAGE_SIZE) {
TraceEvent(SevWarnAlways, "LargeMessage").detail("Size", tagData->version_messages.back().second.expectedSize()); TraceEvent(SevWarnAlways, "LargeMessage").detail("Size", tagData->versionMessages.back().second.expectedSize());
} }
if (tag != txsTag) { if (tag != txsTag) {
expectedBytes += tagData->version_messages.back().second.expectedSize(); expectedBytes += tagData->versionMessages.back().second.expectedSize();
} }
// The factor of VERSION_MESSAGES_OVERHEAD is intended to be an overestimate of the actual memory used to store this data in a std::deque. // The factor of VERSION_MESSAGES_OVERHEAD is intended to be an overestimate of the actual memory used to store this data in a std::deque.
@ -845,7 +845,7 @@ void commitMessages( Reference<LogData> self, Version version, const std::vector
self->bytesInput += addedBytes; self->bytesInput += addedBytes;
bytesInput += addedBytes; bytesInput += addedBytes;
//TraceEvent("TLogPushed", self->dbgid).detail("Bytes", addedBytes).detail("MessageBytes", messages.size()).detail("Tags", tags.size()).detail("expectedBytes", expectedBytes).detail("mCount", mCount).detail("tCount", tCount); //TraceEvent("TLogPushed", self->dbgid).detail("Bytes", addedBytes).detail("MessageBytes", messages.size()).detail("Tags", tags.size()).detail("ExpectedBytes", expectedBytes).detail("MCount", mCount).detail("TCount", tCount);
} }
void commitMessages( Reference<LogData> self, Version version, Arena arena, StringRef messages, int64_t& bytesInput ) { void commitMessages( Reference<LogData> self, Version version, Arena arena, StringRef messages, int64_t& bytesInput ) {
@ -878,13 +878,13 @@ Version poppedVersion( Reference<LogData> self, Tag tag) {
return tagData->popped; return tagData->popped;
} }
std::deque<std::pair<Version, LengthPrefixedStringRef>> & get_version_messages( Reference<LogData> self, Tag tag ) { std::deque<std::pair<Version, LengthPrefixedStringRef>> & getVersionMessages( Reference<LogData> self, Tag tag ) {
auto tagData = self->getTagData(tag); auto tagData = self->getTagData(tag);
if (!tagData) { if (!tagData) {
static std::deque<std::pair<Version, LengthPrefixedStringRef>> empty; static std::deque<std::pair<Version, LengthPrefixedStringRef>> empty;
return empty; return empty;
} }
return tagData->version_messages; return tagData->versionMessages;
}; };
ACTOR Future<Void> tLogPop( TLogData* self, TLogPopRequest req, Reference<LogData> logData ) { ACTOR Future<Void> tLogPop( TLogData* self, TLogPopRequest req, Reference<LogData> logData ) {
@ -893,12 +893,12 @@ ACTOR Future<Void> tLogPop( TLogData* self, TLogPopRequest req, Reference<LogDat
tagData = logData->createTagData(req.tag, req.to, true, true, false); tagData = logData->createTagData(req.tag, req.to, true, true, false);
} else if (req.to > tagData->popped) { } else if (req.to > tagData->popped) {
tagData->popped = req.to; tagData->popped = req.to;
tagData->popped_recently = true; tagData->poppedRecently = true;
if(tagData->unpoppedRecovered && req.to > logData->recoveredAt) { if(tagData->unpoppedRecovered && req.to > logData->recoveredAt) {
tagData->unpoppedRecovered = false; tagData->unpoppedRecovered = false;
logData->unpoppedRecoveredTags--; logData->unpoppedRecoveredTags--;
TraceEvent("TLogPoppedTag", logData->logId).detail("tags", logData->unpoppedRecoveredTags).detail("tag", req.tag.toString()).detail("durableKCVer", logData->durableKnownCommittedVersion).detail("recoveredAt", logData->recoveredAt); TraceEvent("TLogPoppedTag", logData->logId).detail("Tags", logData->unpoppedRecoveredTags).detail("Tag", req.tag.toString()).detail("DurableKCVer", logData->durableKnownCommittedVersion).detail("RecoveredAt", logData->recoveredAt);
if(logData->unpoppedRecoveredTags == 0 && logData->durableKnownCommittedVersion >= logData->recoveredAt && logData->recoveryComplete.canBeSet()) { if(logData->unpoppedRecoveredTags == 0 && logData->durableKnownCommittedVersion >= logData->recoveredAt && logData->recoveryComplete.canBeSet()) {
logData->recoveryComplete.send(Void()); logData->recoveryComplete.send(Void());
} }
@ -916,8 +916,8 @@ ACTOR Future<Void> tLogPop( TLogData* self, TLogPopRequest req, Reference<LogDat
void peekMessagesFromMemory( Reference<LogData> self, TLogPeekRequest const& req, BinaryWriter& messages, Version& endVersion ) { void peekMessagesFromMemory( Reference<LogData> self, TLogPeekRequest const& req, BinaryWriter& messages, Version& endVersion ) {
ASSERT( !messages.getLength() ); ASSERT( !messages.getLength() );
auto& deque = get_version_messages(self, req.tag); auto& deque = getVersionMessages(self, req.tag);
//TraceEvent("tLogPeekMem", self->dbgid).detail("Tag", printable(req.tag1)).detail("pDS", self->persistentDataSequence).detail("pDDS", self->persistentDataDurableSequence).detail("Oldest", map1.empty() ? 0 : map1.begin()->key ).detail("OldestMsgCount", map1.empty() ? 0 : map1.begin()->value.size()); //TraceEvent("TLogPeekMem", self->dbgid).detail("Tag", printable(req.tag1)).detail("PDS", self->persistentDataSequence).detail("PDDS", self->persistentDataDurableSequence).detail("Oldest", map1.empty() ? 0 : map1.begin()->key ).detail("OldestMsgCount", map1.empty() ? 0 : map1.begin()->value.size());
Version begin = std::max( req.begin, self->persistentDataDurableVersion+1 ); Version begin = std::max( req.begin, self->persistentDataDurableVersion+1 );
auto it = std::lower_bound(deque.begin(), deque.end(), std::make_pair(begin, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>()); auto it = std::lower_bound(deque.begin(), deque.end(), std::make_pair(begin, LengthPrefixedStringRef()), CompareFirst<std::pair<Version, LengthPrefixedStringRef>>());
@ -927,7 +927,7 @@ void peekMessagesFromMemory( Reference<LogData> self, TLogPeekRequest const& req
if(it->first != currentVersion) { if(it->first != currentVersion) {
if (messages.getLength() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) { if (messages.getLength() >= SERVER_KNOBS->DESIRED_TOTAL_BYTES) {
endVersion = currentVersion + 1; endVersion = currentVersion + 1;
//TraceEvent("tLogPeekMessagesReached2", self->dbgid); //TraceEvent("TLogPeekMessagesReached2", self->dbgid);
break; break;
} }
@ -971,7 +971,7 @@ ACTOR Future<Void> tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere
return Void(); return Void();
} }
//TraceEvent("tLogPeekMessages0", self->dbgid).detail("reqBeginEpoch", req.begin.epoch).detail("reqBeginSeq", req.begin.sequence).detail("epoch", self->epoch()).detail("persistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2)); //TraceEvent("TLogPeekMessages0", self->dbgid).detail("ReqBeginEpoch", req.begin.epoch).detail("ReqBeginSeq", req.begin.sequence).detail("Epoch", self->epoch()).detail("PersistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2));
// Wait until we have something to return that the caller doesn't already have // Wait until we have something to return that the caller doesn't already have
if( logData->version.get() < req.begin ) { if( logData->version.get() < req.begin ) {
Void _ = wait( logData->version.whenAtLeast( req.begin ) ); Void _ = wait( logData->version.whenAtLeast( req.begin ) );
@ -1007,7 +1007,7 @@ ACTOR Future<Void> tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere
state Version endVersion = logData->version.get() + 1; state Version endVersion = logData->version.get() + 1;
//grab messages from disk //grab messages from disk
//TraceEvent("tLogPeekMessages", self->dbgid).detail("reqBeginEpoch", req.begin.epoch).detail("reqBeginSeq", req.begin.sequence).detail("epoch", self->epoch()).detail("persistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2)); //TraceEvent("TLogPeekMessages", self->dbgid).detail("ReqBeginEpoch", req.begin.epoch).detail("ReqBeginSeq", req.begin.sequence).detail("Epoch", self->epoch()).detail("PersistentDataSeq", self->persistentDataSequence).detail("Tag1", printable(req.tag1)).detail("Tag2", printable(req.tag2));
if( req.begin <= logData->persistentDataDurableVersion ) { if( req.begin <= logData->persistentDataDurableVersion ) {
// Just in case the durable version changes while we are waiting for the read, we grab this data from memory. We may or may not actually send it depending on // Just in case the durable version changes while we are waiting for the read, we grab this data from memory. We may or may not actually send it depending on
// whether we get enough data from disk. // whether we get enough data from disk.
@ -1043,7 +1043,7 @@ ACTOR Future<Void> tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere
reply.messages = messages.toStringRef(); reply.messages = messages.toStringRef();
reply.end = endVersion; reply.end = endVersion;
//TraceEvent("TlogPeek", self->dbgid).detail("logId", logData->logId).detail("endVer", reply.end).detail("msgBytes", reply.messages.expectedSize()).detail("ForAddress", req.reply.getEndpoint().address); //TraceEvent("TlogPeek", self->dbgid).detail("LogId", logData->logId).detail("EndVer", reply.end).detail("MsgBytes", reply.messages.expectedSize()).detail("ForAddress", req.reply.getEndpoint().address);
if(req.sequence.present()) { if(req.sequence.present()) {
auto& trackerData = self->peekTracker[peekId]; auto& trackerData = self->peekTracker[peekId];
@ -1087,7 +1087,7 @@ ACTOR Future<Void> doQueueCommit( TLogData* self, Reference<LogData> logData ) {
logData->durableKnownCommittedVersion = knownCommittedVersion; logData->durableKnownCommittedVersion = knownCommittedVersion;
if(logData->unpoppedRecoveredTags == 0 && knownCommittedVersion >= logData->recoveredAt && logData->recoveryComplete.canBeSet()) { if(logData->unpoppedRecoveredTags == 0 && knownCommittedVersion >= logData->recoveredAt && logData->recoveryComplete.canBeSet()) {
TraceEvent("TLogRecoveryComplete", logData->logId).detail("tags", logData->unpoppedRecoveredTags).detail("durableKCVer", logData->durableKnownCommittedVersion).detail("recoveredAt", logData->recoveredAt); TraceEvent("TLogRecoveryComplete", logData->logId).detail("Tags", logData->unpoppedRecoveredTags).detail("DurableKCVer", logData->durableKnownCommittedVersion).detail("RecoveredAt", logData->recoveredAt);
logData->recoveryComplete.send(Void()); logData->recoveryComplete.send(Void());
} }
@ -1121,7 +1121,7 @@ ACTOR Future<Void> commitQueue( TLogData* self ) {
continue; continue;
} }
TraceEvent("commitQueueNewLog", self->dbgid).detail("logId", logData->logId).detail("version", logData->version.get()).detail("committing", logData->queueCommittingVersion).detail("commmitted", logData->queueCommittedVersion.get()); TraceEvent("CommitQueueNewLog", self->dbgid).detail("LogId", logData->logId).detail("Version", logData->version.get()).detail("Committing", logData->queueCommittingVersion).detail("Commmitted", logData->queueCommittedVersion.get());
if(logData->committingQueue.canBeSet()) { if(logData->committingQueue.canBeSet()) {
logData->committingQueue.send(Void()); logData->committingQueue.send(Void());
} }
@ -1268,8 +1268,8 @@ ACTOR Future<Void> rejoinMasters( TLogData* self, TLogInterface tli, DBRecoveryC
} }
if ( isDisplaced ) if ( isDisplaced )
{ {
TraceEvent("TLogDisplaced", tli.id()).detail("Reason", "DBInfoDoesNotContain").detail("recoveryCount", recoveryCount).detail("infRecoveryCount", inf.recoveryCount).detail("recoveryState", inf.recoveryState) TraceEvent("TLogDisplaced", tli.id()).detail("Reason", "DBInfoDoesNotContain").detail("RecoveryCount", recoveryCount).detail("InfRecoveryCount", inf.recoveryCount).detail("RecoveryState", inf.recoveryState)
.detail("logSysConf", describe(inf.logSystemConfig.tLogs)).detail("priorLogs", describe(inf.priorCommittedLogServers)).detail("oldLogGens", inf.logSystemConfig.oldTLogs.size()); .detail("LogSysConf", describe(inf.logSystemConfig.tLogs)).detail("PriorLogs", describe(inf.priorCommittedLogServers)).detail("OldLogGens", inf.logSystemConfig.oldTLogs.size());
if (BUGGIFY) Void _ = wait( delay( SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * g_random->random01() ) ); if (BUGGIFY) Void _ = wait( delay( SERVER_KNOBS->BUGGIFY_WORKER_REMOVED_MAX_LAG * g_random->random01() ) );
throw worker_removed(); throw worker_removed();
} }
@ -1305,7 +1305,7 @@ ACTOR Future<Void> respondToRecovered( TLogInterface tli, Promise<Void> recovery
} }
finishedRecovery = false; finishedRecovery = false;
} }
TraceEvent("TLogRespondToRecovered", tli.id()).detail("finished", finishedRecovery); TraceEvent("TLogRespondToRecovered", tli.id()).detail("Finished", finishedRecovery);
loop { loop {
TLogRecoveryFinishedRequest req = waitNext( tli.recoveryFinished.getFuture() ); TLogRecoveryFinishedRequest req = waitNext( tli.recoveryFinished.getFuture() );
if(finishedRecovery) { if(finishedRecovery) {
@ -1386,7 +1386,7 @@ ACTOR Future<Void> serveTLogInterface( TLogData* self, TLogInterface tli, Refere
logData->addActor.send( tLogPop( self, req, logData ) ); logData->addActor.send( tLogPop( self, req, logData ) );
} }
when( TLogCommitRequest req = waitNext( tli.commit.getFuture() ) ) { when( TLogCommitRequest req = waitNext( tli.commit.getFuture() ) ) {
//TraceEvent("TLogCommitReq", logData->logId).detail("ver", req.version).detail("prevVer", req.prevVersion).detail("logVer", logData->version.get()); //TraceEvent("TLogCommitReq", logData->logId).detail("Ver", req.version).detail("PrevVer", req.prevVersion).detail("LogVer", logData->version.get());
ASSERT(logData->isPrimary); ASSERT(logData->isPrimary);
TEST(logData->stopped); // TLogCommitRequest while stopped TEST(logData->stopped); // TLogCommitRequest while stopped
if (!logData->stopped) if (!logData->stopped)
@ -1415,7 +1415,7 @@ ACTOR Future<Void> serveTLogInterface( TLogData* self, TLogInterface tli, Refere
} }
void removeLog( TLogData* self, Reference<LogData> logData ) { void removeLog( TLogData* self, Reference<LogData> logData ) {
TraceEvent("TLogRemoved", logData->logId).detail("input", logData->bytesInput.getValue()).detail("durable", logData->bytesDurable.getValue()); TraceEvent("TLogRemoved", logData->logId).detail("Input", logData->bytesInput.getValue()).detail("Durable", logData->bytesDurable.getValue());
logData->stopped = true; logData->stopped = true;
if(!logData->recoveryComplete.isSet()) { if(!logData->recoveryComplete.isSet()) {
logData->recoveryComplete.sendError(end_of_stream()); logData->recoveryComplete.sendError(end_of_stream());
@ -1702,7 +1702,7 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
logData->removed = rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture(), false); logData->removed = rejoinMasters(self, recruited, logData->recoveryCount, registerWithMaster.getFuture(), false);
removed.push_back(errorOr(logData->removed)); removed.push_back(errorOr(logData->removed));
TraceEvent("TLogRestorePersistentStateVer", id1).detail("ver", ver); TraceEvent("TLogRestorePersistentStateVer", id1).detail("Ver", ver);
// Restore popped keys. Pop operations that took place after the last (committed) updatePersistentDataVersion might be lost, but // Restore popped keys. Pop operations that took place after the last (committed) updatePersistentDataVersion might be lost, but
// that is fine because we will get the corresponding data back, too. // that is fine because we will get the corresponding data back, too.
@ -1748,8 +1748,8 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
} }
} }
//TraceEvent("TLogRecoveredQE", self->dbgid).detail("logId", qe.id).detail("ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size()) //TraceEvent("TLogRecoveredQE", self->dbgid).detail("LogId", qe.id).detail("Ver", qe.version).detail("MessageBytes", qe.messages.size()).detail("Tags", qe.tags.size())
// .detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("version", logData->version.get()); // .detail("Tag0", qe.tags.size() ? qe.tags[0].tag : invalidTag).detail("Version", logData->version.get());
if(logData) { if(logData) {
logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion); logData->knownCommittedVersion = std::max(logData->knownCommittedVersion, qe.knownCommittedVersion);
@ -1782,7 +1782,7 @@ ACTOR Future<Void> restorePersistentState( TLogData* self, LocalityData locality
for(auto it : self->id_data) { for(auto it : self->id_data) {
if(it.second->queueCommittedVersion.get() == 0) { if(it.second->queueCommittedVersion.get() == 0) {
TraceEvent("TLogZeroVersion", self->dbgid).detail("logId", it.first); TraceEvent("TLogZeroVersion", self->dbgid).detail("LogId", it.first);
it.second->queueCommittedVersion.set(it.second->version.get()); it.second->queueCommittedVersion.set(it.second->version.get());
} }
it.second->recoveryComplete.sendError(end_of_stream()); it.second->recoveryComplete.sendError(end_of_stream());
@ -1835,9 +1835,9 @@ ACTOR Future<Void> updateLogSystem(TLogData* self, Reference<LogData> logData, L
} else { } else {
logData->logSystem->get()->pop(logData->logRouterPoppedVersion, logData->remoteTag, logData->durableKnownCommittedVersion, logData->locality); logData->logSystem->get()->pop(logData->logRouterPoppedVersion, logData->remoteTag, logData->durableKnownCommittedVersion, logData->locality);
} }
TraceEvent("TLogUpdate", self->dbgid).detail("logId", logData->logId).detail("recruitmentID", logData->recruitmentID).detail("dbRecruitmentID", self->dbInfo->get().logSystemConfig.recruitmentID).detail("recoverFrom", recoverFrom.toString()).detail("dbInfo", self->dbInfo->get().logSystemConfig.toString()).detail("found", found).detail("logSystem", (bool) logSystem->get() ).detail("recoveryState", self->dbInfo->get().recoveryState); TraceEvent("TLogUpdate", self->dbgid).detail("LogId", logData->logId).detail("RecruitmentID", logData->recruitmentID).detail("DbRecruitmentID", self->dbInfo->get().logSystemConfig.recruitmentID).detail("RecoverFrom", recoverFrom.toString()).detail("DbInfo", self->dbInfo->get().logSystemConfig.toString()).detail("Found", found).detail("LogSystem", (bool) logSystem->get() ).detail("RecoveryState", self->dbInfo->get().recoveryState);
for(auto it : self->dbInfo->get().logSystemConfig.oldTLogs) { for(auto it : self->dbInfo->get().logSystemConfig.oldTLogs) {
TraceEvent("TLogUpdateOld", self->dbgid).detail("logId", logData->logId).detail("dbInfo", it.toString()); TraceEvent("TLogUpdateOld", self->dbgid).detail("LogId", logData->logId).detail("DbInfo", it.toString());
} }
Void _ = wait( self->dbInfo->onChange() ); Void _ = wait( self->dbInfo->onChange() );
} }
@ -1857,7 +1857,7 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
for(auto it : self->id_data) { for(auto it : self->id_data) {
if( !it.second->stopped ) { if( !it.second->stopped ) {
TraceEvent("TLogStoppedByNewRecruitment", self->dbgid).detail("stoppedId", it.first.toString()).detail("recruitedId", recruited.id()).detail("endEpoch", it.second->logSystem->get().getPtr() != 0); TraceEvent("TLogStoppedByNewRecruitment", self->dbgid).detail("StoppedId", it.first.toString()).detail("RecruitedId", recruited.id()).detail("EndEpoch", it.second->logSystem->get().getPtr() != 0);
if(!it.second->isPrimary && it.second->logSystem->get()) { if(!it.second->isPrimary && it.second->logSystem->get()) {
it.second->removed = it.second->removed && it.second->logSystem->get()->endEpoch(); it.second->removed = it.second->removed && it.second->logSystem->get()->endEpoch();
} }
@ -1898,7 +1898,7 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
logData->unpoppedRecoveredTags = req.allTags.size(); logData->unpoppedRecoveredTags = req.allTags.size();
Void _ = wait( initPersistentState( self, logData, req.allTags ) || logData->removed ); Void _ = wait( initPersistentState( self, logData, req.allTags ) || logData->removed );
TraceEvent("TLogRecover", self->dbgid).detail("logId", logData->logId).detail("at", req.recoverAt).detail("known", req.knownCommittedVersion).detail("unrecovered", logData->unrecoveredBefore).detail("tags", describe(req.recoverTags)).detail("locality", req.locality).detail("logRouterTags", logData->logRouterTags); TraceEvent("TLogRecover", self->dbgid).detail("LogId", logData->logId).detail("At", req.recoverAt).detail("Known", req.knownCommittedVersion).detail("Unrecovered", logData->unrecoveredBefore).detail("Tags", describe(req.recoverTags)).detail("Locality", req.locality).detail("LogRouterTags", logData->logRouterTags);
if(logData->recoveryComplete.isSet()) { if(logData->recoveryComplete.isSet()) {
throw worker_removed(); throw worker_removed();
@ -1916,7 +1916,7 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
Void _ = wait(pullAsyncData(self, logData, tags, logData->unrecoveredBefore, req.knownCommittedVersion, true) || logData->removed); Void _ = wait(pullAsyncData(self, logData, tags, logData->unrecoveredBefore, req.knownCommittedVersion, true) || logData->removed);
} }
TraceEvent("TLogPullComplete", self->dbgid).detail("logId", logData->logId); TraceEvent("TLogPullComplete", self->dbgid).detail("LogId", logData->logId);
if(req.isPrimary && !req.recoverTags.empty() && !logData->stopped && req.knownCommittedVersion < req.recoverAt) { if(req.isPrimary && !req.recoverTags.empty() && !logData->stopped && req.knownCommittedVersion < req.recoverAt) {
Void _ = wait(pullAsyncData(self, logData, req.recoverTags, req.knownCommittedVersion + 1, req.recoverAt, false) || logData->removed); Void _ = wait(pullAsyncData(self, logData, req.recoverTags, req.knownCommittedVersion + 1, req.recoverAt, false) || logData->removed);
@ -1948,11 +1948,11 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
//so we need to pop all tags that did not have data at the recovery version. //so we need to pop all tags that did not have data at the recovery version.
std::vector<Future<Void>> popFutures; std::vector<Future<Void>> popFutures;
std::set<Tag> allTags(req.allTags.begin(), req.allTags.end()); std::set<Tag> allTags(req.allTags.begin(), req.allTags.end());
for(int tag_locality = 0; tag_locality < logData->tag_data.size(); tag_locality++) { for(int tagLocality = 0; tagLocality < logData->tag_data.size(); tagLocality++) {
for(int tag_id = 0; tag_id < logData->tag_data[tag_locality].size(); tag_id++) { for(int tagId = 0; tagId < logData->tag_data[tagLocality].size(); tagId++) {
auto data = logData->tag_data[tag_locality][tag_id]; auto data = logData->tag_data[tagLocality][tagId];
if(data && !allTags.count(data->tag) && data->tag.locality != tagLocalityLogRouter) { if(data && !allTags.count(data->tag) && data->tag.locality != tagLocalityLogRouter) {
TraceEvent("TLogPopOnRecover", self->dbgid).detail("logId", logData->logId).detail("tag", data->tag.toString()).detail("ver", req.recoverAt); TraceEvent("TLogPopOnRecover", self->dbgid).detail("LogId", logData->logId).detail("Tag", data->tag.toString()).detail("Ver", req.recoverAt);
popFutures.push_back(tLogPop(self, TLogPopRequest(req.recoverAt, 0, data->tag), logData)); popFutures.push_back(tLogPop(self, TLogPopRequest(req.recoverAt, 0, data->tag), logData));
} }
} }
@ -1960,7 +1960,7 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
Void _ = wait(waitForAll(popFutures)); Void _ = wait(waitForAll(popFutures));
TraceEvent("TLogPull2Complete", self->dbgid).detail("logId", logData->logId); TraceEvent("TLogPull2Complete", self->dbgid).detail("LogId", logData->logId);
logData->addActor.send( respondToRecovered( recruited, logData->recoveryComplete ) ); logData->addActor.send( respondToRecovered( recruited, logData->recoveryComplete ) );
} else { } else {
// Brand new tlog, initialization has already been done by caller // Brand new tlog, initialization has already been done by caller
@ -1993,7 +1993,7 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
req.reply.send( recruited ); req.reply.send( recruited );
TraceEvent("TLogReady", logData->logId).detail("allTags", describe(req.allTags)); TraceEvent("TLogReady", logData->logId).detail("AllTags", describe(req.allTags));
updater = Void(); updater = Void();
Void _ = wait( tLogCore( self, logData, recruited ) ); Void _ = wait( tLogCore( self, logData, recruited ) );

76
fdbserver/TagPartitionedLogSystem.actor.cpp
View File

@ -428,7 +428,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
if(begin >= lastBegin) { if(begin >= lastBegin) {
TraceEvent("TLogPeekAllCurrentOnly", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).detail("bestLogs", bestSet >= 0 ? localSets[bestSet]->logServerString() : "no best set"); TraceEvent("TLogPeekAllCurrentOnly", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).detail("BestLogs", bestSet >= 0 ? localSets[bestSet]->logServerString() : "no best set");
return Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet, return Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet,
bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, begin, end, parallelGetMore ) ); bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, begin, end, parallelGetMore ) );
} else { } else {
@ -436,7 +436,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
std::vector< LogMessageVersion > epochEnds; std::vector< LogMessageVersion > epochEnds;
if(lastBegin < end) { if(lastBegin < end) {
TraceEvent("TLogPeekAllAddingCurrent", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).detail("bestLogs", bestSet >= 0 ? localSets[bestSet]->logServerString() : "no best set"); TraceEvent("TLogPeekAllAddingCurrent", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).detail("BestLogs", bestSet >= 0 ? localSets[bestSet]->logServerString() : "no best set");
cursors.push_back( Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet, cursors.push_back( Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet,
bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, lastBegin, end, parallelGetMore)) ); bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, lastBegin, end, parallelGetMore)) );
} }
@ -446,7 +446,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
if(tag == txsTag) { if(tag == txsTag) {
break; break;
} }
TraceEvent("TLogPeekAllDead", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).detail("lastBegin", lastBegin).detail("oldLogDataSize", oldLogData.size()); TraceEvent("TLogPeekAllDead", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).detail("LastBegin", lastBegin).detail("OldLogDataSize", oldLogData.size());
if(throwIfDead) { if(throwIfDead) {
throw worker_removed(); throw worker_removed();
} else { } else {
@ -473,8 +473,8 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
if(thisBegin < lastBegin) { if(thisBegin < lastBegin) {
if(thisBegin < end) { if(thisBegin < end) {
TraceEvent("TLogPeekAllAddingOld", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end) TraceEvent("TLogPeekAllAddingOld", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end)
.detail("bestLogs", bestOldSet >= 0 ? localOldSets[bestOldSet]->logServerString() : "no best set").detail("lastBegin", lastBegin).detail("thisBegin", thisBegin); .detail("BestLogs", bestOldSet >= 0 ? localOldSets[bestOldSet]->logServerString() : "no best set").detail("LastBegin", lastBegin).detail("ThisBegin", thisBegin);
cursors.push_back( Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localOldSets, bestOldSet == -1 ? nextBestOldSet : bestOldSet, cursors.push_back( Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localOldSets, bestOldSet == -1 ? nextBestOldSet : bestOldSet,
bestOldSet >= 0 ? localOldSets[bestOldSet]->bestLocationFor( tag ) : -1, tag, thisBegin, std::min(lastBegin, end), parallelGetMore)) ); bestOldSet >= 0 ? localOldSets[bestOldSet]->bestLocationFor( tag ) : -1, tag, thisBegin, std::min(lastBegin, end), parallelGetMore)) );
epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end))); epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end)));
@ -502,21 +502,21 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
} }
if(bestSet == -1) { if(bestSet == -1) {
TraceEvent("TLogPeekRemoteNoBestSet", dbgid).detail("tag", tag.toString()).detail("begin", begin); TraceEvent("TLogPeekRemoteNoBestSet", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
} }
if(begin >= lastBegin) { if(begin >= lastBegin) {
TraceEvent("TLogPeekRemoteBestOnly", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("bestSet", bestSet).detail("bestSetStart", lastBegin).detail("logRouterIds", tLogs[bestSet]->logRouterString()); TraceEvent("TLogPeekRemoteBestOnly", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("BestSet", bestSet).detail("BestSetStart", lastBegin).detail("LogRouterIds", tLogs[bestSet]->logRouterString());
return Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, begin, getPeekEnd(), false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ); return Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, begin, getPeekEnd(), false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) );
} else { } else {
std::vector< Reference<ILogSystem::IPeekCursor> > cursors; std::vector< Reference<ILogSystem::IPeekCursor> > cursors;
std::vector< LogMessageVersion > epochEnds; std::vector< LogMessageVersion > epochEnds;
TraceEvent("TLogPeekRemoteAddingBest", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("bestSet", bestSet).detail("bestSetStart", lastBegin).detail("logRouterIds", tLogs[bestSet]->logRouterString()); TraceEvent("TLogPeekRemoteAddingBest", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("BestSet", bestSet).detail("BestSetStart", lastBegin).detail("LogRouterIds", tLogs[bestSet]->logRouterString());
cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, lastBegin, getPeekEnd(), false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ) ); cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, lastBegin, getPeekEnd(), false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ) );
int i = 0; int i = 0;
while(begin < lastBegin) { while(begin < lastBegin) {
if(i == oldLogData.size()) { if(i == oldLogData.size()) {
TraceEvent("TLogPeekRemoteDead", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("lastBegin", lastBegin).detail("oldLogDataSize", oldLogData.size()); TraceEvent("TLogPeekRemoteDead", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("LastBegin", lastBegin).detail("OldLogDataSize", oldLogData.size());
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
} }
@ -533,13 +533,13 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
} }
if(bestOldSet == -1) { if(bestOldSet == -1) {
TraceEvent("TLogPeekRemoteNoOldBestSet", dbgid).detail("tag", tag.toString()).detail("begin", begin); TraceEvent("TLogPeekRemoteNoOldBestSet", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
} }
if(thisBegin < lastBegin) { if(thisBegin < lastBegin) {
TraceEvent("TLogPeekRemoteAddingOldBest", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("bestOldSet", bestOldSet).detail("logRouterIds", oldLogData[i].tLogs[bestOldSet]->logRouterString()) TraceEvent("TLogPeekRemoteAddingOldBest", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("BestOldSet", bestOldSet).detail("LogRouterIds", oldLogData[i].tLogs[bestOldSet]->logRouterString())
.detail("lastBegin", lastBegin).detail("thisBegin", thisBegin).detail("bestStartVer", oldLogData[i].tLogs[bestOldSet]->startVersion); .detail("LastBegin", lastBegin).detail("ThisBegin", thisBegin).detail("BestStartVer", oldLogData[i].tLogs[bestOldSet]->startVersion);
cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( oldLogData[i].tLogs[bestOldSet]->logRouters, -1, (int)oldLogData[i].tLogs[bestOldSet]->logRouters.size(), tag, cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( oldLogData[i].tLogs[bestOldSet]->logRouters, -1, (int)oldLogData[i].tLogs[bestOldSet]->logRouters.size(), tag,
thisBegin, lastBegin, false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ) ); thisBegin, lastBegin, false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ) );
epochEnds.push_back(LogMessageVersion(lastBegin)); epochEnds.push_back(LogMessageVersion(lastBegin));
@ -554,7 +554,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
virtual Reference<IPeekCursor> peek( UID dbgid, Version begin, Tag tag, bool parallelGetMore ) { virtual Reference<IPeekCursor> peek( UID dbgid, Version begin, Tag tag, bool parallelGetMore ) {
if(!tLogs.size()) { if(!tLogs.size()) {
TraceEvent("TLogPeekNoLogSets", dbgid).detail("tag", tag.toString()).detail("begin", begin); TraceEvent("TLogPeekNoLogSets", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
} }
@ -567,7 +567,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
virtual Reference<IPeekCursor> peek( UID dbgid, Version begin, std::vector<Tag> tags, bool parallelGetMore ) { virtual Reference<IPeekCursor> peek( UID dbgid, Version begin, std::vector<Tag> tags, bool parallelGetMore ) {
if(tags.empty()) { if(tags.empty()) {
TraceEvent("TLogPeekNoTags", dbgid).detail("begin", begin); TraceEvent("TLogPeekNoTags", dbgid).detail("Begin", begin);
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), invalidTag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), invalidTag, begin, getPeekEnd(), false, false ) );
} }
@ -595,26 +595,26 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
} }
if(bestSet == -1) { if(bestSet == -1) {
TraceEvent("TLogPeekLocalNoBestSet", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end); TraceEvent("TLogPeekLocalNoBestSet", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end);
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( Reference<AsyncVar<OptionalInterface<TLogInterface>>>(), tag, begin, getPeekEnd(), false, false ) );
} }
if(begin >= tLogs[bestSet]->startVersion) { if(begin >= tLogs[bestSet]->startVersion) {
TraceEvent("TLogPeekLocalBestOnly", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).detail("bestSet", bestSet).detail("bestSetStart", tLogs[bestSet]->startVersion).detail("logId", tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )]->get().id()); TraceEvent("TLogPeekLocalBestOnly", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).detail("BestSet", bestSet).detail("BestSetStart", tLogs[bestSet]->startVersion).detail("LogId", tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )]->get().id());
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )], tag, begin, end, false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )], tag, begin, end, false, false ) );
} else { } else {
std::vector< Reference<ILogSystem::IPeekCursor> > cursors; std::vector< Reference<ILogSystem::IPeekCursor> > cursors;
std::vector< LogMessageVersion > epochEnds; std::vector< LogMessageVersion > epochEnds;
if(tLogs[bestSet]->startVersion < end) { if(tLogs[bestSet]->startVersion < end) {
TraceEvent("TLogPeekLocalAddingBest", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).detail("bestSet", bestSet).detail("bestSetStart", tLogs[bestSet]->startVersion).detail("logId", tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )]->get().id()); TraceEvent("TLogPeekLocalAddingBest", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).detail("BestSet", bestSet).detail("BestSetStart", tLogs[bestSet]->startVersion).detail("LogId", tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )]->get().id());
cursors.push_back( Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )], tag, tLogs[bestSet]->startVersion, end, false, false ) ) ); cursors.push_back( Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( tLogs[bestSet]->logServers[tLogs[bestSet]->bestLocationFor( tag )], tag, tLogs[bestSet]->startVersion, end, false, false ) ) );
} }
Version lastBegin = tLogs[bestSet]->startVersion; Version lastBegin = tLogs[bestSet]->startVersion;
int i = 0; int i = 0;
while(begin < lastBegin) { while(begin < lastBegin) {
if(i == oldLogData.size()) { if(i == oldLogData.size()) {
TraceEvent("TLogPeekLocalDead", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end).detail("lastBegin", lastBegin).detail("oldLogDataSize", oldLogData.size()); TraceEvent("TLogPeekLocalDead", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end).detail("LastBegin", lastBegin).detail("OldLogDataSize", oldLogData.size());
throw worker_removed(); throw worker_removed();
} }
@ -630,7 +630,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
if(foundSpecial) { if(foundSpecial) {
TraceEvent("TLogPeekLocalFoundSpecial", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end); TraceEvent("TLogPeekLocalFoundSpecial", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end);
cursors.push_back(peekAll(dbgid, begin, std::min(lastBegin, end), tag, false, true)); cursors.push_back(peekAll(dbgid, begin, std::min(lastBegin, end), tag, false, true));
epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end))); epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end)));
break; break;
@ -644,8 +644,8 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
Version thisBegin = std::max(oldLogData[i].tLogs[bestOldSet]->startVersion, begin); Version thisBegin = std::max(oldLogData[i].tLogs[bestOldSet]->startVersion, begin);
if(thisBegin < lastBegin) { if(thisBegin < lastBegin) {
if(thisBegin < end) { if(thisBegin < end) {
TraceEvent("TLogPeekLocalAddingOldBest", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("end", end) TraceEvent("TLogPeekLocalAddingOldBest", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("End", end)
.detail("logServers", oldLogData[i].tLogs[bestOldSet]->logServerString()).detail("thisBegin", thisBegin).detail("lastBegin", lastBegin); .detail("LogServers", oldLogData[i].tLogs[bestOldSet]->logServerString()).detail("ThisBegin", thisBegin).detail("LastBegin", lastBegin);
cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( oldLogData[i].tLogs[bestOldSet]->logServers, oldLogData[i].tLogs[bestOldSet]->bestLocationFor( tag ), oldLogData[i].tLogs[bestOldSet]->logServers.size() + 1 - oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor, tag, cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( oldLogData[i].tLogs[bestOldSet]->logServers, oldLogData[i].tLogs[bestOldSet]->bestLocationFor( tag ), oldLogData[i].tLogs[bestOldSet]->logServers.size() + 1 - oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor, tag,
thisBegin, std::min(lastBegin, end), false, oldLogData[i].tLogs[bestOldSet]->tLogLocalities, oldLogData[i].tLogs[bestOldSet]->tLogPolicy, oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor))); thisBegin, std::min(lastBegin, end), false, oldLogData[i].tLogs[bestOldSet]->tLogLocalities, oldLogData[i].tLogs[bestOldSet]->tLogPolicy, oldLogData[i].tLogs[bestOldSet]->tLogReplicationFactor)));
epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end))); epochEnds.push_back(LogMessageVersion(std::min(lastBegin, end)));
@ -701,7 +701,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
std::vector<Reference<LogSet>> localSets; std::vector<Reference<LogSet>> localSets;
for(auto& log : tLogs) { for(auto& log : tLogs) {
if(log->isLocal && log->logServers.size()) { if(log->isLocal && log->logServers.size()) {
TraceEvent("TLogPeekLogRouterLocalSet", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("logServers", log->logServerString()); TraceEvent("TLogPeekLogRouterLocalSet", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("LogServers", log->logServerString());
localSets.push_back(log); localSets.push_back(log);
if(log->hasBestPolicy && (log->locality == tag.locality || tag.locality == tagLocalitySpecial || log->locality == tagLocalitySpecial || log->locality == tagLocalityUpgraded)) { if(log->hasBestPolicy && (log->locality == tag.locality || tag.locality == tagLocalitySpecial || log->locality == tagLocalitySpecial || log->locality == tagLocalityUpgraded)) {
bestSet = localSets.size()-1; bestSet = localSets.size()-1;
@ -713,14 +713,14 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
} }
TraceEvent("TLogPeekLogRouterSets", dbgid).detail("tag", tag.toString()).detail("begin", begin); TraceEvent("TLogPeekLogRouterSets", dbgid).detail("Tag", tag.toString()).detail("Begin", begin);
//FIXME: do this merge on one of the logs in the other data center to avoid sending multiple copies across the WAN //FIXME: do this merge on one of the logs in the other data center to avoid sending multiple copies across the WAN
return Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet, return Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet,
bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, begin, getPeekEnd(), false ) ); bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, begin, getPeekEnd(), false ) );
} else { } else {
for( auto& log : tLogs ) { for( auto& log : tLogs ) {
if( log->logServers.size() && log->isLocal && log->hasBestPolicy ) { if( log->logServers.size() && log->isLocal && log->hasBestPolicy ) {
TraceEvent("TLogPeekLogRouterBestOnly", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("logId", log->logServers[log->bestLocationFor( tag )]->get().id()); TraceEvent("TLogPeekLogRouterBestOnly", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("LogId", log->logServers[log->bestLocationFor( tag )]->get().id());
return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( log->logServers[log->bestLocationFor( tag )], tag, begin, getPeekEnd(), false, false ) ); return Reference<ILogSystem::ServerPeekCursor>( new ILogSystem::ServerPeekCursor( log->logServers[log->bestLocationFor( tag )], tag, begin, getPeekEnd(), false, false ) );
} }
} }
@ -745,7 +745,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
std::vector<Reference<LogSet>> localSets; std::vector<Reference<LogSet>> localSets;
for(auto& log : old.tLogs) { for(auto& log : old.tLogs) {
if(log->isLocal && log->logServers.size()) { if(log->isLocal && log->logServers.size()) {
TraceEvent("TLogPeekLogRouterOldLocalSet", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("logServers", log->logServerString()); TraceEvent("TLogPeekLogRouterOldLocalSet", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("LogServers", log->logServerString());
localSets.push_back(log); localSets.push_back(log);
if(log->hasBestPolicy && (log->locality == tag.locality || tag.locality == tagLocalitySpecial || log->locality == tagLocalitySpecial || log->locality == tagLocalityUpgraded)) { if(log->hasBestPolicy && (log->locality == tag.locality || tag.locality == tagLocalitySpecial || log->locality == tagLocalitySpecial || log->locality == tagLocalityUpgraded)) {
bestSet = localSets.size()-1; bestSet = localSets.size()-1;
@ -757,7 +757,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
} }
TraceEvent("TLogPeekLogRouterOldSets", dbgid).detail("tag", tag.toString()).detail("begin", begin).detail("oldEpoch", old.epochEnd); TraceEvent("TLogPeekLogRouterOldSets", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("OldEpoch", old.epochEnd);
//FIXME: do this merge on one of the logs in the other data center to avoid sending multiple copies across the WAN //FIXME: do this merge on one of the logs in the other data center to avoid sending multiple copies across the WAN
return Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet, return Reference<ILogSystem::SetPeekCursor>( new ILogSystem::SetPeekCursor( localSets, bestSet == -1 ? nextBestSet : bestSet,
bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, begin, old.epochEnd, false ) ); bestSet >= 0 ? localSets[bestSet]->bestLocationFor( tag ) : -1, tag, begin, old.epochEnd, false ) );
@ -1128,7 +1128,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
TraceEvent("GetDurableResult", dbgid).detail("Required", requiredCount).detail("Present", results.size()).detail("ServerState", sServerState) TraceEvent("GetDurableResult", dbgid).detail("Required", requiredCount).detail("Present", results.size()).detail("ServerState", sServerState)
.detail("RecoveryVersion", ((safe_range_end > 0) && (safe_range_end-1 < results.size())) ? results[ safe_range_end-1 ].end : -1) .detail("RecoveryVersion", ((safe_range_end > 0) && (safe_range_end-1 < results.size())) ? results[ safe_range_end-1 ].end : -1)
.detail("EndVersion", results[ new_safe_range_begin ].end).detail("SafeBegin", safe_range_begin).detail("SafeEnd", safe_range_end) .detail("EndVersion", results[ new_safe_range_begin ].end).detail("SafeBegin", safe_range_begin).detail("SafeEnd", safe_range_end)
.detail("NewSafeBegin", new_safe_range_begin).detail("knownCommittedVersion", knownCommittedVersion).detail("epochEnd", lockInfo.epochEnd); .detail("NewSafeBegin", new_safe_range_begin).detail("KnownCommittedVersion", knownCommittedVersion).detail("EpochEnd", lockInfo.epochEnd);
return std::make_pair(knownCommittedVersion, results[ new_safe_range_begin ].end); return std::make_pair(knownCommittedVersion, results[ new_safe_range_begin ].end);
} }
@ -1259,7 +1259,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
break; break;
} }
if(!lockedLocalities.count(log->locality)) { if(!lockedLocalities.count(log->locality)) {
TraceEvent("EpochEndLockExtra").detail("locality", log->locality); TraceEvent("EpochEndLockExtra").detail("Locality", log->locality);
TEST(true); //locking old generations for version information TEST(true); //locking old generations for version information
lockedLocalities.insert(log->locality); lockedLocalities.insert(log->locality);
LogLockInfo lockResult; LogLockInfo lockResult;
@ -1348,7 +1348,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
if(!found) { if(!found) {
TraceEvent("RecruitingOldLogRoutersAddingLocality").detail("locality", locality).detail("lastStart", lastStart); TraceEvent("RecruitingOldLogRoutersAddingLocality").detail("Locality", locality).detail("LastStart", lastStart);
Reference<LogSet> newLogSet( new LogSet() ); Reference<LogSet> newLogSet( new LogSet() );
newLogSet->locality = locality; newLogSet->locality = locality;
newLogSet->startVersion = lastStart; newLogSet->startVersion = lastStart;
@ -1399,7 +1399,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
if(!found) { if(!found) {
TraceEvent("RecruitingOldLogRoutersAddingLocality").detail("locality", locality).detail("lastStart", lastStart); TraceEvent("RecruitingOldLogRoutersAddingLocality").detail("Locality", locality).detail("LastStart", lastStart);
Reference<LogSet> newLogSet( new LogSet() ); Reference<LogSet> newLogSet( new LogSet() );
newLogSet->locality = locality; newLogSet->locality = locality;
newLogSet->startVersion = lastStart; newLogSet->startVersion = lastStart;
@ -1559,7 +1559,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
for( int i = 0; i < remoteWorkers.remoteTLogs.size(); i++ ) for( int i = 0; i < remoteWorkers.remoteTLogs.size(); i++ )
remoteTLogInitializationReplies.push_back( transformErrors( throwErrorOr( remoteWorkers.remoteTLogs[i].tLog.getReplyUnlessFailedFor( remoteTLogReqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) ); remoteTLogInitializationReplies.push_back( transformErrors( throwErrorOr( remoteWorkers.remoteTLogs[i].tLog.getReplyUnlessFailedFor( remoteTLogReqs[i], SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) );
TraceEvent("RemoteLogRecruitment_InitializingRemoteLogs").detail("startVersion", logSet->startVersion).detail("localStart", self->tLogs[0]->startVersion).detail("logRouterTags", self->logRouterTags); TraceEvent("RemoteLogRecruitment_InitializingRemoteLogs").detail("StartVersion", logSet->startVersion).detail("LocalStart", self->tLogs[0]->startVersion).detail("LogRouterTags", self->logRouterTags);
Void _ = wait( waitForAll(remoteTLogInitializationReplies) && waitForAll(logRouterInitializationReplies) && oldRouterRecruitment ); Void _ = wait( waitForAll(remoteTLogInitializationReplies) && waitForAll(logRouterInitializationReplies) && oldRouterRecruitment );
for( int i = 0; i < logRouterInitializationReplies.size(); i++ ) { for( int i = 0; i < logRouterInitializationReplies.size(); i++ ) {
@ -1660,7 +1660,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
} }
state Future<Void> oldRouterRecruitment = Never(); state Future<Void> oldRouterRecruitment = Never();
TraceEvent("NewEpochStartVersion").detail("startVersion", logSystem->tLogs[0]->startVersion).detail("epochEnd", oldLogSystem->knownCommittedVersion + 1).detail("locality", primaryLocality); TraceEvent("NewEpochStartVersion").detail("StartVersion", logSystem->tLogs[0]->startVersion).detail("EpochEnd", oldLogSystem->knownCommittedVersion + 1).detail("Locality", primaryLocality);
if(logSystem->tLogs[0]->startVersion < oldLogSystem->knownCommittedVersion + 1) { if(logSystem->tLogs[0]->startVersion < oldLogSystem->knownCommittedVersion + 1) {
oldRouterRecruitment = TagPartitionedLogSystem::recruitOldLogRouters(oldLogSystem.getPtr(), recr.oldLogRouters, recoveryCount, primaryLocality, logSystem->tLogs[0]->startVersion, localities, logSystem->tLogs[0]->tLogPolicy, logSystem->tLogs[0]->hasBestPolicy, false); oldRouterRecruitment = TagPartitionedLogSystem::recruitOldLogRouters(oldLogSystem.getPtr(), recr.oldLogRouters, recoveryCount, primaryLocality, logSystem->tLogs[0]->startVersion, localities, logSystem->tLogs[0]->tLogPolicy, logSystem->tLogs[0]->hasBestPolicy, false);
} else { } else {
@ -1823,7 +1823,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
loop { loop {
choose { choose {
when (TLogLockResult data = wait( tlog->get().present() ? brokenPromiseToNever( tlog->get().interf().lock.getReply<TLogLockResult>() ) : Never() )) { when (TLogLockResult data = wait( tlog->get().present() ? brokenPromiseToNever( tlog->get().interf().lock.getReply<TLogLockResult>() ) : Never() )) {
TraceEvent("TLogLocked", myID).detail("TLog", tlog->get().id()).detail("end", data.end); TraceEvent("TLogLocked", myID).detail("TLog", tlog->get().id()).detail("End", data.end);
return data; return data;
} }
when (Void _ = wait(tlog->onChange())) {} when (Void _ = wait(tlog->onChange())) {}
@ -1972,11 +1972,11 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
TEST(can_omit >= prevState.tLogReplicationFactor - prevState.tLogWriteAntiQuorum); // Locking a subset of the TLogs while ending an epoch. TEST(can_omit >= prevState.tLogReplicationFactor - prevState.tLogWriteAntiQuorum); // Locking a subset of the TLogs while ending an epoch.
const bool reboot_a_tlog = g_network->now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration && BUGGIFY && g_random->random01() < 0.25; const bool reboot_a_tlog = g_network->now() - g_simulator.lastConnectionFailure > g_simulator.connectionFailuresDisableDuration && BUGGIFY && g_random->random01() < 0.25;
TraceEvent(SevInfo, "MasterRecoveryTLogLocking", dbgid) TraceEvent(SevInfo, "MasterRecoveryTLogLocking", dbgid)
.detail("locks", tlogs.size() - can_omit) detail("Locks", tlogs.size() - can_omit)
.detail("skipped", can_omit) detail("Skipped", can_omit)
.detail("replication", prevState.tLogReplicationFactor) detail("Replication", prevState.tLogReplicationFactor)
.detail("antiquorum", prevState.tLogWriteAntiQuorum) detail("Antiquorum", prevState.tLogWriteAntiQuorum)
.detail("reboot_buggify", reboot_a_tlog); detail("RebootBuggify", reboot_a_tlog);
for (int i = 0; i < tlogs.size() - can_omit; i++) { for (int i = 0; i < tlogs.size() - can_omit; i++) {
const int index = tlogs[i].second; const int index = tlogs[i].second;
Future<TLogLockResult>& entry = tLogReply->at(index); Future<TLogLockResult>& entry = tLogReply->at(index);

16
fdbserver/VFSAsync.cpp Executable file → Normal file
View File

@ -92,7 +92,7 @@ std::map<std::string, std::pair<uint32_t,int>> VFSAsyncFile::filename_lockCount_
static int asyncClose(sqlite3_file *pFile){ static int asyncClose(sqlite3_file *pFile){
VFSAsyncFile *p = (VFSAsyncFile*)pFile; VFSAsyncFile *p = (VFSAsyncFile*)pFile;
/*TraceEvent("VFSAsyncClose").detail("fd", p->file->debugFD()) /*TraceEvent("VFSAsyncClose").detail("Fd", p->file->debugFD())
.detail("Filename", p->filename).detail("ZCRefs", p->debug_zcrefs) .detail("Filename", p->filename).detail("ZCRefs", p->debug_zcrefs)
.detail("ZCReads", p->debug_zcreads).detail("NormalReads", p->debug_reads).backtrace();*/ .detail("ZCReads", p->debug_zcreads).detail("NormalReads", p->debug_reads).backtrace();*/
//printf("Closing %s: %d zcrefs, %d/%d reads zc\n", filename.c_str(), debug_zcrefs, debug_zcreads, debug_zcreads+debug_reads); //printf("Closing %s: %d zcrefs, %d/%d reads zc\n", filename.c_str(), debug_zcrefs, debug_zcreads, debug_zcreads+debug_reads);
@ -201,7 +201,7 @@ static int asyncSync(sqlite3_file *pFile, int flags){
} catch (Error& e) { } catch (Error& e) {
TraceEvent("VFSSyncError") TraceEvent("VFSSyncError")
.detail("Filename", p->filename) .detail("Filename", p->filename)
.detail("sqlite3_file", (int64_t)pFile) .detail("Sqlite3File", (int64_t)pFile)
.detail("IAsyncFile", (int64_t)p->file.getPtr()) .detail("IAsyncFile", (int64_t)p->file.getPtr())
.error(e); .error(e);
@ -225,7 +225,7 @@ static int VFSAsyncFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
static int asyncLock(sqlite3_file *pFile, int eLock){ static int asyncLock(sqlite3_file *pFile, int eLock){
VFSAsyncFile *p = (VFSAsyncFile*)pFile; VFSAsyncFile *p = (VFSAsyncFile*)pFile;
//TraceEvent("FileLock").detail("File", p->filename).detail("fd", p->file->debugFD()).detail("PrevLockLevel", p->lockLevel).detail("Op", eLock).detail("LockCount", *p->pLockCount); //TraceEvent("FileLock").detail("File", p->filename).detail("Fd", p->file->debugFD()).detail("PrevLockLevel", p->lockLevel).detail("Op", eLock).detail("LockCount", *p->pLockCount);
return eLock == EXCLUSIVE_LOCK ? SQLITE_BUSY : SQLITE_OK; return eLock == EXCLUSIVE_LOCK ? SQLITE_BUSY : SQLITE_OK;
} }
@ -373,7 +373,7 @@ static int asyncDeviceCharacteristics(sqlite3_file *pFile){ return 0; }
} else if (flags & SQLITE_SHM_SHARED) { } else if (flags & SQLITE_SHM_SHARED) {
for(int i=ofst; i<ofst+n; i++) for(int i=ofst; i<ofst+n; i++)
if ( memInfo->exclusiveLocks[i] != ((pDbFd->sharedMemoryExclusiveLocks>>i)&1) ) { if ( memInfo->exclusiveLocks[i] != ((pDbFd->sharedMemoryExclusiveLocks>>i)&1) ) {
//TraceEvent("ShmLocked").detail("File", DEBUG_DETERMINISM ? 0 : (int64_t)pDbFd).detail("Acquiring", "Shared").detail("i", i).detail("Exclusive", memInfo->exclusiveLocks[i]).detail("MyExclusive", pDbFd->sharedMemoryExclusiveLocks); //TraceEvent("ShmLocked").detail("File", DEBUG_DETERMINISM ? 0 : (int64_t)pDbFd).detail("Acquiring", "Shared").detail("I", i).detail("Exclusive", memInfo->exclusiveLocks[i]).detail("MyExclusive", pDbFd->sharedMemoryExclusiveLocks);
return SQLITE_BUSY; return SQLITE_BUSY;
} }
for(int i=ofst; i<ofst+n; i++) for(int i=ofst; i<ofst+n; i++)
@ -386,7 +386,7 @@ static int asyncDeviceCharacteristics(sqlite3_file *pFile){ return 0; }
if ( memInfo->exclusiveLocks[i] != ((pDbFd->sharedMemoryExclusiveLocks>>i)&1) || if ( memInfo->exclusiveLocks[i] != ((pDbFd->sharedMemoryExclusiveLocks>>i)&1) ||
memInfo->sharedLocks[i] != ((pDbFd->sharedMemorySharedLocks>>i)&1) ) memInfo->sharedLocks[i] != ((pDbFd->sharedMemorySharedLocks>>i)&1) )
{ {
//TraceEvent("ShmLocked").detail("File", DEBUG_DETERMINISM ? 0 : (int64_t)pDbFd).detail("Acquiring", "Exclusive").detail("i", i).detail("Exclusive", memInfo->exclusiveLocks[i]).detail("MyExclusive", pDbFd->sharedMemoryExclusiveLocks).detail("Shared", memInfo->sharedLocks[i]).detail("MyShared", pDbFd->sharedMemorySharedLocks); //TraceEvent("ShmLocked").detail("File", DEBUG_DETERMINISM ? 0 : (int64_t)pDbFd).detail("Acquiring", "Exclusive").detail("I", i).detail("Exclusive", memInfo->exclusiveLocks[i]).detail("MyExclusive", pDbFd->sharedMemoryExclusiveLocks).detail("Shared", memInfo->sharedLocks[i]).detail("MyShared", pDbFd->sharedMemorySharedLocks);
return SQLITE_BUSY; return SQLITE_BUSY;
} }
for(int i=ofst; i<ofst+n; i++) for(int i=ofst; i<ofst+n; i++)
@ -448,7 +448,7 @@ static int asyncDeviceCharacteristics(sqlite3_file *pFile){ return 0; }
} }
VFSAsyncFile::~VFSAsyncFile() { VFSAsyncFile::~VFSAsyncFile() {
//TraceEvent("VFSAsyncFileDel").detail("filename", filename); //TraceEvent("VFSAsyncFileDel").detail("Filename", filename);
if (!--filename_lockCount_openCount[filename].second) { if (!--filename_lockCount_openCount[filename].second) {
filename_lockCount_openCount.erase(filename); filename_lockCount_openCount.erase(filename);
@ -523,9 +523,9 @@ static int asyncOpen(
/*TraceEvent("VFSOpened") /*TraceEvent("VFSOpened")
.detail("Filename", p->filename) .detail("Filename", p->filename)
.detail("fd", DEBUG_DETERMINISM ? 0 : p->file->debugFD()) .detail("Fd", DEBUG_DETERMINISM ? 0 : p->file->debugFD())
.detail("Flags", flags) .detail("Flags", flags)
.detail("sqlite3_file", DEBUG_DETERMINISM ? 0 : (int64_t)pFile) .detail("Sqlite3File", DEBUG_DETERMINISM ? 0 : (int64_t)pFile)
.detail("IAsyncFile", DEBUG_DETERMINISM ? 0 : (int64_t)p->file.getPtr());*/ .detail("IAsyncFile", DEBUG_DETERMINISM ? 0 : (int64_t)p->file.getPtr());*/
} catch (Error& e) { } catch (Error& e) {
TraceEvent("SQLiteOpenFail").detail("Filename", p->filename).error(e); TraceEvent("SQLiteOpenFail").detail("Filename", p->filename).error(e);

9
fdbserver/fdbserver.actor.cpp
View File

@ -849,7 +849,7 @@ int main(int argc, char* argv[]) {
std::string testServersStr; std::string testServersStr;
NetworkAddress publicAddress, listenAddress; NetworkAddress publicAddress, listenAddress;
const char *targetKey = NULL; const char *targetKey = NULL;
uint64_t memLimit = 8LL << 30; uint64_t memLimit = 8LL << 30; // Nice to maintain the same default value for memLimit and SERVER_KNOBS->SERVER_MEM_LIMIT and SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT
uint64_t storageMemLimit = 1LL << 30; uint64_t storageMemLimit = 1LL << 30;
bool buggifyEnabled = false, machineIdOverride = false, restarting = false; bool buggifyEnabled = false, machineIdOverride = false, restarting = false;
Optional<Standalone<StringRef>> zoneId; Optional<Standalone<StringRef>> zoneId;
@ -1390,7 +1390,9 @@ int main(int argc, char* argv[]) {
CLIENT_KNOBS = clientKnobs; CLIENT_KNOBS = clientKnobs;
if (!serverKnobs->setKnob( "log_directory", logFolder )) ASSERT(false); if (!serverKnobs->setKnob( "log_directory", logFolder )) ASSERT(false);
if (role != Simulation) {
if (!serverKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(memLimit))) ASSERT(false);
}
for(auto k=knobs.begin(); k!=knobs.end(); ++k) { for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
try { try {
if (!flowKnobs->setKnob( k->first, k->second ) && if (!flowKnobs->setKnob( k->first, k->second ) &&
@ -1408,6 +1410,7 @@ int main(int argc, char* argv[]) {
throw; throw;
} }
} }
if (!serverKnobs->setKnob("server_mem_limit", std::to_string(memLimit))) ASSERT(false);
if (role == SkipListTest) { if (role == SkipListTest) {
skipListTest(); skipListTest();
@ -1775,7 +1778,7 @@ int main(int argc, char* argv[]) {
flushAndExit(FDB_EXIT_MAIN_ERROR); flushAndExit(FDB_EXIT_MAIN_ERROR);
} catch (std::exception& e) { } catch (std::exception& e) {
fprintf(stderr, "std::exception: %s\n", e.what()); fprintf(stderr, "std::exception: %s\n", e.what());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("std::exception", e.what()); TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
//printf("\n%d tests passed; %d tests failed\n", passCount, failCount); //printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(FDB_EXIT_MAIN_EXCEPTION); flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
} }

30
fdbserver/masterserver.actor.cpp
View File

@ -260,7 +260,7 @@ ACTOR Future<Void> newProxies( Reference<MasterData> self, RecruitFromConfigurat
req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; req.recoveryCount = self->cstate.myDBState.recoveryCount + 1;
req.recoveryTransactionVersion = self->recoveryTransactionVersion; req.recoveryTransactionVersion = self->recoveryTransactionVersion;
req.firstProxy = i == 0; req.firstProxy = i == 0;
TraceEvent("ProxyReplies",self->dbgid).detail("workerID", recr.proxies[i].id()); TraceEvent("ProxyReplies",self->dbgid).detail("WorkerID", recr.proxies[i].id());
initializationReplies.push_back( transformErrors( throwErrorOr( recr.proxies[i].masterProxy.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) ); initializationReplies.push_back( transformErrors( throwErrorOr( recr.proxies[i].masterProxy.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) );
} }
@ -278,7 +278,7 @@ ACTOR Future<Void> newResolvers( Reference<MasterData> self, RecruitFromConfigur
req.recoveryCount = self->cstate.myDBState.recoveryCount + 1; req.recoveryCount = self->cstate.myDBState.recoveryCount + 1;
req.proxyCount = recr.proxies.size(); req.proxyCount = recr.proxies.size();
req.resolverCount = recr.resolvers.size(); req.resolverCount = recr.resolvers.size();
TraceEvent("ResolverReplies",self->dbgid).detail("workerID", recr.resolvers[i].id()); TraceEvent("ResolverReplies",self->dbgid).detail("WorkerID", recr.resolvers[i].id());
initializationReplies.push_back( transformErrors( throwErrorOr( recr.resolvers[i].resolver.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) ); initializationReplies.push_back( transformErrors( throwErrorOr( recr.resolvers[i].resolver.getReplyUnlessFailedFor( req, SERVER_KNOBS->TLOG_TIMEOUT, SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY ) ), master_recovery_failed() ) );
} }
@ -292,7 +292,7 @@ ACTOR Future<Void> newTLogServers( Reference<MasterData> self, RecruitFromConfig
if(self->configuration.remoteTLogReplicationFactor > 0) { if(self->configuration.remoteTLogReplicationFactor > 0) {
state Optional<Key> remoteDcId = self->remoteDcIds.size() ? self->remoteDcIds[0] : Optional<Key>(); state Optional<Key> remoteDcId = self->remoteDcIds.size() ? self->remoteDcIds[0] : Optional<Key>();
if( !self->dcId_locality.count(recr.dcId) ) { if( !self->dcId_locality.count(recr.dcId) ) {
TraceEvent(SevWarn, "UnknownPrimaryDCID", self->dbgid).detail("primaryId", printable(recr.dcId)); TraceEvent(SevWarn, "UnknownPrimaryDCID", self->dbgid).detail("PrimaryId", printable(recr.dcId));
int8_t loc = self->getNextLocality(); int8_t loc = self->getNextLocality();
Standalone<CommitTransactionRef> tr; Standalone<CommitTransactionRef> tr;
tr.set(tr.arena(), tagLocalityListKeyFor(recr.dcId), tagLocalityListValue(loc)); tr.set(tr.arena(), tagLocalityListKeyFor(recr.dcId), tagLocalityListValue(loc));
@ -301,7 +301,7 @@ ACTOR Future<Void> newTLogServers( Reference<MasterData> self, RecruitFromConfig
} }
if( !self->dcId_locality.count(remoteDcId) ) { if( !self->dcId_locality.count(remoteDcId) ) {
TraceEvent(SevWarn, "UnknownRemoteDCID", self->dbgid).detail("remoteId", printable(remoteDcId)); TraceEvent(SevWarn, "UnknownRemoteDCID", self->dbgid).detail("RemoteId", printable(remoteDcId));
int8_t loc = self->getNextLocality(); int8_t loc = self->getNextLocality();
Standalone<CommitTransactionRef> tr; Standalone<CommitTransactionRef> tr;
tr.set(tr.arena(), tagLocalityListKeyFor(remoteDcId), tagLocalityListValue(loc)); tr.set(tr.arena(), tagLocalityListKeyFor(remoteDcId), tagLocalityListValue(loc));
@ -456,7 +456,7 @@ ACTOR Future<Void> updateRegistration( Reference<MasterData> self, Reference<ILo
trigger = self->registrationTrigger.onTrigger(); trigger = self->registrationTrigger.onTrigger();
TraceEvent("MasterUpdateRegistration", self->dbgid).detail("RecoveryCount", self->cstate.myDBState.recoveryCount).detail("logs", describe(logSystem->getLogSystemConfig().tLogs)); TraceEvent("MasterUpdateRegistration", self->dbgid).detail("RecoveryCount", self->cstate.myDBState.recoveryCount).detail("Logs", describe(logSystem->getLogSystemConfig().tLogs));
if (!self->cstateUpdated.isSet()) { if (!self->cstateUpdated.isSet()) {
Void _ = wait(sendMasterRegistration(self.getPtr(), logSystem->getLogSystemConfig(), self->provisionalProxies, self->resolvers, self->cstate.myDBState.recoveryCount, self->cstate.prevDBState.getPriorCommittedLogServers() )); Void _ = wait(sendMasterRegistration(self.getPtr(), logSystem->getLogSystemConfig(), self->provisionalProxies, self->resolvers, self->cstate.myDBState.recoveryCount, self->cstate.prevDBState.getPriorCommittedLogServers() ));
@ -543,7 +543,7 @@ ACTOR Future<Void> recruitEverything( Reference<MasterData> self, vector<Storage
.detail("DesiredProxies", self->configuration.getDesiredProxies()) .detail("DesiredProxies", self->configuration.getDesiredProxies())
.detail("RequiredResolvers", 1) .detail("RequiredResolvers", 1)
.detail("DesiredResolvers", self->configuration.getDesiredResolvers()) .detail("DesiredResolvers", self->configuration.getDesiredResolvers())
.detail("storeType", self->configuration.storageServerStoreType) .detail("StoreType", self->configuration.storageServerStoreType)
.trackLatest("MasterRecoveryState"); .trackLatest("MasterRecoveryState");
//FIXME: we only need log routers for the same locality as the master //FIXME: we only need log routers for the same locality as the master
@ -614,13 +614,13 @@ ACTOR Future<Void> readTransactionSystemState( Reference<MasterData> self, Refer
if ( self->recoveryTransactionVersion < minRequiredCommitVersion ) self->recoveryTransactionVersion = minRequiredCommitVersion; if ( self->recoveryTransactionVersion < minRequiredCommitVersion ) self->recoveryTransactionVersion = minRequiredCommitVersion;
} }
TraceEvent("MasterRecovering", self->dbgid).detail("lastEpochEnd", self->lastEpochEnd).detail("recoveryTransactionVersion", self->recoveryTransactionVersion); TraceEvent("MasterRecovering", self->dbgid).detail("LastEpochEnd", self->lastEpochEnd).detail("RecoveryTransactionVersion", self->recoveryTransactionVersion);
Standalone<VectorRef<KeyValueRef>> rawConf = wait( self->txnStateStore->readRange( configKeys ) ); Standalone<VectorRef<KeyValueRef>> rawConf = wait( self->txnStateStore->readRange( configKeys ) );
self->configuration.fromKeyValues( rawConf ); self->configuration.fromKeyValues( rawConf );
self->originalConfiguration = self->configuration; self->originalConfiguration = self->configuration;
self->hasConfiguration = true; self->hasConfiguration = true;
TraceEvent("MasterRecoveredConfig", self->dbgid).detail("conf", self->configuration.toString()).trackLatest("RecoveredConfig"); TraceEvent("MasterRecoveredConfig", self->dbgid).detail("Conf", self->configuration.toString()).trackLatest("RecoveredConfig");
Standalone<VectorRef<KeyValueRef>> rawLocalities = wait( self->txnStateStore->readRange( tagLocalityListKeys ) ); Standalone<VectorRef<KeyValueRef>> rawLocalities = wait( self->txnStateStore->readRange( tagLocalityListKeys ) );
self->dcId_locality.clear(); self->dcId_locality.clear();
@ -930,7 +930,7 @@ ACTOR Future<Void> resolutionBalancing(Reference<MasterData> self) {
for (int i = 0; i < futures.size(); i++) { for (int i = 0; i < futures.size(); i++) {
total += futures[i].get(); total += futures[i].get();
metrics.insert(std::make_pair(futures[i].get(), i), NoMetric()); metrics.insert(std::make_pair(futures[i].get(), i), NoMetric());
//TraceEvent("ResolverMetric").detail("i", i).detail("metric", futures[i].get()); //TraceEvent("ResolverMetric").detail("I", i).detail("Metric", futures[i].get());
} }
if( metrics.lastItem()->first - metrics.begin()->first > SERVER_KNOBS->MIN_BALANCE_DIFFERENCE ) { if( metrics.lastItem()->first - metrics.begin()->first > SERVER_KNOBS->MIN_BALANCE_DIFFERENCE ) {
try { try {
@ -950,7 +950,7 @@ ACTOR Future<Void> resolutionBalancing(Reference<MasterData> self) {
ResolutionSplitReply split = wait( brokenPromiseToNever(self->resolvers[metrics.lastItem()->second].split.getReply(req, TaskResolutionMetrics)) ); ResolutionSplitReply split = wait( brokenPromiseToNever(self->resolvers[metrics.lastItem()->second].split.getReply(req, TaskResolutionMetrics)) );
KeyRangeRef moveRange = range.second ? KeyRangeRef( range.first.begin, split.key ) : KeyRangeRef( split.key, range.first.end ); KeyRangeRef moveRange = range.second ? KeyRangeRef( range.first.begin, split.key ) : KeyRangeRef( split.key, range.first.end );
movedRanges.push_back_deep(movedRanges.arena(), ResolverMoveRef(moveRange, dest)); movedRanges.push_back_deep(movedRanges.arena(), ResolverMoveRef(moveRange, dest));
TraceEvent("MovingResolutionRange").detail("src", src).detail("dest", dest).detail("amount", amount).detail("startRange", printable(range.first)).detail("moveRange", printable(moveRange)).detail("used", split.used).detail("KeyResolverRanges", key_resolver.size()); TraceEvent("MovingResolutionRange").detail("Src", src).detail("Dest", dest).detail("Amount", amount).detail("StartRange", printable(range.first)).detail("MoveRange", printable(moveRange)).detail("Used", split.used).detail("KeyResolverRanges", key_resolver.size());
amount -= split.used; amount -= split.used;
if(moveRange != range.first || amount <= 0 ) if(moveRange != range.first || amount <= 0 )
break; break;
@ -958,7 +958,7 @@ ACTOR Future<Void> resolutionBalancing(Reference<MasterData> self) {
for(auto& it : movedRanges) for(auto& it : movedRanges)
key_resolver.insert(it.range, it.dest); key_resolver.insert(it.range, it.dest);
//for(auto& it : key_resolver.ranges()) //for(auto& it : key_resolver.ranges())
// TraceEvent("KeyResolver").detail("range", printable(it.range())).detail("value", it.value()); // TraceEvent("KeyResolver").detail("Range", printable(it.range())).detail("Value", it.value());
self->resolverChangesVersion = self->version + 1; self->resolverChangesVersion = self->version + 1;
for (auto& p : self->proxies) for (auto& p : self->proxies)
@ -1204,7 +1204,7 @@ ACTOR Future<Void> masterCore( Reference<MasterData> self ) {
self->addActor.send( waitResolverFailure( self->resolvers ) ); self->addActor.send( waitResolverFailure( self->resolvers ) );
self->addActor.send( waitProxyFailure( self->proxies ) ); self->addActor.send( waitProxyFailure( self->proxies ) );
self->addActor.send( provideVersions(self) ); self->addActor.send( provideVersions(self) );
self->addActor.send( reportErrors(updateRegistration(self, self->logSystem), "updateRegistration", self->dbgid) ); self->addActor.send( reportErrors(updateRegistration(self, self->logSystem), "UpdateRegistration", self->dbgid) );
self->registrationTrigger.trigger(); self->registrationTrigger.trigger();
Void _ = wait(discardCommit(self->txnStateStore, self->txnStateLogAdapter)); Void _ = wait(discardCommit(self->txnStateStore, self->txnStateLogAdapter));
@ -1255,14 +1255,14 @@ ACTOR Future<Void> masterCore( Reference<MasterData> self ) {
double recoveryDuration = now() - recoverStartTime; double recoveryDuration = now() - recoverStartTime;
TraceEvent((recoveryDuration > 4 && !g_network->isSimulated()) ? SevWarnAlways : SevInfo, "MasterRecoveryDuration", self->dbgid) TraceEvent((recoveryDuration > 4 && !g_network->isSimulated()) ? SevWarnAlways : SevInfo, "MasterRecoveryDuration", self->dbgid)
.detail("recoveryDuration", recoveryDuration) .detail("RecoveryDuration", recoveryDuration)
.trackLatest("MasterRecoveryDuration"); .trackLatest("MasterRecoveryDuration");
TraceEvent("MasterRecoveryState", self->dbgid) TraceEvent("MasterRecoveryState", self->dbgid)
.detail("StatusCode", RecoveryStatus::fully_recovered) .detail("StatusCode", RecoveryStatus::fully_recovered)
.detail("Status", RecoveryStatus::names[RecoveryStatus::fully_recovered]) .detail("Status", RecoveryStatus::names[RecoveryStatus::fully_recovered])
.detail("storeType", self->configuration.storageServerStoreType) .detail("StoreType", self->configuration.storageServerStoreType)
.detail("recoveryDuration", recoveryDuration) .detail("RecoveryDuration", recoveryDuration)
.trackLatest("MasterRecoveryState"); .trackLatest("MasterRecoveryState");
// Now that the master is recovered we can start auxiliary services that happen to run here // Now that the master is recovered we can start auxiliary services that happen to run here

92
fdbserver/storageserver.actor.cpp
View File

@ -407,38 +407,38 @@ public:
Counters(StorageServer* self) Counters(StorageServer* self)
: cc("StorageServer", self->thisServerID.toString()), : cc("StorageServer", self->thisServerID.toString()),
getKeyQueries("getKeyQueries", cc), getKeyQueries("GetKeyQueries", cc),
getValueQueries("getValueQueries",cc), getValueQueries("GetValueQueries",cc),
getRangeQueries("getRangeQueries", cc), getRangeQueries("GetRangeQueries", cc),
allQueries("QueryQueue", cc), allQueries("QueryQueue", cc),
finishedQueries("finishedQueries", cc), finishedQueries("FinishedQueries", cc),
rowsQueried("rowsQueried", cc), rowsQueried("RowsQueried", cc),
bytesQueried("bytesQueried", cc), bytesQueried("BytesQueried", cc),
bytesInput("bytesInput", cc), bytesInput("BytesInput", cc),
bytesDurable("bytesDurable", cc), bytesDurable("BytesDurable", cc),
bytesFetched("bytesFetched", cc), bytesFetched("BytesFetched", cc),
mutationBytes("mutationBytes", cc), mutationBytes("MutationBytes", cc),
updateBatches("updateBatches", cc), updateBatches("UpdateBatches", cc),
updateVersions("updateVersions", cc), updateVersions("UpdateVersions", cc),
loops("loops", cc) loops("Loops", cc)
{ {
specialCounter(cc, "lastTLogVersion", [self](){return self->lastTLogVersion; }); specialCounter(cc, "LastTLogVersion", [self](){ return self->lastTLogVersion; });
specialCounter(cc, "version", [self](){return self->version.get(); }); specialCounter(cc, "Version", [self](){ return self->version.get(); });
specialCounter(cc, "storageVersion", [self](){return self->storageVersion(); }); specialCounter(cc, "StorageVersion", [self](){ return self->storageVersion(); });
specialCounter(cc, "durableVersion", [self](){return self->durableVersion.get(); }); specialCounter(cc, "DurableVersion", [self](){ return self->durableVersion.get(); });
specialCounter(cc, "desiredOldestVersion", [self](){return self->desiredOldestVersion.get(); }); specialCounter(cc, "DesiredOldestVersion", [self](){ return self->desiredOldestVersion.get(); });
specialCounter(cc, "FetchKeysFetchActive", [self](){return self->fetchKeysParallelismLock.activePermits(); }); specialCounter(cc, "FetchKeysFetchActive", [self](){ return self->fetchKeysParallelismLock.activePermits(); });
specialCounter(cc, "FetchKeysWaiting", [self](){return self->fetchKeysParallelismLock.waiters(); }); specialCounter(cc, "FetchKeysWaiting", [self](){ return self->fetchKeysParallelismLock.waiters(); });
specialCounter(cc, "QueryQueueMax", [self](){return self->getAndResetMaxQueryQueueSize(); }); specialCounter(cc, "QueryQueueMax", [self](){ return self->getAndResetMaxQueryQueueSize(); });
specialCounter(cc, "bytesStored", [self](){return self->metrics.byteSample.getEstimate(allKeys); }); specialCounter(cc, "BytesStored", [self](){ return self->metrics.byteSample.getEstimate(allKeys); });
specialCounter(cc, "kvstoreBytesUsed", [self](){ return self->storage.getStorageBytes().used; }); specialCounter(cc, "KvstoreBytesUsed", [self](){ return self->storage.getStorageBytes().used; });
specialCounter(cc, "kvstoreBytesFree", [self](){ return self->storage.getStorageBytes().free; }); specialCounter(cc, "KvstoreBytesFree", [self](){ return self->storage.getStorageBytes().free; });
specialCounter(cc, "kvstoreBytesAvailable", [self](){ return self->storage.getStorageBytes().available; }); specialCounter(cc, "KvstoreBytesAvailable", [self](){ return self->storage.getStorageBytes().available; });
specialCounter(cc, "kvstoreBytesTotal", [self](){ return self->storage.getStorageBytes().total; }); specialCounter(cc, "KvstoreBytesTotal", [self](){ return self->storage.getStorageBytes().total; });
} }
} counters; } counters;
@ -773,7 +773,7 @@ ACTOR Future<Void> watchValue_impl( StorageServer* data, WatchValueRequest req )
GetValueRequest getReq( req.key, latest, req.debugID ); GetValueRequest getReq( req.key, latest, req.debugID );
state Future<Void> getValue = getValueQ( data, getReq ); //we are relying on the delay zero at the top of getValueQ, if removed we need one here state Future<Void> getValue = getValueQ( data, getReq ); //we are relying on the delay zero at the top of getValueQ, if removed we need one here
GetValueReply reply = wait( getReq.reply.getFuture() ); GetValueReply reply = wait( getReq.reply.getFuture() );
//TraceEvent("watcherCheckValue").detail("key", printable( req.key ) ).detail("value", printable( req.value ) ).detail("currentValue", printable( v ) ).detail("ver", latest); //TraceEvent("WatcherCheckValue").detail("Key", printable( req.key ) ).detail("Value", printable( req.value ) ).detail("CurrentValue", printable( v ) ).detail("Ver", latest);
debugMutation("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("<null>") ) ); debugMutation("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("<null>") ) );
@ -999,10 +999,10 @@ ACTOR Future<GetKeyValuesReply> readRange( StorageServer* data, Version version,
if (more) { // if there might be more data, begin reading right after what we already found to find out if (more) { // if there might be more data, begin reading right after what we already found to find out
//if (track) printf("more\n"); //if (track) printf("more\n");
if (!(limit<=0 || *pLimitBytes<=0 || result.data.end()[-1].key == atStorageVersion.end()[-1].key)) if (!(limit<=0 || *pLimitBytes<=0 || result.data.end()[-1].key == atStorageVersion.end()[-1].key))
TraceEvent(SevError, "readRangeIssue", data->thisServerID).detail("rB", printable(readBegin)).detail("rE", printable(readEnd)) TraceEvent(SevError, "ReadRangeIssue", data->thisServerID).detail("ReadBegin", printable(readBegin)).detail("ReadEnd", printable(readEnd))
.detail("vStart", vStart ? printable(vStart.key()) : "nil").detail("vEnd", vEnd ? printable(vEnd.key()) : "nil") .detail("VStart", vStart ? printable(vStart.key()) : "nil").detail("VEnd", vEnd ? printable(vEnd.key()) : "nil")
.detail("atStorageVersionBack", printable(atStorageVersion.end()[-1].key)).detail("resultBack", printable(result.data.end()[-1].key)) .detail("AtStorageVersionBack", printable(atStorageVersion.end()[-1].key)).detail("ResultBack", printable(result.data.end()[-1].key))
.detail("limit", limit).detail("*pLimitBytes", *pLimitBytes).detail("resultSize", result.data.size()).detail("prevSize", prevSize); .detail("Limit", limit).detail("LimitBytes", *pLimitBytes).detail("ResultSize", result.data.size()).detail("PrevSize", prevSize);
readBegin = readBeginTemp = keyAfter( result.data.end()[-1].key ); readBegin = readBeginTemp = keyAfter( result.data.end()[-1].key );
ASSERT( limit<=0 || *pLimitBytes<=0 || result.data.end()[-1].key == atStorageVersion.end()[-1].key ); ASSERT( limit<=0 || *pLimitBytes<=0 || result.data.end()[-1].key == atStorageVersion.end()[-1].key );
} else if (vStart && vStart->isClearTo()){ // if vStart is a clear, skip it. } else if (vStart && vStart->isClearTo()){ // if vStart is a clear, skip it.
@ -1018,11 +1018,11 @@ ACTOR Future<GetKeyValuesReply> readRange( StorageServer* data, Version version,
ASSERT( result.data.size() == 0 || *pLimitBytes + result.data.end()[-1].expectedSize() + sizeof(KeyValueRef) > 0 ); ASSERT( result.data.size() == 0 || *pLimitBytes + result.data.end()[-1].expectedSize() + sizeof(KeyValueRef) > 0 );
/*if (*pLimitBytes <= 0) /*if (*pLimitBytes <= 0)
TraceEvent(SevWarn, "readRangeLimitExceeded") TraceEvent(SevWarn, "readRangeLimitExceeded")
.detail("version", version) .detail("Version", version)
.detail("begin", printable(range.begin) ) .detail("Begin", printable(range.begin) )
.detail("end", printable(range.end) ) .detail("End", printable(range.end) )
.detail("limitReamin", limit) .detail("LimitReamin", limit)
.detail("lBytesRemain", *pLimitBytes); */ .detail("LimitBytesRemain", *pLimitBytes); */
/*GetKeyValuesReply correct = wait( readRangeOld(data, version, range, originalLimit, originalLimitBytes) ); /*GetKeyValuesReply correct = wait( readRangeOld(data, version, range, originalLimit, originalLimitBytes) );
bool prefix_equal = true; bool prefix_equal = true;
@ -1203,7 +1203,7 @@ ACTOR Future<Void> getKeyValues( StorageServer* data, GetKeyValuesRequest req )
if( req.debugID.present() ) if( req.debugID.present() )
g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterVersion"); g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterVersion");
//.detail("shardBegin", printable(shard.begin)).detail("shardEnd", printable(shard.end)); //.detail("ShardBegin", printable(shard.begin)).detail("ShardEnd", printable(shard.end));
//} catch (Error& e) { TraceEvent("WrongShardServer", data->thisServerID).detail("Begin", req.begin.toString()).detail("End", req.end.toString()).detail("Version", version).detail("Shard", "None").detail("In", "getKeyValues>getShardKeyRange"); throw e; } //} catch (Error& e) { TraceEvent("WrongShardServer", data->thisServerID).detail("Begin", req.begin.toString()).detail("End", req.end.toString()).detail("Version", version).detail("Shard", "None").detail("In", "getKeyValues>getShardKeyRange"); throw e; }
if ( !selectorInRange(req.end, shard) && !(req.end.isFirstGreaterOrEqual() && req.end.getKey() == shard.end) ) { if ( !selectorInRange(req.end, shard) && !(req.end.isFirstGreaterOrEqual() && req.end.getKey() == shard.end) ) {
@ -1219,7 +1219,7 @@ ACTOR Future<Void> getKeyValues( StorageServer* data, GetKeyValuesRequest req )
state Key end = wait(fEnd); state Key end = wait(fEnd);
if( req.debugID.present() ) if( req.debugID.present() )
g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterKeys"); g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterKeys");
//.detail("off1",offset1).detail("off2",offset2).detail("reqBegin",printable(req.begin.getKey())).detail("reqEnd",printable(req.end.getKey())); //.detail("Off1",offset1).detail("Off2",offset2).detail("ReqBegin",printable(req.begin.getKey())).detail("ReqEnd",printable(req.end.getKey()));
// Offsets of zero indicate begin/end keys in this shard, which obviously means we can answer the query // Offsets of zero indicate begin/end keys in this shard, which obviously means we can answer the query
// An end offset of 1 is also OK because the end key is exclusive, so if the first key of the next shard is the end the last actual key returned must be from this shard. // An end offset of 1 is also OK because the end key is exclusive, so if the first key of the next shard is the end the last actual key returned must be from this shard.
@ -1236,7 +1236,7 @@ ACTOR Future<Void> getKeyValues( StorageServer* data, GetKeyValuesRequest req )
if (begin >= end) { if (begin >= end) {
if( req.debugID.present() ) if( req.debugID.present() )
g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Send"); g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.Send");
//.detail("begin",printable(begin)).detail("end",printable(end)); //.detail("Begin",printable(begin)).detail("End",printable(end));
GetKeyValuesReply none; GetKeyValuesReply none;
none.version = version; none.version = version;
@ -1253,7 +1253,7 @@ ACTOR Future<Void> getKeyValues( StorageServer* data, GetKeyValuesRequest req )
if( req.debugID.present() ) if( req.debugID.present() )
g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterReadRange"); g_traceBatch.addEvent("TransactionDebug", req.debugID.get().first(), "storageserver.getKeyValues.AfterReadRange");
//.detail("begin",printable(begin)).detail("end",printable(end)).detail("sizeOf",r.data.size()); //.detail("Begin",printable(begin)).detail("End",printable(end)).detail("SizeOf",r.data.size());
data->checkChangeCounter( changeCounter, KeyRangeRef( std::min<KeyRef>(begin, std::min<KeyRef>(req.begin.getKey(), req.end.getKey())), std::max<KeyRef>(end, std::max<KeyRef>(req.begin.getKey(), req.end.getKey())) ) ); data->checkChangeCounter( changeCounter, KeyRangeRef( std::min<KeyRef>(begin, std::min<KeyRef>(req.begin.getKey(), req.end.getKey())), std::max<KeyRef>(end, std::max<KeyRef>(req.begin.getKey(), req.end.getKey())) ) );
if (EXPENSIVE_VALIDATION) { if (EXPENSIVE_VALIDATION) {
for (int i = 0; i < r.data.size(); i++) for (int i = 0; i < r.data.size(); i++)
@ -1890,7 +1890,7 @@ ACTOR Future<Void> fetchKeys( StorageServer *data, AddingShard* shard ) {
debug_getRangeRetries++; debug_getRangeRetries++;
if (debug_nextRetryToLog==debug_getRangeRetries){ if (debug_nextRetryToLog==debug_getRangeRetries){
debug_nextRetryToLog += std::min(debug_nextRetryToLog, 1024); debug_nextRetryToLog += std::min(debug_nextRetryToLog, 1024);
TraceEvent(SevWarn, "FetchPast", data->thisServerID).detail("totalAttempts", debug_getRangeRetries).detail("FKID", interval.pairID).detail("V", lastFV).detail("N", fetchVersion).detail("E", data->version.get()); TraceEvent(SevWarn, "FetchPast", data->thisServerID).detail("TotalAttempts", debug_getRangeRetries).detail("FKID", interval.pairID).detail("V", lastFV).detail("N", fetchVersion).detail("E", data->version.get());
} }
} else if (e.code() == error_code_future_version) { } else if (e.code() == error_code_future_version) {
TEST(true); // fetchKeys got future_version, so there must be a huge storage lag somewhere. Keep trying. TEST(true); // fetchKeys got future_version, so there must be a huge storage lag somewhere. Keep trying.
@ -2298,7 +2298,7 @@ private:
.detail("FromVersion", fromVersion) .detail("FromVersion", fromVersion)
.detail("ToVersion", rollbackVersion) .detail("ToVersion", rollbackVersion)
.detail("AtVersion", currentVersion) .detail("AtVersion", currentVersion)
.detail("storageVersion", data->storageVersion()); .detail("StorageVersion", data->storageVersion());
ASSERT( rollbackVersion >= data->storageVersion() ); ASSERT( rollbackVersion >= data->storageVersion() );
} }
// Don't let oldestVersion (and thus storageVersion) go into the rolled back range of versions // Don't let oldestVersion (and thus storageVersion) go into the rolled back range of versions
@ -2508,8 +2508,8 @@ ACTOR Future<Void> update( StorageServer* data, bool* pReceivedUpdate )
data->version.set( ver ); // Triggers replies to waiting gets for new version(s) data->version.set( ver ); // Triggers replies to waiting gets for new version(s)
if (data->otherError.getFuture().isReady()) data->otherError.getFuture().get(); if (data->otherError.getFuture().isReady()) data->otherError.getFuture().get();
//TraceEvent("StorageServerUpdated", data->thisServerID).detail("ver", ver).detail("dataVersion", data->version.get()) //TraceEvent("StorageServerUpdated", data->thisServerID).detail("Ver", ver).detail("DataVersion", data->version.get())
// .detail("lastTLogVersion", data->lastTLogVersion).detail("newOldest", updater.newOldestVersion).detail("desiredOldest",data->desiredOldestVersion.get()) // .detail("LastTLogVersion", data->lastTLogVersion).detail("NewOldest", updater.newOldestVersion).detail("DesiredOldest",data->desiredOldestVersion.get())
// .detail("MAX_READ_TRANSACTION_LIFE_VERSIONS", SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS); // .detail("MAX_READ_TRANSACTION_LIFE_VERSIONS", SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS);
// Trigger updateStorage if necessary // Trigger updateStorage if necessary
@ -3324,9 +3324,9 @@ ACTOR Future<Void> replaceInterface( StorageServer* self, StorageServerInterface
} }
self->allHistory = self->history; self->allHistory = self->history;
TraceEvent("SSTag", self->thisServerID).detail("myTag", self->tag.toString()); TraceEvent("SSTag", self->thisServerID).detail("MyTag", self->tag.toString());
for(auto it : self->history) { for(auto it : self->history) {
TraceEvent("SSHistory", self->thisServerID).detail("ver", it.first).detail("tag", it.second.toString()); TraceEvent("SSHistory", self->thisServerID).detail("Ver", it.first).detail("Tag", it.second.toString());
} }
if(self->history.size() && BUGGIFY) { if(self->history.size() && BUGGIFY) {

14
fdbserver/tester.actor.cpp
View File

@ -965,7 +965,7 @@ vector<TestSpec> readTests( ifstream& ifs ) {
if(value == "true") if(value == "true")
spec.phases = TestWorkload::CHECK; spec.phases = TestWorkload::CHECK;
} else if( attrib == "StderrSeverity" ) { } else if( attrib == "StderrSeverity" ) {
TraceEvent("StderrSeverity").detail("newSeverity", value); TraceEvent("StderrSeverity").detail("NewSeverity", value);
} }
else if (attrib == "ClientInfoLogging") { else if (attrib == "ClientInfoLogging") {
if (value == "false") { if (value == "false") {
@ -1127,9 +1127,9 @@ ACTOR Future<Void> runTests( Reference<ClusterConnectionFile> connFile, test_typ
Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc( new AsyncVar<Optional<ClusterControllerFullInterface>> ); Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc( new AsyncVar<Optional<ClusterControllerFullInterface>> );
Reference<AsyncVar<Optional<ClusterInterface>>> ci( new AsyncVar<Optional<ClusterInterface>> ); Reference<AsyncVar<Optional<ClusterInterface>>> ci( new AsyncVar<Optional<ClusterInterface>> );
vector<Future<Void>> actors; vector<Future<Void>> actors;
actors.push_back( reportErrors(monitorLeader( connFile, cc ), "monitorLeader") ); actors.push_back( reportErrors(monitorLeader( connFile, cc ), "MonitorLeader") );
actors.push_back( reportErrors(extractClusterInterface( cc,ci ),"extractClusterInterface") ); actors.push_back( reportErrors(extractClusterInterface( cc,ci ),"ExtractClusterInterface") );
actors.push_back( reportErrors(failureMonitorClient( ci, false ),"failureMonitorClient") ); actors.push_back( reportErrors(failureMonitorClient( ci, false ),"FailureMonitorClient") );
if(whatToRun == TEST_TYPE_CONSISTENCY_CHECK) { if(whatToRun == TEST_TYPE_CONSISTENCY_CHECK) {
TestSpec spec; TestSpec spec;
@ -1166,11 +1166,11 @@ ACTOR Future<Void> runTests( Reference<ClusterConnectionFile> connFile, test_typ
if (at == TEST_HERE) { if (at == TEST_HERE) {
Reference<AsyncVar<ServerDBInfo>> db( new AsyncVar<ServerDBInfo> ); Reference<AsyncVar<ServerDBInfo>> db( new AsyncVar<ServerDBInfo> );
vector<TesterInterface> iTesters(1); vector<TesterInterface> iTesters(1);
actors.push_back( reportErrors(monitorServerDBInfo( cc, Reference<ClusterConnectionFile>(), LocalityData(), db ), "monitorServerDBInfo") ); // FIXME: Locality actors.push_back( reportErrors(monitorServerDBInfo( cc, Reference<ClusterConnectionFile>(), LocalityData(), db ), "MonitorServerDBInfo") ); // FIXME: Locality
actors.push_back( reportErrors(testerServerCore( iTesters[0], connFile, db, locality ), "testerServerCore") ); actors.push_back( reportErrors(testerServerCore( iTesters[0], connFile, db, locality ), "TesterServerCore") );
tests = runTests( cc, ci, iTesters, testSpecs, startingConfiguration, locality ); tests = runTests( cc, ci, iTesters, testSpecs, startingConfiguration, locality );
} else { } else {
tests = reportErrors(runTests(cc, ci, testSpecs, at, minTestersExpected, startingConfiguration, locality), "runTests"); tests = reportErrors(runTests(cc, ci, testSpecs, at, minTestersExpected, startingConfiguration, locality), "RunTests");
} }
choose { choose {

10
fdbserver/worker.actor.cpp
View File

@ -973,7 +973,7 @@ ACTOR Future<Void> fdbd(
{ {
try { try {
ServerCoordinators coordinators( connFile ); ServerCoordinators coordinators( connFile );
TraceEvent("StartingFDBD").detailext("ZoneID", localities.zoneId()).detailext("machineId", localities.machineId()).detail("DiskPath", dataFolder).detail("CoordPath", coordFolder); TraceEvent("StartingFDBD").detailext("ZoneID", localities.zoneId()).detailext("MachineId", localities.machineId()).detail("DiskPath", dataFolder).detail("CoordPath", coordFolder);
// SOMEDAY: start the services on the machine in a staggered fashion in simulation? // SOMEDAY: start the services on the machine in a staggered fashion in simulation?
Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc( new AsyncVar<Optional<ClusterControllerFullInterface>> ); Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> cc( new AsyncVar<Optional<ClusterControllerFullInterface>> );
@ -984,10 +984,10 @@ ACTOR Future<Void> fdbd(
state Promise<Void> recoveredDiskFiles; state Promise<Void> recoveredDiskFiles;
if ( coordFolder.size() ) if ( coordFolder.size() )
v.push_back( fileNotFoundToNever( coordinationServer( coordFolder ) ) ); //SOMEDAY: remove the fileNotFound wrapper and make DiskQueue construction safe from errors setting up their files v.push_back( fileNotFoundToNever( coordinationServer( coordFolder ) ) ); //SOMEDAY: remove the fileNotFound wrapper and make DiskQueue construction safe from errors setting up their files
v.push_back( reportErrors( processClass == ProcessClass::TesterClass ? monitorLeader( connFile, cc ) : clusterController( connFile, cc , asyncPriorityInfo, recoveredDiskFiles.getFuture()), "clusterController") ); v.push_back( reportErrors( processClass == ProcessClass::TesterClass ? monitorLeader( connFile, cc ) : clusterController( connFile, cc , asyncPriorityInfo, recoveredDiskFiles.getFuture()), "ClusterController") );
v.push_back( reportErrors(extractClusterInterface( cc, ci ), "extractClusterInterface") ); v.push_back( reportErrors(extractClusterInterface( cc, ci ), "ExtractClusterInterface") );
v.push_back( reportErrors(failureMonitorClient( ci, true ), "failureMonitorClient") ); v.push_back( reportErrors(failureMonitorClient( ci, true ), "FailureMonitorClient") );
v.push_back( reportErrorsExcept(workerServer(connFile, cc, localities, asyncPriorityInfo, processClass, dataFolder, memoryLimit, metricsConnFile, metricsPrefix, recoveredDiskFiles), "workerServer", UID(), &normalWorkerErrors()) ); v.push_back( reportErrorsExcept(workerServer(connFile, cc, localities, asyncPriorityInfo, processClass, dataFolder, memoryLimit, metricsConnFile, metricsPrefix, recoveredDiskFiles), "WorkerServer", UID(), &normalWorkerErrors()) );
state Future<Void> firstConnect = reportErrors( printOnFirstConnected(ci), "ClusterFirstConnectedError" ); state Future<Void> firstConnect = reportErrors( printOnFirstConnected(ci), "ClusterFirstConnectedError" );
Void _ = wait( quorum(v,1) ); Void _ = wait( quorum(v,1) );

14
fdbserver/workloads/AsyncFileRead.actor.cpp
View File

@ -60,13 +60,13 @@ struct IOLog {
void dumpMetrics(std::string name){ void dumpMetrics(std::string name){
double elapsed = now() - startTime; double elapsed = now() - startTime;
TraceEvent("ProcessLog") TraceEvent("ProcessLog")
.detail("name", name) .detail("Name", name)
.detail("hz", count / elapsed) .detail("Hz", count / elapsed)
.detail("latency_ms", 1e3 * sumSq / elapsed / 2.0) .detail("Latency", sumSq / elapsed / 2.0)
.detail("avg_latency_ms", 1e3 * sum / count) .detail("AvgLatency", sum / count)
.detail("max_latency_ms", 1e3 * max) .detail("MaxLatency", max)
.detail("startTime", startTime) .detail("StartTime", startTime)
.detail("elapsed", elapsed); .detail("Elapsed", elapsed);
} }
}; };

8
fdbserver/workloads/AtomicOps.actor.cpp
View File

@ -45,7 +45,7 @@ struct AtomicOpsWorkload : TestWorkload {
// Atomic OPs Min and And have modified behavior from api version 510. Hence allowing testing for older version (500) with a 10% probability // Atomic OPs Min and And have modified behavior from api version 510. Hence allowing testing for older version (500) with a 10% probability
// Actual change of api Version happens in setup // Actual change of api Version happens in setup
apiVersion500 = ((sharedRandomNumber % 10) == 0); apiVersion500 = ((sharedRandomNumber % 10) == 0);
TraceEvent("AtomicOpsApiVersion500").detail("apiVersion500", apiVersion500); TraceEvent("AtomicOpsApiVersion500").detail("ApiVersion500", apiVersion500);
int64_t randNum = sharedRandomNumber / 10; int64_t randNum = sharedRandomNumber / 10;
if(opType == -1) if(opType == -1)
@ -87,7 +87,7 @@ struct AtomicOpsWorkload : TestWorkload {
default: default:
ASSERT(false); ASSERT(false);
} }
TraceEvent("AtomicWorkload").detail("opType", opType); TraceEvent("AtomicWorkload").detail("OpType", opType);
} }
virtual std::string description() { return "AtomicOps"; } virtual std::string description() { return "AtomicOps"; }
@ -188,7 +188,7 @@ struct AtomicOpsWorkload : TestWorkload {
} }
if(tr.get(LiteralStringRef("xlogResult")).get() != tr.get(LiteralStringRef("xopsResult")).get()) { if(tr.get(LiteralStringRef("xlogResult")).get() != tr.get(LiteralStringRef("xopsResult")).get()) {
TraceEvent(SevError, "LogMismatch").detail("logResult", printable(tr.get(LiteralStringRef("xlogResult")).get())).detail("opsResult", printable(tr.get(LiteralStringRef("xopsResult")).get().get())); TraceEvent(SevError, "LogMismatch").detail("LogResult", printable(tr.get(LiteralStringRef("xlogResult")).get())).detail("OpsResult", printable(tr.get(LiteralStringRef("xopsResult")).get().get()));
} }
if( self->opType == MutationRef::AddValue ) { if( self->opType == MutationRef::AddValue ) {
@ -202,7 +202,7 @@ struct AtomicOpsWorkload : TestWorkload {
logResult += intValue; logResult += intValue;
} }
if(logResult != opsResult) { if(logResult != opsResult) {
TraceEvent(SevError, "LogAddMismatch").detail("logResult", logResult).detail("opResult", opsResult).detail("opsResultStr", printable(opsResultStr)).detail("size", opsResultStr.size()); TraceEvent(SevError, "LogAddMismatch").detail("LogResult", logResult).detail("OpResult", opsResult).detail("OpsResultStr", printable(opsResultStr)).detail("Size", opsResultStr.size());
} }
} }
break; break;

28
fdbserver/workloads/AtomicOpsApiCorrectness.actor.cpp
View File

@ -111,7 +111,7 @@ public:
break; break;
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(SevInfo, "AtomicOpApiThrow").detail("errCode", e.code()); TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
Void _ = wait(delay(1)); Void _ = wait(delay(1));
} }
} }
@ -120,7 +120,7 @@ public:
ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t)); ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
memcpy(&output, outputVal.get().begin(), outputVal.get().size()); memcpy(&output, outputVal.get().begin(), outputVal.get().size());
if (output != intValue) { if (output != intValue) {
TraceEvent(SevError, "AtomicOpSetOnNonExistingKeyUnexpectedOutput").detail("opOn", "StorageServer").detail("op", opType).detail("ExpectedOutput", intValue).detail("ActualOutput", output); TraceEvent(SevError, "AtomicOpSetOnNonExistingKeyUnexpectedOutput").detail("OpOn", "StorageServer").detail("Op", opType).detail("ExpectedOutput", intValue).detail("ActualOutput", output);
self->testFailed = true; self->testFailed = true;
} }
@ -130,7 +130,7 @@ public:
ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t)); ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
memcpy(&output, outputVal.get().begin(), outputVal.get().size()); memcpy(&output, outputVal.get().begin(), outputVal.get().size());
if (output != intValue) { if (output != intValue) {
TraceEvent(SevError, "AtomicOpSetOnNonExistingKeyUnexpectedOutput").detail("opOn", "RYWLayer").detail("op", opType).detail("ExpectedOutput", intValue).detail("ActualOutput", output); TraceEvent(SevError, "AtomicOpSetOnNonExistingKeyUnexpectedOutput").detail("OpOn", "RYWLayer").detail("Op", opType).detail("ExpectedOutput", intValue).detail("ActualOutput", output);
self->testFailed = true; self->testFailed = true;
} }
return Void(); return Void();
@ -149,7 +149,7 @@ public:
break; break;
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(SevInfo, "AtomicOpApiThrow").detail("errCode", e.code()); TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
Void _ = wait(delay(1)); Void _ = wait(delay(1));
} }
} }
@ -158,7 +158,7 @@ public:
ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t)); ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
memcpy(&output, outputVal.get().begin(), outputVal.get().size()); memcpy(&output, outputVal.get().begin(), outputVal.get().size());
if (output != 0) { if (output != 0) {
TraceEvent(SevError, "AtomicOpUnsetOnNonExistingKeyUnexpectedOutput").detail("opOn", "StorageServer").detail("op", opType).detail("ExpectedOutput", 0).detail("ActualOutput", output); TraceEvent(SevError, "AtomicOpUnsetOnNonExistingKeyUnexpectedOutput").detail("OpOn", "StorageServer").detail("Op", opType).detail("ExpectedOutput", 0).detail("ActualOutput", output);
self->testFailed = true; self->testFailed = true;
} }
@ -168,7 +168,7 @@ public:
ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t)); ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
memcpy(&output, outputVal.get().begin(), outputVal.get().size()); memcpy(&output, outputVal.get().begin(), outputVal.get().size());
if (output != 0) { if (output != 0) {
TraceEvent(SevError, "AtomicOpUnsetOnNonExistingKeyUnexpectedOutput").detail("opOn", "RYWLayer").detail("op", opType).detail("ExpectedOutput", 0).detail("ActualOutput", output); TraceEvent(SevError, "AtomicOpUnsetOnNonExistingKeyUnexpectedOutput").detail("OpOn", "RYWLayer").detail("Op", opType).detail("ExpectedOutput", 0).detail("ActualOutput", output);
self->testFailed = true; self->testFailed = true;
} }
return Void(); return Void();
@ -197,7 +197,7 @@ public:
break; break;
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(SevInfo, "AtomicOpApiThrow").detail("errCode", e.code()); TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
Void _ = wait(delay(1)); Void _ = wait(delay(1));
} }
} }
@ -205,7 +205,7 @@ public:
ASSERT(outputVal.present()); ASSERT(outputVal.present());
Value output = outputVal.get(); Value output = outputVal.get();
if (output != opFunc(existingVal, otherVal)) { if (output != opFunc(existingVal, otherVal)) {
TraceEvent(SevError, "AtomicOpOnEmptyValueUnexpectedOutput").detail("opOn", "StorageServer").detail("op", opType).detail("ExpectedOutput", opFunc(existingVal, otherVal).toString()).detail("ActualOutput", output.toString()); TraceEvent(SevError, "AtomicOpOnEmptyValueUnexpectedOutput").detail("OpOn", "StorageServer").detail("Op", opType).detail("ExpectedOutput", opFunc(existingVal, otherVal).toString()).detail("ActualOutput", output.toString());
self->testFailed = true; self->testFailed = true;
} }
@ -214,7 +214,7 @@ public:
ASSERT(outputVal.present()); ASSERT(outputVal.present());
Value output = outputVal.get(); Value output = outputVal.get();
if (output != opFunc(existingVal, otherVal)) { if (output != opFunc(existingVal, otherVal)) {
TraceEvent(SevError, "AtomicOpOnEmptyValueUnexpectedOutput").detail("opOn", "RYWLayer").detail("op", opType).detail("ExpectedOutput", opFunc(existingVal, otherVal).toString()).detail("ActualOutput", output.toString()); TraceEvent(SevError, "AtomicOpOnEmptyValueUnexpectedOutput").detail("OpOn", "RYWLayer").detail("Op", opType).detail("ExpectedOutput", opFunc(existingVal, otherVal).toString()).detail("ActualOutput", output.toString());
self->testFailed = true; self->testFailed = true;
} }
return Void(); return Void();
@ -240,7 +240,7 @@ public:
break; break;
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(SevInfo, "AtomicOpApiThrow").detail("errCode", e.code()); TraceEvent(SevInfo, "AtomicOpApiThrow").detail("ErrCode", e.code());
Void _ = wait(delay(1)); Void _ = wait(delay(1));
} }
} }
@ -250,7 +250,7 @@ public:
ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t)); ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
memcpy(&output, outputVal.get().begin(), outputVal.get().size()); memcpy(&output, outputVal.get().begin(), outputVal.get().size());
if (output != opFunc(intValue1, intValue2)) { if (output != opFunc(intValue1, intValue2)) {
TraceEvent(SevError, "AtomicOpApiCorrectnessUnexpectedOutput").detail("opOn", "StorageServer").detail("InValue1", intValue1).detail("InValue2", intValue2).detail("AtomicOp", opType).detail("ExpectedOutput", opFunc(intValue1, intValue2)).detail("ActualOutput", output); TraceEvent(SevError, "AtomicOpApiCorrectnessUnexpectedOutput").detail("OpOn", "StorageServer").detail("InValue1", intValue1).detail("InValue2", intValue2).detail("AtomicOp", opType).detail("ExpectedOutput", opFunc(intValue1, intValue2)).detail("ActualOutput", output);
self->testFailed = true; self->testFailed = true;
} }
@ -261,7 +261,7 @@ public:
ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t)); ASSERT(outputVal.present() && outputVal.get().size() == sizeof(uint64_t));
memcpy(&output, outputVal.get().begin(), outputVal.get().size()); memcpy(&output, outputVal.get().begin(), outputVal.get().size());
if (output != opFunc(intValue1, intValue2)) { if (output != opFunc(intValue1, intValue2)) {
TraceEvent(SevError, "AtomicOpApiCorrectnessUnexpectedOutput").detail("opOn", "RYWLayer").detail("InValue1", intValue1).detail("InValue2", intValue2).detail("AtomicOp", opType).detail("ExpectedOutput", opFunc(intValue1, intValue2)).detail("ActualOutput", output); TraceEvent(SevError, "AtomicOpApiCorrectnessUnexpectedOutput").detail("OpOn", "RYWLayer").detail("InValue1", intValue1).detail("InValue2", intValue2).detail("AtomicOp", opType).detail("ExpectedOutput", opFunc(intValue1, intValue2)).detail("ActualOutput", output);
self->testFailed = true; self->testFailed = true;
} }
@ -272,7 +272,7 @@ public:
state int currentApiVersion = getApiVersion(cx); state int currentApiVersion = getApiVersion(cx);
state Key key = self->getTestKey("test_key_min_"); state Key key = self->getTestKey("test_key_min_");
TraceEvent("AtomicOpCorrectnessApiWorkload").detail("opType", "MIN"); TraceEvent("AtomicOpCorrectnessApiWorkload").detail("OpType", "MIN");
// API Version 500 // API Version 500
setApiVersion(&cx, 500); setApiVersion(&cx, 500);
TraceEvent(SevInfo, "Running Atomic Op Min Correctness Test Api Version 500"); TraceEvent(SevInfo, "Running Atomic Op Min Correctness Test Api Version 500");
@ -305,7 +305,7 @@ public:
state int currentApiVersion = getApiVersion(cx); state int currentApiVersion = getApiVersion(cx);
state Key key = self->getTestKey("test_key_and_"); state Key key = self->getTestKey("test_key_and_");
TraceEvent("AtomicOpCorrectnessApiWorkload").detail("opType", "AND"); TraceEvent("AtomicOpCorrectnessApiWorkload").detail("OpType", "AND");
// API Version 500 // API Version 500
setApiVersion(&cx, 500); setApiVersion(&cx, 500);
TraceEvent(SevInfo, "Running Atomic Op AND Correctness Test Api Version 500"); TraceEvent(SevInfo, "Running Atomic Op AND Correctness Test Api Version 500");

10
fdbserver/workloads/AtomicSwitchover.actor.cpp
View File

@ -100,13 +100,13 @@ struct AtomicSwitchoverWorkload : TestWorkload {
while (src != srcFuture.get().end() && bkp != bkpFuture.get().end()) { while (src != srcFuture.get().end() && bkp != bkpFuture.get().end()) {
KeyRef bkpKey = bkp->key.substr(backupPrefix.size()); KeyRef bkpKey = bkp->key.substr(backupPrefix.size());
if (src->key != bkpKey && src->value != bkp->value) { if (src->key != bkpKey && src->value != bkp->value) {
TraceEvent(SevError, "MismatchKeyAndValue").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchKeyAndValue").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
else if (src->key != bkpKey) { else if (src->key != bkpKey) {
TraceEvent(SevError, "MismatchKey").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchKey").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
else if (src->value != bkp->value) { else if (src->value != bkp->value) {
TraceEvent(SevError, "MismatchValue").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchValue").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
begin = std::min(src->key, bkpKey); begin = std::min(src->key, bkpKey);
if (src->key == bkpKey) { if (src->key == bkpKey) {
@ -121,12 +121,12 @@ struct AtomicSwitchoverWorkload : TestWorkload {
} }
} }
while (src != srcFuture.get().end() && !bkpFuture.get().more) { while (src != srcFuture.get().end() && !bkpFuture.get().more) {
TraceEvent(SevError, "MissingBkpKey").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)); TraceEvent(SevError, "MissingBkpKey").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value));
begin = src->key; begin = src->key;
++src; ++src;
} }
while (bkp != bkpFuture.get().end() && !srcFuture.get().more) { while (bkp != bkpFuture.get().end() && !srcFuture.get().more) {
TraceEvent(SevError, "MissingSrcKey").detail("bkpKey", printable(bkp->key.substr(backupPrefix.size()))).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MissingSrcKey").detail("BkpKey", printable(bkp->key.substr(backupPrefix.size()))).detail("BkpVal", printable(bkp->value));
begin = bkp->key; begin = bkp->key;
++bkp; ++bkp;
} }

128
fdbserver/workloads/BackupCorrectness.actor.cpp
View File

@ -83,8 +83,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(start, *i)); backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(start, *i));
// Track the added range // Track the added range
TraceEvent("BARW_BackupCorrectness_Range", randomID).detail("rangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange)) TraceEvent("BARW_BackupCorrectnessRange", randomID).detail("RangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange))
.detail("rangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange)); .detail("RangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange));
} }
} }
} }
@ -101,17 +101,17 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (clientId != 0) if (clientId != 0)
return Void(); return Void();
TraceEvent(SevInfo, "BARW_Param").detail("locked", locked); TraceEvent(SevInfo, "BARW_Param").detail("Locked", locked);
TraceEvent(SevInfo, "BARW_Param").detail("backupAfter", backupAfter); TraceEvent(SevInfo, "BARW_Param").detail("BackupAfter", backupAfter);
TraceEvent(SevInfo, "BARW_Param").detail("restoreAfter", restoreAfter); TraceEvent(SevInfo, "BARW_Param").detail("RestoreAfter", restoreAfter);
TraceEvent(SevInfo, "BARW_Param").detail("performRestore", performRestore); TraceEvent(SevInfo, "BARW_Param").detail("PerformRestore", performRestore);
TraceEvent(SevInfo, "BARW_Param").detail("backupTag", printable(backupTag).c_str()); TraceEvent(SevInfo, "BARW_Param").detail("BackupTag", printable(backupTag).c_str());
TraceEvent(SevInfo, "BARW_Param").detail("backupRangesCount", backupRangesCount); TraceEvent(SevInfo, "BARW_Param").detail("BackupRangesCount", backupRangesCount);
TraceEvent(SevInfo, "BARW_Param").detail("backupRangeLengthMax", backupRangeLengthMax); TraceEvent(SevInfo, "BARW_Param").detail("BackupRangeLengthMax", backupRangeLengthMax);
TraceEvent(SevInfo, "BARW_Param").detail("abortAndRestartAfter", abortAndRestartAfter); TraceEvent(SevInfo, "BARW_Param").detail("AbortAndRestartAfter", abortAndRestartAfter);
TraceEvent(SevInfo, "BARW_Param").detail("differentialBackup", differentialBackup); TraceEvent(SevInfo, "BARW_Param").detail("DifferentialBackup", differentialBackup);
TraceEvent(SevInfo, "BARW_Param").detail("stopDifferentialAfter", stopDifferentialAfter); TraceEvent(SevInfo, "BARW_Param").detail("StopDifferentialAfter", stopDifferentialAfter);
TraceEvent(SevInfo, "BARW_Param").detail("agentRequest", agentRequest); TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest);
return _start(cx, this); return _start(cx, this);
} }
@ -150,19 +150,19 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
Void _ = wait( delay( startDelay )); Void _ = wait( delay( startDelay ));
if (startDelay || BUGGIFY) { if (startDelay || BUGGIFY) {
TraceEvent("BARW_doBackupAbortBackup1", randomID).detail("tag", printable(tag)).detail("startDelay", startDelay); TraceEvent("BARW_DoBackupAbortBackup1", randomID).detail("Tag", printable(tag)).detail("StartDelay", startDelay);
try { try {
Void _ = wait(backupAgent->abortBackup(cx, tag.toString())); Void _ = wait(backupAgent->abortBackup(cx, tag.toString()));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_doBackupAbortBackupException", randomID).detail("tag", printable(tag)).error(e); TraceEvent("BARW_DoBackupAbortBackupException", randomID).detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded) if (e.code() != error_code_backup_unneeded)
throw; throw;
} }
} }
TraceEvent("BARW_doBackupSubmitBackup", randomID).detail("tag", printable(tag)).detail("stopWhenDone", stopDifferentialDelay ? "False" : "True"); TraceEvent("BARW_DoBackupSubmitBackup", randomID).detail("Tag", printable(tag)).detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
state std::string backupContainer = "file://simfdb/backups/"; state std::string backupContainer = "file://simfdb/backups/";
state Future<Void> status = statusLoop(cx, tag.toString()); state Future<Void> status = statusLoop(cx, tag.toString());
@ -171,7 +171,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
Void _ = wait(backupAgent->submitBackup(cx, StringRef(backupContainer), g_random->randomInt(0, 100), tag.toString(), backupRanges, stopDifferentialDelay ? false : true)); Void _ = wait(backupAgent->submitBackup(cx, StringRef(backupContainer), g_random->randomInt(0, 100), tag.toString(), backupRanges, stopDifferentialDelay ? false : true));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_doBackupSubmitBackupException", randomID).detail("tag", printable(tag)).error(e); TraceEvent("BARW_DoBackupSubmitBackupException", randomID).detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
@ -182,12 +182,12 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (stopDifferentialDelay) { if (stopDifferentialDelay) {
TEST(!stopDifferentialFuture.isReady()); //Restore starts at specified time TEST(!stopDifferentialFuture.isReady()); //Restore starts at specified time
Void _ = wait(stopDifferentialFuture); Void _ = wait(stopDifferentialFuture);
TraceEvent("BARW_doBackupWaitToDiscontinue", randomID).detail("tag", printable(tag)).detail("differentialAfter", stopDifferentialDelay); TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID).detail("Tag", printable(tag)).detail("DifferentialAfter", stopDifferentialDelay);
try { try {
if (BUGGIFY) { if (BUGGIFY) {
state KeyBackedTag backupTag = makeBackupTag(tag.toString()); state KeyBackedTag backupTag = makeBackupTag(tag.toString());
TraceEvent("BARW_doBackupWaitForRestorable", randomID).detail("tag", backupTag.tagName); TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", backupTag.tagName);
// Wait until the backup is in a restorable state // Wait until the backup is in a restorable state
state int resultWait = wait(backupAgent->waitBackup(cx, backupTag.tagName, false)); state int resultWait = wait(backupAgent->waitBackup(cx, backupTag.tagName, false));
UidAndAbortedFlagT uidFlag = wait(backupTag.getOrThrow(cx)); UidAndAbortedFlagT uidFlag = wait(backupTag.getOrThrow(cx));
@ -202,56 +202,56 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
restorable = desc.maxRestorableVersion.present(); restorable = desc.maxRestorableVersion.present();
} }
TraceEvent("BARW_lastBackupContainer", randomID) TraceEvent("BARW_LastBackupContainer", randomID)
.detail("backupTag", printable(tag)) .detail("BackupTag", printable(tag))
.detail("lastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "") .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
.detail("logUid", logUid).detail("waitStatus", resultWait).detail("restorable", restorable); .detail("LogUid", logUid).detail("WaitStatus", resultWait).detail("Restorable", restorable);
// Do not check the backup, if aborted // Do not check the backup, if aborted
if (resultWait == BackupAgentBase::STATE_ABORTED) { if (resultWait == BackupAgentBase::STATE_ABORTED) {
} }
// Ensure that a backup container was found // Ensure that a backup container was found
else if (!lastBackupContainer) { else if (!lastBackupContainer) {
TraceEvent("BARW_missingBackupContainer", randomID).detail("logUid", logUid).detail("backupTag", printable(tag)).detail("waitStatus", resultWait); TraceEvent("BARW_MissingBackupContainer", randomID).detail("LogUid", logUid).detail("BackupTag", printable(tag)).detail("WaitStatus", resultWait);
printf("BackupCorrectnessMissingBackupContainer tag: %s status: %d\n", printable(tag).c_str(), resultWait); printf("BackupCorrectnessMissingBackupContainer tag: %s status: %d\n", printable(tag).c_str(), resultWait);
} }
// Check that backup is restorable // Check that backup is restorable
else { else {
if(!restorable) { if(!restorable) {
TraceEvent("BARW_notRestorable", randomID).detail("logUid", logUid).detail("backupTag", printable(tag)) TraceEvent("BARW_NotRestorable", randomID).detail("LogUid", logUid).detail("BackupTag", printable(tag))
.detail("backupFolder", lastBackupContainer->getURL()).detail("waitStatus", resultWait); .detail("BackupFolder", lastBackupContainer->getURL()).detail("WaitStatus", resultWait);
printf("BackupCorrectnessNotRestorable: tag: %s\n", printable(tag).c_str()); printf("BackupCorrectnessNotRestorable: tag: %s\n", printable(tag).c_str());
} }
} }
// Abort the backup, if not the first backup because the second backup may have aborted the backup by now // Abort the backup, if not the first backup because the second backup may have aborted the backup by now
if (startDelay) { if (startDelay) {
TraceEvent("BARW_doBackupAbortBackup2", randomID).detail("tag", printable(tag)) TraceEvent("BARW_DoBackupAbortBackup2", randomID).detail("Tag", printable(tag))
.detail("waitStatus", resultWait) .detail("WaitStatus", resultWait)
.detail("lastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "") .detail("LastBackupContainer", lastBackupContainer ? lastBackupContainer->getURL() : "")
.detail("restorable", restorable); .detail("Restorable", restorable);
Void _ = wait(backupAgent->abortBackup(cx, tag.toString())); Void _ = wait(backupAgent->abortBackup(cx, tag.toString()));
} }
else { else {
TraceEvent("BARW_doBackupDiscontinueBackup", randomID).detail("tag", printable(tag)).detail("differentialAfter", stopDifferentialDelay); TraceEvent("BARW_DoBackupDiscontinueBackup", randomID).detail("Tag", printable(tag)).detail("DifferentialAfter", stopDifferentialDelay);
Void _ = wait(backupAgent->discontinueBackup(cx, tag)); Void _ = wait(backupAgent->discontinueBackup(cx, tag));
} }
} }
else { else {
TraceEvent("BARW_doBackupDiscontinueBackup", randomID).detail("tag", printable(tag)).detail("differentialAfter", stopDifferentialDelay); TraceEvent("BARW_DoBackupDiscontinueBackup", randomID).detail("Tag", printable(tag)).detail("DifferentialAfter", stopDifferentialDelay);
Void _ = wait(backupAgent->discontinueBackup(cx, tag)); Void _ = wait(backupAgent->discontinueBackup(cx, tag));
} }
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_doBackupDiscontinueBackupException", randomID).detail("tag", printable(tag)).error(e); TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
} }
// Wait for the backup to complete // Wait for the backup to complete
TraceEvent("BARW_doBackupWaitBackup", randomID).detail("tag", printable(tag)); TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
state int statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), true)); state int statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), true));
state std::string statusText; state std::string statusText;
@ -260,8 +260,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
statusText = _statusText; statusText = _statusText;
// Can we validate anything about status? // Can we validate anything about status?
TraceEvent("BARW_doBackupComplete", randomID).detail("tag", printable(tag)) TraceEvent("BARW_DoBackupComplete", randomID).detail("Tag", printable(tag))
.detail("status", statusText).detail("statusValue", statusValue); .detail("Status", statusText).detail("StatusValue", statusValue);
return Void(); return Void();
} }
@ -287,7 +287,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (rowCount > 0) { if (rowCount > 0) {
try { try {
Version _ = wait(backupAgent->restore(cx, self->backupTag, KeyRef(lastBackupContainer), true, -1, true, normalKeys, Key(), Key(), self->locked)); Version _ = wait(backupAgent->restore(cx, self->backupTag, KeyRef(lastBackupContainer), true, -1, true, normalKeys, Key(), Key(), self->locked));
TraceEvent(SevError, "BARW_restore_allowed_overwritting_database", randomID); TraceEvent(SevError, "BARW_RestoreAllowedOverwrittingDatabase", randomID);
ASSERT(false); ASSERT(false);
} }
catch (Error &e) { catch (Error &e) {
@ -304,9 +304,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
state FileBackupAgent backupAgent; state FileBackupAgent backupAgent;
state Future<Void> extraBackup; state Future<Void> extraBackup;
state bool extraTasks = false; state bool extraTasks = false;
TraceEvent("BARW_Arguments").detail("backupTag", printable(self->backupTag)).detail("performRestore", self->performRestore) TraceEvent("BARW_Arguments").detail("BackupTag", printable(self->backupTag)).detail("PerformRestore", self->performRestore)
.detail("backupAfter", self->backupAfter).detail("restoreAfter", self->restoreAfter) .detail("BackupAfter", self->backupAfter).detail("RestoreAfter", self->restoreAfter)
.detail("abortAndRestartAfter", self->abortAndRestartAfter).detail("differentialAfter", self->stopDifferentialAfter); .detail("AbortAndRestartAfter", self->abortAndRestartAfter).detail("DifferentialAfter", self->stopDifferentialAfter);
state UID randomID = g_nondeterministic_random->randomUniqueID(); state UID randomID = g_nondeterministic_random->randomUniqueID();
if(self->allowPauses && BUGGIFY) { if(self->allowPauses && BUGGIFY) {
@ -324,17 +324,17 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
// backup // backup
Void _ = wait(delay(self->backupAfter)); Void _ = wait(delay(self->backupAfter));
TraceEvent("BARW_doBackup1", randomID).detail("tag", printable(self->backupTag)); TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
state Promise<Void> submitted; state Promise<Void> submitted;
state Future<Void> b = doBackup(self, 0, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter, submitted); state Future<Void> b = doBackup(self, 0, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter, submitted);
if (self->abortAndRestartAfter) { if (self->abortAndRestartAfter) {
TraceEvent("BARW_doBackup2", randomID).detail("tag", printable(self->backupTag)).detail("abortWait", self->abortAndRestartAfter); TraceEvent("BARW_DoBackup2", randomID).detail("Tag", printable(self->backupTag)).detail("AbortWait", self->abortAndRestartAfter);
Void _ = wait(submitted.getFuture()); Void _ = wait(submitted.getFuture());
b = b && doBackup(self, self->abortAndRestartAfter, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter, Promise<Void>()); b = b && doBackup(self, self->abortAndRestartAfter, &backupAgent, cx, self->backupTag, self->backupRanges, self->stopDifferentialAfter, Promise<Void>());
} }
TraceEvent("BARW_doBackupWait", randomID).detail("backupTag", printable(self->backupTag)).detail("abortAndRestartAfter", self->abortAndRestartAfter); TraceEvent("BARW_DoBackupWait", randomID).detail("BackupTag", printable(self->backupTag)).detail("AbortAndRestartAfter", self->abortAndRestartAfter);
try { try {
Void _ = wait(b); Void _ = wait(b);
} catch( Error &e ) { } catch( Error &e ) {
@ -344,7 +344,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
throw; throw;
return Void(); return Void();
} }
TraceEvent("BARW_doBackupDone", randomID).detail("backupTag", printable(self->backupTag)).detail("abortAndRestartAfter", self->abortAndRestartAfter); TraceEvent("BARW_DoBackupDone", randomID).detail("BackupTag", printable(self->backupTag)).detail("AbortAndRestartAfter", self->abortAndRestartAfter);
state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString()); state KeyBackedTag keyBackedTag = makeBackupTag(self->backupTag.toString());
UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx)); UidAndAbortedFlagT uidFlag = wait(keyBackedTag.getOrThrow(cx));
@ -354,12 +354,12 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
// Occasionally start yet another backup that might still be running when we restore // Occasionally start yet another backup that might still be running when we restore
if (!self->locked && BUGGIFY) { if (!self->locked && BUGGIFY) {
TraceEvent("BARW_submitBackup2", randomID).detail("tag", printable(self->backupTag)); TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
try { try {
extraBackup = backupAgent.submitBackup(cx, LiteralStringRef("file://simfdb/backups/"), g_random->randomInt(0, 100), self->backupTag.toString(), self->backupRanges, true); extraBackup = backupAgent.submitBackup(cx, LiteralStringRef("file://simfdb/backups/"), g_random->randomInt(0, 100), self->backupTag.toString(), self->backupRanges, true);
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_submitBackup2Exception", randomID).detail("backupTag", printable(self->backupTag)).error(e); TraceEvent("BARW_SubmitBackup2Exception", randomID).detail("BackupTag", printable(self->backupTag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
@ -379,7 +379,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
})); }));
// restore database // restore database
TraceEvent("BARW_restore", randomID).detail("lastBackupContainer", lastBackupContainer->getURL()).detail("restoreAfter", self->restoreAfter).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_Restore", randomID).detail("LastBackupContainer", lastBackupContainer->getURL()).detail("RestoreAfter", self->restoreAfter).detail("BackupTag", printable(self->backupTag));
state std::vector<Future<Version>> restores; state std::vector<Future<Version>> restores;
state std::vector<Standalone<StringRef>> restoreTags; state std::vector<Standalone<StringRef>> restoreTags;
@ -417,23 +417,23 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
} }
if (extraBackup.isValid()) { if (extraBackup.isValid()) {
TraceEvent("BARW_waitExtraBackup", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(self->backupTag));
extraTasks = true; extraTasks = true;
try { try {
Void _ = wait(extraBackup); Void _ = wait(extraBackup);
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_extraBackupException", randomID).detail("backupTag", printable(self->backupTag)).error(e); TraceEvent("BARW_ExtraBackupException", randomID).detail("BackupTag", printable(self->backupTag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
TraceEvent("BARW_abortBackupExtra", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(self->backupTag));
try { try {
Void _ = wait(backupAgent.abortBackup(cx, self->backupTag.toString())); Void _ = wait(backupAgent.abortBackup(cx, self->backupTag.toString()));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_abortBackupExtraException", randomID).error(e); TraceEvent("BARW_AbortBackupExtraException", randomID).error(e);
if (e.code() != error_code_backup_unneeded) if (e.code() != error_code_backup_unneeded)
throw; throw;
} }
@ -449,7 +449,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
loop { loop {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx)); state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
TraceEvent("BARW_check_leftoverkeys", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(self->backupTag));
try { try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -459,12 +459,12 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
// Check the left over tasks // Check the left over tasks
// We have to wait for the list to empty since an abort and get status // We have to wait for the list to empty since an abort and get status
// can leave extra tasks in the queue // can leave extra tasks in the queue
TraceEvent("BARW_check_leftovertasks", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(self->backupTag));
state int64_t taskCount = wait( backupAgent.getTaskCount(tr) ); state int64_t taskCount = wait( backupAgent.getTaskCount(tr) );
state int waitCycles = 0; state int waitCycles = 0;
if ((taskCount) && (0)) { if ((taskCount) && (0)) {
TraceEvent("BARW_EndingNonzeroTaskCount", randomID).detail("backupTag", printable(self->backupTag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent("BARW_EndingNonzeroTaskCount", randomID).detail("BackupTag", printable(self->backupTag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("EndingNonZeroTasks: %ld\n", (long) taskCount); printf("EndingNonZeroTasks: %ld\n", (long) taskCount);
Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
} }
@ -472,7 +472,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
loop { loop {
waitCycles ++; waitCycles ++;
TraceEvent("BARW_NonzeroTaskWait", randomID).detail("backupTag", printable(self->backupTag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent("BARW_NonzeroTaskWait", randomID).detail("BackupTag", printable(self->backupTag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("%.6f %-10s Wait #%4d for %lld tasks to end\n", now(), randomID.toString().c_str(), waitCycles, (long long) taskCount); printf("%.6f %-10s Wait #%4d for %lld tasks to end\n", now(), randomID.toString().c_str(), waitCycles, (long long) taskCount);
Void _ = wait(delay(5.0)); Void _ = wait(delay(5.0));
@ -488,7 +488,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (taskCount) { if (taskCount) {
displaySystemKeys ++; displaySystemKeys ++;
TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID).detail("backupTag", printable(self->backupTag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID).detail("BackupTag", printable(self->backupTag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long) taskCount); printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long) taskCount);
} }
@ -500,10 +500,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (agentValues.size() > 0) { if (agentValues.size() > 0) {
displaySystemKeys ++; displaySystemKeys ++;
printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n", agentValues.size(), printable(backupAgentKey).c_str()); printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n", agentValues.size(), printable(backupAgentKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID).detail("backupTag", printable(self->backupTag)) TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID).detail("BackupTag", printable(self->backupTag))
.detail("LeftOverKeys", agentValues.size()).detail("keySpace", printable(backupAgentKey)); .detail("LeftOverKeys", agentValues.size()).detail("KeySpace", printable(backupAgentKey));
for (auto & s : agentValues) { for (auto & s : agentValues) {
TraceEvent("BARW_LeftOverKey", randomID).detail("key", printable(StringRef(s.key.toString()))).detail("value", printable(StringRef(s.value.toString()))); TraceEvent("BARW_LeftOverKey", randomID).detail("Key", printable(StringRef(s.key.toString()))).detail("Value", printable(StringRef(s.value.toString())));
printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str()); printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str());
} }
} }
@ -513,7 +513,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey)); Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
if (latestVersion.present()) { if (latestVersion.present()) {
TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID).detail("backupTag", printable(self->backupTag)).detail("backupLatestVersionsKey", backupLatestVersionsKey.printable()).detail("destUidValue", destUidValue.printable()); TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID).detail("BackupTag", printable(self->backupTag)).detail("BackupLatestVersionsKey", backupLatestVersionsKey.printable()).detail("DestUidValue", destUidValue.printable());
} else { } else {
printf("No left over backup version key\n"); printf("No left over backup version key\n");
} }
@ -526,8 +526,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (logValues.size() > 0) { if (logValues.size() > 0) {
displaySystemKeys ++; displaySystemKeys ++;
printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n", logValues.size(), printable(backupLogValuesKey).c_str()); printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n", logValues.size(), printable(backupLogValuesKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID).detail("backupTag", printable(self->backupTag)) TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID).detail("BackupTag", printable(self->backupTag))
.detail("LeftOverKeys", logValues.size()).detail("keySpace", printable(backupLogValuesKey)); .detail("LeftOverKeys", logValues.size()).detail("KeySpace", printable(backupLogValuesKey));
} }
else { else {
printf("No left over backup log keys\n"); printf("No left over backup log keys\n");
@ -537,7 +537,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
break; break;
} }
catch (Error &e) { catch (Error &e) {
TraceEvent("BARW_checkException", randomID).error(e); TraceEvent("BARW_CheckException", randomID).error(e);
Void _ = wait(tr->onError(e)); Void _ = wait(tr->onError(e));
} }
} }
@ -546,7 +546,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
} }
TraceEvent("BARW_complete", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
// Decrement the backup agent requets // Decrement the backup agent requets
if (self->agentRequest) { if (self->agentRequest) {

2
fdbserver/workloads/BackupToDBAbort.actor.cpp
View File

@ -73,7 +73,7 @@ struct BackupToDBAbort : TestWorkload {
ACTOR static Future<Void> _start(BackupToDBAbort* self, Database cx) { ACTOR static Future<Void> _start(BackupToDBAbort* self, Database cx) {
state DatabaseBackupAgent backupAgent(cx); state DatabaseBackupAgent backupAgent(cx);
TraceEvent("BDBA_Start").detail("delay", self->abortDelay); TraceEvent("BDBA_Start").detail("Delay", self->abortDelay);
Void _ = wait(delay(self->abortDelay)); Void _ = wait(delay(self->abortDelay));
TraceEvent("BDBA_Wait"); TraceEvent("BDBA_Wait");
int _ = wait( backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), false) ); int _ = wait( backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), false) );

108
fdbserver/workloads/BackupToDBCorrectness.actor.cpp
View File

@ -98,8 +98,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
backupRanges.push_back_deep(backupRanges.arena(), (beginRange < endRange) ? KeyRangeRef(beginRange, endRange) : KeyRangeRef(endRange, beginRange)); backupRanges.push_back_deep(backupRanges.arena(), (beginRange < endRange) ? KeyRangeRef(beginRange, endRange) : KeyRangeRef(endRange, beginRange));
// Track the added range // Track the added range
TraceEvent("BackupCorrectness_Range", randomID).detail("rangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange)) TraceEvent("BackupCorrectness_Range", randomID).detail("RangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange))
.detail("rangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange)); .detail("RangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange));
} }
} }
@ -107,7 +107,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1); Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1);
extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get(); extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get();
TraceEvent("BARW_start").detail("locked", locked); TraceEvent("BARW_Start").detail("Locked", locked);
} }
virtual std::string description() { virtual std::string description() {
@ -151,13 +151,13 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
while (src != srcFuture.get().end() && bkp != bkpFuture.get().end()) { while (src != srcFuture.get().end() && bkp != bkpFuture.get().end()) {
KeyRef bkpKey = bkp->key.substr(backupPrefix.size()); KeyRef bkpKey = bkp->key.substr(backupPrefix.size());
if (src->key != bkpKey && src->value != bkp->value) { if (src->key != bkpKey && src->value != bkp->value) {
TraceEvent(SevError, "MismatchKeyAndValue").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchKeyAndValue").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
else if (src->key != bkpKey) { else if (src->key != bkpKey) {
TraceEvent(SevError, "MismatchKey").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchKey").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
else if (src->value != bkp->value) { else if (src->value != bkp->value) {
TraceEvent(SevError, "MismatchValue").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchValue").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
begin = std::min(src->key, bkpKey); begin = std::min(src->key, bkpKey);
if (src->key == bkpKey) { if (src->key == bkpKey) {
@ -172,12 +172,12 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
} }
} }
while (src != srcFuture.get().end() && !bkpFuture.get().more) { while (src != srcFuture.get().end() && !bkpFuture.get().more) {
TraceEvent(SevError, "MissingBkpKey").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)); TraceEvent(SevError, "MissingBkpKey").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value));
begin = src->key; begin = src->key;
++src; ++src;
} }
while (bkp != bkpFuture.get().end() && !srcFuture.get().more) { while (bkp != bkpFuture.get().end() && !srcFuture.get().more) {
TraceEvent(SevError, "MissingSrcKey").detail("bkpKey", printable(bkp->key.substr(backupPrefix.size()))).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MissingSrcKey").detail("BkpKey", printable(bkp->key.substr(backupPrefix.size()))).detail("BkpVal", printable(bkp->value));
begin = bkp->key; begin = bkp->key;
++bkp; ++bkp;
} }
@ -209,13 +209,13 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
Void _ = wait( delay( startDelay )); Void _ = wait( delay( startDelay ));
if (startDelay || BUGGIFY) { if (startDelay || BUGGIFY) {
TraceEvent("BARW_doBackup abortBackup1", randomID).detail("tag", printable(tag)).detail("startDelay", startDelay); TraceEvent("BARW_DoBackupAbortBackup1", randomID).detail("Tag", printable(tag)).detail("StartDelay", startDelay);
try { try {
Void _ = wait(backupAgent->abortBackup(cx, tag)); Void _ = wait(backupAgent->abortBackup(cx, tag));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_doBackup abortBackup Exception", randomID).detail("tag", printable(tag)).error(e); TraceEvent("BARW_DoBackupAbortBackupException", randomID).detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded) if (e.code() != error_code_backup_unneeded)
throw; throw;
} }
@ -226,7 +226,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
// pointless in this test) because separating them causes rare errors where the SubmitBackup commit result is indeterminite but the // pointless in this test) because separating them causes rare errors where the SubmitBackup commit result is indeterminite but the
// submission was in fact successful and the backup actually completes before the retry of SubmitBackup so this second call to submit // submission was in fact successful and the backup actually completes before the retry of SubmitBackup so this second call to submit
// fails because the destination range is no longer empty. // fails because the destination range is no longer empty.
TraceEvent("BARW_doBackup clearAndSubmitBackup", randomID).detail("tag", printable(tag)).detail("stopWhenDone", stopDifferentialDelay ? "False" : "True"); TraceEvent("BARW_DoBackupClearAndSubmitBackup", randomID).detail("Tag", printable(tag)).detail("StopWhenDone", stopDifferentialDelay ? "False" : "True");
try { try {
state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(self->extraDB)); state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(self->extraDB));
@ -250,7 +250,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
} }
} }
catch (Error &e) { catch (Error &e) {
TraceEvent("BARW_doBackup submitBackup Exception", randomID).detail("tag", printable(tag)).error(e); TraceEvent("BARW_DoBackupSubmitBackupException", randomID).detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) { if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) {
throw e; throw e;
} }
@ -264,37 +264,37 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
if (stopDifferentialDelay) { if (stopDifferentialDelay) {
TEST(!stopDifferentialFuture.isReady()); //Restore starts at specified time TEST(!stopDifferentialFuture.isReady()); //Restore starts at specified time
Void _ = wait(stopDifferentialFuture); Void _ = wait(stopDifferentialFuture);
TraceEvent("BARW_doBackup waitToDiscontinue", randomID).detail("tag", printable(tag)).detail("differentialAfter", stopDifferentialDelay); TraceEvent("BARW_DoBackupWaitToDiscontinue", randomID).detail("Tag", printable(tag)).detail("DifferentialAfter", stopDifferentialDelay);
state bool aborted = false; state bool aborted = false;
try { try {
if (BUGGIFY) { if (BUGGIFY) {
TraceEvent("BARW_doBackup waitForRestorable", randomID).detail("tag", printable(tag)); TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", printable(tag));
// Wait until the backup is in a restorable state // Wait until the backup is in a restorable state
state int resultWait = wait(backupAgent->waitBackup(cx, tag, false)); state int resultWait = wait(backupAgent->waitBackup(cx, tag, false));
TraceEvent("BARW_lastBackupFolder", randomID).detail("backupTag", printable(tag)) TraceEvent("BARW_LastBackupFolder", randomID).detail("BackupTag", printable(tag))
.detail("logUid", logUid).detail("waitStatus", resultWait); .detail("LogUid", logUid).detail("WaitStatus", resultWait);
// Abort the backup, if not the first backup because the second backup may have aborted the backup by now // Abort the backup, if not the first backup because the second backup may have aborted the backup by now
if (startDelay) { if (startDelay) {
TraceEvent("BARW_doBackup abortBackup2", randomID).detail("tag", printable(tag)).detail("waitStatus", resultWait); TraceEvent("BARW_DoBackupAbortBackup2", randomID).detail("Tag", printable(tag)).detail("WaitStatus", resultWait);
aborted = true; aborted = true;
Void _ = wait(backupAgent->abortBackup(cx, tag)); Void _ = wait(backupAgent->abortBackup(cx, tag));
} }
else { else {
TraceEvent("BARW_doBackup discontinueBackup", randomID).detail("tag", printable(tag)).detail("differentialAfter", stopDifferentialDelay); TraceEvent("BARW_DoBackupDiscontinueBackup", randomID).detail("Tag", printable(tag)).detail("DifferentialAfter", stopDifferentialDelay);
Void _ = wait(backupAgent->discontinueBackup(cx, tag)); Void _ = wait(backupAgent->discontinueBackup(cx, tag));
} }
} }
else { else {
TraceEvent("BARW_doBackup discontinueBackup", randomID).detail("tag", printable(tag)).detail("differentialAfter", stopDifferentialDelay); TraceEvent("BARW_DoBackupDiscontinueBackup", randomID).detail("Tag", printable(tag)).detail("DifferentialAfter", stopDifferentialDelay);
Void _ = wait(backupAgent->discontinueBackup(cx, tag)); Void _ = wait(backupAgent->discontinueBackup(cx, tag));
} }
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_doBackup discontinueBackup Exception", randomID).detail("tag", printable(tag)).error(e); TraceEvent("BARW_DoBackupDiscontinueBackupException", randomID).detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
@ -305,7 +305,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
} }
// Wait for the backup to complete // Wait for the backup to complete
TraceEvent("BARW_doBackup waitBackup", randomID).detail("tag", printable(tag)); TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag));
UID _destUid = wait(backupAgent->getDestUid(cx, logUid)); UID _destUid = wait(backupAgent->getDestUid(cx, logUid));
self->destUid = _destUid; self->destUid = _destUid;
@ -319,8 +319,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
statusText = _statusText; statusText = _statusText;
// Can we validate anything about status? // Can we validate anything about status?
TraceEvent("BARW_doBackup complete", randomID).detail("tag", printable(tag)) TraceEvent("BARW_DoBackupComplete", randomID).detail("Tag", printable(tag))
.detail("status", statusText).detail("statusValue", statusValue); .detail("Status", statusText).detail("StatusValue", statusValue);
return Void(); return Void();
} }
@ -336,7 +336,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
loop { loop {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx)); state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
TraceEvent("BARW_check_leftoverkeys", randomID).detail("backupTag", printable(tag)); TraceEvent("BARW_CheckLeftoverKeys", randomID).detail("BackupTag", printable(tag));
try { try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -344,12 +344,12 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
// Check the left over tasks // Check the left over tasks
// We have to wait for the list to empty since an abort and get status // We have to wait for the list to empty since an abort and get status
// can leave extra tasks in the queue // can leave extra tasks in the queue
TraceEvent("BARW_check_leftovertasks", randomID).detail("backupTag", printable(tag)); TraceEvent("BARW_CheckLeftoverTasks", randomID).detail("BackupTag", printable(tag));
state int64_t taskCount = wait( backupAgent->getTaskCount(tr) ); state int64_t taskCount = wait( backupAgent->getTaskCount(tr) );
state int waitCycles = 0; state int waitCycles = 0;
if ((taskCount) && (0)) { if ((taskCount) && (0)) {
TraceEvent("BARW_EndingNonzeroTaskCount", randomID).detail("backupTag", printable(tag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent("BARW_EndingNonzeroTaskCount", randomID).detail("BackupTag", printable(tag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("EndingNonZeroTasks: %ld\n", (long) taskCount); printf("EndingNonZeroTasks: %ld\n", (long) taskCount);
Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
} }
@ -357,7 +357,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
loop { loop {
waitCycles ++; waitCycles ++;
TraceEvent("BARW_NonzeroTaskWait", randomID).detail("backupTag", printable(tag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent("BARW_NonzeroTaskWait", randomID).detail("BackupTag", printable(tag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("%.6f %-10s Wait #%4d for %lld tasks to end\n", now(), randomID.toString().c_str(), waitCycles, (long long) taskCount); printf("%.6f %-10s Wait #%4d for %lld tasks to end\n", now(), randomID.toString().c_str(), waitCycles, (long long) taskCount);
Void _ = wait(delay(5.0)); Void _ = wait(delay(5.0));
@ -373,8 +373,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
if (taskCount) { if (taskCount) {
displaySystemKeys ++; displaySystemKeys ++;
TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID).detail("backupTag", printable(tag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent(SevError, "BARW_NonzeroTaskCount", randomID).detail("BackupTag", printable(tag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long) taskCount); printf("BackupCorrectnessLeftoverLogTasks: %ld\n", (long) taskCount);
} }
Standalone<RangeResultRef> agentValues = wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100)); Standalone<RangeResultRef> agentValues = wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
@ -382,11 +382,11 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
// Error if the system keyspace for the backup tag is not empty // Error if the system keyspace for the backup tag is not empty
if (agentValues.size() > 0) { if (agentValues.size() > 0) {
displaySystemKeys++; displaySystemKeys++;
printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n", agentValues.size(), printable(backupAgentKey).c_str()); printf("BackupCorrectnessLeftoverMutationKeys: (%d) %s\n", agentValues.size(), printable(backupAgentKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys", randomID).detail("backupTag", printable(tag)) TraceEvent(SevError, "BackupCorrectnessLeftoverMutationKeys", randomID).detail("BackupTag", printable(tag))
.detail("LeftOverKeys", agentValues.size()).detail("keySpace", printable(backupAgentKey)); .detail("LeftoverKeys", agentValues.size()).detail("KeySpace", printable(backupAgentKey));
for (auto & s : agentValues) { for (auto & s : agentValues) {
TraceEvent("BARW_LeftOverKey", randomID).detail("key", printable(StringRef(s.key.toString()))).detail("value", printable(StringRef(s.value.toString()))); TraceEvent("BARW_LeftoverKey", randomID).detail("Key", printable(StringRef(s.key.toString()))).detail("Value", printable(StringRef(s.value.toString())));
printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str()); printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str());
} }
} }
@ -396,7 +396,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey)); Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
if (latestVersion.present()) { if (latestVersion.present()) {
TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey", randomID).detail("backupTag", printable(tag)).detail("key", backupLatestVersionsKey.printable()).detail("value", BinaryReader::fromStringRef<Version>(latestVersion.get(), Unversioned())); TraceEvent(SevError, "BackupCorrectnessLeftoverVersionKey", randomID).detail("BackupTag", printable(tag)).detail("Key", backupLatestVersionsKey.printable()).detail("Value", BinaryReader::fromStringRef<Version>(latestVersion.get(), Unversioned()));
} else { } else {
printf("No left over backup version key\n"); printf("No left over backup version key\n");
} }
@ -408,11 +408,11 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
// Error if the log/mutation keyspace for the backup tag is not empty // Error if the log/mutation keyspace for the backup tag is not empty
if (logValues.size() > 0) { if (logValues.size() > 0) {
displaySystemKeys++; displaySystemKeys++;
printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n", logValues.size(), printable(backupLogValuesKey).c_str()); printf("BackupCorrectnessLeftoverLogKeys: (%d) %s\n", logValues.size(), printable(backupLogValuesKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys", randomID).detail("backupTag", printable(tag)) TraceEvent(SevError, "BackupCorrectnessLeftoverLogKeys", randomID).detail("BackupTag", printable(tag))
.detail("LeftOverKeys", logValues.size()).detail("keySpace", printable(backupLogValuesKey)).detail("version", decodeBKMutationLogKey(logValues[0].key).first); .detail("LeftoverKeys", logValues.size()).detail("KeySpace", printable(backupLogValuesKey)).detail("Version", decodeBKMutationLogKey(logValues[0].key).first);
for (auto & s : logValues) { for (auto & s : logValues) {
TraceEvent("BARW_LeftOverKey", randomID).detail("key", printable(StringRef(s.key.toString()))).detail("value", printable(StringRef(s.value.toString()))); TraceEvent("BARW_LeftoverKey", randomID).detail("Key", printable(StringRef(s.key.toString()))).detail("Value", printable(StringRef(s.value.toString())));
printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str()); printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str());
} }
} }
@ -424,7 +424,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
break; break;
} }
catch (Error &e) { catch (Error &e) {
TraceEvent("BARW_check Exception", randomID).error(e); TraceEvent("BARW_CheckException", randomID).error(e);
Void _ = wait(tr->onError(e)); Void _ = wait(tr->onError(e));
} }
} }
@ -440,8 +440,8 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
state DatabaseBackupAgent restoreAgent(self->extraDB); state DatabaseBackupAgent restoreAgent(self->extraDB);
state Future<Void> extraBackup; state Future<Void> extraBackup;
state bool extraTasks = false; state bool extraTasks = false;
TraceEvent("BARW_Arguments").detail("backupTag", printable(self->backupTag)).detail("backupAfter", self->backupAfter) TraceEvent("BARW_Arguments").detail("BackupTag", printable(self->backupTag)).detail("BackupAfter", self->backupAfter)
.detail("abortAndRestartAfter", self->abortAndRestartAfter).detail("differentialAfter", self->stopDifferentialAfter); .detail("AbortAndRestartAfter", self->abortAndRestartAfter).detail("DifferentialAfter", self->stopDifferentialAfter);
state UID randomID = g_nondeterministic_random->randomUniqueID(); state UID randomID = g_nondeterministic_random->randomUniqueID();
@ -456,31 +456,31 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
// backup // backup
Void _ = wait(delay(self->backupAfter)); Void _ = wait(delay(self->backupAfter));
TraceEvent("BARW_doBackup1", randomID).detail("tag", printable(self->backupTag)); TraceEvent("BARW_DoBackup1", randomID).detail("Tag", printable(self->backupTag));
state Promise<Void> submitted; state Promise<Void> submitted;
state Future<Void> b = doBackup(self, 0, &backupAgent, self->extraDB, self->backupTag, self->backupRanges, self->stopDifferentialAfter, submitted); state Future<Void> b = doBackup(self, 0, &backupAgent, self->extraDB, self->backupTag, self->backupRanges, self->stopDifferentialAfter, submitted);
if (self->abortAndRestartAfter) { if (self->abortAndRestartAfter) {
TraceEvent("BARW_doBackup2", randomID).detail("tag", printable(self->backupTag)).detail("abortWait", self->abortAndRestartAfter); TraceEvent("BARW_DoBackup2", randomID).detail("Tag", printable(self->backupTag)).detail("AbortWait", self->abortAndRestartAfter);
Void _ = wait(submitted.getFuture()); Void _ = wait(submitted.getFuture());
b = b && doBackup(self, self->abortAndRestartAfter, &backupAgent, self->extraDB, self->backupTag, self->backupRanges, self->stopDifferentialAfter, Promise<Void>()); b = b && doBackup(self, self->abortAndRestartAfter, &backupAgent, self->extraDB, self->backupTag, self->backupRanges, self->stopDifferentialAfter, Promise<Void>());
} }
TraceEvent("BARW_doBackupWait", randomID).detail("backupTag", printable(self->backupTag)).detail("abortAndRestartAfter", self->abortAndRestartAfter); TraceEvent("BARW_DoBackupWait", randomID).detail("BackupTag", printable(self->backupTag)).detail("AbortAndRestartAfter", self->abortAndRestartAfter);
Void _ = wait(b); Void _ = wait(b);
TraceEvent("BARW_doBackupDone", randomID).detail("backupTag", printable(self->backupTag)).detail("abortAndRestartAfter", self->abortAndRestartAfter); TraceEvent("BARW_DoBackupDone", randomID).detail("BackupTag", printable(self->backupTag)).detail("AbortAndRestartAfter", self->abortAndRestartAfter);
state UID logUid = wait(backupAgent.getLogUid(self->extraDB, self->backupTag)); state UID logUid = wait(backupAgent.getLogUid(self->extraDB, self->backupTag));
// Occasionally start yet another backup that might still be running when we restore // Occasionally start yet another backup that might still be running when we restore
if (!self->locked && BUGGIFY) { if (!self->locked && BUGGIFY) {
TraceEvent("BARW_submitBackup2", randomID).detail("tag", printable(self->backupTag)); TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
try { try {
extraBackup = backupAgent.submitBackup(self->extraDB, self->backupTag, self->backupRanges, true, self->extraPrefix, StringRef(), self->locked); extraBackup = backupAgent.submitBackup(self->extraDB, self->backupTag, self->backupRanges, true, self->extraPrefix, StringRef(), self->locked);
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_submitBackup2 Exception", randomID).detail("backupTag", printable(self->backupTag)).error(e); TraceEvent("BARW_SubmitBackup2Exception", randomID).detail("BackupTag", printable(self->backupTag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
@ -491,7 +491,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
if (self->performRestore) { if (self->performRestore) {
// restore database // restore database
TraceEvent("BARW_restore", randomID).detail("restoreAfter", self->restoreAfter).detail("backupTag", printable(self->restoreTag)); TraceEvent("BARW_Restore", randomID).detail("RestoreAfter", self->restoreAfter).detail("BackupTag", printable(self->restoreTag));
//Void _ = wait(diffRanges(self->backupRanges, self->backupPrefix, cx, self->extraDB)); //Void _ = wait(diffRanges(self->backupRanges, self->backupPrefix, cx, self->extraDB));
state Transaction tr3(cx); state Transaction tr3(cx);
@ -520,7 +520,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
Void _ = wait(restoreAgent.submitBackup(cx, self->restoreTag, restoreRange, true, StringRef(), self->backupPrefix, self->locked)); Void _ = wait(restoreAgent.submitBackup(cx, self->restoreTag, restoreRange, true, StringRef(), self->backupPrefix, self->locked));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_doBackup submitBackup Exception", randomID).detail("tag", printable(self->restoreTag)).error(e); TraceEvent("BARW_DoBackupSubmitBackupException", randomID).detail("Tag", printable(self->restoreTag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
@ -530,23 +530,23 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
} }
if (extraBackup.isValid()) { if (extraBackup.isValid()) {
TraceEvent("BARW_wait extraBackup", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_WaitExtraBackup", randomID).detail("BackupTag", printable(self->backupTag));
extraTasks = true; extraTasks = true;
try { try {
Void _ = wait(extraBackup); Void _ = wait(extraBackup);
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_extraBackup Exception", randomID).detail("backupTag", printable(self->backupTag)).error(e); TraceEvent("BARW_ExtraBackupException", randomID).detail("BackupTag", printable(self->backupTag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
TraceEvent("BARW_abortBackup extra", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_AbortBackupExtra", randomID).detail("BackupTag", printable(self->backupTag));
try { try {
Void _ = wait(backupAgent.abortBackup(self->extraDB, self->backupTag)); Void _ = wait(backupAgent.abortBackup(self->extraDB, self->backupTag));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("BARW_abortBackup extra Exception", randomID).error(e); TraceEvent("BARW_AbortBackupExtraException", randomID).error(e);
if (e.code() != error_code_backup_unneeded) if (e.code() != error_code_backup_unneeded)
throw; throw;
} }
@ -559,7 +559,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload {
Void _ = wait( checkData(cx, restoreUid, restoreUid, randomID, self->restoreTag, &restoreAgent, self->shareLogRange) ); Void _ = wait( checkData(cx, restoreUid, restoreUid, randomID, self->restoreTag, &restoreAgent, self->shareLogRange) );
} }
TraceEvent("BARW_complete", randomID).detail("backupTag", printable(self->backupTag)); TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));
// Decrement the backup agent requets // Decrement the backup agent requets
if (self->agentRequest) { if (self->agentRequest) {

78
fdbserver/workloads/BackupToDBUpgrade.actor.cpp
View File

@ -64,8 +64,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
backupRanges.push_back_deep(backupRanges.arena(), (beginRange < endRange) ? KeyRangeRef(beginRange, endRange) : KeyRangeRef(endRange, beginRange)); backupRanges.push_back_deep(backupRanges.arena(), (beginRange < endRange) ? KeyRangeRef(beginRange, endRange) : KeyRangeRef(endRange, beginRange));
// Track the added range // Track the added range
TraceEvent("DRU_backup_range").detail("rangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange)) TraceEvent("DRU_BackupRange").detail("RangeBegin", (beginRange < endRange) ? printable(beginRange) : printable(endRange))
.detail("rangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange)); .detail("RangeEnd", (beginRange < endRange) ? printable(endRange) : printable(beginRange));
} }
} }
@ -73,7 +73,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1); Reference<Cluster> extraCluster = Cluster::createCluster(extraFile, -1);
extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get(); extraDB = extraCluster->createDatabase(LiteralStringRef("DB")).get();
TraceEvent("DRU_start"); TraceEvent("DRU_Start");
} }
virtual std::string description() { virtual std::string description() {
@ -120,9 +120,9 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
} }
} }
TraceEvent("DRU_doBackupInDifferentialMode").detail("tag", printable(tag)); TraceEvent("DRU_DoBackupInDifferentialMode").detail("Tag", printable(tag));
} catch (Error &e) { } catch (Error &e) {
TraceEvent("DRU_doBackupSubmitBackupError").detail("tag", printable(tag)).error(e); TraceEvent("DRU_DoBackupSubmitBackupError").detail("Tag", printable(tag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) { if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) {
throw e; throw e;
} }
@ -146,7 +146,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
loop { loop {
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx)); state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
TraceEvent("DRU_checkLeftoverkeys").detail("backupTag", printable(tag)); TraceEvent("DRU_CheckLeftoverkeys").detail("BackupTag", printable(tag));
try { try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -154,12 +154,12 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
// Check the left over tasks // Check the left over tasks
// We have to wait for the list to empty since an abort and get status // We have to wait for the list to empty since an abort and get status
// can leave extra tasks in the queue // can leave extra tasks in the queue
TraceEvent("DRU_checkLeftovertasks").detail("backupTag", printable(tag)); TraceEvent("DRU_CheckLeftovertasks").detail("BackupTag", printable(tag));
state int64_t taskCount = wait( backupAgent->getTaskCount(tr) ); state int64_t taskCount = wait( backupAgent->getTaskCount(tr) );
state int waitCycles = 0; state int waitCycles = 0;
if ((taskCount) && (0)) { if ((taskCount) && (0)) {
TraceEvent("DRU_EndingNonzeroTaskCount").detail("backupTag", printable(tag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent("DRU_EndingNonzeroTaskCount").detail("BackupTag", printable(tag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("EndingNonZeroTasks: %ld\n", (long) taskCount); printf("EndingNonZeroTasks: %ld\n", (long) taskCount);
Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); Void _ = wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
} }
@ -167,7 +167,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
loop { loop {
waitCycles ++; waitCycles ++;
TraceEvent("DRU_NonzeroTaskWait").detail("backupTag", printable(tag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent("DRU_NonzeroTaskWait").detail("BackupTag", printable(tag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("%.6f Wait #%4d for %lld tasks to end\n", now(), waitCycles, (long long) taskCount); printf("%.6f Wait #%4d for %lld tasks to end\n", now(), waitCycles, (long long) taskCount);
Void _ = wait(delay(20.0)); Void _ = wait(delay(20.0));
@ -183,8 +183,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
if (taskCount) { if (taskCount) {
displaySystemKeys ++; displaySystemKeys ++;
TraceEvent(SevError, "DRU_NonzeroTaskCount").detail("backupTag", printable(tag)).detail("taskCount", taskCount).detail("waitCycles", waitCycles); TraceEvent(SevError, "DRU_NonzeroTaskCount").detail("BackupTag", printable(tag)).detail("TaskCount", taskCount).detail("WaitCycles", waitCycles);
printf("BackupCorrectnessLeftOverLogTasks: %ld\n", (long) taskCount); printf("BackupCorrectnessLeftoverLogTasks: %ld\n", (long) taskCount);
} }
Standalone<RangeResultRef> agentValues = wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100)); Standalone<RangeResultRef> agentValues = wait(tr->getRange(KeyRange(KeyRangeRef(backupAgentKey, strinc(backupAgentKey))), 100));
@ -192,11 +192,11 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
// Error if the system keyspace for the backup tag is not empty // Error if the system keyspace for the backup tag is not empty
if (agentValues.size() > 0) { if (agentValues.size() > 0) {
displaySystemKeys++; displaySystemKeys++;
printf("BackupCorrectnessLeftOverMutationKeys: (%d) %s\n", agentValues.size(), printable(backupAgentKey).c_str()); printf("BackupCorrectnessLeftoverMutationKeys: (%d) %s\n", agentValues.size(), printable(backupAgentKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverMutationKeys").detail("backupTag", printable(tag)) TraceEvent(SevError, "BackupCorrectnessLeftoverMutationKeys").detail("BackupTag", printable(tag))
.detail("LeftOverKeys", agentValues.size()).detail("keySpace", printable(backupAgentKey)); .detail("LeftoverKeys", agentValues.size()).detail("KeySpace", printable(backupAgentKey));
for (auto & s : agentValues) { for (auto & s : agentValues) {
TraceEvent("DRU_LeftOverKey").detail("key", printable(StringRef(s.key.toString()))).detail("value", printable(StringRef(s.value.toString()))); TraceEvent("DRU_LeftoverKey").detail("Key", printable(StringRef(s.key.toString()))).detail("Value", printable(StringRef(s.value.toString())));
printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str()); printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str());
} }
} }
@ -206,7 +206,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey)); Optional<Value> latestVersion = wait(tr->get(backupLatestVersionsKey));
if (latestVersion.present()) { if (latestVersion.present()) {
TraceEvent(SevError, "BackupCorrectnessLeftOverVersionKey").detail("backupTag", printable(tag)).detail("key", backupLatestVersionsKey.printable()).detail("value", BinaryReader::fromStringRef<Version>(latestVersion.get(), Unversioned())); TraceEvent(SevError, "BackupCorrectnessLeftoverVersionKey").detail("BackupTag", printable(tag)).detail("Key", backupLatestVersionsKey.printable()).detail("Value", BinaryReader::fromStringRef<Version>(latestVersion.get(), Unversioned()));
} else { } else {
printf("No left over backup version key\n"); printf("No left over backup version key\n");
} }
@ -218,11 +218,11 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
// Error if the log/mutation keyspace for the backup tag is not empty // Error if the log/mutation keyspace for the backup tag is not empty
if (logValues.size() > 0) { if (logValues.size() > 0) {
displaySystemKeys++; displaySystemKeys++;
printf("BackupCorrectnessLeftOverLogKeys: (%d) %s\n", logValues.size(), printable(backupLogValuesKey).c_str()); printf("BackupCorrectnessLeftoverLogKeys: (%d) %s\n", logValues.size(), printable(backupLogValuesKey).c_str());
TraceEvent(SevError, "BackupCorrectnessLeftOverLogKeys").detail("backupTag", printable(tag)) TraceEvent(SevError, "BackupCorrectnessLeftoverLogKeys").detail("BackupTag", printable(tag))
.detail("LeftOverKeys", logValues.size()).detail("keySpace", printable(backupLogValuesKey)).detail("version", decodeBKMutationLogKey(logValues[0].key).first); .detail("LeftoverKeys", logValues.size()).detail("KeySpace", printable(backupLogValuesKey)).detail("Version", decodeBKMutationLogKey(logValues[0].key).first);
for (auto & s : logValues) { for (auto & s : logValues) {
TraceEvent("DRU_LeftOverKey").detail("key", printable(StringRef(s.key.toString()))).detail("value", printable(StringRef(s.value.toString()))); TraceEvent("DRU_LeftoverKey").detail("Key", printable(StringRef(s.key.toString()))).detail("Value", printable(StringRef(s.value.toString())));
printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str()); printf(" Key: %-50s Value: %s\n", printable(StringRef(s.key.toString())).c_str(), printable(StringRef(s.value.toString())).c_str());
} }
} }
@ -234,7 +234,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
break; break;
} }
catch (Error &e) { catch (Error &e) {
TraceEvent("DRU_checkError").error(e); TraceEvent("DRU_CheckError").error(e);
Void _ = wait(tr->onError(e)); Void _ = wait(tr->onError(e));
} }
} }
@ -252,12 +252,12 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
try{ try{
Void _ = wait(delay(self->backupAfter)); Void _ = wait(delay(self->backupAfter));
TraceEvent("DRU_doBackup").detail("tag", printable(self->backupTag)); TraceEvent("DRU_DoBackup").detail("Tag", printable(self->backupTag));
state Future<Void> b = doBackup(self, &backupAgent, self->extraDB, self->backupTag, self->backupRanges); state Future<Void> b = doBackup(self, &backupAgent, self->extraDB, self->backupTag, self->backupRanges);
TraceEvent("DRU_doBackupWait").detail("backupTag", printable(self->backupTag)); TraceEvent("DRU_DoBackupWait").detail("BackupTag", printable(self->backupTag));
Void _ = wait(b); Void _ = wait(b);
TraceEvent("DRU_doBackupWaitEnd").detail("backupTag", printable(self->backupTag)); TraceEvent("DRU_DoBackupWaitEnd").detail("BackupTag", printable(self->backupTag));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent(SevError, "BackupToDBUpgradeSetuEerror").error(e); TraceEvent(SevError, "BackupToDBUpgradeSetuEerror").error(e);
@ -292,13 +292,13 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
while (src != srcFuture.get().end() && bkp != bkpFuture.get().end()) { while (src != srcFuture.get().end() && bkp != bkpFuture.get().end()) {
KeyRef bkpKey = bkp->key.substr(backupPrefix.size()); KeyRef bkpKey = bkp->key.substr(backupPrefix.size());
if (src->key != bkpKey && src->value != bkp->value) { if (src->key != bkpKey && src->value != bkp->value) {
TraceEvent(SevError, "MismatchKeyAndValue").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchKeyAndValue").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
else if (src->key != bkpKey) { else if (src->key != bkpKey) {
TraceEvent(SevError, "MismatchKey").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchKey").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
else if (src->value != bkp->value) { else if (src->value != bkp->value) {
TraceEvent(SevError, "MismatchValue").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)).detail("bkpKey", printable(bkpKey)).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MismatchValue").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value)).detail("BkpKey", printable(bkpKey)).detail("BkpVal", printable(bkp->value));
} }
begin = std::min(src->key, bkpKey); begin = std::min(src->key, bkpKey);
if (src->key == bkpKey) { if (src->key == bkpKey) {
@ -313,12 +313,12 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
} }
} }
while (src != srcFuture.get().end() && !bkpFuture.get().more) { while (src != srcFuture.get().end() && !bkpFuture.get().more) {
TraceEvent(SevError, "MissingBkpKey").detail("srcKey", printable(src->key)).detail("srcVal", printable(src->value)); TraceEvent(SevError, "MissingBkpKey").detail("SrcKey", printable(src->key)).detail("SrcVal", printable(src->value));
begin = src->key; begin = src->key;
++src; ++src;
} }
while (bkp != bkpFuture.get().end() && !srcFuture.get().more) { while (bkp != bkpFuture.get().end() && !srcFuture.get().more) {
TraceEvent(SevError, "MissingSrcKey").detail("bkpKey", printable(bkp->key.substr(backupPrefix.size()))).detail("bkpVal", printable(bkp->value)); TraceEvent(SevError, "MissingSrcKey").detail("BkpKey", printable(bkp->key.substr(backupPrefix.size()))).detail("BkpVal", printable(bkp->value));
begin = bkp->key; begin = bkp->key;
++bkp; ++bkp;
} }
@ -351,7 +351,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
state Future<Void> stopDifferential = delay(self->stopDifferentialAfter); state Future<Void> stopDifferential = delay(self->stopDifferentialAfter);
state Future<Void> waitUpgrade = backupAgent.waitUpgradeToLatestDrVersion(self->extraDB, self->backupTag); state Future<Void> waitUpgrade = backupAgent.waitUpgradeToLatestDrVersion(self->extraDB, self->backupTag);
Void _ = wait(success(stopDifferential) && success(waitUpgrade)); Void _ = wait(success(stopDifferential) && success(waitUpgrade));
TraceEvent("DRU_waitDifferentialEnd").detail("tag", printable(self->backupTag)); TraceEvent("DRU_WaitDifferentialEnd").detail("Tag", printable(self->backupTag));
try { try {
// Get restore ranges before aborting // Get restore ranges before aborting
@ -379,7 +379,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
} }
} }
TraceEvent("DRU_locked").detail("lockedVersion", commitVersion); TraceEvent("DRU_Locked").detail("LockedVersion", commitVersion);
// Wait for the destination to apply mutations up to the lock commit before switching over. // Wait for the destination to apply mutations up to the lock commit before switching over.
state ReadYourWritesTransaction versionCheckTr(self->extraDB); state ReadYourWritesTransaction versionCheckTr(self->extraDB);
@ -388,7 +388,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
versionCheckTr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); versionCheckTr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
versionCheckTr.setOption(FDBTransactionOptions::LOCK_AWARE); versionCheckTr.setOption(FDBTransactionOptions::LOCK_AWARE);
Optional<Value> v = wait(versionCheckTr.get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin))); Optional<Value> v = wait(versionCheckTr.get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin)));
TraceEvent("DRU_applied").detail("appliedVersion", v.present() ? BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) : -1); TraceEvent("DRU_Applied").detail("AppliedVersion", v.present() ? BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) : -1);
if( v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >= commitVersion) if( v.present() && BinaryReader::fromStringRef<Version>(v.get(), Unversioned()) >= commitVersion)
break; break;
@ -401,16 +401,16 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
} }
} }
TraceEvent("DRU_diffRanges"); TraceEvent("DRU_DiffRanges");
Void _ = wait( diffRanges(prevBackupRanges, self->backupPrefix, cx, self->extraDB ) ); Void _ = wait( diffRanges(prevBackupRanges, self->backupPrefix, cx, self->extraDB ) );
// abort backup // abort backup
TraceEvent("DRU_abortBackup").detail("tag", printable(self->backupTag)); TraceEvent("DRU_AbortBackup").detail("Tag", printable(self->backupTag));
Void _ = wait(backupAgent.abortBackup(self->extraDB, self->backupTag)); Void _ = wait(backupAgent.abortBackup(self->extraDB, self->backupTag));
Void _ = wait( unlockDatabase(self->extraDB, logUid) ); Void _ = wait( unlockDatabase(self->extraDB, logUid) );
// restore database // restore database
TraceEvent("DRU_prepareRestore").detail("restoreTag", printable(self->restoreTag)); TraceEvent("DRU_PrepareRestore").detail("RestoreTag", printable(self->restoreTag));
state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx)); state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(cx));
loop{ loop{
try{ try{
@ -426,7 +426,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
Void _ = wait( tr2->commit() ); Void _ = wait( tr2->commit() );
break; break;
} catch( Error &e ) { } catch( Error &e ) {
TraceEvent("DRU_restoreSetupError").error(e, true); TraceEvent("DRU_RestoreSetupError").error(e, true);
Void _ = wait( tr2->onError(e) ); Void _ = wait( tr2->onError(e) );
} }
} }
@ -438,11 +438,11 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
// start restoring db // start restoring db
try { try {
TraceEvent("DRU_restoreDb").detail("restoreTag", printable(self->restoreTag)); TraceEvent("DRU_RestoreDb").detail("RestoreTag", printable(self->restoreTag));
Void _ = wait(restoreAgent.submitBackup(cx, self->restoreTag, restoreRanges, true, StringRef(), self->backupPrefix)); Void _ = wait(restoreAgent.submitBackup(cx, self->restoreTag, restoreRanges, true, StringRef(), self->backupPrefix));
} }
catch (Error& e) { catch (Error& e) {
TraceEvent("DRU_restoreSubmitBackupError").detail("tag", printable(self->restoreTag)).error(e); TraceEvent("DRU_RestoreSubmitBackupError").detail("Tag", printable(self->restoreTag)).error(e);
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
throw; throw;
} }
@ -454,7 +454,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload {
state UID restoreUid = wait(restoreAgent.getLogUid(cx, self->restoreTag)); state UID restoreUid = wait(restoreAgent.getLogUid(cx, self->restoreTag));
Void _ = wait(checkData(cx, restoreUid, restoreUid, self->restoreTag, &restoreAgent, self->shareLogRange)); Void _ = wait(checkData(cx, restoreUid, restoreUid, self->restoreTag, &restoreAgent, self->shareLogRange));
TraceEvent("DRU_complete").detail("backupTag", printable(self->backupTag)); TraceEvent("DRU_Complete").detail("BackupTag", printable(self->backupTag));
if (g_simulator.drAgents == ISimulator::BackupToDB) { if (g_simulator.drAgents == ISimulator::BackupToDB) {
g_simulator.drAgents = ISimulator::NoBackupAgents; g_simulator.drAgents = ISimulator::NoBackupAgents;

2
fdbserver/workloads/BulkSetup.actor.h
View File

@ -313,7 +313,7 @@ Future<Void> bulkSetup( Database cx, T* workload, uint64_t nodeCount, Promise<do
Void _ = wait( delay( 5.0 ) ); // Wait for the data distribution in a small test to start Void _ = wait( delay( 5.0 ) ); // Wait for the data distribution in a small test to start
loop { loop {
int64_t inFlight = wait( getDataInFlight( cx, workload->dbInfo ) ); int64_t inFlight = wait( getDataInFlight( cx, workload->dbInfo ) );
TraceEvent("DynamicWarming").detail("inFlight", inFlight); TraceEvent("DynamicWarming").detail("InFlight", inFlight);
if( inFlight > 1e6 ) { // Wait for just 1 MB to be in flight if( inFlight > 1e6 ) { // Wait for just 1 MB to be in flight
Void _ = wait( delay( 1.0 ) ); Void _ = wait( delay( 1.0 ) );
} else { } else {

12
fdbserver/workloads/ClientTransactionProfileCorrectness.actor.cpp
View File

@ -105,7 +105,7 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
if (clientId == 0) { if (clientId == 0) {
samplingProbability = getOption(options, LiteralStringRef("samplingProbability"), g_random->random01() / 10); //rand range 0 - 0.1 samplingProbability = getOption(options, LiteralStringRef("samplingProbability"), g_random->random01() / 10); //rand range 0 - 0.1
trInfoSizeLimit = getOption(options, LiteralStringRef("trInfoSizeLimit"), g_random->randomInt(100 * 1024, 10 * 1024 * 1024)); // 100 KB - 10 MB trInfoSizeLimit = getOption(options, LiteralStringRef("trInfoSizeLimit"), g_random->randomInt(100 * 1024, 10 * 1024 * 1024)); // 100 KB - 10 MB
TraceEvent(SevInfo, "ClientTransactionProfilingSetup").detail("samplingProbability", samplingProbability).detail("trInfoSizeLimit", trInfoSizeLimit); TraceEvent(SevInfo, "ClientTransactionProfilingSetup").detail("SamplingProbability", samplingProbability).detail("TrInfoSizeLimit", trInfoSizeLimit);
} }
} }
@ -160,13 +160,13 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
if (trInfoChunks.find(trId) == trInfoChunks.end()) { if (trInfoChunks.find(trId) == trInfoChunks.end()) {
// Some of the earlier chunks for this trId should have been deleted. // Some of the earlier chunks for this trId should have been deleted.
// Discard this chunk as it is of not much use // Discard this chunk as it is of not much use
TraceEvent(SevInfo, "ClientTransactionProfilingSomeChunksMissing").detail("trId", trId); TraceEvent(SevInfo, "ClientTransactionProfilingSomeChunksMissing").detail("TrId", trId);
} }
else { else {
// Check if it is the expected chunk. Otherwise discard the whole transaction entry. // Check if it is the expected chunk. Otherwise discard the whole transaction entry.
// There are scenarios (eg., when deletion is happening) where some chunks get missed. // There are scenarios (eg., when deletion is happening) where some chunks get missed.
if (chunkNum != trInfoChunks.find(trId)->second.size() + 1) { if (chunkNum != trInfoChunks.find(trId)->second.size() + 1) {
TraceEvent(SevInfo, "ClientTransactionProfilingChunksMissing").detail("trId", trId); TraceEvent(SevInfo, "ClientTransactionProfilingChunksMissing").detail("TrId", trId);
trInfoChunks.erase(trId); trInfoChunks.erase(trId);
} }
else { else {
@ -248,11 +248,11 @@ struct ClientTransactionProfileCorrectnessWorkload : TestWorkload {
} }
// FIXME: Find a way to check that contentsSize is not greater than a certain limit. // FIXME: Find a way to check that contentsSize is not greater than a certain limit.
//if (counter != contentsSize) { //if (counter != contentsSize) {
// TraceEvent(SevError, "ClientTransactionProfilingIncorrectCtrVal").detail("counter", counter).detail("contentsSize", contentsSize); // TraceEvent(SevError, "ClientTransactionProfilingIncorrectCtrVal").detail("Counter", counter).detail("ContentsSize", contentsSize);
// return false; // return false;
//} //}
TraceEvent(SevInfo, "ClientTransactionProfilingCtrval").detail("counter", counter); TraceEvent(SevInfo, "ClientTransactionProfilingCtrval").detail("Counter", counter);
TraceEvent(SevInfo, "ClientTransactionProfilingContentsSize").detail("contentsSize", contentsSize); TraceEvent(SevInfo, "ClientTransactionProfilingContentsSize").detail("ContentsSize", contentsSize);
// Check if the data format is as expected // Check if the data format is as expected
return self->checkTxInfoEntriesFormat(txInfoEntries); return self->checkTxInfoEntriesFormat(txInfoEntries);

8
fdbserver/workloads/ConfigureDatabase.actor.cpp
View File

@ -144,7 +144,7 @@ struct ConfigureDatabaseWorkload : TestWorkload {
//TraceEvent("ConfigureTestLoadData").detail("LoadTime", now() - startTime).detail("AmountLoaded",amtLoaded); //TraceEvent("ConfigureTestLoadData").detail("LoadTime", now() - startTime).detail("AmountLoaded",amtLoaded);
} }
else if( randomChoice == 3 ) { else if( randomChoice == 3 ) {
//TraceEvent("ConfigureTestConfigureBegin").detail("newConfig", newConfig); //TraceEvent("ConfigureTestConfigureBegin").detail("NewConfig", newConfig);
int redundancy = g_random->randomInt( 0, sizeof(redundancies)/sizeof(redundancies[0])); int redundancy = g_random->randomInt( 0, sizeof(redundancies)/sizeof(redundancies[0]));
std::string config = redundancies[redundancy]; std::string config = redundancies[redundancy];
if(config == "triple" && g_simulator.physicalDatacenters == 3) { if(config == "triple" && g_simulator.physicalDatacenters == 3) {
@ -156,15 +156,15 @@ struct ConfigureDatabaseWorkload : TestWorkload {
if (g_random->random01() < 0.5) config += " resolvers=" + format("%d", randomRoleNumber()); if (g_random->random01() < 0.5) config += " resolvers=" + format("%d", randomRoleNumber());
ConfigurationResult::Type _ = wait( changeConfig( cx, config ) ); ConfigurationResult::Type _ = wait( changeConfig( cx, config ) );
//TraceEvent("ConfigureTestConfigureEnd").detail("newConfig", newConfig); //TraceEvent("ConfigureTestConfigureEnd").detail("NewConfig", newConfig);
} }
else if( randomChoice == 4 ) { else if( randomChoice == 4 ) {
//TraceEvent("ConfigureTestQuorumBegin").detail("newQuorum", s); //TraceEvent("ConfigureTestQuorumBegin").detail("NewQuorum", s);
auto ch = autoQuorumChange(); auto ch = autoQuorumChange();
if (g_random->randomInt(0,2)) if (g_random->randomInt(0,2))
ch = nameQuorumChange( format("NewName%d", g_random->randomInt(0,100)), ch ); ch = nameQuorumChange( format("NewName%d", g_random->randomInt(0,100)), ch );
CoordinatorsResult::Type _ = wait( changeQuorum( cx, ch ) ); CoordinatorsResult::Type _ = wait( changeQuorum( cx, ch ) );
//TraceEvent("ConfigureTestConfigureEnd").detail("newQuorum", s); //TraceEvent("ConfigureTestConfigureEnd").detail("NewQuorum", s);
} }
else if ( randomChoice == 5) { else if ( randomChoice == 5) {
ConfigurationResult::Type _ = wait( changeConfig( cx, storeTypes[g_random->randomInt( 0, sizeof(storeTypes)/sizeof(storeTypes[0]))] ) ); ConfigurationResult::Type _ = wait( changeConfig( cx, storeTypes[g_random->randomInt( 0, sizeof(storeTypes)/sizeof(storeTypes[0]))] ) );

12
fdbserver/workloads/ConflictRange.actor.cpp
View File

@ -114,7 +114,7 @@ struct ConflictRangeWorkload : TestWorkload {
int clearedB = g_random->randomInt(0, self->maxKeySpace-1); int clearedB = g_random->randomInt(0, self->maxKeySpace-1);
clearedBegin = std::min(clearedA, clearedB); clearedBegin = std::min(clearedA, clearedB);
clearedEnd = std::max(clearedA, clearedB)+1; clearedEnd = std::max(clearedA, clearedB)+1;
TraceEvent("ConflictRangeClear").detail("begin",clearedBegin).detail("end",clearedEnd); TraceEvent("ConflictRangeClear").detail("Begin",clearedBegin).detail("End",clearedEnd);
} }
tr0.clear( KeyRangeRef( StringRef( format( "%010d", 0 ) ), StringRef( format( "%010d", self->maxKeySpace ) ) ) ); tr0.clear( KeyRangeRef( StringRef( format( "%010d", 0 ) ), StringRef( format( "%010d", self->maxKeySpace ) ) ) );
@ -275,8 +275,8 @@ struct ConflictRangeWorkload : TestWorkload {
} }
TraceEvent(SevError, "ConflictRangeError").detail("Info", "Conflict returned, however results are the same") TraceEvent(SevError, "ConflictRangeError").detail("Info", "Conflict returned, however results are the same")
.detail("randomSets",randomSets).detail("myKeyA",myKeyA).detail("myKeyB",myKeyB).detail("onEqualA",onEqualA).detail("onEqualB",onEqualB) .detail("RandomSets",randomSets).detail("MyKeyA",myKeyA).detail("MyKeyB",myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB)
.detail("offsetA",offsetA).detail("offsetB",offsetB).detail("randomLimit",randomLimit).detail("size",originalResults.size()).detail("results", keyStr1).detail("original", keyStr2); .detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("RandomLimit",randomLimit).detail("Size",originalResults.size()).detail("Results", keyStr1).detail("Original", keyStr2);
tr4 = Transaction(cx); tr4 = Transaction(cx);
Standalone<RangeResultRef> res = wait( tr4.getRange( KeyRangeRef( StringRef( format( "%010d", 0 ) ), StringRef( format( "%010d", self->maxKeySpace ) ) ), 200 ) ); Standalone<RangeResultRef> res = wait( tr4.getRange( KeyRangeRef( StringRef( format( "%010d", 0 ) ), StringRef( format( "%010d", self->maxKeySpace ) ) ), 200 ) );
@ -285,7 +285,7 @@ struct ConflictRangeWorkload : TestWorkload {
allKeyEntries += printable( res[i].key ) + " "; allKeyEntries += printable( res[i].key ) + " ";
} }
TraceEvent("ConflictRangeDump").detail("keys", allKeyEntries); TraceEvent("ConflictRangeDump").detail("Keys", allKeyEntries);
} }
throw not_committed(); throw not_committed();
} else { } else {
@ -314,8 +314,8 @@ struct ConflictRangeWorkload : TestWorkload {
TraceEvent(SevError, "ConflictRangeError").detail("Info", "No conflict returned, however result sizes do not match") TraceEvent(SevError, "ConflictRangeError").detail("Info", "No conflict returned, however result sizes do not match")
.detail("OriginalSize", originalResults.size()).detail("NewSize", res.size()) .detail("OriginalSize", originalResults.size()).detail("NewSize", res.size())
.detail("randomSets",randomSets).detail("myKeyA",myKeyA).detail("myKeyB",myKeyB).detail("onEqualA",onEqualA).detail("onEqualB",onEqualB) .detail("RandomSets",randomSets).detail("MyKeyA",myKeyA).detail("MyKeyB",myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB)
.detail("offsetA",offsetA).detail("offsetB",offsetB).detail("randomLimit",randomLimit).detail("size",originalResults.size()).detail("results", keyStr1).detail("original", keyStr2); .detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("RandomLimit",randomLimit).detail("Size",originalResults.size()).detail("Results", keyStr1).detail("Original", keyStr2);
} }
} }
} catch (Error& e) { } catch (Error& e) {

18
fdbserver/workloads/ConsistencyCheck.actor.cpp
View File

@ -280,7 +280,7 @@ struct ConsistencyCheckWorkload : TestWorkload
} }
} }
TraceEvent("ConsistencyCheck_FinishedCheck").detail("repetitions", self->repetitions); TraceEvent("ConsistencyCheck_FinishedCheck").detail("Repetitions", self->repetitions);
return Void(); return Void();
} }
@ -628,7 +628,7 @@ struct ConsistencyCheckWorkload : TestWorkload
//In a quiescent database, check that the team size is the same as the desired team size //In a quiescent database, check that the team size is the same as the desired team size
if(self->firstClient && self->performQuiescentChecks && sourceStorageServers.size() != configuration.storageTeamSize) if(self->firstClient && self->performQuiescentChecks && sourceStorageServers.size() != configuration.storageTeamSize)
{ {
TraceEvent("ConsistencyCheck_InvalidTeamSize").detail("ShardBegin", printable(range.begin)).detail("ShardEnd", printable(range.end)).detail("teamSize", sourceStorageServers.size()).detail("desiredTeamSize", configuration.storageTeamSize); TraceEvent("ConsistencyCheck_InvalidTeamSize").detail("ShardBegin", printable(range.begin)).detail("ShardEnd", printable(range.end)).detail("TeamSize", sourceStorageServers.size()).detail("DesiredTeamSize", configuration.storageTeamSize);
self->testFailure("Invalid team size"); self->testFailure("Invalid team size");
return false; return false;
} }
@ -983,7 +983,7 @@ struct ConsistencyCheckWorkload : TestWorkload
} }
if(bytesReadInRange > 0) { if(bytesReadInRange > 0) {
TraceEvent("ConsistencyCheck_ReadRange").detail("range", printable(range)).detail("bytesRead", bytesReadInRange); TraceEvent("ConsistencyCheck_ReadRange").detail("Range", printable(range)).detail("BytesRead", bytesReadInRange);
} }
} }
@ -1167,7 +1167,7 @@ struct ConsistencyCheckWorkload : TestWorkload
for( auto it : workers ) { for( auto it : workers ) {
ISimulator::ProcessInfo* info = g_simulator.getProcessByAddress(it.first.address()); ISimulator::ProcessInfo* info = g_simulator.getProcessByAddress(it.first.address());
if(!info || info->failed) { if(!info || info->failed) {
TraceEvent("ConsistencyCheck_FailedWorkerInList").detail("addr", it.first.address()); TraceEvent("ConsistencyCheck_FailedWorkerInList").detail("Addr", it.first.address());
return false; return false;
} }
workerAddresses.insert( NetworkAddress(it.first.address().ip, it.first.address().port, true, false) ); workerAddresses.insert( NetworkAddress(it.first.address().ip, it.first.address().port, true, false) );
@ -1177,7 +1177,7 @@ struct ConsistencyCheckWorkload : TestWorkload
for(int i = 0; i < all.size(); i++) { for(int i = 0; i < all.size(); i++) {
if( all[i]->isReliable() && all[i]->name == std::string("Server") && all[i]->startingClass != ProcessClass::TesterClass ) { if( all[i]->isReliable() && all[i]->name == std::string("Server") && all[i]->startingClass != ProcessClass::TesterClass ) {
if(!workerAddresses.count(all[i]->address)) { if(!workerAddresses.count(all[i]->address)) {
TraceEvent("ConsistencyCheck_WorkerMissingFromList").detail("addr", all[i]->address); TraceEvent("ConsistencyCheck_WorkerMissingFromList").detail("Addr", all[i]->address);
return false; return false;
} }
} }
@ -1220,7 +1220,7 @@ struct ConsistencyCheckWorkload : TestWorkload
// Check cluster controller // Check cluster controller
ProcessClass::Fitness bestClusterControllerFitness = getBestAvailableFitness(nonExcludedClassTypes, ProcessClass::ClusterController); ProcessClass::Fitness bestClusterControllerFitness = getBestAvailableFitness(nonExcludedClassTypes, ProcessClass::ClusterController);
if (!nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) || nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].machineClassFitness(ProcessClass::ClusterController) != bestClusterControllerFitness) { if (!nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) || nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].machineClassFitness(ProcessClass::ClusterController) != bestClusterControllerFitness) {
TraceEvent("ConsistencyCheck_ClusterControllerNotBest").detail("bestClusterControllerFitness", bestClusterControllerFitness).detail("existingClusterControllerFit", nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) ? nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].machineClassFitness(ProcessClass::ClusterController) : -1); TraceEvent("ConsistencyCheck_ClusterControllerNotBest").detail("BestClusterControllerFitness", bestClusterControllerFitness).detail("ExistingClusterControllerFit", nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) ? nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].machineClassFitness(ProcessClass::ClusterController) : -1);
return false; return false;
} }
@ -1234,7 +1234,7 @@ struct ConsistencyCheckWorkload : TestWorkload
} }
if (!allWorkerProcessMap.count(db.master.address()) || (!nonExcludedWorkerProcessMap.count(db.master.address()) && bestMasterFitness != ProcessClass::ExcludeFit) || nonExcludedWorkerProcessMap[db.master.address()].machineClassFitness(ProcessClass::Master) != bestMasterFitness) { if (!allWorkerProcessMap.count(db.master.address()) || (!nonExcludedWorkerProcessMap.count(db.master.address()) && bestMasterFitness != ProcessClass::ExcludeFit) || nonExcludedWorkerProcessMap[db.master.address()].machineClassFitness(ProcessClass::Master) != bestMasterFitness) {
TraceEvent("ConsistencyCheck_MasterNotBest").detail("bestMasterFitness", bestMasterFitness).detail("existingMasterFit", nonExcludedWorkerProcessMap.count(db.master.address()) ? nonExcludedWorkerProcessMap[db.master.address()].machineClassFitness(ProcessClass::Master) : -1); TraceEvent("ConsistencyCheck_MasterNotBest").detail("BestMasterFitness", bestMasterFitness).detail("ExistingMasterFit", nonExcludedWorkerProcessMap.count(db.master.address()) ? nonExcludedWorkerProcessMap[db.master.address()].machineClassFitness(ProcessClass::Master) : -1);
return false; return false;
} }
@ -1242,7 +1242,7 @@ struct ConsistencyCheckWorkload : TestWorkload
ProcessClass::Fitness bestMasterProxyFitness = getBestAvailableFitness(nonExcludedClassTypes, ProcessClass::Proxy); ProcessClass::Fitness bestMasterProxyFitness = getBestAvailableFitness(nonExcludedClassTypes, ProcessClass::Proxy);
for (auto masterProxy : db.client.proxies) { for (auto masterProxy : db.client.proxies) {
if (!nonExcludedWorkerProcessMap.count(masterProxy.address()) || nonExcludedWorkerProcessMap[masterProxy.address()].machineClassFitness(ProcessClass::Proxy) != bestMasterProxyFitness) { if (!nonExcludedWorkerProcessMap.count(masterProxy.address()) || nonExcludedWorkerProcessMap[masterProxy.address()].machineClassFitness(ProcessClass::Proxy) != bestMasterProxyFitness) {
TraceEvent("ConsistencyCheck_ProxyNotBest").detail("bestMasterProxyFitness", bestMasterProxyFitness).detail("existingMasterProxyFitness", nonExcludedWorkerProcessMap.count(masterProxy.address()) ? nonExcludedWorkerProcessMap[masterProxy.address()].machineClassFitness(ProcessClass::Proxy) : -1); TraceEvent("ConsistencyCheck_ProxyNotBest").detail("BestMasterProxyFitness", bestMasterProxyFitness).detail("ExistingMasterProxyFitness", nonExcludedWorkerProcessMap.count(masterProxy.address()) ? nonExcludedWorkerProcessMap[masterProxy.address()].machineClassFitness(ProcessClass::Proxy) : -1);
return false; return false;
} }
} }
@ -1251,7 +1251,7 @@ struct ConsistencyCheckWorkload : TestWorkload
ProcessClass::Fitness bestResolverFitness = getBestAvailableFitness(nonExcludedClassTypes, ProcessClass::Resolver); ProcessClass::Fitness bestResolverFitness = getBestAvailableFitness(nonExcludedClassTypes, ProcessClass::Resolver);
for (auto resolver : db.resolvers) { for (auto resolver : db.resolvers) {
if (!nonExcludedWorkerProcessMap.count(resolver.address()) || nonExcludedWorkerProcessMap[resolver.address()].machineClassFitness(ProcessClass::Resolver) != bestResolverFitness) { if (!nonExcludedWorkerProcessMap.count(resolver.address()) || nonExcludedWorkerProcessMap[resolver.address()].machineClassFitness(ProcessClass::Resolver) != bestResolverFitness) {
TraceEvent("ConsistencyCheck_ResolverNotBest").detail("bestResolverFitness", bestResolverFitness).detail("existingResolverFitness", nonExcludedWorkerProcessMap.count(resolver.address()) ? nonExcludedWorkerProcessMap[resolver.address()].machineClassFitness(ProcessClass::Resolver) : -1); TraceEvent("ConsistencyCheck_ResolverNotBest").detail("BestResolverFitness", bestResolverFitness).detail("ExistingResolverFitness", nonExcludedWorkerProcessMap.count(resolver.address()) ? nonExcludedWorkerProcessMap[resolver.address()].machineClassFitness(ProcessClass::Resolver) : -1);
return false; return false;
} }
} }

2
fdbserver/workloads/DDBalance.actor.cpp
View File

@ -202,7 +202,7 @@ struct DDBalanceWorkload : TestWorkload {
} }
if(maxMovedAmount < end-begin) { if(maxMovedAmount < end-begin) {
TraceEvent(SevError, "LostKeys").detail("maxMoved",maxMovedAmount).detail("ShouldHaveMoved",end-begin).detail("ClientId", self->clientId).detail("MoverId", moverId) TraceEvent(SevError, "LostKeys").detail("MaxMoved",maxMovedAmount).detail("ShouldHaveMoved",end-begin).detail("ClientId", self->clientId).detail("MoverId", moverId)
.detail("CurrentBin", sourceBin).detail("NextBin", destinationBin); .detail("CurrentBin", sourceBin).detail("NextBin", destinationBin);
ASSERT( false ); ASSERT( false );
} }

2
fdbserver/workloads/DDMetrics.actor.cpp
View File

@ -50,7 +50,7 @@ struct DDMetricsWorkload : TestWorkload {
ACTOR Future<Void> work( Database cx, DDMetricsWorkload *self ) { ACTOR Future<Void> work( Database cx, DDMetricsWorkload *self ) {
try { try {
TraceEvent("DDMetricsWaiting").detail("startDelay", self->startDelay); TraceEvent("DDMetricsWaiting").detail("StartDelay", self->startDelay);
Void _ = wait( delay( self->startDelay ) ); Void _ = wait( delay( self->startDelay ) );
TraceEvent("DDMetricsStarting"); TraceEvent("DDMetricsStarting");
state double startTime = now(); state double startTime = now();

14
fdbserver/workloads/FastTriggeredWatches.actor.cpp
View File

@ -82,12 +82,12 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
tr.set( key, value.get() ); tr.set( key, value.get() );
else else
tr.clear( key ); tr.clear( key );
//TraceEvent("FTWSetBegin").detail("key", printable(key)).detail("value", printable(value)); //TraceEvent("FTWSetBegin").detail("Key", printable(key)).detail("Value", printable(value));
Void _ = wait( tr.commit() ); Void _ = wait( tr.commit() );
//TraceEvent("FTWSetEnd").detail("key", printable(key)).detail("value", printable(value)).detail("ver", tr.getCommittedVersion()); //TraceEvent("FTWSetEnd").detail("Key", printable(key)).detail("Value", printable(value)).detail("Ver", tr.getCommittedVersion());
return tr.getCommittedVersion(); return tr.getCommittedVersion();
} catch( Error &e ) { } catch( Error &e ) {
//TraceEvent("FTWSetError").detail("key", printable(key)).detail("value", printable(value)).error(e); //TraceEvent("FTWSetError").detail("Key", printable(key)).detail("Value", printable(value)).error(e);
Void _ = wait( tr.onError(e) ); Void _ = wait( tr.onError(e) );
} }
} }
@ -117,23 +117,23 @@ struct FastTriggeredWatchesWorkload : TestWorkload {
getDuration = now() - watchEnd; getDuration = now() - watchEnd;
} }
lastReadVersion = tr.getReadVersion().get(); lastReadVersion = tr.getReadVersion().get();
//TraceEvent("FTWGet").detail("key", printable(setKey)).detail("value", printable(val)).detail("ver", tr.getReadVersion().get()); //TraceEvent("FTWGet").detail("Key", printable(setKey)).detail("Value", printable(val)).detail("Ver", tr.getReadVersion().get());
if( val == setValue ) if( val == setValue )
break; break;
ASSERT( first ); ASSERT( first );
state Future<Void> watchFuture = tr.watch( setKey ); state Future<Void> watchFuture = tr.watch( setKey );
Void _ = wait( tr.commit() ); Void _ = wait( tr.commit() );
//TraceEvent("FTWStartWatch").detail("key", printable(setKey)); //TraceEvent("FTWStartWatch").detail("Key", printable(setKey));
Void _ = wait( watchFuture ); Void _ = wait( watchFuture );
watchEnd = now(); watchEnd = now();
first = false; first = false;
} catch( Error &e ) { } catch( Error &e ) {
//TraceEvent("FTWWatchError").detail("key", printable(setKey)).error(e); //TraceEvent("FTWWatchError").detail("Key", printable(setKey)).error(e);
Void _ = wait( tr.onError(e) ); Void _ = wait( tr.onError(e) );
} }
} }
Version ver = wait( setFuture ); Version ver = wait( setFuture );
//TraceEvent("FTWWatchDone").detail("key", printable(setKey)); //TraceEvent("FTWWatchDone").detail("Key", printable(setKey));
ASSERT( lastReadVersion - ver >= SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT || lastReadVersion - ver < SERVER_KNOBS->VERSIONS_PER_SECOND*(12+getDuration) ); ASSERT( lastReadVersion - ver >= SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT || lastReadVersion - ver < SERVER_KNOBS->VERSIONS_PER_SECOND*(12+getDuration) );
if( now() - testStart > self->testDuration ) if( now() - testStart > self->testDuration )

2
fdbserver/workloads/FileSystem.actor.cpp
View File

@ -310,7 +310,7 @@ struct FileSystemWorkload : TestWorkload {
begin = begin + transfered; begin = begin + transfered;
} }
if( self->loggingQueries ) { if( self->loggingQueries ) {
TraceEvent("DeletionQueryResults").detail("serverID", serverID) TraceEvent("DeletionQueryResults").detail("ServerID", serverID)
.detail("PathBase", base).detail("DeletedFiles", deletedFiles); .detail("PathBase", base).detail("DeletedFiles", deletedFiles);
} }
return Optional<Version>(Version(0)); return Optional<Version>(Version(0));

68
fdbserver/workloads/FuzzApiCorrectness.actor.cpp
View File

@ -68,15 +68,15 @@ struct ExceptionContract {
{ {
Severity s = (i->second == Possible) ? SevWarn : SevInfo; Severity s = (i->second == Possible) ? SevWarn : SevInfo;
TraceEvent evt(s, func.c_str()); TraceEvent evt(s, func.c_str());
evt.error(e).detail("thrown", true) evt.error(e).detail("Thrown", true)
.detail("expected", i->second == Possible ? "possible" : "always").backtrace(); .detail("Expected", i->second == Possible ? "possible" : "always").backtrace();
if (augment) if (augment)
augment(evt); augment(evt);
return; return;
} }
TraceEvent evt(SevError, func.c_str()); TraceEvent evt(SevError, func.c_str());
evt.error(e).detail("thrown", true).detail("expected", "never").backtrace(); evt.error(e).detail("Thrown", true).detail("Expected", "never").backtrace();
if (augment) if (augment)
augment(evt); augment(evt);
throw e; throw e;
@ -87,7 +87,7 @@ struct ExceptionContract {
for (auto i : expected) { for (auto i : expected) {
if (i.second == Always) { if (i.second == Always) {
TraceEvent evt(SevError, func.c_str()); TraceEvent evt(SevError, func.c_str());
evt.detail("thrown", false).detail("expected", "always").error(Error::fromUnvalidatedCode(i.first)).backtrace(); evt.detail("Thrown", false).detail("Expected", "always").error(Error::fromUnvalidatedCode(i.first)).backtrace();
if (augment) if (augment)
augment(evt); augment(evt);
} }
@ -145,13 +145,13 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
maxClearSize = 1<<g_random->randomInt(0, 20); maxClearSize = 1<<g_random->randomInt(0, 20);
conflictRange = KeyRangeRef( LiteralStringRef("\xfe"), LiteralStringRef("\xfe\x00") ); conflictRange = KeyRangeRef( LiteralStringRef("\xfe"), LiteralStringRef("\xfe\x00") );
TraceEvent("FuzzApiCorrectnessConfiguration") TraceEvent("FuzzApiCorrectnessConfiguration")
.detail("nodes", nodes) .detail("Nodes", nodes)
.detail("initialKeyDensity", initialKeyDensity) .detail("InitialKeyDensity", initialKeyDensity)
.detail("adjacentKeys", adjacentKeys) .detail("AdjacentKeys", adjacentKeys)
.detail("valueSizeMin", valueSizeRange.first) .detail("ValueSizeMin", valueSizeRange.first)
.detail("valueSizeRange", valueSizeRange.second) .detail("ValueSizeRange", valueSizeRange.second)
.detail("maxClearSize", maxClearSize) .detail("MaxClearSize", maxClearSize)
.detail("useSystemKeys", useSystemKeys); .detail("UseSystemKeys", useSystemKeys);
TraceEvent("RemapEventSeverity").detail("TargetEvent", "Net2_LargePacket").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo); TraceEvent("RemapEventSeverity").detail("TargetEvent", "Net2_LargePacket").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo);
TraceEvent("RemapEventSeverity").detail("TargetEvent", "LargeTransaction").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo); TraceEvent("RemapEventSeverity").detail("TargetEvent", "LargeTransaction").detail("OriginalSeverity", SevWarnAlways).detail("NewSeverity", SevInfo);
@ -227,7 +227,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
} }
} }
Void _ = wait( unsafeThreadFutureToFuture( tr->commit() ) ); Void _ = wait( unsafeThreadFutureToFuture( tr->commit() ) );
//TraceEvent("WDRInitBatch").detail("i", i).detail("CommittedVersion", tr->getCommittedVersion()); //TraceEvent("WDRInitBatch").detail("I", i).detail("CommittedVersion", tr->getCommittedVersion());
break; break;
} catch( Error &e ) { } catch( Error &e ) {
Void _ = wait( unsafeThreadFutureToFuture( tr->onError( e ) ) ); Void _ = wait( unsafeThreadFutureToFuture( tr->onError( e ) ) );
@ -287,7 +287,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
operations.push_back(testCases[operationType](++self->operationId, self, tr)); operations.push_back(testCases[operationType](++self->operationId, self, tr));
} catch( Error &e ) { } catch( Error &e ) {
TraceEvent(SevWarn, "IgnoredOperation").error(e) TraceEvent(SevWarn, "IgnoredOperation").error(e)
.detail("operation", operationType).detail("id", self->operationId); .detail("Operation", operationType).detail("Id", self->operationId);
} }
} }
@ -373,7 +373,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
virtual ThreadFuture<value_type> createFuture( Reference<ITransaction> tr ) = 0; virtual ThreadFuture<value_type> createFuture( Reference<ITransaction> tr ) = 0;
virtual Void errorCheck(Reference<ITransaction> tr, value_type result) { return Void(); } virtual Void errorCheck(Reference<ITransaction> tr, value_type result) { return Void(); }
virtual void augmentTrace(TraceEvent &e) const { e.detail("id", id); } virtual void augmentTrace(TraceEvent &e) const { e.detail("Id", id); }
protected: protected:
unsigned int id; unsigned int id;
@ -577,7 +577,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("version", v); e.detail("Version", v);
} }
}; };
@ -600,7 +600,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key", printable(key)); e.detail("Key", printable(key));
} }
}; };
@ -623,7 +623,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("keysel", keysel.toString()); e.detail("KeySel", keysel.toString());
} }
}; };
@ -661,7 +661,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("keysel1", keysel1.toString()).detail("keysel2", keysel2.toString()).detail("limit", limit); e.detail("KeySel1", keysel1.toString()).detail("KeySel2", keysel2.toString()).detail("Limit", limit);
} }
}; };
@ -690,10 +690,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("keysel1", keysel1.toString()).detail("keysel2", keysel2.toString()); e.detail("KeySel1", keysel1.toString()).detail("KeySel2", keysel2.toString());
std::stringstream ss; std::stringstream ss;
ss << "(" << limits.rows << ", " << limits.minRows << ", " << limits.bytes << ")"; ss << "(" << limits.rows << ", " << limits.minRows << ", " << limits.bytes << ")";
e.detail("limits", ss.str()); e.detail("Limits", ss.str());
} }
}; };
@ -732,7 +732,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key1", printable(key1)).detail("key2", printable(key2)).detail("limit", limit); e.detail("Key1", printable(key1)).detail("Key2", printable(key2)).detail("Limit", limit);
} }
}; };
@ -762,10 +762,10 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key1", printable(key1)).detail("key2", printable(key2)); e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
std::stringstream ss; std::stringstream ss;
ss << "(" << limits.rows << ", " << limits.minRows << ", " << limits.bytes << ")"; ss << "(" << limits.rows << ", " << limits.minRows << ", " << limits.bytes << ")";
e.detail("limits", ss.str()); e.detail("Limits", ss.str());
} }
}; };
@ -786,7 +786,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key", printable(key)); e.detail("Key", printable(key));
} }
}; };
@ -811,7 +811,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key1", printable(key1)).detail("key2", printable(key2)); e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
} }
}; };
@ -879,7 +879,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key", printable(key)).detail("value", printable(value)).detail("op", op).detail("pos", pos); e.detail("Key", printable(key)).detail("Value", printable(value)).detail("Op", op).detail("Pos", pos);
} }
}; };
@ -908,7 +908,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key", printable(key)).detail("value", printable(value)); e.detail("Key", printable(key)).detail("Value", printable(value));
} }
}; };
@ -937,7 +937,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key1", printable(key1)).detail("key2", printable(key2)); e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
} }
}; };
@ -966,7 +966,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key1", printable(key1)).detail("key2", printable(key2)); e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
} }
}; };
@ -991,7 +991,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key", printable(key)); e.detail("Key", printable(key));
} }
}; };
@ -1017,7 +1017,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key", printable(key)); e.detail("Key", printable(key));
} }
}; };
@ -1042,7 +1042,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("key1", printable(key1)).detail("key2", printable(key2)); e.detail("Key1", printable(key1)).detail("Key2", printable(key2));
} }
}; };
@ -1103,7 +1103,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("op", op).detail("val", printable(val)); e.detail("Op", op).detail("Val", printable(val));
} }
}; };
@ -1132,7 +1132,7 @@ struct FuzzApiCorrectnessWorkload : TestWorkload {
void augmentTrace(TraceEvent &e) const { void augmentTrace(TraceEvent &e) const {
base_type::augmentTrace(e); base_type::augmentTrace(e);
e.detail("errorcode", errorcode); e.detail("ErrorCode", errorcode);
} }
}; };

6
fdbserver/workloads/LockDatabase.actor.cpp
View File

@ -82,13 +82,13 @@ struct LockDatabaseWorkload : TestWorkload {
Void _ = wait( unlockDatabase(&tr, lockID) ); Void _ = wait( unlockDatabase(&tr, lockID) );
state Standalone<RangeResultRef> data2 = wait( tr.getRange(normalKeys, 50000) ); state Standalone<RangeResultRef> data2 = wait( tr.getRange(normalKeys, 50000) );
if(data.size() != data2.size()) { if(data.size() != data2.size()) {
TraceEvent(SevError, "DataChangedWhileLocked").detail("beforeSize", data.size()).detail("afterSize", data2.size()); TraceEvent(SevError, "DataChangedWhileLocked").detail("BeforeSize", data.size()).detail("AfterSize", data2.size());
self->ok = false; self->ok = false;
} else if(data != data2) { } else if(data != data2) {
TraceEvent(SevError, "DataChangedWhileLocked").detail("size", data.size()); TraceEvent(SevError, "DataChangedWhileLocked").detail("Size", data.size());
for(int i = 0; i < data.size(); i++) { for(int i = 0; i < data.size(); i++) {
if( data[i] != data2[i] ) { if( data[i] != data2[i] ) {
TraceEvent(SevError, "DataChangedWhileLocked").detail("i", i).detail("before", printable(data[i])).detail("after", printable(data2[i])); TraceEvent(SevError, "DataChangedWhileLocked").detail("I", i).detail("Before", printable(data[i])).detail("After", printable(data2[i]));
} }
} }
self->ok = false; self->ok = false;

2
fdbserver/workloads/LowLatency.actor.cpp
View File

@ -73,7 +73,7 @@ struct LowLatencyWorkload : TestWorkload {
} }
} }
if(now() - operationStart > self->maxLatency) { if(now() - operationStart > self->maxLatency) {
TraceEvent(SevError, "LatencyTooLarge").detail("maxLatency", self->maxLatency).detail("observedLatency", now() - operationStart); TraceEvent(SevError, "LatencyTooLarge").detail("MaxLatency", self->maxLatency).detail("ObservedLatency", now() - operationStart);
self->ok = false; self->ok = false;
} }
if( now() - testStart > self->testDuration ) if( now() - testStart > self->testDuration )

14
fdbserver/workloads/MachineAttrition.actor.cpp
View File

@ -144,9 +144,9 @@ struct MachineAttritionWorkload : TestWorkload {
g_simulator.killDataCenter( target, kt ); g_simulator.killDataCenter( target, kt );
} else { } else {
while ( killedMachines < self->machinesToKill && self->machines.size() > self->machinesToLeave) { while ( killedMachines < self->machinesToKill && self->machines.size() > self->machinesToLeave) {
TraceEvent("WorkerKillBegin").detail("killedMachines", killedMachines) TraceEvent("WorkerKillBegin").detail("KilledMachines", killedMachines)
.detail("machinesToKill", self->machinesToKill).detail("machinesToLeave", self->machinesToLeave) .detail("MachinesToKill", self->machinesToKill).detail("MachinesToLeave", self->machinesToLeave)
.detail("machines", self->machines.size()); .detail("Machines", self->machines.size());
TEST(true); // Killing a machine TEST(true); // Killing a machine
Void _ = wait( delay( delayBeforeKill ) ); Void _ = wait( delay( delayBeforeKill ) );
@ -170,10 +170,10 @@ struct MachineAttritionWorkload : TestWorkload {
LocalityData targetMachine = self->machines.back(); LocalityData targetMachine = self->machines.back();
TraceEvent("Assassination").detail("TargetMachine", targetMachine.toString()) TraceEvent("Assassination").detail("TargetMachine", targetMachine.toString())
.detailext("zoneId", targetMachine.zoneId()) .detailext("ZoneId", targetMachine.zoneId())
.detail("Reboot", self->reboot).detail("killedMachines", killedMachines) .detail("Reboot", self->reboot).detail("KilledMachines", killedMachines)
.detail("machinesToKill", self->machinesToKill).detail("machinesToLeave", self->machinesToLeave) .detail("MachinesToKill", self->machinesToKill).detail("MachinesToLeave", self->machinesToLeave)
.detail("machines", self->machines.size()).detail("Replace", self->replacement); .detail("Machines", self->machines.size()).detail("Replace", self->replacement);
if (self->reboot) { if (self->reboot) {
if( g_random->random01() > 0.5 ) { if( g_random->random01() > 0.5 ) {

14
fdbserver/workloads/MemoryLifetime.actor.cpp
View File

@ -90,7 +90,7 @@ struct MemoryLifetime : KVWorkload {
state Value getRange_newValue = self->randomValue(); state Value getRange_newValue = self->randomValue();
state bool getRange_isSnapshot = g_random->random01() < 0.5; state bool getRange_isSnapshot = g_random->random01() < 0.5;
//TraceEvent("MemoryLifetimeCheck").detail("isReverse", getRange_isReverse).detail("startKey", printable(getRange_startKey)).detail("randomStart", getRange_randomStart).detail("newValue", getRange_newValue.size()).detail("isSnapshot", getRange_isSnapshot); //TraceEvent("MemoryLifetimeCheck").detail("IsReverse", getRange_isReverse).detail("StartKey", printable(getRange_startKey)).detail("RandomStart", getRange_randomStart).detail("NewValue", getRange_newValue.size()).detail("IsSnapshot", getRange_isSnapshot);
if(getRange_randomStart) if(getRange_randomStart)
tr.set(getRange_startKey, getRange_newValue); tr.set(getRange_startKey, getRange_newValue);
state Standalone<RangeResultRef> getRange_res1 = wait( tr.getRange(getRange_queryRange, GetRangeLimits(4000), getRange_isSnapshot, getRange_isReverse) ); state Standalone<RangeResultRef> getRange_res1 = wait( tr.getRange(getRange_queryRange, GetRangeLimits(4000), getRange_isSnapshot, getRange_isReverse) );
@ -103,16 +103,16 @@ struct MemoryLifetime : KVWorkload {
for(int i = 0; i < getRange_res1.size(); i++) { for(int i = 0; i < getRange_res1.size(); i++) {
if(getRange_res1[i].key != getRange_res2[i].key) { if(getRange_res1[i].key != getRange_res2[i].key) {
TraceEvent(SevError, "MemoryLifetimeCheckKeyError") TraceEvent(SevError, "MemoryLifetimeCheckKeyError")
.detail("key1", printable(getRange_res1[i].key)).detail("key2", printable(getRange_res2[i].key)) .detail("Key1", printable(getRange_res1[i].key)).detail("Key2", printable(getRange_res2[i].key))
.detail("value1", getRange_res1[i].value.size()).detail("value2", getRange_res2[i].value.size()) .detail("Value1", getRange_res1[i].value.size()).detail("Value2", getRange_res2[i].value.size())
.detail("i", i).detail("size", getRange_res2.size()); .detail("I", i).detail("Size", getRange_res2.size());
ASSERT(false); ASSERT(false);
} }
if(getRange_res1[i].value != getRange_res2[i].value) { if(getRange_res1[i].value != getRange_res2[i].value) {
TraceEvent(SevError, "MemoryLifetimeCheckValueError") TraceEvent(SevError, "MemoryLifetimeCheckValueError")
.detail("key1", printable(getRange_res1[i].key)).detail("key2", printable(getRange_res2[i].key)) .detail("Key1", printable(getRange_res1[i].key)).detail("Key2", printable(getRange_res2[i].key))
.detail("value1", getRange_res1[i].value.size()).detail("value2", getRange_res2[i].value.size()) .detail("Value1", getRange_res1[i].value.size()).detail("Value2", getRange_res2[i].value.size())
.detail("i", i).detail("size", getRange_res2.size()); .detail("I", i).detail("Size", getRange_res2.size());
ASSERT(false); ASSERT(false);
} }
} }

2
fdbserver/workloads/Ping.actor.cpp
View File

@ -163,7 +163,7 @@ struct PingWorkload : TestWorkload {
self->totalMessageLatency += elapsed; self->totalMessageLatency += elapsed;
self->maxMessageLatency += std::max(0.0, elapsed*1000.0 - self->maxMessageLatency.getValue()); self->maxMessageLatency += std::max(0.0, elapsed*1000.0 - self->maxMessageLatency.getValue());
++self->messages; ++self->messages;
if (self->logging) TraceEvent("Ping").detail("ms", elapsed*1000.0).detail("To", addr); if (self->logging) TraceEvent("Ping").detail("Elapsed", elapsed).detail("To", addr);
} }
} }

2
fdbserver/workloads/RandomMoveKeys.actor.cpp
View File

@ -67,7 +67,7 @@ struct MoveKeysWorkload : TestWorkload {
state int oldMode = wait( setDDMode( cx, 0 ) ); state int oldMode = wait( setDDMode( cx, 0 ) );
TraceEvent("RMKStartModeSetting"); TraceEvent("RMKStartModeSetting");
Void _ = wait( timeout( reportErrors( self->worker( cx, self ), "moveKeysWorkloadWorkerError" ), self->testDuration, Void() ) ); Void _ = wait( timeout( reportErrors( self->worker( cx, self ), "MoveKeysWorkloadWorkerError" ), self->testDuration, Void() ) );
// Always set the DD mode back, even if we die with an error // Always set the DD mode back, even if we die with an error
TraceEvent("RMKDoneMoving"); TraceEvent("RMKDoneMoving");
int _ = wait( setDDMode( cx, oldMode ) ); int _ = wait( setDDMode( cx, oldMode ) );

10
fdbserver/workloads/RandomSelector.actor.cpp
View File

@ -437,7 +437,7 @@ struct RandomSelectorWorkload : TestWorkload {
randomByteLimit = g_random->randomInt( 0, (self->maxOffset+self->maxKeySpace)*512); randomByteLimit = g_random->randomInt( 0, (self->maxOffset+self->maxKeySpace)*512);
reverse = g_random->random01() > 0.5 ? false : true; reverse = g_random->random01() > 0.5 ? false : true;
//TraceEvent("RYOWgetRange").detail("KeyA", myKeyA).detail("KeyB", myKeyB).detail("onEqualA",onEqualA).detail("onEqualB",onEqualB).detail("offsetA",offsetA).detail("offsetB",offsetB).detail("randomLimit",randomLimit).detail("randomByteLimit", randomByteLimit).detail("reverse", reverse); //TraceEvent("RYOWgetRange").detail("KeyA", myKeyA).detail("KeyB", myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB).detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("RandomLimit",randomLimit).detail("RandomByteLimit", randomByteLimit).detail("Reverse", reverse);
state Standalone<RangeResultRef> getRangeTest1; state Standalone<RangeResultRef> getRangeTest1;
Standalone<RangeResultRef> getRangeTest = wait( trRYOW.getRange(KeySelectorRef(StringRef(clientID + "b/" + myKeyA),onEqualA,offsetA),KeySelectorRef(StringRef(clientID + "b/" + myKeyB),onEqualB,offsetB),randomLimit,false,reverse) ); Standalone<RangeResultRef> getRangeTest = wait( trRYOW.getRange(KeySelectorRef(StringRef(clientID + "b/" + myKeyA),onEqualA,offsetA),KeySelectorRef(StringRef(clientID + "b/" + myKeyB),onEqualB,offsetB),randomLimit,false,reverse) );
@ -449,7 +449,7 @@ struct RandomSelectorWorkload : TestWorkload {
bool fail = false; bool fail = false;
if( getRangeTest1.size() != getRangeTest2.size() ) { if( getRangeTest1.size() != getRangeTest2.size() ) {
TraceEvent(SevError, "RanSelTestFailure").detail("Reason", "The getRange results did not match sizes").detail("size1", getRangeTest1.size()).detail("size2",getRangeTest2.size()).detail("limit",randomLimit).detail("byteLimit", randomByteLimit).detail("bytes1", getRangeTest1.expectedSize()).detail("bytes2", getRangeTest2.expectedSize()).detail("reverse", reverse); TraceEvent(SevError, "RanSelTestFailure").detail("Reason", "The getRange results did not match sizes").detail("Size1", getRangeTest1.size()).detail("Size2",getRangeTest2.size()).detail("Limit",randomLimit).detail("ByteLimit", randomByteLimit).detail("Bytes1", getRangeTest1.expectedSize()).detail("Bytes2", getRangeTest2.expectedSize()).detail("Reverse", reverse);
fail = true; fail = true;
self->fail=true; self->fail=true;
} }
@ -459,7 +459,7 @@ struct RandomSelectorWorkload : TestWorkload {
std::string valueA = printable(getRangeTest1[k].value); std::string valueA = printable(getRangeTest1[k].value);
std::string keyB = printable(getRangeTest2[k].key); std::string keyB = printable(getRangeTest2[k].key);
std::string valueB = printable(getRangeTest2[k].value); std::string valueB = printable(getRangeTest2[k].value);
TraceEvent(SevError, "RanSelTestFailure").detail("Reason", "The getRange results did not match contents").detail("KeyA",keyA).detail("ValueA",valueA).detail("KeyB",keyB).detail("ValueB",valueB).detail("reverse", reverse); TraceEvent(SevError, "RanSelTestFailure").detail("Reason", "The getRange results did not match contents").detail("KeyA",keyA).detail("ValueA",valueA).detail("KeyB",keyB).detail("ValueB",valueB).detail("Reverse", reverse);
fail = true; fail = true;
self->fail=true; self->fail=true;
} }
@ -475,7 +475,7 @@ struct RandomSelectorWorkload : TestWorkload {
outStr2 = outStr2 + printable(getRangeTest2[k].key) + " " + format("%d", getRangeTest2[k].value.size()) + " "; outStr2 = outStr2 + printable(getRangeTest2[k].key) + " " + format("%d", getRangeTest2[k].value.size()) + " ";
} }
TraceEvent("RanSelTestLog").detail("RYOW",outStr1).detail("normal",outStr2); TraceEvent("RanSelTestLog").detail("RYOW",outStr1).detail("Normal",outStr2);
} }
@ -505,7 +505,7 @@ struct RandomSelectorWorkload : TestWorkload {
} }
for(int k = 0; k < finalTest1.size(); k++) for(int k = 0; k < finalTest1.size(); k++)
if(finalTest1[k].value != finalTest2[k].value) { if(finalTest1[k].value != finalTest2[k].value) {
TraceEvent(SevError, "RanSelTestFailure").detail("Reason", "The final results did not match contents").detail("KeyA",printable(finalTest1[k].key)).detail("ValueA",printable(finalTest1[k].value)).detail("KeyB",printable(finalTest2[k].key)).detail("ValueB",printable(finalTest2[k].value)).detail("reverse", reverse); TraceEvent(SevError, "RanSelTestFailure").detail("Reason", "The final results did not match contents").detail("KeyA",printable(finalTest1[k].key)).detail("ValueA",printable(finalTest1[k].value)).detail("KeyB",printable(finalTest2[k].key)).detail("ValueB",printable(finalTest2[k].value)).detail("Reverse", reverse);
self->fail=true; self->fail=true;
} }
break; break;

8
fdbserver/workloads/ReadWrite.actor.cpp
View File

@ -313,10 +313,10 @@ struct ReadWriteWorkload : KVWorkload {
elapsed += self->periodicLoggingInterval; elapsed += self->periodicLoggingInterval;
Void _ = wait( delayUntil(start + elapsed) ); Void _ = wait( delayUntil(start + elapsed) );
TraceEvent("RW_RowReadLatency").detail("mean", self->readLatencies.mean()).detail("median", 1000 * self->readLatencies.median()).detail("percentile5", 1000 * self->readLatencies.percentile(.05)).detail("percentile95", 1000 * self->readLatencies.percentile(.95)).detail("Count", self->readLatencyCount).detail("Elapsed", elapsed); TraceEvent("RW_RowReadLatency").detail("Mean", self->readLatencies.mean()).detail("Median", self->readLatencies.median()).detail("Percentile5", self->readLatencies.percentile(.05)).detail("Percentile95", self->readLatencies.percentile(.95)).detail("Count", self->readLatencyCount).detail("Elapsed", elapsed);
TraceEvent("RW_GRVLatency").detail("mean", 1000 * self->GRVLatencies.mean()).detail("median", 1000 * self->GRVLatencies.median()).detail("percentile5", 1000 * self->GRVLatencies.percentile(.05)).detail("percentile95", 1000 * self->GRVLatencies.percentile(.95)); TraceEvent("RW_GRVLatency").detail("Mean", self->GRVLatencies.mean()).detail("Median", self->GRVLatencies.median()).detail("Percentile5", self->GRVLatencies.percentile(.05)).detail("Percentile95", self->GRVLatencies.percentile(.95));
TraceEvent("RW_CommitLatency").detail("mean", 1000 * self->commitLatencies.mean()).detail("median", 1000 * self->commitLatencies.median()).detail("percentile5", 1000 * self->commitLatencies.percentile(.05)).detail("percentile95", 1000 * self->commitLatencies.percentile(.95)); TraceEvent("RW_CommitLatency").detail("Mean", self->commitLatencies.mean()).detail("Median", self->commitLatencies.median()).detail("Percentile5", self->commitLatencies.percentile(.05)).detail("Percentile95", self->commitLatencies.percentile(.95));
TraceEvent("RW_TotalLatency").detail("mean", 1000 * self->latencies.mean()).detail("median", 1000 * self->latencies.median()).detail("percentile5", 1000 * self->latencies.percentile(.05)).detail("percentile95", 1000 * self->latencies.percentile(.95)); TraceEvent("RW_TotalLatency").detail("Mean", self->latencies.mean()).detail("Median", self->latencies.median()).detail("Percentile5", self->latencies.percentile(.05)).detail("Percentile95", self->latencies.percentile(.95));
int64_t ops = (self->aTransactions.getValue() * (self->readsPerTransactionA+self->writesPerTransactionA)) + int64_t ops = (self->aTransactions.getValue() * (self->readsPerTransactionA+self->writesPerTransactionA)) +
(self->bTransactions.getValue() * (self->readsPerTransactionB+self->writesPerTransactionB)); (self->bTransactions.getValue() * (self->readsPerTransactionB+self->writesPerTransactionB));

42
fdbserver/workloads/RemoveServersSafely.actor.cpp
View File

@ -170,15 +170,15 @@ struct RemoveServersSafelyWorkload : TestWorkload {
auto processNet = AddressExclusion(processInfo->address.ip, processInfo->address.port); auto processNet = AddressExclusion(processInfo->address.ip, processInfo->address.port);
if (processAddrs.find(processNet) != processAddrs.end()) { if (processAddrs.find(processNet) != processAddrs.end()) {
processes.push_back(processInfo); processes.push_back(processInfo);
TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcessItem").detail("ProcessAddress", processInfo->address).detail("Process", describe(*processInfo)).detail("failed", processInfo->failed).detail("excluded", processInfo->excluded).detail("rebooting", processInfo->rebooting).detail("Protected", g_simulator.protectedAddresses.count(processInfo->address)); TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcessItem").detail("ProcessAddress", processInfo->address).detail("Process", describe(*processInfo)).detail("Failed", processInfo->failed).detail("Excluded", processInfo->excluded).detail("Rebooting", processInfo->rebooting).detail("Protected", g_simulator.protectedAddresses.count(processInfo->address));
} }
else { else {
TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcessNoItem").detail("ProcessAddress", processInfo->address).detail("Process", describe(*processInfo)).detail("failed", processInfo->failed).detail("excluded", processInfo->excluded).detail("rebooting", processInfo->rebooting).detail("Protected", g_simulator.protectedAddresses.count(processInfo->address)); TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcessNoItem").detail("ProcessAddress", processInfo->address).detail("Process", describe(*processInfo)).detail("Failed", processInfo->failed).detail("Excluded", processInfo->excluded).detail("Rebooting", processInfo->rebooting).detail("Protected", g_simulator.protectedAddresses.count(processInfo->address));
} }
} }
TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcesses") TraceEvent("RemoveAndKill", functionId).detail("Step", "getProcesses")
.detail("netAddrSize",netAddrs.size()).detail("processAddrSize",processAddrs.size()) .detail("NetAddrSize",netAddrs.size()).detail("ProcessAddrSize",processAddrs.size())
.detail("netAddrs",describe(netAddrs)).detail("processAddrs",describe(processAddrs)) .detail("NetAddrs",describe(netAddrs)).detail("ProcessAddrs",describe(processAddrs))
.detail("Proceses", processes.size()).detail("MachineProcesses", machineProcesses.size()); .detail("Proceses", processes.size()).detail("MachineProcesses", machineProcesses.size());
return processes; return processes;
@ -196,7 +196,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
} }
for (auto& procRecord : procArray) { for (auto& procRecord : procArray) {
procRecord->excluded = true; procRecord->excluded = true;
TraceEvent("RemoveAndKill").detail("Step", "ExcludeAddress").detail("ProcessAddress", procRecord->address).detail("Process", describe(*procRecord)).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "ExcludeAddress").detail("ProcessAddress", procRecord->address).detail("Process", describe(*procRecord)).detail("Failed", procRecord->failed).detail("Rebooting", procRecord->rebooting).detail("ClusterAvailable", g_simulator.isAvailable());
} }
return procArray; return procArray;
} }
@ -215,7 +215,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
// Only change the exclusion member, if not failed since it will require a reboot to revive it // Only change the exclusion member, if not failed since it will require a reboot to revive it
if (!procRecord->failed) if (!procRecord->failed)
procRecord->excluded = false; procRecord->excluded = false;
TraceEvent("RemoveAndKill").detail("Step", "IncludeAddress").detail("ProcessAddress", procRecord->address).detail("Process", describe(*procRecord)).detail("failed", procRecord->failed).detail("rebooting", procRecord->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "IncludeAddress").detail("ProcessAddress", procRecord->address).detail("Process", describe(*procRecord)).detail("Failed", procRecord->failed).detail("Rebooting", procRecord->rebooting).detail("ClusterAvailable", g_simulator.isAvailable());
} }
return procArray; return procArray;
} }
@ -288,23 +288,23 @@ struct RemoveServersSafelyWorkload : TestWorkload {
state std::vector<ISimulator::ProcessInfo*> killProcArray; state std::vector<ISimulator::ProcessInfo*> killProcArray;
state bool bClearedFirst; state bool bClearedFirst;
TraceEvent("RemoveAndKill").detail("Step", "exclude list first").detail("toKill", describe(toKill1)).detail("KillTotal", toKill1.size()).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "exclude list first").detail("ToKill", describe(toKill1)).detail("KillTotal", toKill1.size()).detail("ClusterAvailable", g_simulator.isAvailable());
self->excludeAddresses(toKill1); self->excludeAddresses(toKill1);
Optional<Void> result = wait( timeout( removeAndKill( self, cx, toKill1, NULL), self->kill1Timeout ) ); Optional<Void> result = wait( timeout( removeAndKill( self, cx, toKill1, NULL), self->kill1Timeout ) );
bClearedFirst = result.present(); bClearedFirst = result.present();
TraceEvent("RemoveAndKill").detail("Step", "excluded list first").detail("excluderesult", bClearedFirst ? "succeeded" : "failed").detail("KillTotal", toKill1.size()).detail("Processes", killProcArray.size()).detail("toKill1", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "excluded list first").detail("Excluderesult", bClearedFirst ? "succeeded" : "failed").detail("KillTotal", toKill1.size()).detail("Processes", killProcArray.size()).detail("ToKill1", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable());
bClearedFirst=false; bClearedFirst=false;
// Include the servers, if unable to exclude // Include the servers, if unable to exclude
if (!bClearedFirst) { if (!bClearedFirst) {
// Get the updated list of processes which may have changed due to reboots, deletes, etc // Get the updated list of processes which may have changed due to reboots, deletes, etc
TraceEvent("RemoveAndKill").detail("Step", "include all first").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "include all first").detail("KillTotal", toKill1.size()).detail("ToKill", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable());
Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) ); Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) );
self->includeAddresses(toKill1); self->includeAddresses(toKill1);
TraceEvent("RemoveAndKill").detail("Step", "included all first").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "included all first").detail("KillTotal", toKill1.size()).detail("ToKill", describe(toKill1)).detail("ClusterAvailable", g_simulator.isAvailable());
} }
// Get the list of protected servers // Get the list of protected servers
@ -313,24 +313,24 @@ struct RemoveServersSafelyWorkload : TestWorkload {
// Update the kill networks to the killable processes // Update the kill networks to the killable processes
toKill2 = self->getNetworks(killProcArray); toKill2 = self->getNetworks(killProcArray);
TraceEvent("RemoveAndKill").detail("Step", "exclude list second").detail("KillTotal", toKill2.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "exclude list second").detail("KillTotal", toKill2.size()).detail("ToKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
self->excludeAddresses(toKill2); self->excludeAddresses(toKill2);
// The second set of machines is selected so that we can always make progress without it, even after the permitted number of other permanent failures // The second set of machines is selected so that we can always make progress without it, even after the permitted number of other permanent failures
// so we expect to succeed after a finite amount of time // so we expect to succeed after a finite amount of time
TraceEvent("RemoveAndKill").detail("Step", "exclude second list").detail("toKill2", describe(toKill2)).detail("KillTotal", toKill2.size()) TraceEvent("RemoveAndKill").detail("Step", "exclude second list").detail("ToKill2", describe(toKill2)).detail("KillTotal", toKill2.size())
.detail("Processes", killProcArray.size()).detail("ClusterAvailable", g_simulator.isAvailable()); .detail("Processes", killProcArray.size()).detail("ClusterAvailable", g_simulator.isAvailable());
Void _ = wait( reportErrors( timeoutError( removeAndKill( self, cx, toKill2, bClearedFirst ? &toKill1 : NULL), self->kill2Timeout ), "RemoveServersSafelyError", UID() ) ); Void _ = wait( reportErrors( timeoutError( removeAndKill( self, cx, toKill2, bClearedFirst ? &toKill1 : NULL), self->kill2Timeout ), "RemoveServersSafelyError", UID() ) );
TraceEvent("RemoveAndKill").detail("Step", "excluded second list").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "excluded second list").detail("KillTotal", toKill1.size()).detail("ToKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
// Reinclude all of the machine, if buggified // Reinclude all of the machine, if buggified
if (BUGGIFY) { if (BUGGIFY) {
// Get the updated list of processes which may have changed due to reboots, deletes, etc // Get the updated list of processes which may have changed due to reboots, deletes, etc
TraceEvent("RemoveAndKill").detail("Step", "include all second").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "include all second").detail("KillTotal", toKill1.size()).detail("ToKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) ); Void _ = wait( includeServers( cx, vector<AddressExclusion>(1) ) );
self->includeAddresses(toKill2); self->includeAddresses(toKill2);
TraceEvent("RemoveAndKill").detail("Step", "included all second").detail("KillTotal", toKill1.size()).detail("toKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill").detail("Step", "included all second").detail("KillTotal", toKill1.size()).detail("ToKill", describe(toKill2)).detail("ClusterAvailable", g_simulator.isAvailable());
} }
return Void(); return Void();
@ -352,15 +352,15 @@ struct RemoveServersSafelyWorkload : TestWorkload {
.detail("Processes", killProcArray.size()).detail("ClusterAvailable", g_simulator.isAvailable()).detail("RemoveViaClear", removeViaClear); .detail("Processes", killProcArray.size()).detail("ClusterAvailable", g_simulator.isAvailable()).detail("RemoveViaClear", removeViaClear);
for (auto& killProcess : killProcArray) { for (auto& killProcess : killProcArray) {
if (g_simulator.protectedAddresses.count(killProcess->address)) if (g_simulator.protectedAddresses.count(killProcess->address))
TraceEvent("RemoveAndKill", functionId).detail("Step", "NoKill Process").detail("Process", describe(*killProcess)).detail("failed", killProcess->failed).detail("rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address)); TraceEvent("RemoveAndKill", functionId).detail("Step", "NoKill Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
else if (removeViaClear) { else if (removeViaClear) {
g_simulator.rebootProcess( killProcess, ISimulator::RebootProcessAndDelete); g_simulator.rebootProcess( killProcess, ISimulator::RebootProcessAndDelete);
TraceEvent("RemoveAndKill", functionId).detail("Step", "Clear Process").detail("Process", describe(*killProcess)).detail("failed", killProcess->failed).detail("rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address)); TraceEvent("RemoveAndKill", functionId).detail("Step", "Clear Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
} }
/* /*
else { else {
g_simulator.killProcess( killProcess, ISimulator::KillInstantly ); g_simulator.killProcess( killProcess, ISimulator::KillInstantly );
TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("failed", killProcess->failed).detail("rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address)); TraceEvent("RemoveAndKill", functionId).detail("Step", "Kill Process").detail("Process", describe(*killProcess)).detail("Failed", killProcess->failed).detail("Rebooting", killProcess->rebooting).detail("ClusterAvailable", g_simulator.isAvailable()).detail("Protected", g_simulator.protectedAddresses.count(killProcess->address));
} }
*/ */
} }
@ -400,7 +400,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
std::copy(toKill.begin(), toKill.end(), std::back_inserter(toKillArray)); std::copy(toKill.begin(), toKill.end(), std::back_inserter(toKillArray));
killProcArray = self->getProcesses(toKill); killProcArray = self->getProcesses(toKill);
TraceEvent("RemoveAndKill", functionId).detail("Step", "Activate Server Exclusion").detail("KillAddrs", toKill.size()).detail("KillProcs", killProcArray.size()).detail("MissingProcs", toKill.size()!=killProcArray.size()).detail("toKill", describe(toKill)).detail("Addresses", describe(toKillArray)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill", functionId).detail("Step", "Activate Server Exclusion").detail("KillAddrs", toKill.size()).detail("KillProcs", killProcArray.size()).detail("MissingProcs", toKill.size()!=killProcArray.size()).detail("ToKill", describe(toKill)).detail("Addresses", describe(toKillArray)).detail("ClusterAvailable", g_simulator.isAvailable());
Void _ = wait( excludeServers( cx, toKillArray ) ); Void _ = wait( excludeServers( cx, toKillArray ) );
// We need to skip at least the quorum change if there's nothing to kill, because there might not be enough servers left // We need to skip at least the quorum change if there's nothing to kill, because there might not be enough servers left
@ -410,7 +410,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
TraceEvent("RemoveAndKill", functionId).detail("Step", "Wait For Server Exclusion").detail("Addresses", describe(toKill)).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill", functionId).detail("Step", "Wait For Server Exclusion").detail("Addresses", describe(toKill)).detail("ClusterAvailable", g_simulator.isAvailable());
Void _ = wait( waitForExcludedServers( cx, toKillArray ) ); Void _ = wait( waitForExcludedServers( cx, toKillArray ) );
TraceEvent("RemoveAndKill", functionId).detail("Step", "coordinators auto").detail("desiredCoordinators", g_simulator.desiredCoordinators).detail("ClusterAvailable", g_simulator.isAvailable()); TraceEvent("RemoveAndKill", functionId).detail("Step", "coordinators auto").detail("DesiredCoordinators", g_simulator.desiredCoordinators).detail("ClusterAvailable", g_simulator.isAvailable());
// Setup the coordinators BEFORE the exclusion // Setup the coordinators BEFORE the exclusion
// Otherwise, we may end up with NotEnoughMachinesForCoordinators // Otherwise, we may end up with NotEnoughMachinesForCoordinators
@ -420,7 +420,7 @@ struct RemoveServersSafelyWorkload : TestWorkload {
cycle ++; cycle ++;
nQuorum = ((g_simulator.desiredCoordinators+1)/2)*2-1; nQuorum = ((g_simulator.desiredCoordinators+1)/2)*2-1;
CoordinatorsResult::Type result = wait( changeQuorum( cx, autoQuorumChange(nQuorum) ) ); CoordinatorsResult::Type result = wait( changeQuorum( cx, autoQuorumChange(nQuorum) ) );
TraceEvent(result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES ? SevInfo : SevWarn, "RemoveAndKillQuorumChangeResult").detail("Step", "coordinators auto").detail("Result", (int)result).detail("attempt", cycle).detail("Quorum", nQuorum).detail("desiredCoordinators", g_simulator.desiredCoordinators); TraceEvent(result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES ? SevInfo : SevWarn, "RemoveAndKillQuorumChangeResult").detail("Step", "coordinators auto").detail("Result", (int)result).detail("Attempt", cycle).detail("Quorum", nQuorum).detail("DesiredCoordinators", g_simulator.desiredCoordinators);
if (result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES) if (result==CoordinatorsResult::SUCCESS || result==CoordinatorsResult::SAME_NETWORK_ADDRESSES)
break; break;
} }

2
fdbserver/workloads/Rollback.actor.cpp
View File

@ -47,7 +47,7 @@ struct RollbackWorkload : TestWorkload {
virtual Future<Void> start( Database const& cx ) { virtual Future<Void> start( Database const& cx ) {
if (&g_simulator == g_network && enabled) if (&g_simulator == g_network && enabled)
return timeout( return timeout(
reportErrors( rollbackFailureWorker( cx, this, meanDelay ), "rollbackFailureWorkerError" ), reportErrors( rollbackFailureWorker( cx, this, meanDelay ), "RollbackFailureWorkerError" ),
testDuration, Void() ); testDuration, Void() );
return Void(); return Void();
} }

Some files were not shown because too many files have changed in this diff Show More