wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release-6.1' of https://github.com/apple/foundationdb into reduce-http-error-noise

This commit is contained in:
Stephen Atherton 2019-06-03 16:51:21 -07:00
parent 7ac62fb40f bb22ee7d37
commit d0011c3844
17 changed files with 118 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
FDBLibTLS/verify-test.cpp
View File

@ -52,25 +52,6 @@ struct FDBLibTLSVerifyTest {
std::map<int, Criteria> root_criteria;
};
static std::string printable( std::string const& val ) {
static char const digits[] = "0123456789ABCDEF";
std::string s;
for ( int i = 0; i < val.size(); i++ ) {
uint8_t b = val[i];
if (b >= 32 && b < 127 && b != '\\')
s += (char)b;
else if (b == '\\')
s += "\\\\";
else {
s += "\\x";
s += digits[(b >> 4) & 15];
s += digits[b & 15];
}
}
return s;
}
static std::string criteriaToString(std::map<int, Criteria> const& criteria) {
std::string s;
for (auto &pair: criteria) {

2
bindings/flow/FDBLoanerTypes.h
View File

@ -258,8 +258,6 @@ namespace FDB {
typedef Standalone<KeyRangeRef> KeyRange;
std::string printable( const StringRef& val );
template <class T>
static std::string describe(T const& item) {
return item.toString();

12
bindings/flow/fdb_flow.actor.cpp
View File

@ -427,16 +427,4 @@ namespace FDB {
void TransactionImpl::reset() {
fdb_transaction_reset( tr );
}
std::string printable( const StringRef& val ) {
std::string s;
for(int i=0; i<val.size(); i++) {
uint8_t b = val[i];
if (b >= 32 && b < 127 && b != '\\') s += (char)b;
else if (b == '\\') s += "\\\\";
else s += format("\\x%02x", b);
}
return s;
}
}

16
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -107,7 +107,7 @@ struct DirectoryCreateSubspaceFunc : InstructionFunc {
state Tuple path = wait(popTuple(data));
Tuple rawPrefix = wait(data->stack.waitAndPop());
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), printable(rawPrefix.getString(0)).c_str()));
logOp(format("Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str()));
data->directoryData.push(new Subspace(path, rawPrefix.getString(0)));
return Void();
}
@ -133,7 +133,7 @@ struct DirectoryCreateLayerFunc : InstructionFunc {
else {
Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get();
Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get();
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, printable(nodeSubspace->key()).c_str(), index2, printable(nodeSubspace->key()).c_str(), allowManualPrefixes));
logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, allow_manual_prefixes = %d", index1, nodeSubspace->key().printable().c_str(), index2, nodeSubspace->key().printable().c_str(), allowManualPrefixes));
data->directoryData.push(Reference<IDirectory>(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes)));
}
@ -158,7 +158,7 @@ struct DirectoryChangeFunc : InstructionFunc {
if(LOG_DIRS) {
DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex];
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : printable(d.subspace.get()->key()).c_str());
printf("Changed directory to %d (%s @\'%s\')\n", data->directoryData.directoryListIndex, d.typeString().c_str(), d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() : d.subspace.get()->key().printable().c_str());
fflush(stdout);
}
@ -192,7 +192,7 @@ struct DirectoryCreateOrOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), printable(layer).c_str()));
logOp(format("create_or_open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer] () {
return directory->createOrOpen(instruction->tr, path, layer);
@ -217,7 +217,7 @@ struct DirectoryCreateFunc : InstructionFunc {
Optional<Standalone<StringRef>> prefix = args[1].getType(0) == Tuple::NULL_TYPE ? Optional<Standalone<StringRef>>() : args[1].getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), printable(layer).c_str(), prefix.present() ? printable(prefix.get()).c_str() : "<not present>"));
logOp(format("create %s: layer=%s, prefix=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str(), prefix.present() ? prefix.get().printable().c_str() : "<not present>"));
Reference<DirectorySubspace> dirSubspace = wait(executeMutation(instruction, [this, directory, layer, prefix] () {
return directory->create(instruction->tr, path, layer, prefix);
@ -241,7 +241,7 @@ struct DirectoryOpenFunc : InstructionFunc {
Standalone<StringRef> layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0);
Reference<IDirectory> directory = data->directoryData.directory();
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), printable(layer).c_str()));
logOp(format("open %s: layer=%s", pathToString(combinePaths(directory->getPath(), path)).c_str(), layer.printable().c_str()));
Reference<DirectorySubspace> dirSubspace = wait(directory->open(instruction->tr, path, layer));
data->directoryData.push(dirSubspace);
@ -433,7 +433,7 @@ struct DirectoryUnpackKeyFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop());
Subspace *subspace = data->directoryData.subspace();
logOp(format("Unpack %s in subspace with prefix %s", printable(key.getString(0)).c_str(), printable(subspace->key()).c_str()));
logOp(format("Unpack %s in subspace with prefix %s", key.getString(0).printable().c_str(), subspace->key().printable().c_str()));
Tuple tuple = subspace->unpack(key.getString(0));
for(int i = 0; i < tuple.size(); ++i) {
data->stack.push(tuple.subTuple(i, i+1).pack());
@ -483,7 +483,7 @@ struct DirectoryOpenSubspaceFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple tuple = wait(popTuple(data));
Subspace *subspace = data->directoryData.subspace();
logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), printable(subspace->key()).c_str()));
logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str()));
Subspace *child = new Subspace(subspace->subspace(tuple));
data->directoryData.push(child);

4
bindings/flow/tester/Tester.actor.cpp
View File

@ -97,7 +97,7 @@ std::string tupleToString(Tuple const& tuple) {
if(type == Tuple::UTF8) {
str += "u";
}
str += "\'" + printable(tuple.getString(i)) + "\'";
str += "\'" + tuple.getString(i).printable() + "\'";
}
else if(type == Tuple::INT) {
str += format("%ld", tuple.getInt(i));
@ -1790,7 +1790,7 @@ ACTOR void _test_versionstamp() {
ASSERT(trVersion.compare(dbVersion) == 0);
fprintf(stderr, "%s\n", printable(trVersion).c_str());
fprintf(stderr, "%s\n", trVersion.printable().c_str());
g_network->stop();
}

4
documentation/sphinx/source/downloads.rst
View File

@ -17,8 +17,8 @@ Ubuntu
The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x.
* `foundationdb-clients-6.1.8-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.8/ubuntu/installers/foundationdb-clients_6.1.7-1_amd64.deb>`_
* `foundationdb-server-6.1.8-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.8/ubuntu/installers/foundationdb-server_6.1.7-1_amd64.deb>`_ (depends on the clients package)
* `foundationdb-clients-6.1.8-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.8/ubuntu/installers/foundationdb-clients_6.1.8-1_amd64.deb>`_
* `foundationdb-server-6.1.8-1_amd64.deb <https://www.foundationdb.org/downloads/6.1.8/ubuntu/installers/foundationdb-server_6.1.8-1_amd64.deb>`_ (depends on the clients package)
RHEL/CentOS EL6
---------------

8
documentation/sphinx/source/release-notes.rst
View File

@ -2,6 +2,14 @@
Release Notes
#############
6.1.9
=====
Fixes
-----
* Sometimes a minority of coordinators would not converge to the leader. `(PR #1649) <https://github.com/apple/foundationdb/pull/1649>`_
6.1.8
=====

9
fdbclient/ClusterInterface.h
View File

@ -41,6 +41,15 @@ struct ClusterInterface {
UID id() const { return openDatabase.getEndpoint().token; }
NetworkAddress address() const { return openDatabase.getEndpoint().getPrimaryAddress(); }
bool hasMessage() {
return openDatabase.getFuture().isReady() ||
failureMonitoring.getFuture().isReady() ||
databaseStatus.getFuture().isReady() ||
ping.getFuture().isReady() ||
getClientWorkers.getFuture().isReady() ||
forceRecovery.getFuture().isReady();
}
void initEndpoints() {
openDatabase.getEndpoint( TaskClusterController );
failureMonitoring.getEndpoint( TaskFailureMonitor );

11
fdbclient/NativeAPI.actor.cpp
View File

@ -2687,7 +2687,8 @@ Future<Void> Transaction::commitMutations() {
.detail("Size", transactionSize)
.detail("NumMutations", tr.transaction.mutations.size())
.detail("ReadConflictSize", tr.transaction.read_conflict_ranges.expectedSize())
.detail("WriteConflictSize", tr.transaction.write_conflict_ranges.expectedSize());
.detail("WriteConflictSize", tr.transaction.write_conflict_ranges.expectedSize())
.detail("DebugIdentifier", trLogInfo ? trLogInfo->identifier : "");
}
if(!apiVersionAtLeast(300)) {
@ -2839,15 +2840,15 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional<Stri
if (trLogInfo) {
if (trLogInfo->identifier.empty()) {
trLogInfo->identifier = printable(value.get());
trLogInfo->identifier = value.get().printable();
}
else if (trLogInfo->identifier != printable(value.get())) {
TraceEvent(SevWarn, "CannotChangeDebugTransactionIdentifier").detail("PreviousIdentifier", trLogInfo->identifier).detail("NewIdentifier", printable(value.get()));
else if (trLogInfo->identifier != value.get().printable()) {
TraceEvent(SevWarn, "CannotChangeDebugTransactionIdentifier").detail("PreviousIdentifier", trLogInfo->identifier).detail("NewIdentifier", value.get().printable());
throw client_invalid_operation();
}
}
else {
trLogInfo = Reference<TransactionLogInfo>(new TransactionLogInfo(printable(value.get()), TransactionLogInfo::DONT_LOG));
trLogInfo = Reference<TransactionLogInfo>(new TransactionLogInfo(value.get().printable(), TransactionLogInfo::DONT_LOG));
}
break;

5
fdbrpc/FlowTransport.actor.cpp
View File

@ -392,6 +392,7 @@ struct Peer : NonCopyable {
state ReplyPromise<Void> reply;
FlowTransport::transport().sendUnreliable( SerializeSource<ReplyPromise<Void>>(reply), remotePing.getEndpoint() );
state int64_t startingBytes = peer->bytesReceived;
state int timeouts = 0;
loop {
choose {
when (wait( delay( FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT ) )) {
@ -399,7 +400,11 @@ struct Peer : NonCopyable {
TraceEvent("ConnectionTimeout").suppressFor(1.0).detail("WithAddr", peer->destination);
throw connection_failed();
}
if(timeouts > 1) {
TraceEvent(SevWarnAlways, "ConnectionSlowPing").suppressFor(1.0).detail("WithAddr", peer->destination).detail("Timeouts", timeouts);
}
startingBytes = peer->bytesReceived;
timeouts++;
}
when (wait( reply.getFuture() )) {
break;

24
fdbserver/ClusterController.actor.cpp
View File

@ -2706,6 +2706,16 @@ ACTOR Future<Void> clusterControllerCore( ClusterControllerFullInterface interf,
}
}
ACTOR Future<Void> replaceInterface( ClusterControllerFullInterface interf ) {
loop {
if( interf.hasMessage() ) {
wait(delay(SERVER_KNOBS->REPLACE_INTERFACE_DELAY));
return Void();
}
wait(delay(SERVER_KNOBS->REPLACE_INTERFACE_CHECK_DELAY));
}
}
ACTOR Future<Void> clusterController( ServerCoordinators coordinators, Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> currentCC, bool hasConnected, Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo, LocalityData locality ) {
loop {
state ClusterControllerFullInterface cci;
@ -2714,19 +2724,23 @@ ACTOR Future<Void> clusterController( ServerCoordinators coordinators, Reference
try {
//Register as a possible leader; wait to be elected
state Future<Void> leaderFail = tryBecomeLeader( coordinators, cci, currentCC, hasConnected, asyncPriorityInfo );
state Future<Void> shouldReplace = replaceInterface( cci );
while (!currentCC->get().present() || currentCC->get().get() != cci) {
choose {
when( wait(currentCC->onChange()) ) {}
when( wait(leaderFail) ) { ASSERT(false); throw internal_error(); }
when( wait(shouldReplace) ) { break; }
}
}
if(!shouldReplace.isReady()) {
shouldReplace = Future<Void>();
hasConnected = true;
startRole(Role::CLUSTER_CONTROLLER, cci.id(), UID());
inRole = true;
hasConnected = true;
startRole(Role::CLUSTER_CONTROLLER, cci.id(), UID());
inRole = true;
wait( clusterControllerCore( cci, leaderFail, coordinators, locality ) );
wait( clusterControllerCore( cci, leaderFail, coordinators, locality ) );
}
} catch(Error& e) {
if (inRole)
endRole(Role::CLUSTER_CONTROLLER, cci.id(), "Error", e.code() == error_code_actor_cancelled || e.code() == error_code_coordinators_changed, e);

11
fdbserver/ClusterRecruitmentInterface.h
View File

@ -48,6 +48,17 @@ struct ClusterControllerFullInterface {
bool operator == (ClusterControllerFullInterface const& r) const { return id() == r.id(); }
bool operator != (ClusterControllerFullInterface const& r) const { return id() != r.id(); }
bool hasMessage() {
return clientInterface.hasMessage() ||
recruitFromConfiguration.getFuture().isReady() ||
recruitRemoteFromConfiguration.getFuture().isReady() ||
recruitStorage.getFuture().isReady() ||
registerWorker.getFuture().isReady() ||
getWorkers.getFuture().isReady() ||
registerMaster.getFuture().isReady() ||
getServerDBInfo.getFuture().isReady();
}
void initEndpoints() {
clientInterface.initEndpoints();
recruitFromConfiguration.getEndpoint( TaskClusterController );

11
fdbserver/CompactMap.cpp
View File

@ -355,17 +355,6 @@ struct CompactPreOrderTree {
#endif
};
std::string printable(const StringRef& val) {
std::string s;
for (int i = 0; i<val.size(); i++) {
uint8_t b = val[i];
if (b >= 32 && b < 127 && b != '\\') s += (char)b;
else if (b == '\\') s += "\\\\";
else s += format("\\x%02x", b);
}
return s;
}
void compactMapTests(std::vector<std::string> testData, std::vector<std::string> sampleQueries, std::string prefixTreeDOTFile = "") {
double t1, t2;
int r = 0;

32
fdbserver/Coordination.actor.cpp
View File

@ -271,38 +271,22 @@ ACTOR Future<Void> leaderRegister(LeaderElectionRegInterface interf, Key key) {
return Void();
} else {
Optional<LeaderInfo> nextNominee;
if (availableLeaders.size() && availableCandidates.size()) {
nextNominee = ( *availableLeaders.begin() < *availableCandidates.begin() ) ? *availableLeaders.begin() : *availableCandidates.begin();
} else if (availableLeaders.size()) {
nextNominee = *availableLeaders.begin();
} else if (availableCandidates.size()) {
if( availableCandidates.size() && (!availableLeaders.size() || availableLeaders.begin()->leaderChangeRequired(*availableCandidates.begin())) ) {
nextNominee = *availableCandidates.begin();
} else {
nextNominee = Optional<LeaderInfo>();
} else if( availableLeaders.size() ) {
nextNominee = *availableLeaders.begin();
}
bool foundCurrentNominee = false;
if(currentNominee.present()) {
for(auto& it : availableLeaders) {
if(currentNominee.get().equalInternalId(it)) {
foundCurrentNominee = true;
break;
}
}
}
if ( !nextNominee.present() || !foundCurrentNominee || currentNominee.get().leaderChangeRequired(nextNominee.get()) ) {
TraceEvent("NominatingLeader").detail("Nominee", nextNominee.present() ? nextNominee.get().changeID : UID())
.detail("Changed", nextNominee != currentNominee).detail("Key", printable(key));
if( !currentNominee.present() || !nextNominee.present() || !currentNominee.get().equalInternalId(nextNominee.get()) || nextNominee.get() > currentNominee.get() ) {
TraceEvent("NominatingLeader").detail("NextNominee", nextNominee.present() ? nextNominee.get().changeID : UID())
.detail("CurrentNominee", currentNominee.present() ? currentNominee.get().changeID : UID()).detail("Key", printable(key));
for(unsigned int i=0; i<notify.size(); i++)
notify[i].send( nextNominee );
notify.clear();
currentNominee = nextNominee;
} else if (currentNominee.get().equalInternalId(nextNominee.get())) {
// leader becomes better
currentNominee = nextNominee;
}
currentNominee = nextNominee;
if( availableLeaders.size() ) {
nextInterval = delay( SERVER_KNOBS->POLLING_FREQUENCY );
if(leaderIntervalCount++ > 5) {

2
fdbserver/Knobs.cpp
View File

@ -326,6 +326,8 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( MAX_VERSION_DIFFERENCE, 20 * VERSIONS_PER_SECOND );
init( FORCE_RECOVERY_CHECK_DELAY, 5.0 );
init( RATEKEEPER_FAILURE_TIME, 1.0 );
init( REPLACE_INTERFACE_DELAY, 60.0 );
init( REPLACE_INTERFACE_CHECK_DELAY, 5.0 );
init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
init( EXPECTED_MASTER_FITNESS, ProcessClass::UnsetFit );

2
fdbserver/Knobs.h
View File

@ -269,6 +269,8 @@ public:
int64_t MAX_VERSION_DIFFERENCE;
double FORCE_RECOVERY_CHECK_DELAY;
double RATEKEEPER_FAILURE_TIME;
double REPLACE_INTERFACE_DELAY;
double REPLACE_INTERFACE_CHECK_DELAY;
// Knobs used to select the best policy (via monte carlo)
int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)

42
flow/Arena.h
View File

@ -559,13 +559,43 @@ public:
}
std::string toString() const { return std::string( (const char*)data, length ); }
std::string printable() const {
std::string s;
for (int i = 0; i<length; i++) {
uint8_t b = (*this)[i];
if (b >= 32 && b < 127 && b != '\\') s += (char)b;
else if (b == '\\') s += "\\\\";
else s += format("\\x%02x", b);
constexpr char hex[] = "0123456789abcdef";
int additionalLength = 0;
for (int i = 0; i < length; i++) {
uint8_t b = data[i];
if (b == '\\') additionalLength += 1;
else if (b < 32 || b >= 127) additionalLength += 3;
}
if(additionalLength == 0) {
return std::string((const char*)data, length);
}
std::string s(length + additionalLength, '\\');
int beginCopy = 0;
int resultPos = 0;
for (int i = 0; i < length; i++) {
uint8_t b = data[i];
if (b < 32 || b >= 127 || b == '\\') {
memcpy((char*)s.c_str() + resultPos,(const char*)data + beginCopy, i - beginCopy);
resultPos += i - beginCopy;
beginCopy = i + 1;
if (b == '\\') {
resultPos += 2;
} else {
++resultPos;
s[resultPos++] = 'x';
s[resultPos++] = hex[b >> 4];
s[resultPos++] = hex[b & 0xf];
}
}
}
if(beginCopy < length) {
memcpy((char*)s.c_str() + resultPos,(const char*)data + beginCopy, length - beginCopy);
}
return s;
}