wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release-6.0'

This commit is contained in:
Bhaskar Muppana 2018-09-06 14:24:02 -07:00
parent 8bf67c7afc 5e954a8d57
commit 920fd3fe97
5 changed files with 88 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/setup.py.in
View File

@ -18,7 +18,7 @@ setup(name="foundationdb",
classifiers=[
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'License :: OSI Approved :: Apache v2 License',
'License :: OSI Approved :: Apache Software License',
'Operating System :: MacOS :: MacOS X',
'Operating System :: Microsoft :: Windows',
'Operating System :: POSIX :: Linux',

1
fdbclient/BackupContainer.actor.cpp
View File

@ -1251,6 +1251,7 @@ ACTOR Future<Version> timeKeeperVersionFromDatetime(std::string datetime, Databa
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state std::vector<std::pair<int64_t, Version>> results = wait( versionMap.getRange(tr, 0, time, 1, false, true) );
if (results.size() != 1) {
// No key less than time was found in the database

55
fdbserver/Status.actor.cpp
View File

@ -310,7 +310,7 @@ static JsonString machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair<W
std::string machineId = event.getValue("MachineID");
// If this machine ID does not already exist in the machineMap, add it
if (!machineMap.isPresent(machineId)) {
if (machineJsonMap.count(machineId) == 0) {
statusObj["machine_id"] = machineId;
if (dcIds.count(it->first)){
@ -587,14 +587,12 @@ ACTOR static Future<JsonString> processStatusFetcher(
state std::vector<std::pair<TLogInterface, TraceEventFields>>::iterator log;
state Version maxTLogVersion = 0;
state Version tLogVersion = 0;
// Get largest TLog version
for(log = tLogs.begin(); log != tLogs.end(); ++log) {
Version tLogVersion = 0;
JsonString const& roleStatus = roles.addRole( "log", log->first, log->second, &tLogVersion );
if (roleStatus.isPresent("data_version")) {
maxTLogVersion = std::max(maxTLogVersion, tLogVersion);
}
maxTLogVersion = std::max(maxTLogVersion, tLogVersion);
wait(yield());
}
@ -880,8 +878,9 @@ ACTOR static Future<JsonString> recoveryStateStatusFetcher(std::pair<WorkerInter
}
// If recovery status name is not know, status is incomplete
if (!message.isPresent("name"))
if (message.empty()) {
incomplete_reasons->insert("Recovery Status unavailable.");
}
return message;
}
@ -951,10 +950,13 @@ ACTOR static Future<double> doCommitProbe(Future<double> grvProbe, Transaction *
}
}
ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, const char* prefix, const char* description, JsonString *probeObj, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, const char* prefix, const char* description, JsonString *probeObj, JsonStringArray *messages, std::set<std::string> *incomplete_reasons, bool *isAvailable = nullptr) {
choose {
when(ErrorOr<double> result = wait(errorOr(probe))) {
if(result.isError()) {
if(isAvailable != nullptr) {
*isAvailable = false;
}
incomplete_reasons->insert(format("Unable to retrieve latency probe information (%s: %s).", description, result.getError().what()));
}
else {
@ -962,6 +964,9 @@ ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, cons
}
}
when(wait(delay(timeoutSeconds))) {
if(isAvailable != nullptr) {
*isAvailable = false;
}
messages->push_back(JsonString::makeMessage(format("%s_probe_timeout", prefix).c_str(), format("Unable to %s after %d seconds.", description, timeoutSeconds).c_str()));
}
}
@ -969,7 +974,7 @@ ACTOR static Future<Void> doProbe(Future<double> probe, int timeoutSeconds, cons
return Void();
}
ACTOR static Future<JsonString> latencyProbeFetcher(Database cx, JsonStringArray *messages, std::set<std::string> *incomplete_reasons) {
ACTOR static Future<JsonString> latencyProbeFetcher(Database cx, JsonStringArray *messages, std::set<std::string> *incomplete_reasons, bool *isAvailable) {
state Transaction trImmediate(cx);
state Transaction trDefault(cx);
state Transaction trBatch(cx);
@ -988,11 +993,11 @@ ACTOR static Future<JsonString> latencyProbeFetcher(Database cx, JsonStringArray
int timeoutSeconds = 5;
std::vector<Future<Void>> probes;
probes.push_back(doProbe(immediateGrvProbe, timeoutSeconds, "immediate_priority_transaction_start", "start immediate priority transaction", &statusObj, messages, incomplete_reasons));
probes.push_back(doProbe(immediateGrvProbe, timeoutSeconds, "immediate_priority_transaction_start", "start immediate priority transaction", &statusObj, messages, incomplete_reasons, isAvailable));
probes.push_back(doProbe(defaultGrvProbe, timeoutSeconds, "transaction_start", "start default priority transaction", &statusObj, messages, incomplete_reasons));
probes.push_back(doProbe(batchGrvProbe, timeoutSeconds, "batch_priority_transaction_start", "start batch priority transaction", &statusObj, messages, incomplete_reasons));
probes.push_back(doProbe(readProbe, timeoutSeconds, "read", "read", &statusObj, messages, incomplete_reasons));
probes.push_back(doProbe(commitProbe, timeoutSeconds, "commit", "commit", &statusObj, messages, incomplete_reasons));
probes.push_back(doProbe(readProbe, timeoutSeconds, "read", "read", &statusObj, messages, incomplete_reasons, isAvailable));
probes.push_back(doProbe(commitProbe, timeoutSeconds, "commit", "commit", &statusObj, messages, incomplete_reasons, isAvailable));
wait(waitForAll(probes));
}
@ -1688,7 +1693,7 @@ ACTOR Future<JsonString> layerStatusFetcher(Database cx, JsonStringArray *messag
json.cleanOps();
JsonString statusObj = result.toJsonString();
TraceEvent("LayerStatusFetcher").detail("Duration", now()-tStart).detail("StatusSize",statusObj.getLength()).detail("StatusNameTotal",statusObj.getNameTotal());
TraceEvent("LayerStatusFetcher").detail("Duration", now()-tStart).detail("StatusSize",statusObj.getLength());
return statusObj;
}
@ -1746,7 +1751,9 @@ ACTOR Future<StatusReply> clusterGetStatus(
std::vector<NetworkAddress> incompatibleConnections,
Version datacenterVersionDifference )
{
state double tStart = now();
// since we no longer offer multi-database support, all databases must be named DB
state std::string dbName = "DB";
state double tStart = timer();
// Check if master worker is present
state JsonStringArray messages;
@ -1828,7 +1835,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
state Optional<DatabaseConfiguration> configuration;
state Optional<bool> fullReplication;
if(!(recoveryStateStatus.isPresent("name") && statusCode == RecoveryStatus::configuration_missing)) {
if(statusCode != RecoveryStatus::configuration_missing) {
std::pair<Optional<DatabaseConfiguration>,Optional<bool>> loadResults = wait(loadConfiguration(cx, &messages, &status_incomplete_reasons));
configuration = loadResults.first;
fullReplication = loadResults.second;
@ -1842,9 +1849,10 @@ ACTOR Future<StatusReply> clusterGetStatus(
if (configuration.present()){
// Do the latency probe by itself to avoid interference from other status activities
JsonString latencyProbeResults = wait(latencyProbeFetcher(cx, &messages, &status_incomplete_reasons));
state bool isAvailable = true;
JsonString latencyProbeResults = wait(latencyProbeFetcher(cx, &messages, &status_incomplete_reasons, &isAvailable));
statusObj["database_available"] = latencyProbeResults.isPresent("immediate_priority_transaction_start_seconds") && latencyProbeResults.isPresent("read_seconds") && latencyProbeResults.isPresent("commit_seconds");
statusObj["database_available"] = isAvailable;
if (!latencyProbeResults.empty()) {
statusObj["latency_probe"] = latencyProbeResults;
}
@ -1970,8 +1978,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
statusObj["cluster_controller_timestamp"] = clusterTime;
}
TraceEvent("ClusterGetStatus").detail("Duration", now()-tStart)
.detail("StatusSize",statusObj.getLength()) .detail("StatusNameTotal",statusObj.getNameTotal());
TraceEvent("ClusterGetStatus").detail("Duration", timer()-tStart).detail("StatusSize",statusObj.getLength());
return StatusReply(statusObj);
} catch( Error&e ) {
@ -2005,12 +2012,6 @@ TEST_CASE("status/json/jsonstring") {
ASSERT(jsonString1.equals(jsonString4));
ASSERT(jsonString4.equals(jsonString1));
// Check presence of top level keys
ASSERT(jsonString1.isPresent("_valid"));
ASSERT(jsonString1.isPresent("error"));
ASSERT(!jsonString1.isPresent("configurationMissing"));
ASSERT(jsonString1.getNameTotal() == 2);
// Check empty
ASSERT(!jsonString1.empty());
ASSERT(JsonString().empty());
@ -2027,12 +2028,6 @@ TEST_CASE("status/json/jsonstring") {
testObj["pi"] = 3.14159;
jsonObj["piobj"] = testObj;
// Check presence of top level keys
ASSERT(jsonObj.isPresent("count"));
ASSERT(testObj.isPresent("pi"));
ASSERT(!jsonObj.isPresent("pi"));
ASSERT(jsonObj.getNameTotal() == 3);
testObj.clear();
testObj["val1"] = 7.9;
testObj["val2"] = 34;

89
flow/JsonString.cpp Normal file → Executable file
View File

@ -6,62 +6,58 @@
std::string format( const char* form, ... );
JsonString::JsonString() : _jsonText(), _keyNames() {
JsonString::JsonString() : hasKey(false) {
}
JsonString::JsonString( const JsonString& jsonString) : _jsonText(jsonString._jsonText), _keyNames(jsonString._keyNames) {
JsonString::JsonString( const JsonString& jsonString) : _jsonText(jsonString._jsonText), hasKey(jsonString.hasKey) {
}
JsonString::JsonString( const JsonStringArray& jsonArray) : _jsonText(), _keyNames() {
JsonString::JsonString( const JsonStringArray& jsonArray) : hasKey(false) {
append(jsonArray);
}
JsonString::JsonString( const std::string& value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& value ) : hasKey(false) {
append(value);
}
JsonString::JsonString( const char* value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const char* value ) : hasKey(false) {
append(value);
}
JsonString::JsonString( const std::string& name, const std::string& value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, const std::string& value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, const char* value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, const char* value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, double value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, double value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, long int value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, long int value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, long unsigned int value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, long unsigned int value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, long long int value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, long long int value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, long long unsigned int value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, long long unsigned int value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, int value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, int value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, unsigned value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, unsigned value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, bool value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, bool value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, const JsonString& value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, const JsonString& value ) : hasKey(false) {
append(name, value);
}
JsonString::JsonString( const std::string& name, const JsonStringArray& value ) : _jsonText(), _keyNames() {
JsonString::JsonString( const std::string& name, const JsonStringArray& value ) : hasKey(false) {
append(name, value);
}
bool JsonString::isPresent(const std::string& name) const {
return (_keyNames.find(name) != _keyNames.end());
}
JsonString JsonString::makeMessage(const char *name, const char *description) {
JsonString out;
out["name"] = name;
@ -69,15 +65,8 @@ JsonString JsonString::makeMessage(const char *name, const char *description) {
return out;
}
void JsonString::hashName( const std::string& name) {
if (isPresent(name)) {
TraceEvent(g_network && g_network->isSimulated() ? SevError : SevWarnAlways, "JsonError").detail("KeyPresent", name).backtrace();
}
_keyNames.insert(name);
}
JsonString& JsonString::appendImpl( const std::string& name, const std::string& value, bool quote ) {
hashName(name);
hasKey = true;
_jsonText += (_jsonText.empty() ? "\"" : ",\n \"") + name + (quote ? "\": \"" : "\": ") + value;
if (quote)
_jsonText += "\"";
@ -154,12 +143,12 @@ JsonString& JsonString::append( const std::string& name, bool value ) {
return appendImpl(name, stringify(value), false);
}
JsonString& JsonString::append( const std::string& name, const JsonString& value ) {
hashName(name);
hasKey = true;
_jsonText += (_jsonText.empty() ? "\"" : ",\n \"") + name + "\": { " + value._jsonText + " }";
return *this;
}
JsonString& JsonString::append( const std::string& name, const JsonStringArray& values ) {
hashName(name);
hasKey = true;
_jsonText += (_jsonText.empty() ? "\"" : ",\n \"") + name + "\": [ ";
size_t counter = 0;
for (auto const& value : values) {
@ -208,7 +197,9 @@ JsonString& JsonString::append( const JsonString& value ) {
if (!_jsonText.empty())
_jsonText += ",\n ";
_jsonText += value._jsonText;
_keyNames.insert(value._keyNames.begin(), value._keyNames.end());
if(value.hasKey) {
hasKey = true;
}
}
return *this;
}
@ -226,34 +217,31 @@ JsonString& JsonString::append( const JsonStringArray& values ) {
}
JsonString& JsonString::clear() {
_keyNames.clear();
_jsonText.clear();
hasKey = false;
return *this;
}
bool JsonString::empty() const {
bool JsonString::empty() const {
return _jsonText.empty();
}
const std::string& JsonString::getJsonText() const {
const std::string& JsonString::getJsonText() const {
return _jsonText;
}
size_t JsonString::getLength() const {
return _jsonText.length() + ((!empty() && _keyNames.empty()) ? 0 : 2);
}
size_t JsonString::getNameTotal() const {
return _keyNames.size();
size_t JsonString::getLength() const {
return _jsonText.length() + ((!empty() && !hasKey) ? 0 : 4);
}
std::string JsonString::getJson() const {
std::string JsonString::getJson() const {
// If not empty with no names (only values), don't add brackets because prob in an array
return (!empty() && _keyNames.empty()) ? _jsonText : ("{ " + _jsonText + " }");
return (!empty() && !hasKey) ? _jsonText : ("{ " + _jsonText + " }");
}
JsonString& JsonString::copy( const JsonString& jsonString ) {
_jsonText = jsonString._jsonText;
_keyNames = jsonString._keyNames;
hasKey = jsonString.hasKey;
return *this;
}
@ -262,27 +250,14 @@ JsonString& JsonString::operator=( const JsonString& jsonString ) {
}
//TODO: Populate key names member
void JsonString::setJson(const std::string& jsonText) {
_keyNames.clear();
void JsonString::setJson(const std::string& jsonText) {
_jsonText = jsonText;
}
JsonString& JsonString::swapJsonText(std::string& jsonText) {
_keyNames.clear();
_jsonText.swap(jsonText);
return *this;
}
JsonStringSetter& JsonString::operator[]( const std::string& name ) {
JsonStringSetter* stringSetter = new JsonStringSetter(*this, name);
return *stringSetter;
}
JsonStringSetter& JsonString::operator[]( const char* name ) {
std::string textName(name);
JsonStringSetter* stringSetter = new JsonStringSetter(*this, textName);
return *stringSetter;
}
int JsonString::compare( const JsonString& jsonString ) const {
return jsonString._jsonText.compare(_jsonText);
}

57
flow/JsonString.h Normal file → Executable file
View File

@ -3,7 +3,6 @@
#define JSONSTRING_H
#include <string>
#include <unordered_set>
#include <vector>
class JsonString;
@ -59,9 +58,8 @@ class JsonString {
JsonString& append( bool value );
JsonString& append( const JsonString& value );
JsonStringSetter& operator[]( const std::string& name );
JsonStringSetter& operator[]( const char* name );
JsonStringSetter operator[]( const std::string& name );
JsonStringSetter operator[]( const char* name );
int compare( const JsonString& jsonString ) const;
bool equals( const JsonString& jsonString ) const;
@ -71,41 +69,36 @@ class JsonString {
JsonString& copy( const JsonString& jsonString );
JsonString& operator=( const JsonString& jsonString );
std::string getJson() const;
void setJson(const std::string& jsonText);
JsonString& swapJsonText(std::string& jsonText);
std::string getJson() const;
void setJson(const std::string& jsonText);
JsonString& swapJsonText(std::string& jsonText);
size_t getLength() const;
size_t getNameTotal() const;
size_t getLength() const;
JsonString& clear();
bool empty() const;
bool isPresent(const std::string& name) const;
bool empty() const;
static JsonString makeMessage(const char *name, const char *description);
protected:
void hashName( const std::string& name);
JsonString& appendImpl( const std::string& name, const std::string& value, bool quote);
JsonString& appendImpl( const std::string& value, bool quote);
static std::string stringify(const char* value);
static std::string stringify(double value);
static std::string stringify(long int value);
static std::string stringify(long unsigned int value);
static std::string stringify(long long int value);
static std::string stringify(long long unsigned int value);
static std::string stringify(int value);
static std::string stringify(unsigned value);
static std::string stringify(bool value);
static std::string stringify(const char* value);
static std::string stringify(double value);
static std::string stringify(long int value);
static std::string stringify(long unsigned int value);
static std::string stringify(long long int value);
static std::string stringify(long long unsigned int value);
static std::string stringify(int value);
static std::string stringify(unsigned value);
static std::string stringify(bool value);
protected:
std::string _jsonText;
std::unordered_set<std::string> _keyNames;
std::string _jsonText;
bool hasKey;
// Uneditted text
const std::string& getJsonText() const;
const std::string& getJsonText() const;
};
// Make protected because no virtual destructor
@ -140,8 +133,16 @@ class JsonStringSetter {
}
protected:
JsonString& _jsonString;
std::string _name;
JsonString& _jsonString;
std::string _name;
};
inline JsonStringSetter JsonString::operator[]( const std::string& name ) {
return JsonStringSetter(*this, name);
}
inline JsonStringSetter JsonString::operator[]( const char* name ) {
return JsonStringSetter(*this, name);
}
#endif