wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor Status retrieval

This commit is contained in:
Xiaoxi Wang 2022-05-03 15:26:24 -07:00
parent 940512f208
commit 75a90be0dd
2 changed files with 71 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
fdbclient/FDBTypes.h
View File

@ -33,6 +33,7 @@
#include "flow/FastRef.h" #include "flow/FastRef.h"
#include "flow/ProtocolVersion.h" #include "flow/ProtocolVersion.h"
#include "flow/flow.h" #include "flow/flow.h"
#include "fdbclient/Status.h"
enum class TraceFlags : uint8_t { unsampled = 0b00000000, sampled = 0b00000001 }; enum class TraceFlags : uint8_t { unsampled = 0b00000000, sampled = 0b00000001 };
@ -1473,6 +1474,14 @@ struct StorageMetadataType {
void serialize(Ar& ar) { void serialize(Ar& ar) {
serializer(ar, createdTime, storeType); serializer(ar, createdTime, storeType);
} }
StatusObject toJSON() const {
StatusObject result;
result["created_time_timestamp"] = createdTime;
result["created_time_datetime"] = epochsToGMTString(createdTime);
result["storage_engine"] = storeType.toString();
return result;
}
}; };
// store metadata of wiggle action // store metadata of wiggle action

126
fdbserver/Status.actor.cpp
View File

@ -103,6 +103,14 @@ extern const char* limitReasonDesc[];
typedef std::map<std::string, TraceEventFields> EventMap; typedef std::map<std::string, TraceEventFields> EventMap;
struct StorageServerStatusInfo : public StorageServerInterface {
Optional<StorageMetadataType> metadata;
EventMap eventMap;
StorageServerStatusInfo(const StorageServerInterface& interface,
Optional<StorageMetadataType> metadata = Optional<StorageMetadataType>())
: StorageServerInterface(interface), metadata(metadata) {}
};
ACTOR static Future<Optional<TraceEventFields>> latestEventOnWorker(WorkerInterface worker, std::string eventName) { ACTOR static Future<Optional<TraceEventFields>> latestEventOnWorker(WorkerInterface worker, std::string eventName) {
try { try {
EventLogRequest req = EventLogRequest req =
@ -468,12 +476,13 @@ struct RolesInfo {
obj["role"] = role; obj["role"] = role;
return roles.insert(std::make_pair(address, obj))->second; return roles.insert(std::make_pair(address, obj))->second;
} }
JsonBuilderObject& addRole(std::string const& role, JsonBuilderObject& addRole(std::string const& role,
StorageServerInterface& iface, StorageServerStatusInfo& iface,
EventMap const& metrics,
Version maxTLogVersion, Version maxTLogVersion,
double* pDataLagSeconds) { double* pDataLagSeconds) {
JsonBuilderObject obj; JsonBuilderObject obj;
EventMap const& metrics = iface.eventMap;
double dataLagSeconds = -1.0; double dataLagSeconds = -1.0;
obj["id"] = iface.id().shortString(); obj["id"] = iface.id().shortString();
obj["role"] = role; obj["role"] = role;
@ -584,13 +593,8 @@ struct RolesInfo {
} }
} }
if (!iface.isTss()) { // only storage server has Metadata field if (iface.metadata.present()) {
TraceEventFields const& metadata = metrics.at("Metadata"); obj["storage_metadata"] = iface.metadata.get().toJSON();
JsonBuilderObject metadataObj;
metadataObj["created_time_datetime"] = metadata.getValue("CreatedTimeDatetime");
metadataObj["created_time_timestamp"] = metadata.getDouble("CreatedTimeTimestamp");
metadataObj["storage_engine"] = metadata.getValue("StoreType");
obj["storage_metadata"] = metadataObj;
// printf("%s\n", metadataObj.getJson().c_str()); // printf("%s\n", metadataObj.getJson().c_str());
} }
@ -731,7 +735,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
WorkerEvents traceFileOpenErrors, WorkerEvents traceFileOpenErrors,
WorkerEvents programStarts, WorkerEvents programStarts,
std::map<std::string, std::vector<JsonBuilderObject>> processIssues, std::map<std::string, std::vector<JsonBuilderObject>> processIssues,
std::vector<std::pair<StorageServerInterface, EventMap>> storageServers, std::vector<StorageServerStatusInfo> storageServers,
std::vector<std::pair<TLogInterface, EventMap>> tLogs, std::vector<std::pair<TLogInterface, EventMap>> tLogs,
std::vector<std::pair<CommitProxyInterface, EventMap>> commitProxies, std::vector<std::pair<CommitProxyInterface, EventMap>> commitProxies,
std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies, std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies,
@ -861,13 +865,13 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
wait(yield()); wait(yield());
} }
state std::vector<std::pair<StorageServerInterface, EventMap>>::iterator ss; state std::vector<StorageServerStatusInfo>::iterator ss;
state std::map<NetworkAddress, double> ssLag; state std::map<NetworkAddress, double> ssLag;
state double lagSeconds; state double lagSeconds;
for (ss = storageServers.begin(); ss != storageServers.end(); ++ss) { for (ss = storageServers.begin(); ss != storageServers.end(); ++ss) {
roles.addRole("storage", ss->first, ss->second, maxTLogVersion, &lagSeconds); roles.addRole("storage", *ss, maxTLogVersion, &lagSeconds);
if (lagSeconds != -1.0) { if (lagSeconds != -1.0) {
ssLag[ss->first.address()] = lagSeconds; ssLag[ss->address()] = lagSeconds;
} }
wait(yield()); wait(yield());
} }
@ -1919,74 +1923,69 @@ static Future<std::vector<TraceEventFields>> getServerBusiestWriteTags(
} }
ACTOR ACTOR
static Future<std::vector<Optional<StorageMetadataType>>> getServerMetadata(std::vector<StorageServerInterface> servers, static Future<std::vector<StorageServerStatusInfo>> readStorageInterfaceAndMetadata(Database cx,
Database cx, bool use_system_priority) {
bool use_system_priority) {
state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin, state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
IncludeVersion()); IncludeVersion());
state std::vector<Optional<StorageMetadataType>> res(servers.size());
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx); state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
state std::vector<StorageServerStatusInfo> servers;
loop { loop {
try { try {
servers.clear();
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if (use_system_priority) { if (use_system_priority) {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
} }
state int i = 0; state RangeResult serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
for (i = 0; i < servers.size(); ++i) { ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
Optional<StorageMetadataType> metadata = wait(metadataMap.get(tr, servers[i].id(), Snapshot::True));
// TraceEvent(SevDebug, "MetadataAppear", servers[i].id()).detail("Present", metadata.present()); servers.reserve(serverList.size());
res[i] = metadata; for (int i = 0; i < serverList.size(); i++) {
servers.push_back(StorageServerStatusInfo(decodeServerListValue(serverList[i].value)));
} }
state std::vector<Future<Void>> futures(servers.size());
for (int i = 0; i < servers.size(); ++i) {
auto& info = servers[i];
futures[i] = fmap(
[&info](Optional<StorageMetadataType> meta) -> Void {
info.metadata = meta;
return Void();
},
metadataMap.get(tr, servers[i].id()));
// TraceEvent(SevDebug, "MetadataAppear", servers[i].id()).detail("Present", metadata.present());
}
wait(waitForAll(futures));
wait(tr->commit()); wait(tr->commit());
break; break;
} catch (Error& e) { } catch (Error& e) {
wait(tr->onError(e)); wait(tr->onError(e));
} }
} }
return res; return servers;
} }
ACTOR static Future<std::vector<std::pair<StorageServerInterface, EventMap>>> getStorageServersAndMetrics( ACTOR static Future<std::vector<StorageServerStatusInfo>> getStorageServerStatusInfos(
Database cx, Database cx,
std::unordered_map<NetworkAddress, WorkerInterface> address_workers, std::unordered_map<NetworkAddress, WorkerInterface> address_workers,
WorkerDetails rkWorker) { WorkerDetails rkWorker) {
state std::vector<StorageServerInterface> servers = wait(timeoutError(getStorageServers(cx, true), 5.0)); state std::vector<StorageServerStatusInfo> servers =
state std::vector<std::pair<StorageServerInterface, EventMap>> results; wait(timeoutError(readStorageInterfaceAndMetadata(cx, true), 5.0));
state std::vector<std::pair<StorageServerStatusInfo, EventMap>> results;
state std::vector<TraceEventFields> busiestWriteTags; state std::vector<TraceEventFields> busiestWriteTags;
state std::vector<Optional<StorageMetadataType>> metadata;
wait(store(results, wait(store(results,
getServerMetrics(servers, getServerMetrics(servers,
address_workers, address_workers,
std::vector<std::string>{ std::vector<std::string>{
"StorageMetrics", "ReadLatencyMetrics", "ReadLatencyBands", "BusiestReadTag" })) && "StorageMetrics", "ReadLatencyMetrics", "ReadLatencyBands", "BusiestReadTag" })) &&
store(busiestWriteTags, getServerBusiestWriteTags(servers, address_workers, rkWorker)) && store(busiestWriteTags, getServerBusiestWriteTags(servers, address_workers, rkWorker)));
store(metadata, getServerMetadata(servers, cx, true)));
ASSERT(busiestWriteTags.size() == results.size() && metadata.size() == results.size()); ASSERT(busiestWriteTags.size() == results.size());
for (int i = 0; i < results.size(); ++i) { for (int i = 0; i < results.size(); ++i) {
results[i].second.emplace("BusiestWriteTag", busiestWriteTags[i]); servers[i].eventMap = std::move(results[i].second);
servers[i].eventMap.emplace("BusiestWriteTag", busiestWriteTags[i]);
// FIXME: it's possible that a SS is removed between `getStorageServers` and `getServerMetadata`. Maybe we can
// read StorageServer and Metadata in an atomic transaction?
if (metadata[i].present()) {
TraceEventFields metadataField;
metadataField.addField("CreatedTimeTimestamp", std::to_string(metadata[i].get().createdTime));
metadataField.addField("CreatedTimeDatetime", epochsToGMTString(metadata[i].get().createdTime));
metadataField.addField("StoreType", metadata[i].get().storeType.toString());
results[i].second.emplace("Metadata", metadataField);
} else if (!servers[i].isTss()) {
TraceEventFields metadataField;
metadataField.addField("CreatedTimeTimestamp", "0");
metadataField.addField("CreatedTimeDatetime", "[removed]");
metadataField.addField("StoreType", KeyValueStoreType::getStoreTypeStr(KeyValueStoreType::END));
results[i].second.emplace("Metadata", metadataField);
}
} }
return servers;
return results;
} }
ACTOR static Future<std::vector<std::pair<TLogInterface, EventMap>>> getTLogsAndMetrics( ACTOR static Future<std::vector<std::pair<TLogInterface, EventMap>>> getTLogsAndMetrics(
@ -2103,7 +2102,7 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
JsonBuilderObject* qos, JsonBuilderObject* qos,
JsonBuilderObject* data_overlay, JsonBuilderObject* data_overlay,
std::set<std::string>* incomplete_reasons, std::set<std::string>* incomplete_reasons,
Future<ErrorOr<std::vector<std::pair<StorageServerInterface, EventMap>>>> storageServerFuture) { Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture) {
state JsonBuilderObject statusObj; state JsonBuilderObject statusObj;
state JsonBuilderObject operationsObj; state JsonBuilderObject operationsObj;
state JsonBuilderObject bytesObj; state JsonBuilderObject bytesObj;
@ -2275,7 +2274,7 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
// Reads // Reads
try { try {
ErrorOr<std::vector<std::pair<StorageServerInterface, EventMap>>> storageServers = wait(storageServerFuture); ErrorOr<std::vector<StorageServerStatusInfo>> storageServers = wait(storageServerFuture);
if (!storageServers.present()) { if (!storageServers.present()) {
throw storageServers.getError(); throw storageServers.getError();
} }
@ -2287,7 +2286,7 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
StatusCounter lowPriorityReads; StatusCounter lowPriorityReads;
for (auto& ss : storageServers.get()) { for (auto& ss : storageServers.get()) {
TraceEventFields const& storageMetrics = ss.second.at("StorageMetrics"); TraceEventFields const& storageMetrics = ss.eventMap.at("StorageMetrics");
if (storageMetrics.size() > 0) { if (storageMetrics.size() > 0) {
readRequests.updateValues(StatusCounter(storageMetrics.getValue("QueryQueue"))); readRequests.updateValues(StatusCounter(storageMetrics.getValue("QueryQueue")));
@ -2318,14 +2317,14 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(
ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher( ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher(
WorkerEvents pMetrics, WorkerEvents pMetrics,
Future<ErrorOr<std::vector<std::pair<StorageServerInterface, EventMap>>>> storageServerFuture, Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture,
Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tlogFuture, Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tlogFuture,
std::set<std::string>* incomplete_reasons) { std::set<std::string>* incomplete_reasons) {
state JsonBuilderObject statusObj; state JsonBuilderObject statusObj;
try { try {
state JsonBuilderObject cacheStatistics; state JsonBuilderObject cacheStatistics;
ErrorOr<std::vector<std::pair<StorageServerInterface, EventMap>>> storageServers = wait(storageServerFuture); ErrorOr<std::vector<StorageServerStatusInfo>> storageServers = wait(storageServerFuture);
if (!storageServers.present()) { if (!storageServers.present()) {
throw storageServers.getError(); throw storageServers.getError();
@ -2335,7 +2334,7 @@ ACTOR static Future<JsonBuilderObject> clusterSummaryStatisticsFetcher(
double storageCacheMissesHz = 0; double storageCacheMissesHz = 0;
for (auto& ss : storageServers.get()) { for (auto& ss : storageServers.get()) {
auto processMetrics = pMetrics.find(ss.first.address()); auto processMetrics = pMetrics.find(ss.address());
if (processMetrics != pMetrics.end()) { if (processMetrics != pMetrics.end()) {
int64_t hits = processMetrics->second.getInt64("CacheHits"); int64_t hits = processMetrics->second.getInt64("CacheHits");
int64_t misses = processMetrics->second.getInt64("CacheMisses"); int64_t misses = processMetrics->second.getInt64("CacheMisses");
@ -2947,7 +2946,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
state std::map<std::string, std::vector<JsonBuilderObject>> processIssues = state std::map<std::string, std::vector<JsonBuilderObject>> processIssues =
getProcessIssuesAsMessages(workerIssues); getProcessIssuesAsMessages(workerIssues);
state std::vector<std::pair<StorageServerInterface, EventMap>> storageServers; state std::vector<StorageServerStatusInfo> storageServers;
state std::vector<std::pair<TLogInterface, EventMap>> tLogs; state std::vector<std::pair<TLogInterface, EventMap>> tLogs;
state std::vector<std::pair<CommitProxyInterface, EventMap>> commitProxies; state std::vector<std::pair<CommitProxyInterface, EventMap>> commitProxies;
state std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies; state std::vector<std::pair<GrvProxyInterface, EventMap>> grvProxies;
@ -3021,8 +3020,8 @@ ACTOR Future<StatusReply> clusterGetStatus(
address_workers[worker.interf.address()] = worker.interf; address_workers[worker.interf.address()] = worker.interf;
} }
state Future<ErrorOr<std::vector<std::pair<StorageServerInterface, EventMap>>>> storageServerFuture = state Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture =
errorOr(getStorageServersAndMetrics(cx, address_workers, rkWorker)); errorOr(getStorageServerStatusInfos(cx, address_workers, rkWorker));
state Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tLogFuture = state Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tLogFuture =
errorOr(getTLogsAndMetrics(db, address_workers)); errorOr(getTLogsAndMetrics(db, address_workers));
state Future<ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>>> commitProxyFuture = state Future<ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>>> commitProxyFuture =
@ -3136,8 +3135,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
} }
// Need storage servers now for processStatusFetcher() below. // Need storage servers now for processStatusFetcher() below.
ErrorOr<std::vector<std::pair<StorageServerInterface, EventMap>>> _storageServers = ErrorOr<std::vector<StorageServerStatusInfo>> _storageServers = wait(storageServerFuture);
wait(storageServerFuture);
if (_storageServers.present()) { if (_storageServers.present()) {
storageServers = _storageServers.get(); storageServers = _storageServers.get();
} else { } else {
@ -3225,11 +3223,11 @@ ACTOR Future<StatusReply> clusterGetStatus(
int activeTSSCount = 0; int activeTSSCount = 0;
JsonBuilderArray wiggleServerAddress; JsonBuilderArray wiggleServerAddress;
for (auto& it : storageServers) { for (auto& it : storageServers) {
if (it.first.isTss()) { if (it.isTss()) {
activeTSSCount++; activeTSSCount++;
} }
if (wiggleServers.count(it.first.id())) { if (wiggleServers.count(it.id())) {
wiggleServerAddress.push_back(it.first.address().toString()); wiggleServerAddress.push_back(it.address().toString());
} }
} }
statusObj["active_tss_count"] = activeTSSCount; statusObj["active_tss_count"] = activeTSSCount;