wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10907 from jzhou77/main 

ClusterController watches changes to storage metadata
This commit is contained in:
Jingyu Zhou 2023-09-19 12:21:35 -07:00 committed by GitHub
parent 8706098916 f72eab118d
commit aa1d005cc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 106 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
fdbclient/SystemData.cpp
View File

@ -690,8 +690,21 @@ UID decodeTssQuarantineKey(KeyRef const& key) {
const KeyRangeRef tssMismatchKeys("\xff/tssMismatch/"_sr, "\xff/tssMismatch0"_sr);
const KeyRef serverMetadataChangeKey = "\xff\x02/serverMetadataChanges"_sr;
const KeyRangeRef serverMetadataKeys("\xff/serverMetadata/"_sr, "\xff/serverMetadata0"_sr);
UID decodeServerMetadataKey(const KeyRef& key) {
// Key is packed by KeyBackedObjectMap::packKey
return TupleCodec<UID>::unpack(key.removePrefix(serverMetadataKeys.begin));
}
StorageMetadataType decodeServerMetadataValue(const KeyRef& value) {
StorageMetadataType type;
ObjectReader rd(value.begin(), IncludeVersion());
rd.deserialize(type);
return type;
}
const KeyRangeRef serverTagKeys("\xff/serverTag/"_sr, "\xff/serverTag0"_sr);
const KeyRef serverTagPrefix = serverTagKeys.begin;

8
fdbclient/include/fdbclient/StorageServerInterface.h
View File

@ -260,6 +260,14 @@ struct StorageInfo : NonCopyable, public ReferenceCounted<StorageInfo> {
StorageInfo() : tag(invalidTag) {}
};
struct StorageServerMetaInfo : public StorageServerInterface {
Optional<StorageMetadataType> metadata;
StorageServerMetaInfo(const StorageServerInterface& interface,
Optional<StorageMetadataType> metadata = Optional<StorageMetadataType>())
: StorageServerInterface(interface), metadata(metadata) {}
};
struct ServerCacheInfo {
std::vector<Tag> tags; // all tags in both primary and remote DC for the key-range
std::vector<Reference<StorageInfo>> src_info;

6
fdbclient/include/fdbclient/SystemData.h
View File

@ -215,6 +215,12 @@ extern const KeyRangeRef tssMismatchKeys;
// Note: storageInterfaceUID is the one stated in the file name
extern const KeyRangeRef serverMetadataKeys;
// Any update to serverMetadataKeys will update this key to a random UID.
extern const KeyRef serverMetadataChangeKey;
UID decodeServerMetadataKey(const KeyRef&);
StorageMetadataType decodeServerMetadataValue(const KeyRef&);
// "\xff/serverTag/[[serverID]]" = "[[Tag]]"
// Provides the Tag for the given serverID. Used to access a
// storage server's corresponding TLog in order to apply mutations.

49
fdbserver/ClusterController.actor.cpp
View File

@ -1514,6 +1514,7 @@ ACTOR Future<Void> statusServer(FutureStream<StatusRequest> requests,
self->cx,
workers,
workerIssues,
self->storageStatusInfos,
&self->db.clientStatus,
coordinators,
incompatibleConnections,
@ -1669,6 +1670,53 @@ ACTOR Future<Void> monitorServerInfoConfig(ClusterControllerData::DBInfo* db) {
}
}
// Monitors storage metadata changes and updates to storage servers.
ACTOR Future<Void> monitorStorageMetadata(ClusterControllerData* self) {
state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
IncludeVersion());
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(self->cx);
state std::vector<StorageServerMetaInfo> servers;
loop {
try {
servers.clear();
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
state RangeResult serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
servers.reserve(serverList.size());
for (const auto ss : serverList) {
servers.push_back(StorageServerMetaInfo(decodeServerListValue(ss.value)));
}
state RangeResult serverMetadata = wait(tr->getRange(serverMetadataKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!serverMetadata.more && serverMetadata.size() < CLIENT_KNOBS->TOO_MANY);
std::map<UID, StorageMetadataType> idMetadata;
for (const auto& sm : serverMetadata) {
const UID id = decodeServerMetadataKey(sm.key);
idMetadata[id] = decodeServerMetadataValue(sm.value);
}
for (auto& s : servers) {
if (idMetadata.count(s.id())) {
s.metadata = idMetadata[s.id()];
} else {
TraceEvent(SevWarn, "StorageServerMetadataMissing", self->id).detail("ServerID", s.id());
}
}
state Future<Void> watchFuture = tr->watch(serverMetadataChangeKey);
wait(tr->commit());
self->storageStatusInfos = std::move(servers);
wait(watchFuture);
tr->reset();
} catch (Error& e) {
wait(tr->onError(e));
}
}
}
// Monitors the global configuration version key for changes. When changes are
// made, the global configuration history is read and any updates are sent to
// all processes in the system by updating the ClientDBInfo object. The
@ -3074,6 +3122,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
self.addActor.send(timeKeeper(&self));
self.addActor.send(monitorProcessClasses(&self));
self.addActor.send(monitorServerInfoConfig(&self.db));
self.addActor.send(monitorStorageMetadata(&self));
self.addActor.send(monitorGlobalConfig(&self.db));
self.addActor.send(updatedChangingDatacenters(&self));
self.addActor.send(updatedChangedDatacenters(&self));

13
fdbserver/DDTeamCollection.actor.cpp
View File

@ -18,15 +18,19 @@
* limitations under the License.
*/
#include "fdbserver/DDTeamCollection.h"
#include <climits>
#include "fdbclient/SystemData.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/BlobMigratorInterface.h"
#include "fdbserver/DDTeamCollection.h"
#include "fdbserver/ExclusionTracker.actor.h"
#include "fdbserver/DataDistributionTeam.h"
#include "fdbserver/BlobMigratorInterface.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/actorcompiler.h" // This must be the last #include.
#include "flow/network.h"
#include <climits>
#include "flow/actorcompiler.h" // This must be the last #include.
namespace {
@ -3078,6 +3082,7 @@ public:
data.createdTime = metadata.get().createdTime;
}
metadataMap.set(tr, server->getId(), data);
tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
wait(tr->commit());
break;
} catch (Error& e) {

7
fdbserver/MoveKeys.actor.cpp
View File

@ -1159,10 +1159,6 @@ ACTOR Future<Void> waitForShardReady(StorageServerInterface server,
try {
GetShardStateReply rep =
wait(server.getShardState.getReply(GetShardStateRequest(keys, mode), TaskPriority::MoveKeys));
TraceEvent("GetShardStateReadyDD", server.id())
.detail("RepVersion", rep.first)
.detail("MinVersion", rep.second)
.log();
if (rep.first >= minVersion) {
return Void();
}
@ -2483,6 +2479,7 @@ ACTOR Future<std::pair<Version, Tag>> addStorageServer(Database cx, StorageServe
StorageMetadataType metadata(StorageMetadataType::currentTime());
metadataMap.set(tr, server.id(), metadata);
tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
tr->set(serverListKeyFor(server.id()), serverListValue(server));
wait(tr->commit());
@ -2639,6 +2636,7 @@ ACTOR Future<Void> removeStorageServer(Database cx,
}
metadataMap.erase(tr, serverID);
tr->set(serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
retry = true;
wait(tr->commit());
@ -3262,6 +3260,7 @@ void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vector<Storag
tr.set(arena, uidRef.withPrefix(tssMappingKeys.begin), uidRef);
}
}
tr.set(arena, serverMetadataChangeKey, deterministicRandom()->randomUniqueID().toString());
std::vector<Tag> serverTags;
std::vector<UID> serverSrcUID;

56
fdbserver/Status.actor.cpp
View File

@ -114,12 +114,9 @@ extern const char* limitReasonDesc[];
typedef std::map<std::string, TraceEventFields> EventMap;
struct StorageServerStatusInfo : public StorageServerInterface {
Optional<StorageMetadataType> metadata;
struct StorageServerStatusInfo : public StorageServerMetaInfo {
EventMap eventMap;
StorageServerStatusInfo(const StorageServerInterface& interface,
Optional<StorageMetadataType> metadata = Optional<StorageMetadataType>())
: StorageServerInterface(interface), metadata(metadata) {}
StorageServerStatusInfo(const StorageServerMetaInfo& info) : StorageServerMetaInfo(info, info.metadata) {}
};
ACTOR static Future<Optional<TraceEventFields>> latestEventOnWorker(WorkerInterface worker, std::string eventName) {
@ -1979,43 +1976,6 @@ static Future<std::vector<std::pair<iface, EventMap>>> getServerMetrics(
return results;
}
ACTOR
static Future<std::vector<StorageServerStatusInfo>> readStorageInterfaceAndMetadata(Database cx,
bool use_system_priority) {
state KeyBackedObjectMap<UID, StorageMetadataType, decltype(IncludeVersion())> metadataMap(serverMetadataKeys.begin,
IncludeVersion());
state Reference<ReadYourWritesTransaction> tr = makeReference<ReadYourWritesTransaction>(cx);
state std::vector<StorageServerStatusInfo> servers;
loop {
try {
servers.clear();
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
if (use_system_priority) {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
}
state RangeResult serverList = wait(tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
servers.reserve(serverList.size());
for (int i = 0; i < serverList.size(); i++) {
servers.push_back(StorageServerStatusInfo(decodeServerListValue(serverList[i].value)));
}
state std::vector<Future<Void>> futures(servers.size());
for (int i = 0; i < servers.size(); ++i) {
futures[i] = store(servers[i].metadata, metadataMap.get(tr, servers[i].id()));
// TraceEvent(SevDebug, "MetadataAppear", servers[i].id()).detail("Present", metadata.present());
}
wait(waitForAll(futures));
wait(tr->commit());
break;
} catch (Error& e) {
wait(tr->onError(e));
}
}
return servers;
}
namespace {
const std::vector<std::string> STORAGE_SERVER_METRICS_LIST{ "StorageMetrics",
@ -2027,11 +1987,14 @@ const std::vector<std::string> STORAGE_SERVER_METRICS_LIST{ "StorageMetrics",
} // namespace
ACTOR static Future<std::vector<StorageServerStatusInfo>> getStorageServerStatusInfos(
Database cx,
std::vector<StorageServerMetaInfo> storageMetadatas,
std::unordered_map<NetworkAddress, WorkerInterface> address_workers,
WorkerDetails rkWorker) {
state std::vector<StorageServerStatusInfo> servers =
wait(timeoutError(readStorageInterfaceAndMetadata(cx, true), 5.0));
state std::vector<StorageServerStatusInfo> servers;
servers.reserve(storageMetadatas.size());
for (const auto& meta : storageMetadatas) {
servers.push_back(StorageServerStatusInfo(meta));
}
state std::vector<std::pair<StorageServerStatusInfo, EventMap>> results;
wait(store(results, getServerMetrics(servers, address_workers, STORAGE_SERVER_METRICS_LIST)));
for (int i = 0; i < results.size(); ++i) {
@ -3062,6 +3025,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
Database cx,
std::vector<WorkerDetails> workers,
std::vector<ProcessIssues> workerIssues,
std::vector<StorageServerMetaInfo> storageMetadatas,
std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* clientStatus,
ServerCoordinators coordinators,
std::vector<NetworkAddress> incompatibleConnections,
@ -3326,7 +3290,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
}
state Future<ErrorOr<std::vector<StorageServerStatusInfo>>> storageServerFuture =
errorOr(getStorageServerStatusInfos(cx, address_workers, rkWorker));
errorOr(getStorageServerStatusInfos(storageMetadatas, address_workers, rkWorker));
state Future<ErrorOr<std::vector<std::pair<TLogInterface, EventMap>>>> tLogFuture =
errorOr(getTLogsAndMetrics(db, address_workers));
state Future<ErrorOr<std::vector<std::pair<CommitProxyInterface, EventMap>>>> commitProxyFuture =

12
fdbserver/include/fdbserver/ClusterController.actor.h
View File

@ -18,24 +18,23 @@
* limitations under the License.
*/
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#include "fdbclient/StorageServerInterface.h"
#include "fdbserver/BlobMigratorInterface.h"
#include <utility>
#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERCONTROLLER_ACTOR_G_H)
#define FDBSERVER_CLUSTERCONTROLLER_ACTOR_G_H
#include "fdbserver/ClusterController.actor.g.h"
#elif !defined(FDBSERVER_CLUSTERCONTROLLER_ACTOR_H)
#define FDBSERVER_CLUSTERCONTROLLER_ACTOR_H
#pragma once
#include <utility>
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/MetaclusterRegistration.h"
#include "fdbclient/StorageServerInterface.h"
#include "fdbrpc/Replication.h"
#include "fdbrpc/ReplicationUtils.h"
#include "fdbserver/BlobMigratorInterface.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "flow/SystemMonitor.h"
@ -3339,6 +3338,7 @@ public:
AsyncTrigger updateDBInfo;
std::set<Endpoint> updateDBInfoEndpoints;
std::set<Endpoint> removedDBInfoEndpoints;
std::vector<StorageServerMetaInfo> storageStatusInfos;
DBInfo db;
Database cx;

1
fdbserver/include/fdbserver/Status.actor.h
View File

@ -48,6 +48,7 @@ Future<StatusReply> clusterGetStatus(
Database const& cx,
std::vector<WorkerDetails> const& workers,
std::vector<ProcessIssues> const& workerIssues,
std::vector<StorageServerMetaInfo> const& storageMetadatas,
std::map<NetworkAddress, std::pair<double, OpenDatabaseRequest>>* const& clientStatus,
ServerCoordinators const& coordinators,
std::vector<NetworkAddress> const& incompatibleConnections,

2
fdbserver/worker.actor.cpp
View File

@ -3801,7 +3801,7 @@ TEST_CASE("/fdbserver/worker/swversion/runNewer") {
namespace {
KeyValueStoreType randomStoreType() {
int type = deterministicRandom()->randomInt(0, (int)KeyValueStoreType::END);
if (type == KeyValueStoreType::NONE) {
if (type == KeyValueStoreType::NONE || type == KeyValueStoreType::SSD_REDWOOD_V1) {
type = KeyValueStoreType::SSD_BTREE_V2;
}
#ifndef WITH_ROCKSDB