wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1960 from mpilman/remove-objser-flag 

Remove --object-serializer flag from executables
This commit is contained in:
Evan Tschannen 2019-08-06 16:37:38 -07:00 committed by GitHub
parent cf95a7b2d7 f453c12f0f
commit 7022dabcdd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 31 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
fdbbackup/backup.actor.cpp
View File

@ -118,7 +118,6 @@ enum {
OPT_CLEANUP,
OPT_TRACE_FORMAT,
OPT_USE_OBJECT_SERIALIZER
};
CSimpleOpt::SOption g_rgAgentOptions[] = {
@ -135,8 +134,6 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_LOCALITY, "--locality_", SO_REQ_SEP },
@ -176,8 +173,6 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -245,8 +240,6 @@ CSimpleOpt::SOption g_rgBackupStatusOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
@ -277,8 +270,6 @@ CSimpleOpt::SOption g_rgBackupAbortOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -310,8 +301,6 @@ CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -343,8 +332,6 @@ CSimpleOpt::SOption g_rgBackupWaitOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -372,8 +359,6 @@ CSimpleOpt::SOption g_rgBackupPauseOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -403,8 +388,6 @@ CSimpleOpt::SOption g_rgBackupExpireOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -442,8 +425,6 @@ CSimpleOpt::SOption g_rgBackupDeleteOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -475,8 +456,6 @@ CSimpleOpt::SOption g_rgBackupDescribeOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -541,8 +520,6 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -585,8 +562,6 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -624,8 +599,6 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_LOCALITY, "--locality_", SO_REQ_SEP },
@ -656,8 +629,6 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -691,8 +662,6 @@ CSimpleOpt::SOption g_rgDBStatusOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
@ -724,8 +693,6 @@ CSimpleOpt::SOption g_rgDBSwitchOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -759,8 +726,6 @@ CSimpleOpt::SOption g_rgDBAbortOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -790,8 +755,6 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
{ OPT_TRACE, "--log", SO_NONE },
{ OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
{ OPT_QUIET, "-q", SO_NONE },
{ OPT_QUIET, "--quiet", SO_NONE },
@ -2740,7 +2703,6 @@ int main(int argc, char* argv[]) {
bool dryRun = false;
std::string traceDir = "";
std::string traceFormat = "";
bool useObjectSerializer = true;
std::string traceLogGroup;
uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE;
uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
@ -2851,18 +2813,6 @@ int main(int argc, char* argv[]) {
}
traceFormat = args->OptionArg();
break;
case OPT_USE_OBJECT_SERIALIZER: {
std::string s = args->OptionArg();
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "on" || s == "true" || s == "1") {
useObjectSerializer = true;
} else if (s == "off" || s == "false" || s == "0") {
useObjectSerializer = false;
} else {
fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str());
}
break;
}
case OPT_TRACE_LOG_GROUP:
traceLogGroup = args->OptionArg();
break;
@ -3209,11 +3159,6 @@ int main(int argc, char* argv[]) {
setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING);
}
setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
// The USE_OBJECT_SERIALIZER network option expects an 8 byte little endian integer which is interpreted as
// zero = false, non-zero = true.
setNetworkOption(FDBNetworkOptions::USE_OBJECT_SERIALIZER,
useObjectSerializer ? LiteralStringRef("\x01\x00\x00\x00\x00\x00\x00\x00")
: LiteralStringRef("\x00\x00\x00\x00\x00\x00\x00\x00"));
// deferred TLS options
if (tlsCertPath.size()) {

26
fdbcli/fdbcli.actor.cpp
View File

@ -70,8 +70,7 @@ enum {
OPT_NO_STATUS,
OPT_STATUS_FROM_JSON,
OPT_VERSION,
OPT_TRACE_FORMAT,
OPT_USE_OBJECT_SERIALIZER
OPT_TRACE_FORMAT
};
CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
@ -89,8 +88,6 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
@ -2415,7 +2412,6 @@ struct CLIOptions {
bool trace;
std::string traceDir;
std::string traceFormat;
bool useObjectSerializer = true;
int exit_timeout;
Optional<std::string> exec;
bool initialStatusCheck;
@ -2517,20 +2513,6 @@ struct CLIOptions {
}
traceFormat = args.OptionArg();
break;
case OPT_USE_OBJECT_SERIALIZER: {
std::string s = args.OptionArg();
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "on" || s == "true" || s == "1") {
useObjectSerializer = true;
} else if (s == "off" || s == "false" || s == "0") {
useObjectSerializer = false;
} else {
fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str());
printProgramUsage(program_name.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_VERSION:
printVersion();
return FDB_EXIT_SUCCESS;
@ -3647,12 +3629,6 @@ int main(int argc, char **argv) {
}
setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING);
}
// The USE_OBJECT_SERIALIZER network option expects an 8 byte little endian integer which is interpreted as zero =
// false, non-zero = true.
setNetworkOption(FDBNetworkOptions::USE_OBJECT_SERIALIZER,
opt.useObjectSerializer ? LiteralStringRef("\x01\x00\x00\x00\x00\x00\x00\x00")
: LiteralStringRef("\x00\x00\x00\x00\x00\x00\x00\x00"));
initHelp();
// deferred TLS options

4
fdbclient/MonitorLeader.actor.cpp
View File

@ -511,7 +511,7 @@ ACTOR Future<Void> asyncDeserializeClusterInterface(Reference<AsyncVar<Value>> s
Reference<AsyncVar<Optional<ClusterInterface>>> outKnownLeader) {
state Reference<AsyncVar<Optional<ClusterControllerClientInterface>>> knownLeader(
new AsyncVar<Optional<ClusterControllerClientInterface>>{});
state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader, g_network->useObjectSerializer());
state Future<Void> deserializer = asyncDeserialize(serializedInfo, knownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER);
loop {
choose {
when(wait(deserializer)) { UNSTOPPABLE_ASSERT(false); }
@ -645,7 +645,7 @@ ACTOR Future<Void> monitorLeaderForProxies( Key clusterKey, vector<NetworkAddres
}
if (leader.get().first.serializedInfo.size()) {
if (g_network->useObjectSerializer()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
ObjectReader reader(leader.get().first.serializedInfo.begin(), IncludeVersion());
ClusterControllerClientInterface res;
reader.deserialize(res);

2
fdbclient/MonitorLeader.h
View File

@ -67,7 +67,7 @@ template <class LeaderInterface>
struct LeaderDeserializer {
Future<Void> operator()(const Reference<AsyncVar<Value>>& serializedInfo,
const Reference<AsyncVar<Optional<LeaderInterface>>>& outKnownLeader) {
return asyncDeserialize(serializedInfo, outKnownLeader, g_network->useObjectSerializer());
return asyncDeserialize(serializedInfo, outKnownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER);
}
};

5
fdbclient/NativeAPI.actor.cpp
View File

@ -837,9 +837,6 @@ const UniqueOrderedOptionList<FDBTransactionOptions>& Database::getTransactionDe
void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
switch(option) {
// SOMEDAY: If the network is already started, should these three throw an error?
case FDBNetworkOptions::USE_OBJECT_SERIALIZER:
networkOptions.useObjectSerializer = extractIntOption(value) != 0;
break;
case FDBNetworkOptions::TRACE_ENABLE:
networkOptions.traceDirectory = value.present() ? value.get().toString() : "";
break;
@ -990,7 +987,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) {
if (!networkOptions.logClientInfo.present())
networkOptions.logClientInfo = true;
g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present(), networkOptions.useObjectSerializer);
g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present());
FlowTransport::createInstance(true, transportId);
Net2FileSystem::newFileSystem();

3
fdbclient/NativeAPI.actor.h
View File

@ -61,13 +61,12 @@ struct NetworkOptions {
Optional<bool> logClientInfo;
Standalone<VectorRef<ClientVersionRef>> supportedVersions;
bool slowTaskProfilingEnabled;
bool useObjectSerializer;
// The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h.
NetworkOptions()
: localAddress(""), clusterFile(""), traceDirectory(Optional<std::string>()),
traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
traceFormat("xml"), slowTaskProfilingEnabled(false), useObjectSerializer(true) {}
traceFormat("xml"), slowTaskProfilingEnabled(false) {}
};
class Database {

3
fdbclient/vexillographer/fdb.options
View File

@ -33,9 +33,6 @@ description is not currently required but encouraged.
<Option name="local_address" code="10"
paramType="String" paramDescription="IP:PORT"
description="Deprecated"/>
<Option name="use_object_serializer" code="11"
paramType="Int" paramDescription="0 is false, every other value is true"
description="enable the object serializer for network communication"/>
<Option name="cluster_file" code="20"
paramType="String" paramDescription="path to cluster file"
description="Deprecated"/>

1
fdbrpc/FlowTests.actor.cpp
View File

@ -201,7 +201,6 @@ struct YieldMockNetwork : INetwork, ReferenceCounted<YieldMockNetwork> {
virtual void run() { return baseNetwork->run(); }
virtual void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) { return baseNetwork->getDiskBytes(directory,free,total); }
virtual bool isAddressOnThisHost(NetworkAddress const& addr) { return baseNetwork->isAddressOnThisHost(addr); }
virtual bool useObjectSerializer() const { return baseNetwork->useObjectSerializer(); }
};
struct NonserializableThing {};

13
fdbrpc/FlowTransport.actor.cpp
View File

@ -337,7 +337,7 @@ struct Peer : NonCopyable {
pkt.connectPacketLength = sizeof(pkt) - sizeof(pkt.connectPacketLength);
pkt.protocolVersion = currentProtocolVersion;
if (g_network->useObjectSerializer()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
pkt.protocolVersion.addObjectSerializerFlag();
}
pkt.connectionId = transport->transportId;
@ -654,7 +654,7 @@ ACTOR static void deliver(TransportData* self, Endpoint destination, ArenaReader
if (receiver) {
try {
g_currentDeliveryPeerAddress = destination.addresses;
if (g_network->useObjectSerializer()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
StringRef data = reader.arenaReadAll();
ASSERT(data.size() > 8);
ArenaObjectReader objReader(reader.arena(), reader.arenaReadAll(), AssumeVersion(reader.protocolVersion()));
@ -858,7 +858,7 @@ ACTOR static Future<Void> connectionReader(
serializer(pktReader, pkt);
uint64_t connectionId = pkt.connectionId;
if(g_network->useObjectSerializer() != pkt.protocolVersion.hasObjectSerializerFlag() ||
if((FLOW_KNOBS->USE_OBJECT_SERIALIZER != 0) != pkt.protocolVersion.hasObjectSerializerFlag() ||
!pkt.protocolVersion.isCompatible(currentProtocolVersion)) {
incompatibleProtocolVersionNewer = pkt.protocolVersion > currentProtocolVersion;
NetworkAddress addr = pkt.canonicalRemotePort
@ -896,8 +896,7 @@ ACTOR static Future<Void> connectionReader(
TraceEvent("ConnectionEstablished", conn->getDebugID())
.suppressFor(1.0)
.detail("Peer", conn->getPeerAddress())
.detail("ConnectionId", connectionId)
.detail("UseObjectSerializer", g_network->useObjectSerializer());
.detail("ConnectionId", connectionId);
}
if(connectionId > 1) {
@ -1149,7 +1148,7 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
// SOMEDAY: Would it be better to avoid (de)serialization by doing this check in flow?
Standalone<StringRef> copy;
if (g_network->useObjectSerializer()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
ObjectWriter wr(AssumeVersion(currentProtocolVersion));
what.serializeObjectWriter(wr);
copy = wr.toStringRef();
@ -1198,7 +1197,7 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c
wr.writeAhead(packetInfoSize , &packetInfoBuffer);
wr << destination.token;
what.serializePacketWriter(wr, g_network->useObjectSerializer());
what.serializePacketWriter(wr, FLOW_KNOBS->USE_OBJECT_SERIALIZER);
pb = wr.finish();
len = wr.size() - packetInfoSize;

4
fdbrpc/networksender.actor.h
View File

@ -34,7 +34,7 @@ ACTOR template <class T>
void networkSender(Future<T> input, Endpoint endpoint) {
try {
T value = wait(input);
if (g_network->useObjectSerializer()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<EnsureTable<T>>>(value), endpoint);
} else {
FlowTransport::transport().sendUnreliable(SerializeBoolAnd<T>(true, value), endpoint, false);
@ -42,7 +42,7 @@ void networkSender(Future<T> input, Endpoint endpoint) {
} catch (Error& err) {
// if (err.code() == error_code_broken_promise) return;
ASSERT(err.code() != error_code_actor_cancelled);
if (g_network->useObjectSerializer()) {
if (FLOW_KNOBS->USE_OBJECT_SERIALIZER) {
FlowTransport::transport().sendUnreliable(SerializeSource<ErrorOr<EnsureTable<T>>>(err), endpoint);
} else {
FlowTransport::transport().sendUnreliable(SerializeBoolAnd<Error>(false, err), endpoint, false);

12
fdbrpc/sim2.actor.cpp
View File

@ -1092,10 +1092,6 @@ public:
return primaryTLogsDead || primaryProcessesDead.validate(storagePolicy);
}
virtual bool useObjectSerializer() const {
return net2->useObjectSerializer();
}
// The following function will determine if the specified configuration of available and dead processes can allow the cluster to survive
virtual bool canKillProcesses(std::vector<ProcessInfo*> const& availableProcesses, std::vector<ProcessInfo*> const& deadProcesses, KillType kt, KillType* newKillType) const
{
@ -1588,10 +1584,10 @@ public:
machines.erase(machineId);
}
Sim2(bool objSerializer) : time(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(TaskPriority::Zero) {
Sim2() : time(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(TaskPriority::Zero) {
// Not letting currentProcess be NULL eliminates some annoying special cases
currentProcess = new ProcessInfo("NoMachine", LocalityData(Optional<Standalone<StringRef>>(), StringRef(), StringRef(), StringRef()), ProcessClass(), {NetworkAddress()}, this, "", "");
g_network = net2 = newNet2(false, true, objSerializer);
g_network = net2 = newNet2(false, true);
Net2FileSystem::newFileSystem();
check_yield(TaskPriority::Zero);
}
@ -1699,9 +1695,9 @@ public:
int yield_limit; // how many more times yield may return false before next returning true
};
void startNewSimulator(bool objSerializer) {
void startNewSimulator() {
ASSERT( !g_network );
g_network = g_pSimulator = new Sim2(objSerializer);
g_network = g_pSimulator = new Sim2();
g_simulator.connectionFailuresDisableDuration = deterministicRandom()->random01() < 0.5 ? 0 : 1e6;
}

3
fdbrpc/simulator.h
View File

@ -155,7 +155,6 @@ public:
virtual bool isAvailable() const = 0;
virtual bool datacenterDead(Optional<Standalone<StringRef>> dcId) const = 0;
virtual void displayWorkers() const;
virtual bool useObjectSerializer() const = 0;
virtual void addRole(NetworkAddress const& address, std::string const& role) {
roleAddresses[address][role] ++;
@ -341,7 +340,7 @@ private:
extern ISimulator* g_pSimulator;
#define g_simulator (*g_pSimulator)
void startNewSimulator(bool useObjectSerializer);
void startNewSimulator();
//Parameters used to simulate disk performance
struct DiskParameters : ReferenceCounted<DiskParameters> {

4
fdbserver/LeaderElection.h
View File

@ -61,9 +61,9 @@ Future<Void> tryBecomeLeader( ServerCoordinators const& coordinators,
Reference<AsyncVar<Value>> serializedInfo(new AsyncVar<Value>);
Future<Void> m = tryBecomeLeaderInternal(
coordinators,
g_network->useObjectSerializer() ? ObjectWriter::toValue(proposedInterface, IncludeVersion()) : BinaryWriter::toValue(proposedInterface, IncludeVersion()),
FLOW_KNOBS->USE_OBJECT_SERIALIZER ? ObjectWriter::toValue(proposedInterface, IncludeVersion()) : BinaryWriter::toValue(proposedInterface, IncludeVersion()),
serializedInfo, hasConnected, asyncPriorityInfo);
return m || asyncDeserialize(serializedInfo, outKnownLeader, g_network->useObjectSerializer());
return m || asyncDeserialize(serializedInfo, outKnownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER);
}
#pragma endregion

28
fdbserver/fdbserver.actor.cpp
View File

@ -87,7 +87,7 @@ enum {
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK,
OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
OPT_TRACE_FORMAT, OPT_USE_OBJECT_SERIALIZER, OPT_WHITELIST_BINPATH
OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH
};
CSimpleOpt::SOption g_rgOptions[] = {
@ -165,8 +165,6 @@ CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_IO_TRUST_SECONDS, "--io_trust_seconds", SO_REQ_SEP },
{ OPT_IO_TRUST_WARN_ONLY, "--io_trust_warn_only", SO_NONE },
{ OPT_TRACE_FORMAT , "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_WHITELIST_BINPATH, "--whitelist_binpath", SO_REQ_SEP },
#ifndef TLS_DISABLED
@ -603,10 +601,6 @@ static void printUsage( const char *name, bool devhelp ) {
" Machine class (valid options are storage, transaction,\n"
" resolution, proxy, master, test, unset, stateless, log, router,\n"
" and cluster_controller).\n");
printf(" -S ON|OFF, --object-serializer ON|OFF\n"
" Use object serializer for sending messages. The object serializer\n"
" is currently a beta feature and it allows fdb processes to talk to\n"
" each other even if they don't have the same version\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
@ -970,7 +964,6 @@ int main(int argc, char* argv[]) {
double fileIoTimeout = 0.0;
bool fileIoWarnOnly = false;
uint64_t rsssize = -1;
bool useObjectSerializer = true;
if( argc == 1 ) {
printUsage(argv[0], false);
@ -1331,21 +1324,6 @@ int main(int argc, char* argv[]) {
fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg());
}
break;
case OPT_USE_OBJECT_SERIALIZER:
{
std::string s = args.OptionArg();
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "on" || s == "true" || s == "1") {
useObjectSerializer = true;
} else if (s == "off" || s == "false" || s == "0") {
useObjectSerializer = false;
} else {
fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_WHITELIST_BINPATH:
whitelistBinPaths = args.OptionArg();
break;
@ -1562,10 +1540,10 @@ int main(int argc, char* argv[]) {
if (role == Simulation || role == CreateTemplateDatabase) {
//startOldSimulator();
startNewSimulator(useObjectSerializer);
startNewSimulator();
openTraceFile(NetworkAddress(), rollsize, maxLogsSize, logFolder, "trace", logGroup);
} else {
g_network = newNet2(useThreadPool, true, useObjectSerializer);
g_network = newNet2(useThreadPool, true);
FlowTransport::createInstance(false, 1);
const bool expectsPublicAddress = (role == FDBD || role == NetworkTestServer || role == Restore);

1
flow/Knobs.cpp
View File

@ -68,6 +68,7 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
init( RECONNECTION_TIME_GROWTH_RATE, 1.2 );
init( RECONNECTION_RESET_TIME, 5.0 );
init( CONNECTION_ACCEPT_DELAY, 0.01 );
init( USE_OBJECT_SERIALIZER, 1 );
init( TOO_MANY_CONNECTIONS_CLOSED_RESET_DELAY, 5.0 );
init( TOO_MANY_CONNECTIONS_CLOSED_TIMEOUT, 20.0 );

1
flow/Knobs.h
View File

@ -88,6 +88,7 @@ public:
double RECONNECTION_TIME_GROWTH_RATE;
double RECONNECTION_RESET_TIME;
double CONNECTION_ACCEPT_DELAY;
int USE_OBJECT_SERIALIZER;
int TLS_CERT_REFRESH_DELAY_SECONDS;

11
flow/Net2.actor.cpp
View File

@ -111,7 +111,7 @@ thread_local INetwork* thread_network = 0;
class Net2 sealed : public INetwork, public INetworkConnections {
public:
Net2(bool useThreadPool, bool useMetrics, bool useObjectSerializer);
Net2(bool useThreadPool, bool useMetrics);
void run();
void initMetrics();
@ -148,11 +148,9 @@ public:
virtual flowGlobalType global(int id) { return (globals.size() > id) ? globals[id] : NULL; }
virtual void setGlobal(size_t id, flowGlobalType v) { globals.resize(std::max(globals.size(),id+1)); globals[id] = v; }
virtual bool useObjectSerializer() const { return _useObjectSerializer; }
std::vector<flowGlobalType> globals;
bool useThreadPool;
bool _useObjectSerializer = false;
//private:
ASIOReactor reactor;
@ -483,9 +481,8 @@ struct PromiseTask : public Task, public FastAllocated<PromiseTask> {
}
};
Net2::Net2(bool useThreadPool, bool useMetrics, bool useObjectSerializer)
Net2::Net2(bool useThreadPool, bool useMetrics)
: useThreadPool(useThreadPool),
_useObjectSerializer(useObjectSerializer),
network(this),
reactor(this),
stopped(false),
@ -1027,9 +1024,9 @@ void ASIOReactor::wake() {
} // namespace net2
INetwork* newNet2(bool useThreadPool, bool useMetrics, bool useObjectSerializer) {
INetwork* newNet2(bool useThreadPool, bool useMetrics) {
try {
N2::g_net2 = new N2::Net2(useThreadPool, useMetrics, useObjectSerializer);
N2::g_net2 = new N2::Net2(useThreadPool, useMetrics);
}
catch(boost::system::system_error e) {
TraceEvent("Net2InitError").detail("Message", e.what());

5
flow/network.h
View File

@ -372,7 +372,7 @@ typedef NetworkAddressList (*NetworkAddressesFuncPtr)();
class INetwork;
extern INetwork* g_network;
extern INetwork* newNet2(bool useThreadPool = false, bool useMetrics = false, bool useObjectSerializer = false);
extern INetwork* newNet2(bool useThreadPool = false, bool useMetrics = false);
class INetwork {
public:
@ -447,9 +447,6 @@ public:
virtual bool isAddressOnThisHost( NetworkAddress const& addr ) = 0;
// Returns true if it is reasonably certain that a connection to the given address would be a fast loopback connection
virtual bool useObjectSerializer() const = 0;
// Whether or not the object serializer should be used when sending packets
// Shorthand for transport().getLocalAddress()
static NetworkAddress getLocalAddress()
{

3
tests/TestRunner/TestRunner.py
View File

@ -279,9 +279,6 @@ def run_simulation_test(basedir, options):
if options.testtype == 'test':
pargs.append('-C')
pargs.append(os.path.join(args.builddir, 'fdb.cluster'))
else:
pargs.append('-S')
pargs.append('on' if seed % 2 == 0 else 'off')
td = TestDirectory(basedir)
if options.buggify:
pargs.append('-b')