wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbserver/fdbserver.actor.cpp

2040 lines
78 KiB
C++
Raw Blame History

/*
* fdbserver.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// There's something in one of the files below that defines a macros
// a macro that makes boost interprocess break on Windows.
#define BOOST_DATE_TIME_NO_LIB
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/algorithm/string.hpp>
#include "fdbrpc/simulator.h"
#include "flow/DeterministicRandom.h"
#include "fdbrpc/PerfMetric.h"
#include "flow/Platform.h"
#include "flow/SystemMonitor.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/FailureMonitorClient.h"
#include "fdbserver/CoordinationInterface.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/RestoreWorkerInterface.actor.h"
#include "fdbserver/ClusterRecruitmentInterface.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/MoveKeys.actor.h"
#include "fdbserver/ConflictSet.h"
#include "fdbserver/DataDistribution.actor.h"
#include "fdbserver/NetworkTest.h"
#include "fdbserver/IKeyValueStore.h"
#include <algorithm>
#include <stdarg.h>
#include <stdio.h>
#include <fstream>
#include "fdbserver/pubsub.h"
#include "fdbserver/SimulatedCluster.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbserver/workloads/workloads.actor.h"
#include <time.h>
#include "fdbserver/Status.h"
#include "fdbrpc/TLSConnection.h"
#include "fdbrpc/Net2FileSystem.h"
#include "fdbrpc/Platform.h"
#include "fdbrpc/AsyncFileCached.actor.h"
#include "fdbserver/CoroFlow.h"
#include "flow/SignalSafeUnwind.h"
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif
#include "fdbmonitor/SimpleIni.h"
#ifdef __linux__
#include <execinfo.h>
#include <signal.h>
#ifdef ALLOC_INSTRUMENTATION
#include <cxxabi.h>
#endif
#endif
#ifdef WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif
#include "flow/SimpleOpt.h"
#include "flow/actorcompiler.h" // This must be the last #include.
enum {
OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_NEWCONSOLE,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACECLOCK,
OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
OPT_TRACE_FORMAT, OPT_USE_OBJECT_SERIALIZER, OPT_WHITELIST_BINPATH
};
CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_CONNFILE, "-C", SO_REQ_SEP },
{ OPT_CONNFILE, "--cluster_file", SO_REQ_SEP },
{ OPT_SEEDCONNFILE, "--seed_cluster_file", SO_REQ_SEP },
{ OPT_SEEDCONNSTRING, "--seed_connection_string", SO_REQ_SEP },
{ OPT_ROLE, "-r", SO_REQ_SEP },
{ OPT_ROLE, "--role", SO_REQ_SEP },
{ OPT_PUBLICADDR, "-p", SO_REQ_SEP },
{ OPT_PUBLICADDR, "--public_address", SO_REQ_SEP },
{ OPT_LISTEN, "-l", SO_REQ_SEP },
{ OPT_LISTEN, "--listen_address", SO_REQ_SEP },
#ifdef __linux__
{ OPT_FILESYSTEM, "--data_filesystem", SO_REQ_SEP },
{ OPT_PROFILER_RSS_SIZE, "--rsssize", SO_REQ_SEP },
#endif
{ OPT_DATAFOLDER, "-d", SO_REQ_SEP },
{ OPT_DATAFOLDER, "--datadir", SO_REQ_SEP },
{ OPT_LOGFOLDER, "-L", SO_REQ_SEP },
{ OPT_LOGFOLDER, "--logdir", SO_REQ_SEP },
{ OPT_ROLLSIZE, "-Rs", SO_REQ_SEP },
{ OPT_ROLLSIZE, "--logsize", SO_REQ_SEP },
{ OPT_MAXLOGS, "--maxlogs", SO_REQ_SEP },
{ OPT_MAXLOGSSIZE, "--maxlogssize", SO_REQ_SEP },
{ OPT_LOGGROUP, "--loggroup", SO_REQ_SEP },
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
#ifdef _WIN32
{ OPT_NEWCONSOLE, "-n", SO_NONE },
{ OPT_NEWCONSOLE, "--newconsole", SO_NONE },
{ OPT_NOBOX, "-q", SO_NONE },
{ OPT_NOBOX, "--no_dialog", SO_NONE },
#endif
{ OPT_KVFILE, "--kvfile", SO_REQ_SEP },
{ OPT_TESTFILE, "-f", SO_REQ_SEP },
{ OPT_TESTFILE, "--testfile", SO_REQ_SEP },
{ OPT_RESTARTING, "-R", SO_NONE },
{ OPT_RESTARTING, "--restarting", SO_NONE },
{ OPT_RANDOMSEED, "-s", SO_REQ_SEP },
{ OPT_RANDOMSEED, "--seed", SO_REQ_SEP },
{ OPT_KEY, "-k", SO_REQ_SEP },
{ OPT_KEY, "--key", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_STORAGEMEMLIMIT, "-M", SO_REQ_SEP },
{ OPT_STORAGEMEMLIMIT, "--storage_memory", SO_REQ_SEP },
{ OPT_CACHEMEMLIMIT, "--cache_memory", SO_REQ_SEP },
{ OPT_MACHINEID, "-i", SO_REQ_SEP },
{ OPT_MACHINEID, "--machine_id", SO_REQ_SEP },
{ OPT_DCID, "-a", SO_REQ_SEP },
{ OPT_DCID, "--datacenter_id", SO_REQ_SEP },
{ OPT_MACHINE_CLASS, "-c", SO_REQ_SEP },
{ OPT_MACHINE_CLASS, "--class", SO_REQ_SEP },
{ OPT_BUGGIFY, "-b", SO_REQ_SEP },
{ OPT_BUGGIFY, "--buggify", SO_REQ_SEP },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_NETWORKIMPL, "-N", SO_REQ_SEP },
{ OPT_NETWORKIMPL, "--network", SO_REQ_SEP },
{ OPT_NOBUFSTDOUT, "--unbufferedout", SO_NONE },
{ OPT_BUFSTDOUTERR, "--bufferedout", SO_NONE },
{ OPT_TRACECLOCK, "--traceclock", SO_REQ_SEP },
{ OPT_NUMTESTERS, "--num_testers", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_LOCALITY, "--locality_", SO_REQ_SEP },
{ OPT_TESTSERVERS, "--testservers", SO_REQ_SEP },
{ OPT_TEST_ON_SERVERS, "--testonservers", SO_NONE },
{ OPT_METRICSCONNFILE, "--metrics_cluster", SO_REQ_SEP },
{ OPT_METRICSPREFIX, "--metrics_prefix", SO_REQ_SEP },
{ OPT_IO_TRUST_SECONDS, "--io_trust_seconds", SO_REQ_SEP },
{ OPT_IO_TRUST_WARN_ONLY, "--io_trust_warn_only", SO_NONE },
{ OPT_TRACE_FORMAT , "--trace_format", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "-S", SO_REQ_SEP },
{ OPT_USE_OBJECT_SERIALIZER, "--object-serializer", SO_REQ_SEP },
{ OPT_WHITELIST_BINPATH, "--whitelist_binpath", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
GlobalCounters g_counters;
extern void dsltest();
extern void pingtest();
extern void copyTest();
extern void versionedMapTest();
extern void createTemplateDatabase();
// FIXME: this really belongs in a header somewhere since it is actually used.
extern IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs);
extern const char* getHGVersion();
extern void flushTraceFileVoid();
extern bool noUnseed;
extern const int MAX_CLUSTER_FILE_BYTES;
#ifdef ALLOC_INSTRUMENTATION
extern uint8_t *g_extra_memory;
#endif
bool enableFailures = true;
#define test_assert(x) if (!(x)) { cout << "Test failed: " #x << endl; return false; }
template <class X> vector<X> vec( X x ) { vector<X> v; v.push_back(x); return v; }
template <class X> vector<X> vec( X x, X y ) { vector<X> v; v.push_back(x); v.push_back(y); return v; }
template <class X> vector<X> vec( X x, X y, X z ) { vector<X> v; v.push_back(x); v.push_back(y); v.push_back(z); return v; }
//KeyRange keyRange( const Key& a, const Key& b ) { return std::make_pair(a,b); }
vector< Standalone<VectorRef<DebugEntryRef>> > debugEntries;
int64_t totalDebugEntriesSize = 0;
#if CENABLED(0, NOT_IN_CLEAN)
StringRef debugKey = LiteralStringRef( "" );
StringRef debugKey2 = LiteralStringRef( "\xff\xff\xff\xff" );
bool debugMutation( const char* context, Version version, MutationRef const& mutation ) {
if ((mutation.type == mutation.SetValue || mutation.type == mutation.AddValue || mutation.type==mutation.DebugKey) && (mutation.param1 == debugKey || mutation.param1 == debugKey2))
;//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", "SetValue").detail("Key", mutation.param1).detail("Value", mutation.param2);
else if ((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) && ((mutation.param1<=debugKey && mutation.param2>debugKey) || (mutation.param1<=debugKey2 && mutation.param2>debugKey2)))
;//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", "ClearRange").detail("KeyBegin", mutation.param1).detail("KeyEnd", mutation.param2);
else
return false;
const char* type =
mutation.type == MutationRef::SetValue ? "SetValue" :
mutation.type == MutationRef::ClearRange ? "ClearRange" :
mutation.type == MutationRef::AddValue ? "AddValue" :
mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" :
mutation.type == MutationRef::DebugKey ? "DebugKey" :
"UnknownMutation";
printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%lld\t%s\t%s\t%s\n", now(), g_network->getLocalAddress().toString().c_str(), context, version, type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str());
return true;
}
bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) {
if (keys.contains(debugKey) || keys.contains(debugKey2)) {
debugMutation(context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) );
//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("KeyBegin", keys.begin).detail("KeyEnd", keys.end);
return true;
} else
return false;
}
#elif CENABLED(0, NOT_IN_CLEAN)
bool debugMutation( const char* context, Version version, MutationRef const& mutation ) {
if (!debugEntries.size() || debugEntries.back().size() >= 1000) {
if (debugEntries.size()) totalDebugEntriesSize += debugEntries.back().arena().getSize() + sizeof(debugEntries.back());
debugEntries.push_back(Standalone<VectorRef<DebugEntryRef>>());
TraceEvent("DebugMutationBuffer").detail("Bytes", totalDebugEntriesSize);
}
auto& v = debugEntries.back();
v.push_back_deep( v.arena(), DebugEntryRef(context, version, mutation) );
return false; // No auxiliary logging
}
bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) {
return debugMutation( context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) );
}
#else // Default implementation.
bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { return false; }
bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return false; }
#endif
#ifdef _WIN32
#include <sddl.h>
// It is your
// responsibility to properly initialize the
// structure and to free the structure's
// lpSecurityDescriptor member when you have
// finished using it. To free the structure's
// lpSecurityDescriptor member, call the
// LocalFree function.
BOOL CreatePermissiveReadWriteDACL(SECURITY_ATTRIBUTES * pSA)
{
UNSTOPPABLE_ASSERT( pSA != NULL );
TCHAR * szSD = TEXT("D:") // Discretionary ACL
TEXT("(A;OICI;GR;;;AU)") // Allow read/write/execute to authenticated users
TEXT("(A;OICI;GA;;;BA)"); // Allow full control to administrators
return ConvertStringSecurityDescriptorToSecurityDescriptor(
szSD,
SDDL_REVISION_1,
&(pSA->lpSecurityDescriptor),
NULL);
}
#endif
class WorldReadablePermissions {
public:
WorldReadablePermissions() {
#ifdef _WIN32
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.bInheritHandle = FALSE;
if( !CreatePermissiveReadWriteDACL(&sa) ) {
TraceEvent("Win32DACLCreationFail").GetLastError();
throw platform_error();
}
permission.set_permissions( &sa );
#elif (defined(__linux__) || defined(__APPLE__))
// There is nothing to do here, since the default permissions are fine
#else
#error Port me!
#endif
}
virtual ~WorldReadablePermissions() {
#ifdef _WIN32
LocalFree( sa.lpSecurityDescriptor );
#elif (defined(__linux__) || defined(__APPLE__))
// There is nothing to do here, since the default permissions are fine
#else
#error Port me!
#endif
}
boost::interprocess::permissions permission;
private:
WorldReadablePermissions(const WorldReadablePermissions &rhs) {}
#ifdef _WIN32
SECURITY_ATTRIBUTES sa;
#endif
};
UID getSharedMemoryMachineId() {
UID *machineId = NULL;
int numTries = 0;
// Permissions object defaults to 0644 on *nix, but on windows defaults to allowing access to only the creator.
// On windows, this means that we have to create an elaborate workaround for DACLs
WorldReadablePermissions p;
std::string sharedMemoryIdentifier = "fdbserver_shared_memory_id";
loop {
try {
// "0" is the default parameter "addr"
boost::interprocess::managed_shared_memory segment(boost::interprocess::open_or_create, sharedMemoryIdentifier.c_str(), 1000, 0, p.permission);
machineId = segment.find_or_construct<UID>("machineId")(deterministicRandom()->randomUniqueID());
if (!machineId)
criticalError(FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate or create shared memory - 'machineId'");
return *machineId;
}
catch (boost::interprocess::interprocess_exception& ) {
try {
//If the shared memory already exists, open it read-only in case it was created by another user
boost::interprocess::managed_shared_memory segment(boost::interprocess::open_read_only, sharedMemoryIdentifier.c_str());
machineId = segment.find<UID>("machineId").first;
if (!machineId)
criticalError(FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate shared memory - 'machineId'");
return *machineId;
}
catch (boost::interprocess::interprocess_exception &ex) {
//Retry in case the shared memory was deleted in between the call to open_or_create and open_read_only
//Don't keep trying forever in case this is caused by some other problem
if (++numTries == 10)
criticalError(FDB_EXIT_ERROR, "SharedMemoryError", format("Could not open shared memory - %s", ex.what()).c_str());
}
}
}
}
ACTOR void failAfter( Future<Void> trigger, ISimulator::ProcessInfo* m = g_simulator.getCurrentProcess() ) {
wait( trigger );
if (enableFailures) {
printf("Killing machine: %s at %f\n", m->address.toString().c_str(), now());
g_simulator.killProcess( m, ISimulator::KillInstantly );
}
}
void failAfter( Future<Void> trigger, Endpoint e ) {
if (g_network == &g_simulator)
failAfter( trigger, g_simulator.getProcess( e ) );
}
void testSerializationSpeed() {
double tstart;
double build = 0, serialize = 0, deserialize = 0, copy = 0, deallocate = 0;
double bytes = 0;
double testBegin = timer();
for(int a=0; a<10000; a++) {
{
tstart = timer();
Arena batchArena;
VectorRef< CommitTransactionRef > batch;
batch.resize( batchArena, 1000 );
for(int t=0; t<batch.size(); t++) {
CommitTransactionRef &tr = batch[t];
tr.read_snapshot = 0;
for(int i=0; i<2; i++)
tr.mutations.push_back_deep( batchArena,
MutationRef( MutationRef::SetValue, LiteralStringRef("KeyABCDE"), LiteralStringRef("SomeValu") ) );
tr.mutations.push_back_deep( batchArena,
MutationRef( MutationRef::ClearRange, LiteralStringRef("BeginKey"), LiteralStringRef("EndKeyAB") ));
}
build += timer()-tstart;
tstart = timer();
BinaryWriter wr( IncludeVersion() );
wr << batch;
bytes += wr.getLength();
serialize += timer() - tstart;
for(int i=0; i<1; i++) {
tstart = timer();
Arena arena;
StringRef data( arena, StringRef( (const uint8_t*)wr.getData(), wr.getLength() ) );
copy += timer() - tstart;
tstart = timer();
ArenaReader rd( arena, data, IncludeVersion() );
VectorRef<CommitTransactionRef> batch2;
rd >> arena >> batch2;
deserialize += timer() - tstart;
}
tstart = timer();
}
deallocate += timer() - tstart;
}
double elapsed = (timer()-testBegin);
printf("Test speed: %0.1f MB/sec (%0.0f/sec)\n", bytes/1e6/elapsed, 1000000/elapsed);
printf(" Build: %0.1f MB/sec\n", bytes/1e6/build);
printf(" Serialize: %0.1f MB/sec\n", bytes/1e6/serialize);
printf(" Copy: %0.1f MB/sec\n", bytes/1e6/copy);
printf(" Deserialize: %0.1f MB/sec\n", bytes/1e6/deserialize);
printf(" Deallocate: %0.1f MB/sec\n", bytes/1e6/deallocate);
printf(" Bytes: %0.1f MB\n", bytes/1e6);
printf("\n");
}
std::string toHTML( const StringRef& binaryString ) {
std::string s;
for(int i=0; i<binaryString.size(); i++) {
uint8_t c = binaryString[i];
if (c == '<') s += "&lt;";
else if (c == '>') s += "&gt;";
else if (c == '&') s += "&amp;";
else if (c == '"') s += "&quot;";
else if (c == ' ') s += "&nbsp;";
else if (c > 32 && c < 127) s += c;
else s += format("<span class=\"binary\">[%02x]</span>", c);
}
return s;
}
ACTOR Future<Void> dumpDatabase( Database cx, std::string outputFilename, KeyRange range = allKeys ) {
try {
state Transaction tr( cx );
loop {
state FILE* output = fopen(outputFilename.c_str(), "wt");
try {
state KeySelectorRef iter = firstGreaterOrEqual( range.begin );
state Arena arena;
fprintf(output, "<html><head><style type=\"text/css\">.binary {color:red}</style></head><body>\n");
Version ver = wait( tr.getReadVersion() );
fprintf(output, "<h3>Database version: %" PRId64 "</h3>", ver);
loop {
Standalone<RangeResultRef> results = wait(
tr.getRange( iter, firstGreaterOrEqual( range.end ), 1000 ) );
for(int r=0; r<results.size(); r++) {
std::string key = toHTML( results[r].key ), value = toHTML( results[r].value );
fprintf( output, "<p>%s <b>:=</b> %s</p>\n", key.c_str(), value.c_str() );
}
if (results.size() < 1000) break;
iter = firstGreaterThan( KeyRef(arena, results[ results.size()-1 ].key) );
}
fprintf(output, "</body></html>");
fclose(output);
TraceEvent("DatabaseDumped").detail("Filename", outputFilename);
return Void();
} catch (Error& e) {
fclose(output);
wait( tr.onError(e) );
}
}
} catch (Error& e) {
TraceEvent(SevError,"DumpDatabaseError").error(e).detail("Filename", outputFilename);
throw;
}
}
void memoryTest();
void skipListTest();
Future<Void> startSystemMonitor(std::string dataFolder, Optional<Standalone<StringRef>> zoneId, Optional<Standalone<StringRef>> machineId) {
initializeSystemMonitorMachineState(SystemMonitorMachineState(dataFolder, zoneId, machineId, g_network->getLocalAddress().ip));
systemMonitor();
return recurring( &systemMonitor, 5.0, TaskPriority::FlushTrace );
}
void testIndexedSet();
#ifdef _WIN32
void parentWatcher(void *parentHandle) {
HANDLE parent = (HANDLE)parentHandle;
int signal = WaitForSingleObject( parent, INFINITE );
CloseHandle( parentHandle );
if( signal == WAIT_OBJECT_0 )
criticalError( FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited" );
TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError();
}
#else
void* parentWatcher(void *arg) {
int *parent_pid = (int*) arg;
while(1) {
sleep(1);
if(getppid() != *parent_pid)
criticalError( FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited" );
}
}
#endif
static void printVersion() {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("source version %s\n", getHGVersion());
printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
}
static void printHelpTeaser( const char *name ) {
fprintf(stderr, "Try `%s --help' for more information.\n", name);
}
static void printUsage( const char *name, bool devhelp ) {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("Usage: %s -p ADDRESS [OPTIONS]\n\n", name);
printf(" -p ADDRESS, --public_address ADDRESS\n"
" Public address, specified as `IP_ADDRESS:PORT' or `auto:PORT'.\n");
printf(" -l ADDRESS, --listen_address ADDRESS\n"
" Listen address, specified as `IP_ADDRESS:PORT' (defaults to\n");
printf(" public address).\n");
printf(" -C CONNFILE, --cluster_file CONNFILE\n"
" The path of a file containing the connection string for the\n"
" FoundationDB cluster. The default is first the value of the\n"
" FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
" then `%s'.\n", platform::getDefaultClusterFilePath().c_str());
printf(" --seed_cluster_file SEEDCONNFILE\n"
" The path of a seed cluster file which will be used to connect\n"
" if the -C cluster file does not exist. If the server connects\n"
" successfully using the seed file, then it copies the file to\n"
" the -C file location.\n");
printf(" --seed_connection_string SEEDCONNSTRING\n"
" The path of a seed connection string which will be used to connect\n"
" if the -C cluster file does not exist. If the server connects\n"
" successfully using the seed string, then it copies the string to\n"
" the -C file location.\n");
#ifdef __linux__
printf(" --data_filesystem PATH\n"
" Turns on validation that all data files are written to a drive\n"
" mounted at the specified PATH. This checks that the device at PATH\n"
" is currently mounted and that any data files get written to the\n"
" same device.\n");
#endif
printf(" -d PATH, --datadir PATH\n"
" Store data files in the given folder (must be unique for each\n");
printf(" fdbserver instance on a given machine).\n");
printf(" -L PATH, --logdir PATH\n"
" Store log files in the given folder (default is `.').\n");
printf(" --logsize SIZE Roll over to a new log file after the current log file\n"
" exceeds SIZE bytes. The default value is 10MiB.\n");
printf(" --maxlogs SIZE, --maxlogssize SIZE\n"
" Delete the oldest log file when the total size of all log\n"
" files exceeds SIZE bytes. If set to 0, old log files will not\n"
" be deleted. The default value is 100MiB.\n");
printf(" --trace_format FORMAT\n"
" Select the format of the log files. xml (the default) and json\n"
" are supported.\n");
printf(" -i ID, --machine_id ID\n"
" Machine and zone identifier key (up to 16 hex characters).\n"
" Defaults to a random value shared by all fdbserver processes\n"
" on this machine.\n");
printf(" -a ID, --datacenter_id ID\n"
" Data center identifier key (up to 16 hex characters).\n");
printf(" --locality_LOCALITYKEY LOCALITYVALUE\n"
" Define a locality key. LOCALITYKEY is case-insensitive though\n"
" LOCALITYVALUE is not.\n");
printf(" -m SIZE, --memory SIZE\n"
" Memory limit. The default value is 8GiB. When specified\n"
" without a unit, MiB is assumed.\n");
printf(" -M SIZE, --storage_memory SIZE\n"
" Maximum amount of memory used for storage. The default\n"
" value is 1GiB. When specified without a unit, MB is\n"
" assumed.\n");
printf(" --cache_memory SIZE\n"
" The amount of memory to use for caching disk pages.\n"
" The default value is 2GiB. When specified without a unit,\n"
" MiB is assumed.\n");
printf(" -c CLASS, --class CLASS\n"
" Machine class (valid options are storage, transaction,\n"
" resolution, proxy, master, test, unset, stateless, log, router,\n"
" and cluster_controller).\n");
printf(" -S ON|OFF, --object-serializer ON|OFF\n"
" Use object serializer for sending messages. The object serializer\n"
" is currently a beta feature and it allows fdb processes to talk to\n"
" each other even if they don't have the same version\n");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printf(" -v, --version Print version information and exit.\n");
printf(" -h, -?, --help Display this help and exit.\n");
if( devhelp ) {
printf(" -r ROLE, --role ROLE\n"
" Server role (valid options are fdbd, test, multitest,\n");
printf(" simulation, networktestclient, networktestserver, restore\n");
printf(" consistencycheck, kvfileintegritycheck, kvfilegeneratesums). The default is `fdbd'.\n");
#ifdef _WIN32
printf(" -n, --newconsole\n"
" Create a new console.\n");
printf(" -q, --no_dialog\n"
" Disable error dialog on crash.\n");
printf(" --parentpid PID\n");
printf(" Specify a process after whose termination to exit.\n");
#endif
printf(" -f TESTFILE, --testfile\n"
" Testfile to run, defaults to `tests/default.txt'.\n");
printf(" -R, --restarting\n");
printf(" Restart a previous simulation that was cleanly shut down.\n");
printf(" -s SEED, --seed SEED\n"
" Random seed.\n");
printf(" -k KEY, --key KEY Target key for search role.\n");
printf(" --kvfile FILE Input file (SQLite database file) for use by the 'kvfilegeneratesums' and 'kvfileintegritycheck' roles.\n");
printf(" -b [on,off], --buggify [on,off]\n"
" Sets Buggify system state, defaults to `off'.\n");
printf(" --crash Crash on serious errors instead of continuing.\n");
printf(" -N NETWORKIMPL, --network NETWORKIMPL\n"
" Select network implementation, `net2' (default),\n");
printf(" `net2-threadpool'.\n");
printf(" --unbufferedout\n");
printf(" Do not buffer stdout and stderr.\n");
printf(" --bufferedout\n");
printf(" Buffer stdout and stderr.\n");
printf(" --traceclock CLOCKIMPL\n");
printf(" Select clock source for trace files, `now' (default) or\n");
printf(" `realtime'.\n");
printf(" --num_testers NUM\n");
printf(" A multitester will wait for NUM testers before starting\n");
printf(" (defaults to 1).\n");
#ifdef __linux__
printf(" --rsssize SIZE\n"
" Turns on automatic heap profiling when RSS memory size exceeds\n"
" the given threshold. fdbserver needs to be compiled with\n"
" USE_GPERFTOOLS flag in order to use this feature.\n");
#endif
printf(" --testservers ADDRESSES\n");
printf(" The addresses of networktestservers\n");
printf(" specified as ADDRESS:PORT,ADDRESS:PORT...\n");
printf(" --testonservers\n");
printf(" Testers are recruited on servers.\n");
printf(" --metrics_cluster CONNFILE\n");
printf(" The cluster file designating where this process will\n");
printf(" store its metric data. By default metrics will be stored\n");
printf(" in the same database the process is participating in.\n");
printf(" --metrics_prefix PREFIX\n");
printf(" The prefix where this process will store its metric data.\n");
printf(" Must be specified if using a different database for metrics.\n");
printf(" --knob_KNOBNAME KNOBVALUE\n");
printf(" Changes a database knob. KNOBNAME should be lowercase.\n");
printf(" --io_trust_seconds SECONDS\n");
printf(" Sets the time in seconds that a read or write operation is allowed to take\n"
" before timing out with an error. If an operation times out, all future\n"
" operations on that file will fail with an error as well. Only has an effect\n"
" when using AsyncFileKAIO in Linux.\n");
printf(" --io_trust_warn_only\n");
printf(" Instead of failing when an I/O operation exceeds io_trust_seconds, just\n"
" log a warning to the trace log. Has no effect if io_trust_seconds is unspecified.\n");
} else {
printf(" --dev-help Display developer-specific help and exit.\n");
}
printf("\n"
"SIZE parameters may use one of the multiplicative suffixes B=1, KB=10^3,\n"
"KiB=2^10, MB=10^6, MiB=2^20, GB=10^9, GiB=2^30, TB=10^12, or TiB=2^40.\n");
}
extern bool g_crashOnError;
#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
void* operator new (std::size_t size) throw(std::bad_alloc) {
void* p = malloc(size);
if(!p)
throw std::bad_alloc();
recordAllocation( p, size );
return p;
}
void operator delete (void* ptr) throw() {
recordDeallocation( ptr );
free( ptr );
}
//scalar, nothrow new and it matching delete
void* operator new (std::size_t size,const std::nothrow_t&) throw() {
void* p = malloc(size);
recordAllocation( p, size );
return p;
}
void operator delete (void* ptr, const std::nothrow_t&) throw() {
recordDeallocation( ptr );
free( ptr );
}
//array throwing new and matching delete[]
void* operator new [](std::size_t size) throw(std::bad_alloc) {
void* p = malloc(size);
if(!p)
throw std::bad_alloc();
recordAllocation( p, size );
return p;
}
void operator delete[](void* ptr) throw() {
recordDeallocation( ptr );
free( ptr );
}
//array, nothrow new and matching delete[]
void* operator new [](std::size_t size, const std::nothrow_t&) throw() {
void* p = malloc(size);
recordAllocation( p, size );
return p;
}
void operator delete[](void* ptr, const std::nothrow_t&) throw() {
recordDeallocation( ptr );
free( ptr );
}
#endif
Optional<bool> checkBuggifyOverride(const char *testFile) {
std::ifstream ifs;
ifs.open(testFile, std::ifstream::in);
if (!ifs.good())
return 0;
std::string cline;
while (ifs.good()) {
getline(ifs, cline);
std::string line = removeWhitespace(std::string(cline));
if (!line.size() || line.find(';') == 0)
continue;
size_t found = line.find('=');
if (found == std::string::npos)
// hmmm, not good
continue;
std::string attrib = removeWhitespace(line.substr(0, found));
std::string value = removeWhitespace(line.substr(found + 1));
if (attrib == "buggify") {
if( !strcmp( value.c_str(), "on" ) ) {
ifs.close();
return true;
} else if( !strcmp( value.c_str(), "off" ) ) {
ifs.close();
return false;
} else {
fprintf(stderr, "ERROR: Unknown buggify override state `%s'\n", value.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
}
ifs.close();
return Optional<bool>();
}
// Takes a vector of public and listen address strings given via command line, and returns vector of NetworkAddress objects.
std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(const ClusterConnectionFile& connectionFile,
const vector<std::string>& publicAddressStrs,
vector<std::string>& listenAddressStrs) {
if (listenAddressStrs.size() > 0 && publicAddressStrs.size() != listenAddressStrs.size()) {
fprintf(stderr,
"ERROR: Listen addresses (if provided) should be equal to the number of public addresses in order.\n");
flushAndExit(FDB_EXIT_ERROR);
}
listenAddressStrs.resize(publicAddressStrs.size(), "public");
if (publicAddressStrs.size() > 2) {
fprintf(stderr, "ERROR: maximum 2 public/listen addresses are allowed\n");
flushAndExit(FDB_EXIT_ERROR);
}
NetworkAddressList publicNetworkAddresses;
NetworkAddressList listenNetworkAddresses;
auto& coordinators = connectionFile.getConnectionString().coordinators();
ASSERT(coordinators.size() > 0);
for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
const std::string& publicAddressStr = publicAddressStrs[ii];
bool autoPublicAddress = StringRef(publicAddressStr).startsWith(LiteralStringRef("auto:"));
NetworkAddress currentPublicAddress;
if (autoPublicAddress) {
try {
const NetworkAddress& parsedAddress = NetworkAddress::parse("0.0.0.0:" + publicAddressStr.substr(5));
const IPAddress publicIP = determinePublicIPAutomatically(connectionFile.getConnectionString());
currentPublicAddress = NetworkAddress(publicIP, parsedAddress.port, true, parsedAddress.isTLS());
} catch (Error& e) {
fprintf(stderr, "ERROR: could not determine public address automatically from `%s': %s\n", publicAddressStr.c_str(), e.what());
throw;
}
} else {
try {
currentPublicAddress = NetworkAddress::parse(publicAddressStr);
} catch (Error&) {
fprintf(stderr, "ERROR: Could not parse network address `%s' (specify as IP_ADDRESS:PORT)\n", publicAddressStr.c_str());
throw;
}
}
if(ii == 0) {
publicNetworkAddresses.address = currentPublicAddress;
} else {
publicNetworkAddresses.secondaryAddress = currentPublicAddress;
}
if (!currentPublicAddress.isValid()) {
fprintf(stderr, "ERROR: %s is not a valid IP address\n", currentPublicAddress.toString().c_str());
flushAndExit(FDB_EXIT_ERROR);
}
const std::string& listenAddressStr = listenAddressStrs[ii];
NetworkAddress currentListenAddress;
if (listenAddressStr == "public") {
currentListenAddress = currentPublicAddress;
} else {
try {
currentListenAddress = NetworkAddress::parse(listenAddressStr);
} catch (Error&) {
fprintf(stderr, "ERROR: Could not parse network address `%s' (specify as IP_ADDRESS:PORT)\n", listenAddressStr.c_str());
throw;
}
if (currentListenAddress.isTLS() != currentPublicAddress.isTLS()) {
fprintf(stderr,
"ERROR: TLS state of listen address: %s is not equal to the TLS state of public address: %s.\n",
listenAddressStr.c_str(), publicAddressStr.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
if(ii == 0) {
listenNetworkAddresses.address = currentListenAddress;
} else {
listenNetworkAddresses.secondaryAddress = currentListenAddress;
}
bool hasSameCoord =
std::all_of(coordinators.begin(), coordinators.end(), [&](const NetworkAddress& address) {
if (address.ip == currentPublicAddress.ip && address.port == currentPublicAddress.port) {
return address.isTLS() == currentPublicAddress.isTLS();
}
return true;
});
if (!hasSameCoord) {
fprintf(stderr, "ERROR: TLS state of public address %s does not match in coordinator list.\n",
publicAddressStr.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
if (publicNetworkAddresses.secondaryAddress.present() && publicNetworkAddresses.address.isTLS() == publicNetworkAddresses.secondaryAddress.get().isTLS()) {
fprintf(stderr, "ERROR: only one public address of each TLS state is allowed.\n");
flushAndExit(FDB_EXIT_ERROR);
}
return std::make_pair(publicNetworkAddresses, listenNetworkAddresses);
}
// moves files from 'dirSrc' to 'dirToMove' if their name contains 'role'
void restoreRoleFilesHelper(std::string dirSrc, std::string dirToMove, std::string role) {
std::vector<std::string> returnFiles = platform::listFiles(dirSrc, "");
for (const auto & fileEntry: returnFiles) {
if (fileEntry != "fdb.cluster" && fileEntry.find(role) != std::string::npos) {
//rename files
TraceEvent("RenamingSnapFile")
.detail("Oldname", dirSrc + "/" + fileEntry)
.detail("Newname", dirToMove + "/" + fileEntry);
renameFile(dirSrc + "/" + fileEntry, dirToMove + "/" + fileEntry);
}
}
}
int main(int argc, char* argv[]) {
try {
platformInit();
initSignalSafeUnwind();
#ifdef ALLOC_INSTRUMENTATION
g_extra_memory = new uint8_t[1000000];
#endif
registerCrashHandler();
// Set default of line buffering standard out and error
setvbuf(stdout, NULL, _IOLBF, BUFSIZ);
setvbuf(stderr, NULL, _IOLBF, BUFSIZ);
//Enables profiling on this thread (but does not start it)
registerThreadForProfiling();
std::string commandLine;
for (int a = 0; a<argc; a++) {
if (a) commandLine += ' ';
commandLine += argv[a];
}
CSimpleOpt args(argc, argv, g_rgOptions, SO_O_EXACT);
enum Role {
Simulation,
FDBD,
Test,
MultiTester,
SkipListTest,
SearchMutations,
DSLTest,
VersionedMapTest,
CreateTemplateDatabase,
NetworkTestClient,
NetworkTestServer,
Restore,
KVFileIntegrityCheck,
KVFileGenerateIOLogChecksums,
ConsistencyCheck
};
std::string fileSystemPath = "", dataFolder, connFile = "", seedConnFile = "", seedConnString = "", logFolder = ".", metricsConnFile = "", metricsPrefix = "";
std::string logGroup = "default";
Role role = FDBD;
uint32_t randomSeed = platform::getRandomSeed();
const char *testFile = "tests/default.txt";
std::string kvFile;
std::string testServersStr;
std::string whitelistBinPaths;
std::vector<std::string> publicAddressStrs, listenAddressStrs;
const char *targetKey = NULL;
uint64_t memLimit = 8LL << 30; // Nice to maintain the same default value for memLimit and SERVER_KNOBS->SERVER_MEM_LIMIT and SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT
uint64_t storageMemLimit = 1LL << 30;
bool buggifyEnabled = false, restarting = false;
Optional<Standalone<StringRef>> zoneId;
Optional<Standalone<StringRef>> dcId;
ProcessClass processClass = ProcessClass( ProcessClass::UnsetClass, ProcessClass::CommandLineSource );
bool useNet2 = true;
bool useThreadPool = false;
uint64_t rollsize = TRACE_DEFAULT_ROLL_SIZE;
uint64_t maxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
bool maxLogsSizeSet = false;
int maxLogs = 0;
bool maxLogsSet = false;
std::vector<std::pair<std::string, std::string>> knobs;
LocalityData localities;
int minTesterCount = 1;
bool testOnServers = false;
Reference<TLSOptions> tlsOptions = Reference<TLSOptions>( new TLSOptions );
std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword;
std::vector<std::string> tlsVerifyPeers;
double fileIoTimeout = 0.0;
bool fileIoWarnOnly = false;
std::vector<std::string> blobCredentials; // used for fast restore workers
// const char *blobCredsFromENV = nullptr;
uint64_t rsssize = -1;
bool useObjectSerializer = true;
if( argc == 1 ) {
printUsage(argv[0], false);
flushAndExit(FDB_EXIT_ERROR);
}
#ifdef _WIN32
// Windows needs a gentle nudge to format floats correctly
//_set_output_format(_TWO_DIGIT_EXPONENT);
#endif
while (args.Next()) {
if (args.LastError() == SO_ARG_INVALID_DATA) {
fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() == SO_ARG_INVALID) {
fprintf(stderr, "ERROR: argument given for option `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() == SO_ARG_MISSING) {
fprintf(stderr, "ERROR: missing argument for option `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() == SO_OPT_INVALID) {
fprintf(stderr, "ERROR: unknown option: `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() != SO_SUCCESS) {
fprintf(stderr, "ERROR: error parsing options\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
const char *sRole;
Optional<uint64_t> ti;
std::string argStr;
std::vector<std::string> tmpStrings;
switch (args.OptionId()) {
case OPT_HELP:
printUsage(argv[0], false);
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_DEVHELP:
printUsage(argv[0], true);
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_KNOB: {
std::string syn = args.OptionSyntax();
if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) {
fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
syn = syn.substr(7);
knobs.push_back(std::make_pair(syn, args.OptionArg()));
break;
}
case OPT_LOCALITY: {
std::string syn = args.OptionSyntax();
if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) {
fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
syn = syn.substr(11);
std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower);
localities.set(Standalone<StringRef>(syn), Standalone<StringRef>(std::string(args.OptionArg())));
break;
}
case OPT_VERSION:
printVersion();
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_NOBUFSTDOUT:
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
break;
case OPT_BUFSTDOUTERR:
setvbuf(stdout, NULL, _IOFBF, BUFSIZ);
setvbuf(stderr, NULL, _IOFBF, BUFSIZ);
break;
case OPT_ROLE:
sRole = args.OptionArg();
if (!strcmp(sRole, "fdbd")) role = FDBD;
else if (!strcmp(sRole, "simulation")) role = Simulation;
else if (!strcmp(sRole, "test")) role = Test;
else if (!strcmp(sRole, "multitest")) role = MultiTester;
else if (!strcmp(sRole, "skiplisttest")) role = SkipListTest;
else if (!strcmp(sRole, "search")) role = SearchMutations;
else if (!strcmp(sRole, "dsltest")) role = DSLTest;
else if (!strcmp(sRole, "versionedmaptest")) role = VersionedMapTest;
else if (!strcmp(sRole, "createtemplatedb")) role = CreateTemplateDatabase;
else if (!strcmp(sRole, "networktestclient")) role = NetworkTestClient;
else if (!strcmp(sRole, "networktestserver")) role = NetworkTestServer;
else if (!strcmp(sRole, "restore")) role = Restore;
else if (!strcmp(sRole, "kvfileintegritycheck")) role = KVFileIntegrityCheck;
else if (!strcmp(sRole, "kvfilegeneratesums")) role = KVFileGenerateIOLogChecksums;
else if (!strcmp(sRole, "consistencycheck")) role = ConsistencyCheck;
else {
fprintf(stderr, "ERROR: Unknown role `%s'\n", sRole);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_PUBLICADDR:
argStr = args.OptionArg();
boost::split(tmpStrings, argStr, [](char c){return c == ',';});
publicAddressStrs.insert(publicAddressStrs.end(), tmpStrings.begin(), tmpStrings.end());
break;
case OPT_LISTEN:
argStr = args.OptionArg();
boost::split(tmpStrings, argStr, [](char c){return c == ',';});
listenAddressStrs.insert(listenAddressStrs.end(), tmpStrings.begin(), tmpStrings.end());
break;
case OPT_CONNFILE:
connFile = args.OptionArg();
break;
case OPT_LOGGROUP:
logGroup = args.OptionArg();
break;
case OPT_SEEDCONNFILE:
seedConnFile = args.OptionArg();
break;
case OPT_SEEDCONNSTRING:
seedConnString = args.OptionArg();
break;
#ifdef __linux__
case OPT_FILESYSTEM: {
fileSystemPath = args.OptionArg();
break;
}
case OPT_PROFILER_RSS_SIZE:{
const char *a = args.OptionArg();
char *end;
rsssize = strtoull(a, &end, 10);
if(*end) {
fprintf(stderr, "ERROR: Unrecognized memory size `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
#endif
case OPT_DATAFOLDER:
dataFolder = args.OptionArg();
break;
case OPT_LOGFOLDER:
logFolder = args.OptionArg();
break;
case OPT_NETWORKIMPL: {
const char *a = args.OptionArg();
if (!strcmp(a, "net2")) useNet2 = true;
else if (!strcmp(a, "net2-threadpool")) {
useNet2 = true;
useThreadPool = true;
}
else {
fprintf(stderr, "ERROR: Unknown network implementation `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_TRACECLOCK: {
const char *a = args.OptionArg();
if (!strcmp(a, "realtime")) g_trace_clock = TRACE_CLOCK_REALTIME;
else if (!strcmp(a, "now")) g_trace_clock = TRACE_CLOCK_NOW;
else {
fprintf(stderr, "ERROR: Unknown clock source `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_NUMTESTERS: {
const char *a = args.OptionArg();
if (!sscanf(a, "%d", &minTesterCount)) {
fprintf(stderr, "ERROR: Could not parse numtesters `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_ROLLSIZE: {
const char *a = args.OptionArg();
ti = parse_with_suffix(a);
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse logsize `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
rollsize = ti.get();
break;
}
case OPT_MAXLOGSSIZE: {
const char *a = args.OptionArg();
ti = parse_with_suffix(a);
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse maxlogssize `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
maxLogsSize = ti.get();
maxLogsSizeSet = true;
break;
}
case OPT_MAXLOGS: {
const char *a = args.OptionArg();
char *end;
maxLogs = strtoull(a, &end, 10);
if (*end) {
fprintf(stderr, "ERROR: Unrecognized maximum number of logs `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
maxLogsSet = true;
break;
}
#ifdef _WIN32
case OPT_PARENTPID: {
auto pid_str = args.OptionArg();
int parent_pid = atoi(pid_str);
auto pHandle = OpenProcess( SYNCHRONIZE, FALSE, parent_pid );
if( !pHandle ) {
TraceEvent("ParentProcessOpenError").GetLastError();
fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError());
throw platform_error();
}
startThread(&parentWatcher, pHandle);
break;
}
case OPT_NEWCONSOLE:
FreeConsole();
AllocConsole();
freopen("CONIN$","rb",stdin);
freopen("CONOUT$","wb",stdout);
freopen("CONOUT$","wb",stderr);
break;
case OPT_NOBOX:
SetErrorMode(SetErrorMode(0) | SEM_NOGPFAULTERRORBOX);
break;
#else
case OPT_PARENTPID: {
auto pid_str = args.OptionArg();
int *parent_pid = new(int);
*parent_pid = atoi(pid_str);
startThread(&parentWatcher, parent_pid);
break;
}
#endif
case OPT_TESTFILE:
testFile = args.OptionArg();
break;
case OPT_KVFILE:
kvFile = args.OptionArg();
break;
case OPT_RESTARTING:
restarting = true;
break;
case OPT_RANDOMSEED: {
char* end;
randomSeed = (uint32_t)strtoul( args.OptionArg(), &end, 0 );
if( *end ) {
fprintf(stderr, "ERROR: Could not parse random seed `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_MACHINEID: {
zoneId = std::string(args.OptionArg());
break;
}
case OPT_DCID: {
dcId = std::string(args.OptionArg());
break;
}
case OPT_MACHINE_CLASS:
sRole = args.OptionArg();
processClass = ProcessClass(sRole, ProcessClass::CommandLineSource);
if (processClass == ProcessClass::InvalidClass) {
fprintf(stderr, "ERROR: Unknown machine class `%s'\n", sRole);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_BLOB_CREDENTIAL_FILE: {
//Add blob credential following backup agent example
blobCredentials.push_back(args.OptionArg());
printf("blob credential file:%s\n", blobCredentials.back().c_str());
//
// blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS");
// if (blobCredsFromENV != nullptr) {
// printf("[WARNING] set blob credetial via env variable is not tested\n");
// StringRef t((uint8_t *) blobCredsFromENV, strlen(blobCredsFromENV));
// do {
// StringRef file = t.eat(":");
// if (file.size() != 0)
// blobCredentials.push_back(file.toString());
// } while (t.size() != 0);
// }
break;
}
case OPT_KEY:
targetKey = args.OptionArg();
break;
case OPT_MEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
memLimit = ti.get();
break;
case OPT_STORAGEMEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse storage memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
storageMemLimit = ti.get();
break;
case OPT_CACHEMEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse cache memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
// SOMEDAY: ideally we'd have some better way to express that a knob should be elevated to formal parameter
knobs.push_back(std::make_pair("page_cache_4k", format("%ld", ti.get() / 4096 * 4096))); // The cache holds 4K pages, so we can truncate this to the next smaller multiple of 4K.
break;
case OPT_BUGGIFY:
if( !strcmp( args.OptionArg(), "on" ) )
buggifyEnabled = true;
else if( !strcmp( args.OptionArg(), "off" ) )
buggifyEnabled = false;
else {
fprintf(stderr, "ERROR: Unknown buggify state `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_CRASHONERROR:
g_crashOnError = true;
break;
case OPT_TESTSERVERS:
testServersStr = args.OptionArg();
break;
case OPT_TEST_ON_SERVERS:
testOnServers = true;
break;
case OPT_METRICSCONNFILE:
metricsConnFile = args.OptionArg();
break;
case OPT_METRICSPREFIX:
metricsPrefix = args.OptionArg();
break;
case OPT_IO_TRUST_SECONDS: {
const char* a = args.OptionArg();
if( !sscanf(a, "%lf", &fileIoTimeout) ) {
fprintf(stderr, "ERROR: Could not parse io_trust_seconds `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_IO_TRUST_WARN_ONLY:
fileIoWarnOnly = true;
break;
case OPT_TRACE_FORMAT:
if (!selectTraceFormatter(args.OptionArg())) {
fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg());
}
break;
case OPT_USE_OBJECT_SERIALIZER:
{
std::string s = args.OptionArg();
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "on" || s == "true" || s == "1") {
useObjectSerializer = true;
} else if (s == "off" || s == "false" || s == "0") {
useObjectSerializer = false;
} else {
fprintf(stderr, "ERROR: Could not parse object serializer option: `%s'\n", s.c_str());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_WHITELIST_BINPATH:
whitelistBinPaths = args.OptionArg();
break;
#ifndef TLS_DISABLED
case TLSOptions::OPT_TLS_PLUGIN:
args.OptionArg();
break;
case TLSOptions::OPT_TLS_CERTIFICATES:
tlsCertPath = args.OptionArg();
break;
case TLSOptions::OPT_TLS_PASSWORD:
tlsPassword = args.OptionArg();
break;
case TLSOptions::OPT_TLS_CA_FILE:
tlsCAPath = args.OptionArg();
break;
case TLSOptions::OPT_TLS_KEY:
tlsKeyPath = args.OptionArg();
break;
case TLSOptions::OPT_TLS_VERIFY_PEERS:
tlsVerifyPeers.push_back(args.OptionArg());
break;
#endif
}
}
if (seedConnString.length() && seedConnFile.length()) {
fprintf(stderr, "%s\n", "--seed_cluster_file and --seed_connection_string may not both be specified at once.");
return FDB_EXIT_ERROR;
}
bool seedSpecified = seedConnFile.length() || seedConnString.length();
if (seedSpecified && !connFile.length()){
fprintf(stderr, "%s\n", "If -seed_cluster_file or --seed_connection_string is specified, -C must be specified as well.");
return FDB_EXIT_ERROR;
}
if( metricsConnFile == connFile )
metricsConnFile = "";
if( metricsConnFile != "" && metricsPrefix == "" ) {
fprintf(stderr, "If a metrics cluster file is specified, a metrics prefix is required.\n");
return FDB_EXIT_ERROR;
}
bool autoPublicAddress = std::any_of(publicAddressStrs.begin(), publicAddressStrs.end(),
[](const std::string& addr) {
return StringRef(addr).startsWith(LiteralStringRef("auto:"));
});
Reference<ClusterConnectionFile> connectionFile;
if ( (role != Simulation && role != CreateTemplateDatabase && role != KVFileIntegrityCheck && role != KVFileGenerateIOLogChecksums) || autoPublicAddress ) {
if (seedSpecified && !fileExists(connFile)){
std::string connectionString = seedConnString.length() ? seedConnString : "";
ClusterConnectionString ccs;
if(seedConnFile.length()) {
try {
connectionString = readFileBytes(seedConnFile, MAX_CLUSTER_FILE_BYTES);
}
catch (Error& e) {
fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(std::make_pair(seedConnFile, false), e).c_str());
throw;
}
}
try {
ccs = ClusterConnectionString(connectionString);
}
catch (Error& e) {
fprintf(stderr, "%s\n", ClusterConnectionString::getErrorString(connectionString, e).c_str());
throw;
}
connectionFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(connFile, ccs));
}
else {
std::pair<std::string, bool> resolvedClusterFile;
try {
resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(connFile);
connectionFile = Reference<ClusterConnectionFile>(new ClusterConnectionFile(resolvedClusterFile.first));
} catch (Error& e) {
fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
throw;
}
}
// failmon?
}
NetworkAddressList publicAddresses, listenAddresses;
try {
if (!publicAddressStrs.empty()) {
std::tie(publicAddresses, listenAddresses) = buildNetworkAddresses(*connectionFile, publicAddressStrs, listenAddressStrs);
}
} catch (Error&) {
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if(role == ConsistencyCheck) {
if(!publicAddressStrs.empty()) {
fprintf(stderr, "ERROR: Public address cannot be specified for consistency check processes\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
auto publicIP = determinePublicIPAutomatically(connectionFile->getConnectionString());
publicAddresses.address = NetworkAddress(publicIP, ::getpid());
}
if (role==Simulation)
printf("Random seed is %u...\n", randomSeed);
if( zoneId.present() )
printf("ZoneId set to %s, dcId to %s\n", printable(zoneId).c_str(), printable(dcId).c_str());
setThreadLocalDeterministicRandomSeed(randomSeed);
if(role==Simulation) {
Optional<bool> buggifyOverride = checkBuggifyOverride(testFile);
if(buggifyOverride.present())
buggifyEnabled = buggifyOverride.get();
}
enableBuggify(buggifyEnabled, BuggifyType::General);
delete FLOW_KNOBS;
delete SERVER_KNOBS;
delete CLIENT_KNOBS;
FlowKnobs* flowKnobs = new FlowKnobs(true, role == Simulation);
ClientKnobs* clientKnobs = new ClientKnobs(true);
ServerKnobs* serverKnobs = new ServerKnobs(true, clientKnobs);
FLOW_KNOBS = flowKnobs;
SERVER_KNOBS = serverKnobs;
CLIENT_KNOBS = clientKnobs;
if (!serverKnobs->setKnob( "log_directory", logFolder )) ASSERT(false);
if (role != Simulation) {
if (!serverKnobs->setKnob("commit_batches_mem_bytes_hard_limit", std::to_string(memLimit))) ASSERT(false);
}
for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
try {
if (!flowKnobs->setKnob( k->first, k->second ) &&
!clientKnobs->setKnob( k->first, k->second ) &&
!serverKnobs->setKnob( k->first, k->second ))
{
fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str());
TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first));
}
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
fprintf(stderr, "WARNING: Invalid value '%s' for option '%s'\n", k->second.c_str(), k->first.c_str());
TraceEvent(SevWarnAlways, "InvalidKnobValue").detail("Knob", printable(k->first)).detail("Value", printable(k->second));
} else {
throw;
}
}
}
if (!serverKnobs->setKnob("server_mem_limit", std::to_string(memLimit))) ASSERT(false);
// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
EvictablePageCache::evictionPolicyStringToEnum(flowKnobs->CACHE_EVICTION_POLICY);
if (role == SkipListTest) {
skipListTest();
flushAndExit(FDB_EXIT_SUCCESS);
}
if (role == DSLTest) {
dsltest();
flushAndExit(FDB_EXIT_SUCCESS);
}
if (role == VersionedMapTest) {
versionedMapTest();
flushAndExit(FDB_EXIT_SUCCESS);
}
if (role == SearchMutations && !targetKey) {
fprintf(stderr, "ERROR: please specify a target key\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (role == NetworkTestClient && !testServersStr.size() ) {
fprintf(stderr, "ERROR: please specify --testservers\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
// Interpret legacy "maxLogs" option in the most sensible and unsurprising way we can while eliminating its code path
if (maxLogsSet) {
if (maxLogsSizeSet) {
// This is the case where both options are set and we must deconflict.
auto maxLogsAsSize = maxLogs * rollsize;
// If either was unlimited, then the safe option here is to take the larger one.
// This means that is one of the two options specified a limited amount of logging
// then the option that specified "unlimited" will be ignored.
if( maxLogsSize == 0 || maxLogs == 0 )
maxLogsSize = std::max(maxLogsSize, maxLogsAsSize);
else
maxLogsSize = std::min(maxLogsSize, maxLogs * rollsize);
}
else {
maxLogsSize = maxLogs * rollsize;
}
}
// Initialize the thread pool
CoroThreadPool::init();
// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now.
TraceEvent::setNetworkThread();
std::vector<Future<Void>> listenErrors;
if (role == Simulation || role == CreateTemplateDatabase) {
//startOldSimulator();
startNewSimulator(useObjectSerializer);
openTraceFile(NetworkAddress(), rollsize, maxLogsSize, logFolder, "trace", logGroup);
} else {
g_network = newNet2(useThreadPool, true, useObjectSerializer);
FlowTransport::createInstance(false, 1);
const bool expectsPublicAddress = (role == FDBD || role == NetworkTestServer || role == Restore);
if (publicAddressStrs.empty()) {
if (expectsPublicAddress) {
fprintf(stderr, "ERROR: The -p or --public_address option is required\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
}
openTraceFile(publicAddresses.address, rollsize, maxLogsSize, logFolder, "trace", logGroup);
#ifndef TLS_DISABLED
if ( tlsCertPath.size() )
tlsOptions->set_cert_file( tlsCertPath );
if (tlsCAPath.size())
tlsOptions->set_ca_file(tlsCAPath);
if (tlsKeyPath.size()) {
if (tlsPassword.size())
tlsOptions->set_key_password(tlsPassword);
tlsOptions->set_key_file(tlsKeyPath);
}
if ( tlsVerifyPeers.size() )
tlsOptions->set_verify_peers( tlsVerifyPeers );
tlsOptions->register_network();
#endif
if (expectsPublicAddress) {
for (int ii = 0; ii < (publicAddresses.secondaryAddress.present() ? 2 : 1); ++ii) {
const NetworkAddress& publicAddress = ii==0 ? publicAddresses.address : publicAddresses.secondaryAddress.get();
const NetworkAddress& listenAddress = ii==0 ? listenAddresses.address : listenAddresses.secondaryAddress.get();
try {
const Future<Void>& errorF = FlowTransport::transport().bind(publicAddress, listenAddress);
listenErrors.push_back(errorF);
if (errorF.isReady()) errorF.get();
} catch (Error& e) {
TraceEvent("BindError").error(e);
fprintf(stderr, "Error initializing networking with public address %s and listen address %s (%s)\n",
publicAddress.toString().c_str(), listenAddress.toString().c_str(), e.what());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
}
}
// Use a negative ioTimeout to indicate warn-only
Net2FileSystem::newFileSystem(fileIoWarnOnly ? -fileIoTimeout : fileIoTimeout, fileSystemPath);
g_network->initMetrics();
FlowTransport::transport().initMetrics();
initTraceEventMetrics();
}
double start = timer(), startNow = now();
std::string cwd = "<unknown>";
try {
cwd = platform::getWorkingDirectory();
} catch(Error &e) {
// Allow for platform error by rethrowing all _other_ errors
if( e.code() != error_code_platform_error )
throw;
}
TraceEvent("ProgramStart")
.setMaxEventLength(12000)
.detail("RandomSeed", randomSeed)
.detail("SourceVersion", getHGVersion())
.detail("Version", FDB_VT_VERSION )
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detail("FileSystem", fileSystemPath)
.detail("DataFolder", dataFolder)
.detail("WorkingDirectory", cwd)
.detail("ClusterFile", connectionFile ? connectionFile->getFilename().c_str() : "")
.detail("ConnectionString", connectionFile ? connectionFile->getConnectionString().toString() : "")
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
.setMaxFieldLength(10000)
.detail("CommandLine", commandLine)
.setMaxFieldLength(0)
.detail("BuggifyEnabled", buggifyEnabled)
.detail("MemoryLimit", memLimit)
.trackLatest("ProgramStart");
// Test for TraceEvent length limits
/*std::string foo(4096, 'x');
TraceEvent("TooLongDetail").detail("Contents", foo);
TraceEvent("TooLongEvent")
.detail("Contents1", foo)
.detail("Contents2", foo)
.detail("Contents3", foo)
.detail("Contents4", foo)
.detail("Contents5", foo)
.detail("Contents6", foo)
.detail("Contents7", foo)
.detail("Contents8", foo)
.detail("ExtraTest", 1776);*/
Error::init();
std::set_new_handler( &platform::outOfMemory );
setMemoryQuota( memLimit );
Future<Optional<Void>> f;
Standalone<StringRef> machineId(getSharedMemoryMachineId().toString());
if (!localities.isPresent(LocalityData::keyZoneId))
localities.set(LocalityData::keyZoneId, zoneId.present() ? zoneId : machineId);
if (!localities.isPresent(LocalityData::keyMachineId))
localities.set(LocalityData::keyMachineId, zoneId.present() ? zoneId : machineId);
if (!localities.isPresent(LocalityData::keyDcId) && dcId.present())
localities.set(LocalityData::keyDcId, dcId);
if (role == Simulation) {
TraceEvent("Simulation").detail("TestFile", testFile);
clientKnobs->trace();
flowKnobs->trace();
serverKnobs->trace();
if (!dataFolder.size())
dataFolder = "simfdb";
std::vector<std::string> directories = platform::listDirectories( dataFolder );
for(int i = 0; i < directories.size(); i++)
if (directories[i].size() != 32 && directories[i] != "." && directories[i] != ".." &&
directories[i] != "backups" && directories[i].find("snap") == std::string::npos) {
TraceEvent(SevError, "IncompatibleDirectoryFound").detail("DataFolder", dataFolder).detail("SuspiciousFile", directories[i]);
fprintf(stderr, "ERROR: Data folder `%s' had non fdb file `%s'; please use clean, fdb-only folder\n", dataFolder.c_str(), directories[i].c_str());
flushAndExit(FDB_EXIT_ERROR);
}
std::vector<std::string> files = platform::listFiles( dataFolder );
if( (files.size()>1 || (files.size()==1 && files[0] != "restartInfo.ini" )) && !restarting ) {
TraceEvent(SevError, "IncompatibleFileFound").detail("DataFolder", dataFolder);
fprintf(stderr, "ERROR: Data folder `%s' is non-empty; please use clean, fdb-only folder\n", dataFolder.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
else if ( files.empty() && restarting ) {
TraceEvent(SevWarnAlways, "FileNotFound").detail("DataFolder", dataFolder);
printf("ERROR: Data folder `%s' is empty, but restarting option selected. Run Phase 1 test first\n", dataFolder.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
int isRestoring = 0;
if (!restarting) {
platform::eraseDirectoryRecursive( dataFolder );
platform::createDirectory( dataFolder );
} else {
CSimpleIni ini;
ini.SetUnicode();
std::string absDataFolder = abspath(dataFolder);
ini.LoadFile(joinPath(absDataFolder, "restartInfo.ini").c_str());
int backupFailed = true;
const char* isRestoringStr = ini.GetValue("RESTORE", "isRestoring", NULL);
if (isRestoringStr) {
isRestoring = atoi(isRestoringStr);
const char* backupFailedStr = ini.GetValue("RESTORE", "BackupFailed", NULL);
if (isRestoring && backupFailedStr) {
backupFailed = atoi(backupFailedStr);
}
}
if (isRestoring && !backupFailed) {
std::vector<std::string> returnList;
std::string ext = "";
returnList = platform::listDirectories(absDataFolder);
std::string snapStr = ini.GetValue("RESTORE", "RestoreSnapUID");
TraceEvent("RestoringDataFolder").detail("DataFolder", absDataFolder);
TraceEvent("RestoreSnapUID").detail("UID", snapStr);
// delete all files (except fdb.cluster) in non-snap directories
for (const auto & dirEntry : returnList) {
if (dirEntry == "." || dirEntry == "..") {
continue;
}
if (dirEntry.find(snapStr) != std::string::npos) {
continue;
}
std::string childf = absDataFolder + "/" + dirEntry;
std::vector<std::string> returnFiles = platform::listFiles(childf, ext);
for (const auto & fileEntry : returnFiles) {
if (fileEntry != "fdb.cluster" && fileEntry != "fitness") {
TraceEvent("DeletingNonSnapfiles")
.detail("FileBeingDeleted", childf + "/" + fileEntry);
deleteFile(childf + "/" + fileEntry);
}
}
}
// cleanup unwanted and partial directories
for (const auto & dirEntry : returnList) {
if (dirEntry == "." || dirEntry == "..") {
continue;
}
std::string dirSrc = absDataFolder + "/" + dirEntry;
// delete snap directories which are not part of restoreSnapUID
if (dirEntry.find(snapStr) == std::string::npos) {
if (dirEntry.find("snap") != std::string::npos) {
platform::eraseDirectoryRecursive(dirSrc);
}
continue;
}
// remove empty/partial snap directories
std::vector<std::string> childrenList = platform::listFiles(dirSrc);
if (childrenList.size() == 0) {
TraceEvent("RemovingEmptySnapDirectory").detail("DirBeingDeleted", dirSrc);
platform::eraseDirectoryRecursive(dirSrc);
continue;
}
}
// move snapshotted files to appropriate locations
for (const auto & dirEntry : returnList) {
if (dirEntry == "." || dirEntry == "..") {
continue;
}
std::string dirSrc = absDataFolder + "/" + dirEntry;
std::string origDir = dirEntry.substr(0, 32);
std::string dirToMove = absDataFolder + "/" + origDir;
if ((dirEntry.find("snap") != std::string::npos) &&
(dirEntry.find("tlog") != std::string::npos)) {
// restore tlog files
restoreRoleFilesHelper(dirSrc, dirToMove, "log");
} else if ((dirEntry.find("snap") != std::string::npos) &&
(dirEntry.find("storage") != std::string::npos)) {
// restore storage files
restoreRoleFilesHelper(dirSrc, dirToMove, "storage");
} else if ((dirEntry.find("snap") != std::string::npos) &&
(dirEntry.find("coord") != std::string::npos)) {
// restore coordinator files
restoreRoleFilesHelper(dirSrc, dirToMove, "coordination");
}
}
}
}
setupAndRun( dataFolder, testFile, restarting, (isRestoring >= 1), whitelistBinPaths, tlsOptions);
g_simulator.run();
} else if (role == FDBD) {
// Call fast restore for the class FastRestoreClass. This is a short-cut to run fast restore in circus
if ( processClass == ProcessClass::FastRestoreClass) {
printf("Run as fast restore worker\n");
// Update the global blob credential files list
std::vector<std::string> *pFiles = (std::vector<std::string> *) g_network->global(INetwork::enBlobCredentialFiles);
if (pFiles != nullptr) {
for (auto &f : blobCredentials) {
pFiles->push_back(f);
}
}
f = stopAfter( restoreWorker(connectionFile, localities) );
g_network->run();
} else {
ASSERT( connectionFile );
setupSlowTaskProfiler();
if (!dataFolder.size())
dataFolder = format("fdb/%d/", publicAddresses.address.port); // SOMEDAY: Better default
vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
actors.push_back( fdbd(connectionFile, localities, processClass, dataFolder, dataFolder, storageMemLimit, metricsConnFile, metricsPrefix, rsssize, whitelistBinPaths) );
//actors.push_back( recurring( []{}, .001 ) ); // for ASIO latency measurement
f = stopAfter( waitForAll(actors) );
g_network->run();
}
} else if (role == MultiTester) {
f = stopAfter( runTests( connectionFile, TEST_TYPE_FROM_FILE, testOnServers ? TEST_ON_SERVERS : TEST_ON_TESTERS, minTesterCount, testFile, StringRef(), localities ) );
g_network->run();
} else if (role == Test) {
auto m = startSystemMonitor(dataFolder, zoneId, zoneId);
f = stopAfter( runTests( connectionFile, TEST_TYPE_FROM_FILE, TEST_HERE, 1, testFile, StringRef(), localities ) );
g_network->run();
} else if (role == ConsistencyCheck) {
setupSlowTaskProfiler();
auto m = startSystemMonitor(dataFolder, zoneId, zoneId);
f = stopAfter( runTests( connectionFile, TEST_TYPE_CONSISTENCY_CHECK, TEST_HERE, 1, testFile, StringRef(), localities ) );
g_network->run();
} else if (role == CreateTemplateDatabase) {
createTemplateDatabase();
} else if (role == NetworkTestClient) {
f = stopAfter( networkTestClient( testServersStr ) );
g_network->run();
} else if (role == NetworkTestServer) {
f = stopAfter( networkTestServer() );
g_network->run();
} else if (role == Restore) {
f = stopAfter( restoreWorker(connectionFile, localities) );
g_network->run();
} else if (role == KVFileIntegrityCheck) {
f = stopAfter( KVFileCheck(kvFile, true) );
g_network->run();
} else if (role == KVFileGenerateIOLogChecksums) {
Optional<Void> result;
try {
GenerateIOLogChecksumFile(kvFile);
result = Void();
}
catch(Error &e) {
fprintf(stderr, "Fatal Error: %s\n", e.what());
}
f = result;
}
int rc = FDB_EXIT_SUCCESS;
if(f.isValid() && f.isReady() && !f.isError() && !f.get().present()) {
rc = FDB_EXIT_ERROR;
}
int unseed = noUnseed ? 0 : deterministicRandom()->randomInt(0, 100001);
TraceEvent("ElapsedTime").detail("SimTime", now()-startNow).detail("RealTime", timer()-start)
.detail("RandomUnseed", unseed);
if (role==Simulation){
printf("Unseed: %d\n", unseed);
printf("Elapsed: %f simsec, %f real seconds\n", now()-startNow, timer()-start);
//cout << format(" %d endpoints left\n", transport().getEndpointCount());
}
//IFailureMonitor::failureMonitor().address_info.clear();
// we should have shut down ALL actors associated with this machine; let's list all of the ones still live
/*{
auto living = Actor::all;
printf("%d surviving actors:\n", living.size());
for(auto a = living.begin(); a != living.end(); ++a)
printf(" #%lld %s %p\n", (*a)->creationIndex, (*a)->getName(), (*a));
}
{
auto living = DatabaseContext::all;
printf("%d surviving DatabaseContexts:\n", living.size());
for(auto a = living.begin(); a != living.end(); ++a)
printf(" #%lld %p\n", (*a)->creationIndex, (*a));
}
{
auto living = TransactionData::all;
printf("%d surviving TransactionData(s):\n", living.size());
for(auto a = living.begin(); a != living.end(); ++a)
printf(" #%lld %p\n", (*a)->creationIndex, (*a));
}*/
/*cout << Actor::allActors.size() << " surviving actors:" << endl;
std::map<std::string,int> actorCount;
for(int i=0; i<Actor::allActors.size(); i++)
++actorCount[Actor::allActors[i]->getName()];
for(auto i = actorCount.rbegin(); !(i == actorCount.rend()); ++i)
cout << " " << i->second << " " << i->first << endl;*/
// cout << " " << Actor::allActors[i]->getName() << endl;
int total = 0;
for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i)
total += i->second;
if (total)
printf("%d errors:\n", total);
for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i)
if (i->second > 0)
printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what());
if (&g_simulator == g_network) {
auto processes = g_simulator.getAllProcesses();
for(auto i = processes.begin(); i != processes.end(); ++i)
printf("%s %s: %0.3f Mclocks\n", (*i)->name, (*i)->address.toString().c_str(), (*i)->cpuTicks / 1e6);
}
if (role == Simulation) {
unsigned long sevErrorEventsLogged = TraceEvent::CountEventsLoggedAt(SevError);
if (sevErrorEventsLogged > 0) {
printf("%lu SevError events logged\n", sevErrorEventsLogged);
rc = FDB_EXIT_ERROR;
}
}
//g_simulator.run();
#ifdef ALLOC_INSTRUMENTATION
{
std::cout << "Page Counts: "
<< FastAllocator<16>::pageCount << " "
<< FastAllocator<32>::pageCount << " "
<< FastAllocator<64>::pageCount << " "
<< FastAllocator<128>::pageCount << " "
<< FastAllocator<256>::pageCount << " "
<< FastAllocator<512>::pageCount << " "
<< FastAllocator<1024>::pageCount << " "
<< FastAllocator<2048>::pageCount << " "
<< FastAllocator<4096>::pageCount << " "
<< FastAllocator<8192>::pageCount << std::endl;
vector< std::pair<std::string, const char*> > typeNames;
for( auto i = allocInstr.begin(); i != allocInstr.end(); ++i ) {
std::string s;
#ifdef __linux__
char *demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL);
if (demangled) {
s = demangled;
if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
free(demangled);
} else
s = i->first;
#else
s = i->first;
if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
else if (StringRef(s).startsWith(LiteralStringRef("class ")))
s = s.substr(LiteralStringRef("class ").size());
else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
s = s.substr(LiteralStringRef("struct ").size());
#endif
typeNames.push_back( std::make_pair(s, i->first) );
}
std::sort(typeNames.begin(), typeNames.end());
for(int i=0; i<typeNames.size(); i++) {
const char* n = typeNames[i].second;
auto& f = allocInstr[n];
printf("%+d\t%+d\t%d\t%d\t%s\n", f.allocCount, -f.deallocCount, f.allocCount-f.deallocCount, f.maxAllocated, typeNames[i].first.c_str());
}
// We're about to exit and clean up data structures, this will wreak havoc on allocation recording
memSample_entered = true;
}
#endif
//printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(rc);
} catch (Error& e) {
fprintf(stderr, "Error: %s\n", e.what());
TraceEvent(SevError, "MainError").error(e);
//printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(FDB_EXIT_MAIN_ERROR);
} catch (std::exception& e) {
fprintf(stderr, "std::exception: %s\n", e.what());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
//printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
}
static_assert( LBLocalityData<StorageServerInterface>::Present, "Storage server interface should be load balanced" );
static_assert( LBLocalityData<MasterProxyInterface>::Present, "Master proxy interface should be load balanced" );
static_assert( LBLocalityData<TLogInterface>::Present, "TLog interface should be load balanced" );
static_assert( !LBLocalityData<MasterInterface>::Present, "Master interface should not be load balanced" );
}
Reference in New Issue View Git Blame Copy Permalink