wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2275 from dongxinEric/bugfix/2273/fix-read-key-sampling 

Resolves #2273: Use a large value for read sampling size threshold. Also at sampling …
This commit is contained in:
Evan Tschannen 2019-10-31 10:21:58 -07:00 committed by GitHub
parent 988e88338b 199a34b827
commit 71dfaa3f95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fdbserver/Knobs.cpp
View File

@ -130,8 +130,8 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( MAX_SHARD_BYTES, 500000000 );
init( KEY_SERVER_SHARD_BYTES, 500000000 );
bool buggifySmallReadBandwidth = randomize && BUGGIFY;
init( SHARD_MAX_BYTES_READ_PER_KSEC, 100LL*1000000*1000 ); if( buggifySmallReadBandwidth ) SHARD_MAX_BYTES_READ_PER_KSEC = 100LL*1000*1000;
/* 100*1MB/sec * 1000sec/ksec
init( SHARD_MAX_BYTES_READ_PER_KSEC, 8LL*1000000*1000 ); if( buggifySmallReadBandwidth ) SHARD_MAX_BYTES_READ_PER_KSEC = 100LL*1000*1000;
/* 8*1MB/sec * 1000sec/ksec
Shards with more than this read bandwidth will be considered as a read cache candidate
*/
init( SHARD_MAX_BYTES_READ_PER_KSEC_JITTER, 0.1 );
@ -457,7 +457,8 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( SPLIT_JITTER_AMOUNT, 0.05 ); if( randomize && BUGGIFY ) SPLIT_JITTER_AMOUNT = 0.2;
init( IOPS_UNITS_PER_SAMPLE, 10000 * 1000 / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 100 );
init( BANDWIDTH_UNITS_PER_SAMPLE, SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 );
init( BYTES_READ_UNITS_PER_SAMPLE, 100); // Effectively weight up read on small or non-existing key/values.
init( BYTES_READ_UNITS_PER_SAMPLE, 100000 ); // 100K bytes
init( EMPTY_READ_PENALTY, 20 ); // 20 bytes
//Storage Server
init( STORAGE_LOGGING_DELAY, 5.0 );

1
fdbserver/Knobs.h
View File

@ -395,6 +395,7 @@ public:
int64_t IOPS_UNITS_PER_SAMPLE;
int64_t BANDWIDTH_UNITS_PER_SAMPLE;
int64_t BYTES_READ_UNITS_PER_SAMPLE;
int64_t EMPTY_READ_PENALTY;
//Storage Server
double STORAGE_LOGGING_DELAY;

18
fdbserver/storageserver.actor.cpp
View File

@ -516,6 +516,8 @@ public:
specialCounter(cc, "VersionLag", [self](){ return self->versionLag; });
specialCounter(cc, "LocalRate", [self]{ return self->currentRate() * 100; });
specialCounter(cc, "BytesReadSampleCount", [self]() { return self->metrics.bytesReadSample.queue.size(); });
specialCounter(cc, "FetchKeysFetchActive", [self](){ return self->fetchKeysParallelismLock.activePermits(); });
specialCounter(cc, "FetchKeysWaiting", [self](){ return self->fetchKeysParallelismLock.waiters(); });
@ -890,9 +892,10 @@ ACTOR Future<Void> getValueQ( StorageServer* data, GetValueRequest req ) {
}
StorageMetrics metrics;
metrics.bytesReadPerKSecond = v.present() ? std::max((int64_t)(req.key.size() + v.get().size()),
SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE)
: SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
// If the read yields no value, randomly sample the empty read.
metrics.bytesReadPerKSecond =
v.present() ? std::max((int64_t)(req.key.size() + v.get().size()), SERVER_KNOBS->EMPTY_READ_PENALTY)
: SERVER_KNOBS->EMPTY_READ_PENALTY;
data->metrics.notify(req.key, metrics);
if( req.debugID.present() )
@ -1271,7 +1274,7 @@ ACTOR Future<GetKeyValuesReply> readRange( StorageServer* data, Version version,
result.more = limit == 0 || *pLimitBytes<=0; // FIXME: Does this have to be exact?
result.version = version;
StorageMetrics metrics;
metrics.bytesReadPerKSecond = std::max(readSize, SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE);
metrics.bytesReadPerKSecond = std::max(readSize, SERVER_KNOBS->EMPTY_READ_PENALTY);
data->metrics.notify(limit >= 0 ? range.begin : range.end, metrics);
return result;
}
@ -1327,14 +1330,13 @@ ACTOR Future<Key> findKey( StorageServer* data, KeySelectorRef sel, Version vers
*pOffset = 0;
StorageMetrics metrics;
metrics.bytesReadPerKSecond =
std::max((int64_t)rep.data[index].key.size(), SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE);
metrics.bytesReadPerKSecond = std::max((int64_t)rep.data[index].key.size(), SERVER_KNOBS->EMPTY_READ_PENALTY);
data->metrics.notify(sel.getKey(), metrics);
return rep.data[ index ].key;
} else {
StorageMetrics metrics;
metrics.bytesReadPerKSecond = SERVER_KNOBS->BYTES_READ_UNITS_PER_SAMPLE;
metrics.bytesReadPerKSecond = SERVER_KNOBS->EMPTY_READ_PENALTY;
data->metrics.notify(sel.getKey(), metrics);
// FIXME: If range.begin=="" && !forward, return success?
@ -1466,7 +1468,7 @@ ACTOR Future<Void> getKeyValues( StorageServer* data, GetKeyValuesRequest req )
for (int i = 0; i < r.data.size(); i++) {
StorageMetrics m;
m.bytesReadPerKSecond = r.data[i].expectedSize();
m.bytesReadPerKSecond = std::max((int64_t)r.data[i].expectedSize(), SERVER_KNOBS->EMPTY_READ_PENALTY);
data->metrics.notify(r.data[i].key, m);
}