wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3447 from sfc-gh-tclinkenbeard/remove-limit-knobs 

Remove ROW_LIMIT_UNLIMITED and BYTE_LIMIT_UNLIMITED from CLIENT_KNOBS
This commit is contained in:
Meng Xu 2020-07-04 20:01:39 -07:00 committed by GitHub
parent 63b4c1a16f bcbca81dfe
commit 1ce10a8db5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 32 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bindings/c/fdb_c.cpp
View File

@ -438,18 +438,17 @@ FDBFuture* fdb_transaction_get_range_impl(
} }
/* Zero at the C API maps to "infinity" at lower levels */ /* Zero at the C API maps to "infinity" at lower levels */
if (!limit) if (!limit) limit = GetRangeLimits::ROW_LIMIT_UNLIMITED;
limit = CLIENT_KNOBS->ROW_LIMIT_UNLIMITED; if (!target_bytes) target_bytes = GetRangeLimits::BYTE_LIMIT_UNLIMITED;
if (!target_bytes)
target_bytes = CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED;
/* Unlimited/unlimited with mode _EXACT isn't permitted */ /* Unlimited/unlimited with mode _EXACT isn't permitted */
if (limit == CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && mode == FDB_STREAMING_MODE_EXACT) if (limit == GetRangeLimits::ROW_LIMIT_UNLIMITED && target_bytes == GetRangeLimits::BYTE_LIMIT_UNLIMITED &&
mode == FDB_STREAMING_MODE_EXACT)
return TSAV_ERROR(Standalone<RangeResultRef>, exact_mode_without_limits); return TSAV_ERROR(Standalone<RangeResultRef>, exact_mode_without_limits);
/* _ITERATOR mode maps to one of the known streaming modes /* _ITERATOR mode maps to one of the known streaming modes
depending on iteration */ depending on iteration */
const int mode_bytes_array[] = { CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED, 256, 1000, 4096, 80000 }; const int mode_bytes_array[] = { GetRangeLimits::BYTE_LIMIT_UNLIMITED, 256, 1000, 4096, 80000 };
/* The progression used for FDB_STREAMING_MODE_ITERATOR. /* The progression used for FDB_STREAMING_MODE_ITERATOR.
Goes from small -> medium -> large. Then 1.5 * previous until serial. */ Goes from small -> medium -> large. Then 1.5 * previous until serial. */
@ -474,9 +473,9 @@ FDBFuture* fdb_transaction_get_range_impl(
else else
return TSAV_ERROR(Standalone<RangeResultRef>, client_invalid_operation); return TSAV_ERROR(Standalone<RangeResultRef>, client_invalid_operation);
if(target_bytes == CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) if (target_bytes == GetRangeLimits::BYTE_LIMIT_UNLIMITED)
target_bytes = mode_bytes; target_bytes = mode_bytes;
else if(mode_bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED) else if (mode_bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
target_bytes = std::min(target_bytes, mode_bytes); target_bytes = std::min(target_bytes, mode_bytes);
return (FDBFuture*)( TXN(tr)->getRange( return (FDBFuture*)( TXN(tr)->getRange(

3
fdbbackup/backup.actor.cpp
View File

@ -1554,7 +1554,8 @@ ACTOR Future<json_spirit::mObject> getLayerStatus(Database src, std::string root
try { try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::LOCK_AWARE); tr.setOption(FDBTransactionOptions::LOCK_AWARE);
state Standalone<RangeResultRef> kvPairs = wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), CLIENT_KNOBS->ROW_LIMIT_UNLIMITED)); state Standalone<RangeResultRef> kvPairs =
wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), GetRangeLimits::ROW_LIMIT_UNLIMITED));
json_spirit::mObject statusDoc; json_spirit::mObject statusDoc;
JSONDoc modifier(statusDoc); JSONDoc modifier(statusDoc);
for(auto &kv : kvPairs) { for(auto &kv : kvPairs) {

10
fdbclient/BackupAgentBase.actor.cpp
View File

@ -349,7 +349,10 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RangeResultWithVersi
loop{ loop{
try { try {
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, (g_network->isSimulated() && !g_simulator.speedUpSimulation) ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES); state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
: CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
if (systemAccess) if (systemAccess)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
@ -413,7 +416,10 @@ ACTOR Future<Void> readCommitted(Database cx, PromiseStream<RCGroup> results, Fu
loop{ loop{
try { try {
state GetRangeLimits limits(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, (g_network->isSimulated() && !g_simulator.speedUpSimulation) ? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES : CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES); state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED,
(g_network->isSimulated() && !g_simulator.speedUpSimulation)
? CLIENT_KNOBS->BACKUP_SIMULATED_LIMIT_BYTES
: CLIENT_KNOBS->BACKUP_GET_RANGE_LIMIT_BYTES);
if (systemAccess) if (systemAccess)
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);

3
fdbclient/Knobs.cpp
View File

@ -33,9 +33,6 @@ ClientKnobs::ClientKnobs() {
void ClientKnobs::initialize(bool randomize) { void ClientKnobs::initialize(bool randomize) {
// clang-format off // clang-format off
// FIXME: These are not knobs, get them out of ClientKnobs!
BYTE_LIMIT_UNLIMITED = GetRangeLimits::BYTE_LIMIT_UNLIMITED;
ROW_LIMIT_UNLIMITED = GetRangeLimits::ROW_LIMIT_UNLIMITED;
init( TOO_MANY, 1000000 ); init( TOO_MANY, 1000000 );

21
fdbclient/NativeAPI.actor.cpp
View File

@ -351,7 +351,7 @@ ACTOR static Future<Void> delExcessClntTxnEntriesActor(Transaction *tr, int64_t
if (txInfoSize < clientTxInfoSizeLimit) if (txInfoSize < clientTxInfoSizeLimit)
return Void(); return Void();
int getRangeByteLimit = (txInfoSize - clientTxInfoSizeLimit) < CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT ? (txInfoSize - clientTxInfoSizeLimit) : CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT; int getRangeByteLimit = (txInfoSize - clientTxInfoSizeLimit) < CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT ? (txInfoSize - clientTxInfoSizeLimit) : CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
GetRangeLimits limit(CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, getRangeByteLimit); GetRangeLimits limit(GetRangeLimits::ROW_LIMIT_UNLIMITED, getRangeByteLimit);
Standalone<RangeResultRef> txEntries = wait(tr->getRange(KeyRangeRef(clientLatencyName, strinc(clientLatencyName)), limit)); Standalone<RangeResultRef> txEntries = wait(tr->getRange(KeyRangeRef(clientLatencyName, strinc(clientLatencyName)), limit));
state int64_t numBytesToDel = 0; state int64_t numBytesToDel = 0;
KeyRef endKey; KeyRef endKey;
@ -1426,23 +1426,21 @@ Future<Reference<ProxyInfo>> DatabaseContext::getMasterProxiesFuture(bool usePro
} }
void GetRangeLimits::decrement( VectorRef<KeyValueRef> const& data ) { void GetRangeLimits::decrement( VectorRef<KeyValueRef> const& data ) {
if( rows != CLIENT_KNOBS->ROW_LIMIT_UNLIMITED ) { if (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED) {
ASSERT(data.size() <= rows); ASSERT(data.size() <= rows);
rows -= data.size(); rows -= data.size();
} }
minRows = std::max(0, minRows - data.size()); minRows = std::max(0, minRows - data.size());
if( bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED ) if (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
bytes = std::max( 0, bytes - (int)data.expectedSize() - (8-(int)sizeof(KeyValueRef))*data.size() ); bytes = std::max( 0, bytes - (int)data.expectedSize() - (8-(int)sizeof(KeyValueRef))*data.size() );
} }
void GetRangeLimits::decrement( KeyValueRef const& data ) { void GetRangeLimits::decrement( KeyValueRef const& data ) {
minRows = std::max(0, minRows - 1); minRows = std::max(0, minRows - 1);
if( rows != CLIENT_KNOBS->ROW_LIMIT_UNLIMITED ) if (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED) rows--;
rows--; if (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED) bytes = std::max(0, bytes - (int)8 - (int)data.expectedSize());
if( bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED )
bytes = std::max( 0, bytes - (int)8 - (int)data.expectedSize() );
} }
// True if either the row or byte limit has been reached // True if either the row or byte limit has been reached
@ -1452,16 +1450,17 @@ bool GetRangeLimits::isReached() {
// True if data would cause the row or byte limit to be reached // True if data would cause the row or byte limit to be reached
bool GetRangeLimits::reachedBy( VectorRef<KeyValueRef> const& data ) { bool GetRangeLimits::reachedBy( VectorRef<KeyValueRef> const& data ) {
return ( rows != CLIENT_KNOBS->ROW_LIMIT_UNLIMITED && data.size() >= rows ) return (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED && data.size() >= rows) ||
|| ( bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED && (int)data.expectedSize() + (8-(int)sizeof(KeyValueRef))*data.size() >= bytes && data.size() >= minRows ); (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED &&
(int)data.expectedSize() + (8 - (int)sizeof(KeyValueRef)) * data.size() >= bytes && data.size() >= minRows);
} }
bool GetRangeLimits::hasByteLimit() { bool GetRangeLimits::hasByteLimit() {
return bytes != CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED; return bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED;
} }
bool GetRangeLimits::hasRowLimit() { bool GetRangeLimits::hasRowLimit() {
return rows != CLIENT_KNOBS->ROW_LIMIT_UNLIMITED; return rows != GetRangeLimits::ROW_LIMIT_UNLIMITED;
} }
bool GetRangeLimits::hasSatisfiedMinRows() { bool GetRangeLimits::hasSatisfiedMinRows() {

2
fdbclient/ReadYourWrites.actor.cpp
View File

@ -451,7 +451,7 @@ public:
// Calculating request byte limit // Calculating request byte limit
if(requestLimit.bytes==0) { if(requestLimit.bytes==0) {
requestLimit.bytes = CLIENT_KNOBS->BYTE_LIMIT_UNLIMITED; requestLimit.bytes = GetRangeLimits::BYTE_LIMIT_UNLIMITED;
if(!requestLimit.hasRowLimit()) { if(!requestLimit.hasRowLimit()) {
requestLimit.rows = (int)std::min(std::max(std::max(1,requestLimit.rows) + additionalRows, (int64_t)offset), (int64_t)std::numeric_limits<int>::max()); requestLimit.rows = (int)std::min(std::max(std::max(1,requestLimit.rows) + additionalRows, (int64_t)offset), (int64_t)std::numeric_limits<int>::max());
} }

4
fdbserver/storageserver.actor.cpp
View File

@ -2177,7 +2177,9 @@ ACTOR Future<Void> fetchKeys( StorageServer *data, AddingShard* shard ) {
try { try {
TEST(true); // Fetching keys for transferred shard TEST(true); // Fetching keys for transferred shard
state Standalone<RangeResultRef> this_block = wait( tryGetRange( data->cx, fetchVersion, keys, GetRangeLimits( CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, fetchBlockBytes ), &isTooOld ) ); state Standalone<RangeResultRef> this_block =
wait(tryGetRange(data->cx, fetchVersion, keys,
GetRangeLimits(GetRangeLimits::ROW_LIMIT_UNLIMITED, fetchBlockBytes), &isTooOld));
int expectedSize = (int)this_block.expectedSize() + (8-(int)sizeof(KeyValueRef))*this_block.size(); int expectedSize = (int)this_block.expectedSize() + (8-(int)sizeof(KeyValueRef))*this_block.size();

2
fdbserver/workloads/IndexScan.actor.cpp
View File

@ -108,7 +108,7 @@ struct IndexScanWorkload : KVWorkload {
state int startNode = deterministicRandom()->randomInt(0, self->nodeCount / 2); //start in the first half of the database state int startNode = deterministicRandom()->randomInt(0, self->nodeCount / 2); //start in the first half of the database
state KeySelector begin = firstGreaterOrEqual( self->keyForIndex( startNode ) ); state KeySelector begin = firstGreaterOrEqual( self->keyForIndex( startNode ) );
state KeySelector end = firstGreaterThan( self->keyForIndex( self->nodeCount ) ); state KeySelector end = firstGreaterThan( self->keyForIndex( self->nodeCount ) );
state GetRangeLimits limits( CLIENT_KNOBS->ROW_LIMIT_UNLIMITED, self->bytesPerRead ); state GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, self->bytesPerRead);
state int rowsRead; state int rowsRead;
state int chunks; state int chunks;