wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbserver/Knobs.cpp

445 lines
33 KiB
C++
Raw Normal View History

Initial repository commit
2017-05-26 04:48:44 +08:00
/*
* Knobs.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
remove trailing whitespace from our copyright headers ; fixed formatting of python setup.py
2018-02-22 02:25:11 +08:00
*
Initial repository commit
2017-05-26 04:48:44 +08:00
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
remove trailing whitespace from our copyright headers ; fixed formatting of python setup.py
2018-02-22 02:25:11 +08:00
*
Initial repository commit
2017-05-26 04:48:44 +08:00
* http://www.apache.org/licenses/LICENSE-2.0
remove trailing whitespace from our copyright headers ; fixed formatting of python setup.py
2018-02-22 02:25:11 +08:00
*
Initial repository commit
2017-05-26 04:48:44 +08:00
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Adjust all includes to be relative to the root. Remove the use of relative paths. A header at foo/bar.h could be included by files under foo/ with "bar.h", but would be included everywhere else as "foo/bar.h". Adjust so that every include references such a header with the latter form. Signed-off-by: Robert Escriva <rescriva@dropbox.com>
2018-10-20 01:30:13 +08:00
#include "fdbserver/Knobs.h"
Initial repository commit
2017-05-26 04:48:44 +08:00
#include "fdbrpc/Locality.h"
Several small compilation fixes for new versions of gcc There are several missing includes for cmath in the code, I added those. Next, Coro returns a reference to a stack variable and this causes a warning. As this is probably ok for Coro, I disabled the warning in that file for GCC. I want to have this warning in the build system as it is generally a very useful warning to have. Another change is that major and minor are deprecated for a while now. I replaced those with gnu_dev_major and gnu_dev_minor. ErrorOr currently implements operators ==, !=, and <. These do not compile because Error does not implement ==. This compiles on older versions of gcc and clang because ErrorOr<T>::operator== is not used anywhere. It is still wrong though and newer gcc versions complain. I simply removed these methods. The most interesting fix is that TraceEvent::~TraceEvent is currently throwing exceptions. This is illegal behavior in C++11 and a idea in older versions of C++. For now I simply removed the throw, but this might need some more thought.
2018-05-06 00:38:09 +08:00
#include <cmath>
Initial repository commit
2017-05-26 04:48:44 +08:00
ServerKnobs const* SERVER_KNOBS = new ServerKnobs();
#define init( knob, value ) initKnob( knob, value, #knob )
ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
added a buffered cursor, which efficiently merges lots of peek cursors
2018-07-13 03:09:48 +08:00
// Versions
init( VERSIONS_PER_SECOND, 1e6 );
init( MAX_VERSIONS_IN_FLIGHT, 100 * VERSIONS_PER_SECOND );
init( MAX_VERSIONS_IN_FLIGHT_FORCED, 6e5 * VERSIONS_PER_SECOND ); //one week of versions
init( MAX_READ_TRANSACTION_LIFE_VERSIONS, 5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_READ_TRANSACTION_LIFE_VERSIONS = VERSIONS_PER_SECOND; else if (randomize && BUGGIFY) MAX_READ_TRANSACTION_LIFE_VERSIONS = std::max<int>(1, 0.1 * VERSIONS_PER_SECOND); else if( randomize && BUGGIFY ) MAX_READ_TRANSACTION_LIFE_VERSIONS = 10 * VERSIONS_PER_SECOND;
init( MAX_WRITE_TRANSACTION_LIFE_VERSIONS, 5 * VERSIONS_PER_SECOND ); if (randomize && BUGGIFY) MAX_WRITE_TRANSACTION_LIFE_VERSIONS=std::max<int>(1, 1 * VERSIONS_PER_SECOND);
init( MAX_COMMIT_BATCH_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 2.0; // Each master proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly
MAX_COMMIT_BATCH_INTERVAL = std::min(MAX_COMMIT_BATCH_INTERVAL, MAX_READ_TRANSACTION_LIFE_VERSIONS/double(2*VERSIONS_PER_SECOND)); // Ensure that the proxy commits 2 times every MAX_READ_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough
Initial repository commit
2017-05-26 04:48:44 +08:00
// TLogs
init( TLOG_TIMEOUT, 0.4 ); //cannot buggify because of availability
init( RECOVERY_TLOG_SMART_QUORUM_DELAY, 0.25 ); if( randomize && BUGGIFY ) RECOVERY_TLOG_SMART_QUORUM_DELAY = 0.0; // smaller might be better for bug amplification
init( TLOG_STORAGE_MIN_UPDATE_INTERVAL, 0.5 );
init( BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL, 30 );
init( UNFLUSHED_DATA_RATIO, 0.05 ); if( randomize && BUGGIFY ) UNFLUSHED_DATA_RATIO = 0.0;
init( DESIRED_TOTAL_BYTES, 150000 ); if( randomize && BUGGIFY ) DESIRED_TOTAL_BYTES = 10000;
The storage server would block the main thread when processing a single version with a large amount of data
2018-10-19 04:37:31 +08:00
init( DESIRED_UPDATE_BYTES, 2*DESIRED_TOTAL_BYTES );
init( UPDATE_DELAY, 0.001 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MAXIMUM_PEEK_BYTES, 10e6 );
init( APPLY_MUTATION_BYTES, 1e6 );
init( RECOVERY_DATA_BYTE_LIMIT, 100000 );
init( BUGGIFY_RECOVERY_DATA_LIMIT, 1000 );
init( LONG_TLOG_COMMIT_TIME, 0.25 ); //cannot buggify because of recovery time
init( LARGE_TLOG_COMMIT_BYTES, 4<<20 );
init( BUGGIFY_RECOVER_MEMORY_LIMIT, 1e6 );
init( BUGGIFY_WORKER_REMOVED_MAX_LAG, 30 );
init( UPDATE_STORAGE_BYTE_LIMIT, 1e6 );
Batch more when spilling data.
2019-03-04 04:57:35 +08:00
init( REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT, 20e6 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( TLOG_PEEK_DELAY, 0.00005 );
init( LEGACY_TLOG_UPGRADE_ENTRIES_PER_VERSION, 100 );
init( VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS, 1072 ); // Based on a naive interpretation of the gcc version of std::deque, we would expect this to be 16 bytes overhead per 512 bytes data. In practice, it seems to be 24 bytes overhead per 512.
The tlog uses the tags embedded in the message instead of a separate vector of locations optimized remote tlog committing to avoid re-serializing the message
2018-03-17 07:47:05 +08:00
init( VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD, std::ceil(16.0 * VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS / 1024) );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE, 1e5 );
init( MAX_MESSAGE_SIZE, std::max<int>(LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE, 1e5 + 2e4 + 1) + 8 ); // VALUE_SIZE_LIMIT + SYSTEM_KEY_SIZE_LIMIT + 9 bytes (4 bytes for length, 4 bytes for sequence number, and 1 byte for mutation type)
init( TLOG_MESSAGE_BLOCK_BYTES, 10e6 );
updated release notes for 5.0.1
2017-06-29 07:52:45 +08:00
init( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR, double(TLOG_MESSAGE_BLOCK_BYTES) / (TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE) ); //1.0121466709838096006362758832473
all timeouts longer than two minutes have been can be lowered to 60.0 with buggification added a workload that tries for a 50 second maximum latency in the presence of one failure with both buggification and connection failures
2017-09-19 02:04:51 +08:00
init( PEEK_TRACKER_EXPIRATION_TIME, 600 ); if( randomize && BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = g_random->coinflip() ? 0.1 : 60;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( PARALLEL_GET_MORE_REQUESTS, 32 ); if( randomize && BUGGIFY ) PARALLEL_GET_MORE_REQUESTS = 2;
multi cursors start a get more for the first 10 cursors to hide latency
2018-10-04 04:57:45 +08:00
init( MULTI_CURSOR_PRE_FETCH_LIMIT, 10 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MAX_QUEUE_COMMIT_BYTES, 15e6 ); if( randomize && BUGGIFY ) MAX_QUEUE_COMMIT_BYTES = 5000;
added more yields to avoid slow tasks
2018-07-13 08:47:35 +08:00
init( VERSIONS_PER_BATCH, VERSIONS_PER_SECOND/20 ); if( randomize && BUGGIFY ) VERSIONS_PER_BATCH = std::max<int64_t>(1,VERSIONS_PER_SECOND/1000);
tlogs serve reads to log routers at a low priority, to prevent them from using all their resources catching up a remote dc that has been down for a long time increase the amount of memory ratekeeper budgets for tlogs so that there is a gap after the spill threshold to prevent temporarily overshooting the budget
2018-08-05 01:31:30 +08:00
init( CONCURRENT_LOG_ROUTER_READS, 1 );
fix: if the disk queue adapter peek hangs for a while, switch to a peek from a different locality
2018-10-04 04:58:55 +08:00
init( DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME, 1.0 );
init( DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME, 5.0 );
Implement hard limiting of memory used to serve peek requests.
2019-03-04 04:57:38 +08:00
init( TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES, 2e9 ); if ( randomize && BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES = 2e6;
Allow the disk queue to shrink if it has unneeded slack space.
2019-03-04 04:57:44 +08:00
init( DISK_QUEUE_FILE_EXTENSION_BYTES, 10<<20 ); // BUGGIFYd per file within the DiskQueue
when a tlog queue commit takes longer than 5 seconds, its process is marked as degraded
2019-03-09 00:46:34 +08:00
init( DISK_QUEUE_FILE_SHRINK_BYTES, 100<<20 ); // BUGGIFYd per file within the DiskQueue
init( TLOG_DEGRADED_DELAY_COUNT, 5 );
init( TLOG_DEGRADED_DURATION, 5.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
// Data distribution queue
init( HEALTH_POLL_TIME, 1.0 );
init( BEST_TEAM_STUCK_DELAY, 1.0 );
init( BG_DD_POLLING_INTERVAL, 10.0 );
init( DD_QUEUE_LOGGING_INTERVAL, 5.0 );
init( RELOCATION_PARALLELISM_PER_SOURCE_SERVER, 4 ); if( randomize && BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1;
init( DD_QUEUE_MAX_KEY_SERVERS, 100 ); if( randomize && BUGGIFY ) DD_QUEUE_MAX_KEY_SERVERS = 1;
init( DD_REBALANCE_PARALLELISM, 50 );
init( DD_REBALANCE_RESET_AMOUNT, 30 );
init( BG_DD_MAX_WAIT, 120.0 );
init( BG_DD_MIN_WAIT, 0.1 );
init( BG_DD_INCREASE_RATE, 1.10 );
init( BG_DD_DECREASE_RATE, 1.02 );
init( BG_DD_SATURATION_DELAY, 1.0 );
init( INFLIGHT_PENALTY_HEALTHY, 1.0 );
init( INFLIGHT_PENALTY_UNHEALTHY, 10.0 );
init( INFLIGHT_PENALTY_ONE_LEFT, 1000.0 );
// Data distribution
init( RETRY_RELOCATESHARD_DELAY, 0.1 );
init( DATA_DISTRIBUTION_FAILURE_REACTION_TIME, 10.0 ); if( randomize && BUGGIFY ) DATA_DISTRIBUTION_FAILURE_REACTION_TIME = 1.0;
bool buggifySmallShards = randomize && BUGGIFY;
init( MIN_SHARD_BYTES, 200000 ); if( buggifySmallShards ) MIN_SHARD_BYTES = 40000; //FIXME: data distribution tracker (specifically StorageMetrics) relies on this number being larger than the maximum size of a key value pair
init( SHARD_BYTES_RATIO, 4 );
init( SHARD_BYTES_PER_SQRT_BYTES, 45 ); if( buggifySmallShards ) SHARD_BYTES_PER_SQRT_BYTES = 0;//Approximately 10000 bytes per shard
init( MAX_SHARD_BYTES, 500000000 );
the keyservers shards are always a fixed large size
2017-10-28 02:52:11 +08:00
init( KEY_SERVER_SHARD_BYTES, 500000000 );
Initial repository commit
2017-05-26 04:48:44 +08:00
bool buggifySmallBandwidthSplit = randomize && BUGGIFY;
init( SHARD_MAX_BYTES_PER_KSEC, 1LL*1000000*1000 ); if( buggifySmallBandwidthSplit ) SHARD_MAX_BYTES_PER_KSEC = 10LL*1000*1000;
/* 10*1MB/sec * 1000sec/ksec
Shards with more than this bandwidth will be split immediately.
For a large shard (100MB), splitting it costs ~100MB of work or about 10MB/sec over a 10 sec sampling window.
If the sampling window is too much longer, the MVCC window will fill up while we wait.
If SHARD_MAX_BYTES_PER_KSEC is too much lower, we could do a lot of data movement work in response to a small impulse of bandwidth.
If SHARD_MAX_BYTES_PER_KSEC is too high relative to the I/O bandwidth of a given server, a workload can remain concentrated on a single
team indefinitely, limiting performance.
*/
init( SHARD_MIN_BYTES_PER_KSEC, 100 * 1000 * 1000 ); if( buggifySmallBandwidthSplit ) SHARD_MIN_BYTES_PER_KSEC = 200*1*1000;
/* 200*1KB/sec * 1000sec/ksec
Shards with more than this bandwidth will not be merged.
Obviously this needs to be significantly less than SHARD_MAX_BYTES_PER_KSEC, else we will repeatedly merge and split.
It should probably be significantly less than SHARD_SPLIT_BYTES_PER_KSEC, else we will merge right after splitting.
The number of extra shards in the database because of bandwidth splitting can't be more than about W/SHARD_MIN_BYTES_PER_KSEC, where
W is the maximum bandwidth of the entire database in bytes/ksec. For 250MB/sec write bandwidth, (250MB/sec)/(200KB/sec) = 1250 extra
shards.
The bandwidth sample maintained by the storage server needs to be accurate enough to reliably measure this minimum bandwidth. See
BANDWIDTH_UNITS_PER_SAMPLE. If this number is too low, the storage server needs to spend more memory and time on sampling.
*/
init( SHARD_SPLIT_BYTES_PER_KSEC, 250 * 1000 * 1000 ); if( buggifySmallBandwidthSplit ) SHARD_SPLIT_BYTES_PER_KSEC = 50 * 1000 * 1000;
/* 500*1KB/sec * 1000sec/ksec
When splitting a shard, it is split into pieces with less than this bandwidth.
Obviously this should be less than half of SHARD_MAX_BYTES_PER_KSEC.
Smaller values mean that high bandwidth shards are split into more pieces, more quickly utilizing large numbers of servers to handle the
bandwidth.
Too many pieces (too small a value) may stress data movement mechanisms (see e.g. RELOCATION_PARALLELISM_PER_SOURCE_SERVER).
If this value is too small relative to SHARD_MIN_BYTES_PER_KSEC immediate merging work will be generated.
*/
all timeouts longer than two minutes have been can be lowered to 60.0 with buggification added a workload that tries for a 50 second maximum latency in the presence of one failure with both buggification and connection failures
2017-09-19 02:04:51 +08:00
init( STORAGE_METRIC_TIMEOUT, 600.0 ); if( randomize && BUGGIFY ) STORAGE_METRIC_TIMEOUT = g_random->coinflip() ? 10.0 : 60.0;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( METRIC_DELAY, 0.1 ); if( randomize && BUGGIFY ) METRIC_DELAY = 1.0;
init( ALL_DATA_REMOVED_DELAY, 1.0 );
init( INITIAL_FAILURE_REACTION_DELAY, 30.0 ); if( randomize && BUGGIFY ) INITIAL_FAILURE_REACTION_DELAY = 0.0;
fix: give time to detect failed servers before building teams
2018-06-11 11:21:39 +08:00
init( CHECK_TEAM_DELAY, 30.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( LOG_ON_COMPLETION_DELAY, DD_QUEUE_LOGGING_INTERVAL );
init( BEST_TEAM_MAX_TEAM_TRIES, 10 );
init( BEST_TEAM_OPTION_COUNT, 4 );
init( BEST_OF_AMT, 4 );
init( SERVER_LIST_DELAY, 1.0 );
init( RECRUITMENT_IDLE_DELAY, 1.0 );
do not spin on a failed storage server recruitment
2018-10-03 08:31:07 +08:00
init( STORAGE_RECRUITMENT_DELAY, 10.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( DATA_DISTRIBUTION_LOGGING_INTERVAL, 5.0 );
init( DD_ENABLED_CHECK_DELAY, 1.0 );
fix: remove bad teams needed to use dd_stall_check delay, because in simulation the buggified delay time could make us remove bad teams before they submit their ranges to the queue
2019-02-21 06:18:36 +08:00
init( DD_STALL_CHECK_DELAY, 0.4 ); //Must be larger than 2*MAX_BUGGIFIED_DELAY
Initial repository commit
2017-05-26 04:48:44 +08:00
init( DD_MERGE_COALESCE_DELAY, 120.0 ); if( randomize && BUGGIFY ) DD_MERGE_COALESCE_DELAY = 0.001;
init( STORAGE_METRICS_POLLING_DELAY, 2.0 ); if( randomize && BUGGIFY ) STORAGE_METRICS_POLLING_DELAY = 15.0;
init( STORAGE_METRICS_RANDOM_DELAY, 0.2 );
init( FREE_SPACE_RATIO_CUTOFF, 0.1 );
init( FREE_SPACE_RATIO_DD_CUTOFF, 0.2 );
init( DESIRED_TEAMS_PER_SERVER, 5 ); if( randomize && BUGGIFY ) DESIRED_TEAMS_PER_SERVER = 1;
fix: do not attempt data movement to an unhealthy destination team allow building more teams than desired if all teams are unhealthy bestTeamStuck is an error in simulation again
2018-07-08 07:51:16 +08:00
init( MAX_TEAMS_PER_SERVER, 3*DESIRED_TEAMS_PER_SERVER );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( DD_SHARD_SIZE_GRANULARITY, 5000000 );
init( DD_SHARD_SIZE_GRANULARITY_SIM, 500000 ); if( randomize && BUGGIFY ) DD_SHARD_SIZE_GRANULARITY_SIM = 0;
init( DD_MOVE_KEYS_PARALLELISM, 20 ); if( randomize && BUGGIFY ) DD_MOVE_KEYS_PARALLELISM = 1;
init( DD_MERGE_LIMIT, 2000 ); if( randomize && BUGGIFY ) DD_MERGE_LIMIT = 2;
init( DD_SHARD_METRICS_TIMEOUT, 60.0 ); if( randomize && BUGGIFY ) DD_SHARD_METRICS_TIMEOUT = 0.1;
init( DD_LOCATION_CACHE_SIZE, 2000000 ); if( randomize && BUGGIFY ) DD_LOCATION_CACHE_SIZE = 3;
init( MOVEKEYS_LOCK_POLLING_DELAY, 5.0 );
init( DEBOUNCE_RECRUITING_DELAY, 5.0 );
Add a failure timeout knob for data distributor. Set default time to 1.0s.
2019-01-19 07:43:43 +08:00
init( DD_FAILURE_TIME, 1.0 ); if( randomize && BUGGIFY ) DD_FAILURE_TIME = 10.0;
Fix another build team bug The buildTeam() can create teams with undesired storage servers, which are considered unhealthy. As a result, the data movement can become stuck. Fix this by adding an ACTOR monitorHealthyTeams that builds team every one second whenever there is no healthy teams. Clean up storageServerTracker() interface.
2019-02-13 06:02:21 +08:00
init( DD_ZERO_HEALTHY_TEAM_DELAY, 1.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
TeamCollection: Add knobs for team remover Added three knobs to control team remover bool TR_FLAG_DISABLE_TEAM_REMOVER: Disable the teamRemover actor double TR_REMOVE_MACHINE_TEAM_DELAY: Wait for the specified time before try to remove next machine team double TR_WAIT_FOR_ALL_MACHINES_HEALTHY_DELAY: Wait before checking if all machines are healthy
2019-02-14 07:11:53 +08:00
// TeamRemover
TR_FLAG_DISABLE_TEAM_REMOVER = false; if( randomize && BUGGIFY ) TR_FLAG_DISABLE_TEAM_REMOVER = g_random->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true
TeamRemover: Change it to run periodically This simplifies the problem of when we should invoke the teamRemover
2019-02-21 07:55:07 +08:00
init( TR_REMOVE_MACHINE_TEAM_DELAY, 60.0 ); if( randomize && BUGGIFY ) TR_REMOVE_MACHINE_TEAM_DELAY = g_random->random01() * 60.0;
TeamCollection: Add knobs for team remover Added three knobs to control team remover bool TR_FLAG_DISABLE_TEAM_REMOVER: Disable the teamRemover actor double TR_REMOVE_MACHINE_TEAM_DELAY: Wait for the specified time before try to remove next machine team double TR_WAIT_FOR_ALL_MACHINES_HEALTHY_DELAY: Wait before checking if all machines are healthy
2019-02-14 07:11:53 +08:00
VersionedBTree now uses PrefixTree based pages (with bugs). This required significant changes to both classes because the interface and semantics for building, seeking in, and iterating through pages is very different from the previous trivial approach which was based on serialized vectors. PrefixTree node format rewritten to support optional values without increasing overhead for common node scenarios. PrefixTree::Cursor rewritten to reuse path prefix memory instead of allocating new memory on each movement which is then 'leaked' until destruction. PrefixTree::Cursor movement modified to work better with VersionedBTree::InternalCursor, which was also heavily modified. Added knobs related to key arrangement in PrefixTree nodes. Added StringRef::toHexString() as an alternative to printable() to make reading raw PrefixTree data easier. PrefixTree performance is temporarily worse with this update and VersionedBtree fails its unit test.
2018-06-08 18:32:34 +08:00
// Redwood Storage Engine
init( PREFIX_TREE_IMMEDIATE_KEY_SIZE_LIMIT, 30 );
init( PREFIX_TREE_IMMEDIATE_KEY_SIZE_MIN, 0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
// KeyValueStore SQLITE
init( CLEAR_BUFFER_SIZE, 20000 );
init( READ_VALUE_TIME_ESTIMATE, .00005 );
init( READ_RANGE_TIME_ESTIMATE, .00005 );
init( SET_TIME_ESTIMATE, .00005 );
init( CLEAR_TIME_ESTIMATE, .00005 );
init( COMMIT_TIME_ESTIMATE, .005 );
init( CHECK_FREE_PAGE_AMOUNT, 100 ); if( randomize && BUGGIFY ) CHECK_FREE_PAGE_AMOUNT = 5;
init( DISK_METRIC_LOGGING_INTERVAL, 5.0 );
init( SOFT_HEAP_LIMIT, 300e6 );
init( SQLITE_PAGE_SCAN_ERROR_LIMIT, 10000 );
init( SQLITE_BTREE_PAGE_USABLE, 4096 - 8); // pageSize - reserveSize for page checksum
// Maximum and minimum cell payload bytes allowed on primary page as calculated in SQLite.
// These formulas are copied from SQLite, using its hardcoded constants, so if you are
// changing this you should also be changing SQLite.
init( SQLITE_BTREE_CELL_MAX_LOCAL, (SQLITE_BTREE_PAGE_USABLE - 12) * 64/255 - 23 );
init( SQLITE_BTREE_CELL_MIN_LOCAL, (SQLITE_BTREE_PAGE_USABLE - 12) * 32/255 - 23 );
// Maximum FDB fragment key and value bytes that can fit in a primary btree page
init( SQLITE_FRAGMENT_PRIMARY_PAGE_USABLE,
SQLITE_BTREE_CELL_MAX_LOCAL
- 1 // vdbeRecord header length size
- 2 // max key length size
- 4 // max index length size
- 2 // max value fragment length size
);
// Maximum FDB fragment value bytes in an overflow page
init( SQLITE_FRAGMENT_OVERFLOW_PAGE_USABLE,
SQLITE_BTREE_PAGE_USABLE
- 4 // next pageNumber size
);
init( SQLITE_FRAGMENT_MIN_SAVINGS, 0.20 );
// KeyValueStoreSqlite spring cleaning
init( CLEANING_INTERVAL, 1.0 );
init( SPRING_CLEANING_TIME_ESTIMATE, .010 );
init( SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE, 0.0 ); if( randomize && BUGGIFY ) SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE = g_random->coinflip() ? 1e9 : g_random->random01() * 5;
init( SPRING_CLEANING_MIN_LAZY_DELETE_PAGES, 0 ); if( randomize && BUGGIFY ) SPRING_CLEANING_MIN_LAZY_DELETE_PAGES = g_random->randomInt(1, 100);
init( SPRING_CLEANING_MAX_LAZY_DELETE_PAGES, 1e9 ); if( randomize && BUGGIFY ) SPRING_CLEANING_MAX_LAZY_DELETE_PAGES = g_random->coinflip() ? 0 : g_random->randomInt(1, 1e4);
init( SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE, 100 ); if( randomize && BUGGIFY ) SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE = g_random->randomInt(1, 1000);
init( SPRING_CLEANING_MIN_VACUUM_PAGES, 1 ); if( randomize && BUGGIFY ) SPRING_CLEANING_MIN_VACUUM_PAGES = g_random->randomInt(0, 100);
init( SPRING_CLEANING_MAX_VACUUM_PAGES, 1e9 ); if( randomize && BUGGIFY ) SPRING_CLEANING_MAX_VACUUM_PAGES = g_random->coinflip() ? 0 : g_random->randomInt(1, 1e4);
knobify replace contents bytes
2018-05-02 10:43:35 +08:00
// KeyValueStoreMemory
init( REPLACE_CONTENTS_BYTES, 1e5 ); if( randomize && BUGGIFY ) REPLACE_CONTENTS_BYTES = 1e3;
Initial repository commit
2017-05-26 04:48:44 +08:00
// Leader election
bool longLeaderElection = randomize && BUGGIFY;
fix: fixed a memory leak where leaderInfo notifications are not cleared out
2018-07-07 08:40:29 +08:00
init( MAX_NOTIFICATIONS, 100000 );
slowly send notifications to clients to clear the list of dead clients
2018-08-09 08:29:32 +08:00
init( MIN_NOTIFICATIONS, 100 );
init( NOTIFICATION_FULL_CLEAR_TIME, 10000.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( CANDIDATE_MIN_DELAY, 0.05 );
init( CANDIDATE_MAX_DELAY, 1.0 );
init( CANDIDATE_GROWTH_RATE, 1.2 );
init( POLLING_FREQUENCY, 1.0 ); if( longLeaderElection ) POLLING_FREQUENCY = 8.0;
init( HEARTBEAT_FREQUENCY, 0.25 ); if( longLeaderElection ) HEARTBEAT_FREQUENCY = 1.0;
// Master Proxy
init( START_TRANSACTION_BATCH_INTERVAL_MIN, 1e-6 );
init( START_TRANSACTION_BATCH_INTERVAL_MAX, 0.010 );
init( START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION, 0.5 );
init( START_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA, 0.1 );
init( START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL, 0.001 );
init( START_TRANSACTION_MAX_TRANSACTIONS_TO_START, 10000 );
init( START_TRANSACTION_MAX_BUDGET_SIZE, 20 ); // Currently set to match CLIENT_KNOBS->MAX_BATCH_SIZE
init( COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, 0.0005 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE = 0.005;
init( COMMIT_TRANSACTION_BATCH_INTERVAL_MIN, 0.001 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_MIN = 0.1;
init( COMMIT_TRANSACTION_BATCH_INTERVAL_MAX, 0.020 );
init( COMMIT_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION, 0.1 );
init( COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA, 0.1 );
init( COMMIT_TRANSACTION_BATCH_COUNT_MAX, 32768 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_COUNT_MAX = 1000; // Do NOT increase this number beyond 32768, as CommitIds only budget 2 bytes for storing transaction id within each batch
Proxy going OOM when applying high volumes of writes to a proxy, particular in a sudden fashion before ratekeeper can control the workload. Address this issue by proactively monitoring the memory used by commit batches and dropping requests if a certain memory limit is exceeded.
2018-06-02 06:21:40 +08:00
init( COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT, 8LL << 30 ); if (randomize && BUGGIFY) COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT = g_random->randomInt64(100LL << 20, 8LL << 30);
added a workload which tests killing an entire region, and recovering from the failure with data loss. fix: we cannot pop the txs tag from remote logs until they have a full copy of the txnStateStore fix: we have to modify all of history, we cannot stop after finding a local remote
2018-09-18 09:32:39 +08:00
init( COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL, 0.5 );
init( COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR, 10.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
// these settings disable batch bytes scaling. Try COMMIT_TRANSACTION_BATCH_BYTES_MAX=1e6, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE=50000, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER=0.5?
init( COMMIT_TRANSACTION_BATCH_BYTES_MIN, 100000 );
init( COMMIT_TRANSACTION_BATCH_BYTES_MAX, 100000 ); if( randomize && BUGGIFY ) { COMMIT_TRANSACTION_BATCH_BYTES_MIN = COMMIT_TRANSACTION_BATCH_BYTES_MAX = 1000000; }
init( COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE, 100000 );
init( COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER, 0.0 );
init( TRANSACTION_BUDGET_TIME, 0.050 ); if( randomize && BUGGIFY ) TRANSACTION_BUDGET_TIME = 0.0;
init( RESOLVER_COALESCE_TIME, 1.0 );
init( BUGGIFIED_ROW_LIMIT, APPLY_MUTATION_BYTES ); if( randomize && BUGGIFY ) BUGGIFIED_ROW_LIMIT = g_random->randomInt(3, 30);
init( PROXY_SPIN_DELAY, 0.01 );
added a workload which tests killing an entire region, and recovering from the failure with data loss. fix: we cannot pop the txs tag from remote logs until they have a full copy of the txnStateStore fix: we have to modify all of history, we cannot stop after finding a local remote
2018-09-18 09:32:39 +08:00
init( UPDATE_REMOTE_LOG_VERSION_INTERVAL, 2.0 );
init( MAX_TXS_POP_VERSION_HISTORY, 1e5 );
Initial repository commit
2017-05-26 04:48:44 +08:00
// Master Server
// masterCommitter() in the master server will allow lower priority tasks (e.g. DataDistibution)
// by delay()ing for this amount of time between accepted batches of TransactionRequests.
buggify resolution balancing so that it still happens in simulation
2018-11-12 16:03:07 +08:00
bool fastBalancing = randomize && BUGGIFY;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( COMMIT_SLEEP_TIME, 0.0001 ); if( randomize && BUGGIFY ) COMMIT_SLEEP_TIME = 0;
updated resolution balancing knobs to be a little more aggressive
2018-11-13 11:11:28 +08:00
init( KEY_BYTES_PER_SAMPLE, 2e4 ); if( fastBalancing ) KEY_BYTES_PER_SAMPLE = 1e3;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MIN_BALANCE_TIME, 0.2 );
updated resolution balancing knobs to be a little more aggressive
2018-11-13 11:11:28 +08:00
init( MIN_BALANCE_DIFFERENCE, 1e6 ); if( fastBalancing ) MIN_BALANCE_DIFFERENCE = 1e4;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( SECONDS_BEFORE_NO_FAILURE_DELAY, 8 * 3600 );
init( MAX_TXS_SEND_MEMORY, 1e7 ); if( randomize && BUGGIFY ) MAX_TXS_SEND_MEMORY = 1e5;
fix: we cannot make MAX_RECOVERY_VERSIONS lower than MAX_VERSIONS_IN_FLIGHT because we can mark a recovery as stalled before finishing the recovery, leading to an infinite loop of recoveries
2018-07-08 08:41:20 +08:00
init( MAX_RECOVERY_VERSIONS, 200 * VERSIONS_PER_SECOND );
The master will tell the cluster controller if it is going to take a long time to recruit new logs in its DC; the cluster controller can determine if the other DC would be better and recruit there. The cluster controller will not switch to the other data center if remote logs are too far behind. We will not recruit in DCs with negative priority.
2018-06-14 09:14:14 +08:00
init( MAX_RECOVERY_TIME, 20.0 ); if( randomize && BUGGIFY ) MAX_RECOVERY_TIME = 1.0;
Initial repository commit
2017-05-26 04:48:44 +08:00
// Resolver
init( SAMPLE_OFFSET_PER_KEY, 100 );
init( SAMPLE_EXPIRATION_TIME, 1.0 );
init( SAMPLE_POLL_TIME, 0.1 );
init( RESOLVER_STATE_MEMORY_LIMIT, 1e6 );
init( LAST_LIMITED_RATIO, 0.6 );
//Cluster Controller
added logging for the datacenter version difference
2018-06-22 07:31:52 +08:00
init( CLUSTER_CONTROLLER_LOGGING_DELAY, 5.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MASTER_FAILURE_REACTION_TIME, 0.4 ); if( randomize && BUGGIFY ) MASTER_FAILURE_REACTION_TIME = 10.0;
init( MASTER_FAILURE_SLOPE_DURING_RECOVERY, 0.1 );
init( WORKER_COORDINATION_PING_DELAY, 60 );
init( SIM_SHUTDOWN_TIMEOUT, 10 );
all timeouts longer than two minutes have been can be lowered to 60.0 with buggification added a workload that tries for a 50 second maximum latency in the presence of one failure with both buggification and connection failures
2017-09-19 02:04:51 +08:00
init( SHUTDOWN_TIMEOUT, 600 ); if( randomize && BUGGIFY ) SHUTDOWN_TIMEOUT = 60.0;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MASTER_SPIN_DELAY, 1.0 ); if( randomize && BUGGIFY ) MASTER_SPIN_DELAY = 10.0;
re-enabled better master exists the cluster controller can choose a better data center for itself and let the workers know where the next cluster controller should be recruited
2018-02-10 08:48:55 +08:00
init( CC_CHANGE_DELAY, 0.1 );
update better machine classes first to give them a higher chance of becoming the next cluster controller
2018-06-29 16:11:59 +08:00
init( CC_CLASS_DELAY, 0.01 );
Swapped OkayFit and UnsetFit, because generally if machine classes are set on one machine they are set everywhere and it helps with wait_for_good_recruitment logic wait_for_good_recruitment now requires that you have the desired count of each roll remote recruitment is given a much longer wait_for_good_recruitment time interval, which does not start until enough remote machines have registered
2018-06-23 01:15:24 +08:00
init( WAIT_FOR_GOOD_RECRUITMENT_DELAY, 1.0 );
init( WAIT_FOR_GOOD_REMOTE_RECRUITMENT_DELAY, 5.0 );
fix: knob was set to incorrect value
2018-02-07 10:11:45 +08:00
init( ATTEMPT_RECRUITMENT_DELAY, 0.035 );
Per review comments Add a knob for starting distributor delay. Move distributor failed variable to a local loop.
2019-02-01 02:51:25 +08:00
init( WAIT_FOR_DISTRIBUTOR_JOIN_DELAY, 1.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( WORKER_FAILURE_TIME, 1.0 ); if( randomize && BUGGIFY ) WORKER_FAILURE_TIME = 10.0;
checking outstanding requests can be CPU intensive, so rate limit checking requests
2018-06-28 14:02:08 +08:00
init( CHECK_OUTSTANDING_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) CHECK_OUTSTANDING_INTERVAL = 0.001;
The master will tell the cluster controller if it is going to take a long time to recruit new logs in its DC; the cluster controller can determine if the other DC would be better and recruit there. The cluster controller will not switch to the other data center if remote logs are too far behind. We will not recruit in DCs with negative priority.
2018-06-14 09:14:14 +08:00
init( VERSION_LAG_METRIC_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) VERSION_LAG_METRIC_INTERVAL = 10.0;
init( MAX_VERSION_DIFFERENCE, 20 * VERSIONS_PER_SECOND );
knobify force recovery recovery check delay
2019-02-20 08:05:20 +08:00
init( FORCE_RECOVERY_CHECK_DELAY, 5.0 );
The master will tell the cluster controller if it is going to take a long time to recruit new logs in its DC; the cluster controller can determine if the other DC would be better and recruit there. The cluster controller will not switch to the other data center if remote logs are too far behind. We will not recruit in DCs with negative priority.
2018-06-14 09:14:14 +08:00
all timeouts longer than two minutes have been can be lowered to 60.0 with buggification added a workload that tries for a 50 second maximum latency in the presence of one failure with both buggification and connection failures
2017-09-19 02:04:51 +08:00
init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
Swapped OkayFit and UnsetFit, because generally if machine classes are set on one machine they are set everywhere and it helps with wait_for_good_recruitment logic wait_for_good_recruitment now requires that you have the desired count of each roll remote recruitment is given a much longer wait_for_good_recruitment time interval, which does not start until enough remote machines have registered
2018-06-23 01:15:24 +08:00
init( EXPECTED_MASTER_FITNESS, ProcessClass::UnsetFit );
init( EXPECTED_TLOG_FITNESS, ProcessClass::UnsetFit );
init( EXPECTED_LOG_ROUTER_FITNESS, ProcessClass::UnsetFit );
init( EXPECTED_PROXY_FITNESS, ProcessClass::UnsetFit );
init( EXPECTED_RESOLVER_FITNESS, ProcessClass::UnsetFit );
all timeouts longer than two minutes have been can be lowered to 60.0 with buggification added a workload that tries for a 50 second maximum latency in the presence of one failure with both buggification and connection failures
2017-09-19 02:04:51 +08:00
init( RECRUITMENT_TIMEOUT, 600 ); if( randomize && BUGGIFY ) RECRUITMENT_TIMEOUT = g_random->coinflip() ? 60.0 : 1.0;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( POLICY_RATING_TESTS, 200 ); if( randomize && BUGGIFY ) POLICY_RATING_TESTS = 20;
init( POLICY_GENERATIONS, 100 ); if( randomize && BUGGIFY ) POLICY_GENERATIONS = 10;
//Move Keys
init( SHARD_READY_DELAY, 0.25 );
init( SERVER_READY_QUORUM_INTERVAL, std::min(1.0, std::min(MAX_READ_TRANSACTION_LIFE_VERSIONS, MAX_WRITE_TRANSACTION_LIFE_VERSIONS)/(5.0*VERSIONS_PER_SECOND)) );
init( SERVER_READY_QUORUM_TIMEOUT, 15.0 ); if( randomize && BUGGIFY ) SERVER_READY_QUORUM_TIMEOUT = 1.0;
init( REMOVE_RETRY_DELAY, 1.0 );
init( MOVE_KEYS_KRM_LIMIT, 2000 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT = 2;
init( MOVE_KEYS_KRM_LIMIT_BYTES, 1e5 ); if( randomize && BUGGIFY ) MOVE_KEYS_KRM_LIMIT_BYTES = 5e4; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map
optimized the tlog to use a vector for tags instead of a map
2018-03-18 01:36:19 +08:00
init( MAX_SKIP_TAGS, 1 ); //The TLogs require tags to be densely packed to be memory efficient, so be careful increasing this knob
fix: start move keys will only move destination servers to become source servers if less than destination servers are healthy and the total number of sources is less than 2x the number of destinations
2018-09-01 03:43:14 +08:00
init( MAX_ADDED_SOURCES_MULTIPLIER, 2.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
//FdbServer
bool longReboots = randomize && BUGGIFY;
init( MIN_REBOOT_TIME, 4.0 ); if( longReboots ) MIN_REBOOT_TIME = 10.0;
init( MAX_REBOOT_TIME, 5.0 ); if( longReboots ) MAX_REBOOT_TIME = 20.0;
Add a string knob to hold the Log directory, and write profiles to it. This is the combination of two small changes. 1. Add support for a string knob type. 2. Change profiles to be written to the log directory instead of the working directory. We have three options of where to write files: the working directory, the data directory, and the log directory. The working directory may be set to a non-writable location, and likely contains the fdb binaries. Allowing these files to be overwritten would likely not be a wise idea. The data directory hosts our sqlite b-trees. It would also be very unfortunate if these were ever overwritten by an unfortunate profile name. The log directory contains logs. Out of the three, these matter the least if they disappear or become corrupted. Thus, we write to the log directory.
2017-10-13 08:49:41 +08:00
init( LOG_DIRECTORY, "."); // Will be set to the command line flag.
Proxy going OOM when applying high volumes of writes to a proxy, particular in a sudden fashion before ratekeeper can control the workload. Address this issue by proactively monitoring the memory used by commit batches and dropping requests if a certain memory limit is exceeded.
2018-06-02 06:21:40 +08:00
init(SERVER_MEM_LIMIT, 8LL << 30);
Initial repository commit
2017-05-26 04:48:44 +08:00
//Ratekeeper
bool slowRateKeeper = randomize && BUGGIFY;
init( SMOOTHING_AMOUNT, 1.0 ); if( slowRateKeeper ) SMOOTHING_AMOUNT = 5.0;
init( SLOW_SMOOTHING_AMOUNT, 10.0 ); if( slowRateKeeper ) SLOW_SMOOTHING_AMOUNT = 50.0;
init( METRIC_UPDATE_RATE, .1 ); if( slowRateKeeper ) METRIC_UPDATE_RATE = 0.5;
Track health metrics in Ratekeeper and send these metrics to proxies in GetRateInfoReply messages
2019-02-01 04:56:58 +08:00
init( DETAILED_METRIC_UPDATE_RATE, 5.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
bool smallStorageTarget = randomize && BUGGIFY;
fix: tripled the smallStorageTarget to prevent simulations which do a lot of work from timing out
2018-05-08 08:26:44 +08:00
init( TARGET_BYTES_PER_STORAGE_SERVER, 1000e6 ); if( smallStorageTarget ) TARGET_BYTES_PER_STORAGE_SERVER = 3000e3;
init( SPRING_BYTES_STORAGE_SERVER, 100e6 ); if( smallStorageTarget ) SPRING_BYTES_STORAGE_SERVER = 300e3;
Initial implementation of adding separate limits for batch priority in ratekeeper
2019-02-28 02:31:56 +08:00
init( TARGET_BYTES_PER_STORAGE_SERVER_BATCH, 500e6 ); if( smallStorageTarget ) TARGET_BYTES_PER_STORAGE_SERVER_BATCH = 1500e3;
init( SPRING_BYTES_STORAGE_SERVER_BATCH, 50e6 ); if( smallStorageTarget ) SPRING_BYTES_STORAGE_SERVER_BATCH = 150e3;
fix: tripled the smallStorageTarget to prevent simulations which do a lot of work from timing out
2018-05-08 08:26:44 +08:00
init( STORAGE_HARD_LIMIT_BYTES, 1500e6 ); if( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES = 4500e3;
Initial repository commit
2017-05-26 04:48:44 +08:00
bool smallTlogTarget = randomize && BUGGIFY;
tlogs serve reads to log routers at a low priority, to prevent them from using all their resources catching up a remote dc that has been down for a long time increase the amount of memory ratekeeper budgets for tlogs so that there is a gap after the spill threshold to prevent temporarily overshooting the budget
2018-08-05 01:31:30 +08:00
init( TARGET_BYTES_PER_TLOG, 2400e6 ); if( smallTlogTarget ) TARGET_BYTES_PER_TLOG = 2000e3;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( SPRING_BYTES_TLOG, 400e6 ); if( smallTlogTarget ) SPRING_BYTES_TLOG = 200e3;
Initial implementation of adding separate limits for batch priority in ratekeeper
2019-02-28 02:31:56 +08:00
init( TARGET_BYTES_PER_TLOG_BATCH, 1000e6 ); if( smallTlogTarget ) TARGET_BYTES_PER_TLOG_BATCH = 1000e3;
init( SPRING_BYTES_TLOG_BATCH, 200e6 ); if( smallTlogTarget ) SPRING_BYTES_TLOG_BATCH = 100e3;
Initial repository commit
2017-05-26 04:48:44 +08:00
init( TLOG_SPILL_THRESHOLD, 1500e6 ); if( smallTlogTarget ) TLOG_SPILL_THRESHOLD = 1500e3; if( randomize && BUGGIFY ) TLOG_SPILL_THRESHOLD = 0;
added a hard memory limit to the TLog to prevent it from running out of memory. Because remote logs are not ratekeeper controlled this is their only protection
2018-06-19 08:22:40 +08:00
init( TLOG_HARD_LIMIT_BYTES, 3000e6 ); if( smallTlogTarget ) TLOG_HARD_LIMIT_BYTES = 3000e3;
added a separate knob for tlog’s recoverMemoryLimit
2018-08-22 12:11:23 +08:00
init( TLOG_RECOVER_MEMORY_LIMIT, TARGET_BYTES_PER_TLOG + SPRING_BYTES_TLOG );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MAX_TRANSACTIONS_PER_BYTE, 1000 );
init( MIN_FREE_SPACE, 1e8 );
init( MIN_FREE_SPACE_RATIO, 0.05 );
init( MAX_TL_SS_VERSION_DIFFERENCE, 1e99 ); // if( randomize && BUGGIFY ) MAX_TL_SS_VERSION_DIFFERENCE = std::max(1.0, 0.25 * VERSIONS_PER_SECOND); // spring starts at half this value //FIXME: this knob causes ratekeeper to clamp on idle cluster in simulation that have a large number of logs
Initial implementation of adding separate limits for batch priority in ratekeeper
2019-02-28 02:31:56 +08:00
init( MAX_TL_SS_VERSION_DIFFERENCE_BATCH, 1e99 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( MAX_MACHINES_FALLING_BEHIND, 1 );
//Storage Metrics
init( STORAGE_METRICS_AVERAGE_INTERVAL, 120.0 );
init( STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS, 1000.0 / STORAGE_METRICS_AVERAGE_INTERVAL ); // milliHz!
init( SPLIT_JITTER_AMOUNT, 0.05 ); if( randomize && BUGGIFY ) SPLIT_JITTER_AMOUNT = 0.2;
init( IOPS_UNITS_PER_SAMPLE, 10000 * 1000 / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 100 );
init( BANDWIDTH_UNITS_PER_SAMPLE, SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 );
//Storage Server
init( STORAGE_LOGGING_DELAY, 5.0 );
init( STORAGE_SERVER_POLL_METRICS_DELAY, 1.0 );
do not buggify future version delay, because remote storage servers will be delayed getting data so they need additional time
2018-06-30 02:29:22 +08:00
init( FUTURE_VERSION_DELAY, 1.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( STORAGE_LIMIT_BYTES, 500000 );
init( BUGGIFY_LIMIT_BYTES, 1000 );
init( FETCH_BLOCK_BYTES, 2e6 );
init( FETCH_KEYS_PARALLELISM_BYTES, 5e6 ); if( randomize && BUGGIFY ) FETCH_KEYS_PARALLELISM_BYTES = 4e6;
init( BUGGIFY_BLOCK_BYTES, 10000 );
init( STORAGE_COMMIT_BYTES, 10000000 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000;
init( STORAGE_COMMIT_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) STORAGE_COMMIT_INTERVAL = 2.0;
init( UPDATE_SHARD_VERSION_INTERVAL, 0.25 ); if( randomize && BUGGIFY ) UPDATE_SHARD_VERSION_INTERVAL = 1.0;
init( BYTE_SAMPLING_FACTOR, 250 ); //cannot buggify because of differences in restarting tests
init( BYTE_SAMPLING_OVERHEAD, 100 );
init( MAX_STORAGE_SERVER_WATCH_BYTES, 100e6 ); if( randomize && BUGGIFY ) MAX_STORAGE_SERVER_WATCH_BYTES = 10e3;
init( MAX_BYTE_SAMPLE_CLEAR_MAP_SIZE, 1e9 ); if( randomize && BUGGIFY ) MAX_BYTE_SAMPLE_CLEAR_MAP_SIZE = 1e3;
init( LONG_BYTE_SAMPLE_RECOVERY_DELAY, 60.0 );
byte sample recovery did not use limits for its range reads, leading to slow tasks
2019-01-05 02:32:31 +08:00
init( BYTE_SAMPLE_LOAD_PARALLELISM, 32 ); if( randomize && BUGGIFY ) BYTE_SAMPLE_LOAD_PARALLELISM = 1;
init( BYTE_SAMPLE_LOAD_DELAY, 0.0 ); if( randomize && BUGGIFY ) BYTE_SAMPLE_LOAD_DELAY = 0.1;
Storage server periodically calculates cpuUsage and diskUsage metrics. These metrics (as well as all other metrics necessary for health metrics calculation) are sent in the StorageQueuingMetricsReply message.
2019-02-01 03:49:23 +08:00
init( UPDATE_STORAGE_PROCESS_STATS_INTERVAL, 5.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
//Wait Failure
init( BUGGIFY_OUTSTANDING_WAIT_FAILURE_REQUESTS, 2 );
init( MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS, 250 ); if( randomize && BUGGIFY ) MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS = 2;
init( WAIT_FAILURE_DELAY_LIMIT, 1.0 ); if( randomize && BUGGIFY ) WAIT_FAILURE_DELAY_LIMIT = 5.0;
//Worker
init( WORKER_LOGGING_INTERVAL, 5.0 );
init( INCOMPATIBLE_PEER_DELAY_BEFORE_LOGGING, 5.0 );
// Test harness
init( WORKER_POLL_DELAY, 1.0 );
// Coordination
init( COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL, 1.0 ); if( randomize && BUGGIFY ) COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL = 10.0;
// Buggification
init( BUGGIFIED_EVENTUAL_CONSISTENCY, 1.0 );
BUGGIFY_ALL_COORDINATION = false; if( randomize && BUGGIFY ) { BUGGIFY_ALL_COORDINATION = true; TraceEvent("BuggifyAllCoordination"); }
// Status
init( STATUS_MIN_TIME_BETWEEN_REQUESTS, 0.0 );
Throttle status requests with MAX_STATUS_REQUESTS_PER_SECOND knob, whenever status batching is used.
2019-01-29 07:37:30 +08:00
init( MAX_STATUS_REQUESTS_PER_SECOND, 256.0 );
Initial repository commit
2017-05-26 04:48:44 +08:00
init( CONFIGURATION_ROWS_TO_FETCH, 20000 );
Merge branch 'master' into feature-redwood # Conflicts: # fdbserver/fdbserver.vcxproj # fdbserver/fdbserver.vcxproj.filters
2017-06-10 05:56:41 +08:00
// IPager
init( PAGER_RESERVED_PAGES, 1 );
// IndirectShadowPager
init( FREE_PAGE_VACUUM_THRESHOLD, 1 );
init( VACUUM_QUEUE_SIZE, 100000 );
init( VACUUM_BYTES_PER_SECOND, 1e6 );
<rdar://problem/34557380> Need a way to map real time to version
2017-09-26 03:40:24 +08:00
// Timekeeper
Fixing PR comments <rdar://problem/34557380> Need a way to map real time to version
2017-09-28 07:31:38 +08:00
init( TIME_KEEPER_DELAY, 10 );
<rdar://problem/34557380> Need a way to map real time to version
2017-09-26 03:40:24 +08:00
init( TIME_KEEPER_MAX_ENTRIES, 3600 * 24 * 30 * 6); if( randomize && BUGGIFY ) { TIME_KEEPER_MAX_ENTRIES = 2; }
Initial repository commit
2017-05-26 04:48:44 +08:00
if(clientKnobs)
clientKnobs->IS_ACCEPTABLE_DELAY = clientKnobs->IS_ACCEPTABLE_DELAY*std::min(MAX_READ_TRANSACTION_LIFE_VERSIONS, MAX_WRITE_TRANSACTION_LIFE_VERSIONS)/(5.0*VERSIONS_PER_SECOND);
}