2017-05-26 04:48:44 +08:00
/*
2021-06-01 18:12:47 +08:00
* ServerKnobs . cpp
2017-05-26 04:48:44 +08:00
*
* This source file is part of the FoundationDB open source project
*
2022-03-22 04:36:23 +08:00
* Copyright 2013 - 2022 Apple Inc . and the FoundationDB project authors
2018-02-22 02:25:11 +08:00
*
2017-05-26 04:48:44 +08:00
* Licensed under the Apache License , Version 2.0 ( the " License " ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
2018-02-22 02:25:11 +08:00
*
2017-05-26 04:48:44 +08:00
* http : //www.apache.org/licenses/LICENSE-2.0
2018-02-22 02:25:11 +08:00
*
2017-05-26 04:48:44 +08:00
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an " AS IS " BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
*/
2021-06-01 18:12:47 +08:00
# include "fdbclient/ServerKnobs.h"
2022-09-23 05:31:49 +08:00
# include "flow/CompressionUtils.h"
2021-10-07 11:34:40 +08:00
# include "flow/IRandom.h"
2022-06-30 12:17:11 +08:00
# include "flow/flow.h"
2017-05-26 04:48:44 +08:00
2021-08-19 06:47:56 +08:00
# define init(...) KNOB_FN(__VA_ARGS__, INIT_ATOMIC_KNOB, INIT_KNOB)(__VA_ARGS__)
2021-06-10 11:50:00 +08:00
ServerKnobs : : ServerKnobs ( Randomize randomize , ClientKnobs * clientKnobs , IsSimulated isSimulated ) {
initialize ( randomize , clientKnobs , isSimulated ) ;
2020-04-02 04:59:06 +08:00
}
2021-07-03 12:41:50 +08:00
void ServerKnobs : : initialize ( Randomize randomize , ClientKnobs * clientKnobs , IsSimulated isSimulated ) {
2019-07-31 13:35:34 +08:00
// clang-format off
2022-06-01 07:04:28 +08:00
init ( ALLOW_DANGEROUS_KNOBS , isSimulated ) ;
2018-07-13 03:09:48 +08:00
// Versions
init ( VERSIONS_PER_SECOND , 1e6 ) ;
init ( MAX_VERSIONS_IN_FLIGHT , 100 * VERSIONS_PER_SECOND ) ;
init ( MAX_VERSIONS_IN_FLIGHT_FORCED , 6e5 * VERSIONS_PER_SECOND ) ; //one week of versions
2022-05-21 04:02:07 +08:00
init ( ENABLE_VERSION_VECTOR , false ) ;
2022-07-01 00:16:23 +08:00
init ( ENABLE_VERSION_VECTOR_TLOG_UNICAST , false ) ;
2022-10-20 00:45:56 +08:00
bool buggifyShortReadWindow = randomize & & BUGGIFY & & ! ENABLE_VERSION_VECTOR ;
2022-05-21 04:02:07 +08:00
init ( MAX_READ_TRANSACTION_LIFE_VERSIONS , 5 * VERSIONS_PER_SECOND ) ; if ( randomize & & BUGGIFY ) MAX_READ_TRANSACTION_LIFE_VERSIONS = VERSIONS_PER_SECOND ; else if ( buggifyShortReadWindow ) MAX_READ_TRANSACTION_LIFE_VERSIONS = std : : max < int > ( 1 , 0.1 * VERSIONS_PER_SECOND ) ; else if ( randomize & & BUGGIFY ) MAX_READ_TRANSACTION_LIFE_VERSIONS = 10 * VERSIONS_PER_SECOND ;
2018-07-13 03:09:48 +08:00
init ( MAX_WRITE_TRANSACTION_LIFE_VERSIONS , 5 * VERSIONS_PER_SECOND ) ; if ( randomize & & BUGGIFY ) MAX_WRITE_TRANSACTION_LIFE_VERSIONS = std : : max < int > ( 1 , 1 * VERSIONS_PER_SECOND ) ;
2020-09-11 08:44:15 +08:00
init ( MAX_COMMIT_BATCH_INTERVAL , 2.0 ) ; if ( randomize & & BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 0.5 ; // Each commit proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly
2018-07-13 03:09:48 +08:00
MAX_COMMIT_BATCH_INTERVAL = std : : min ( MAX_COMMIT_BATCH_INTERVAL , MAX_READ_TRANSACTION_LIFE_VERSIONS / double ( 2 * VERSIONS_PER_SECOND ) ) ; // Ensure that the proxy commits 2 times every MAX_READ_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough
2022-10-20 00:45:56 +08:00
MAX_COMMIT_BATCH_INTERVAL = std : : min ( MAX_COMMIT_BATCH_INTERVAL , MAX_WRITE_TRANSACTION_LIFE_VERSIONS / double ( 2 * VERSIONS_PER_SECOND ) ) ; // Ensure that the proxy commits 2 times every MAX_WRITE_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough
Add fdbcli command to read/write version epoch (#6480)
* Initialize cluster version at wall-clock time
Previously, new clusters would begin at version 0. After this change,
clusters will initialize at a version matching wall-clock time. Instead
of using the Unix epoch (or Windows epoch), FDB clusters will use a new
epoch, defaulting to January 1, 2010, 01:00:00+00:00. In the future,
this base epoch will be modifiable through fdbcli, allowing
administrators to advance the cluster version.
Basing the version off of time allows different FDB clusters to share
data without running into version issues.
* Send version epoch to master
* Cleanup
* Update fdbserver/storageserver.actor.cpp
Co-authored-by: A.J. Beamon <aj.beamon@snowflake.com>
* Jump directly to expected version if possible
* Fix initial version issue on storage servers
* Add random recovery offset to start version in simulation
* Type fixes
* Disable reference time by default
Enable on a cluster using the fdbcli command `versionepoch add 0`.
* Use correct recoveryTransactionVersion when recovering
* Allow version epoch to be adjusted forwards (to decrease the version)
* Set version epoch in simulation
* Add quiet database check to ensure small version offset
* Fix initial version issue on storage servers
* Disable reference time by default
Enable on a cluster using the fdbcli command `versionepoch add 0`.
* Add fdbcli command to read/write version epoch
* Cause recovery when version epoch is set
* Handle optional version epoch key
* Add ability to clear the version epoch
This causes version advancement to revert to the old methodology whereas
versions attempt to advance by about a million versions per second,
instead of trying to match the clock.
* Update transaction access
* Modify version epoch to use microseconds instead of seconds
* Modify fdbcli version target API
Move commands from `versionepoch` to `targetversion` top level command.
* Add fdbcli tests for
* Temporarily disable targetversion cli tests
* Fix version epoch fetch issue
* Fix Arena issue
* Reduce max version jump in simulation to 1,000,000
* Rework fdbcli API
It now requires two commands to fully switch a cluster to using the
version epoch. First, enable the version epoch with `versionepoch
enable` or `versionepoch set <versionepoch>`. At this point, versions
will be given out at a faster or slower rate in an attempt to reach the
expected version. Then, run `versionepoch commit` to perform a one time
jump to the expected version. This is essentially irreversible.
* Temporarily disable old targetversion tests
* Cleanup
* Move version epoch buggify to sequencer
This will cause some issues with the QuietDatabase check for the version
offset - namely, it won't do anything, since the version epoch is not
being written to the txnStateStore in simulation. This will get fixed in
the future.
Co-authored-by: A.J. Beamon <aj.beamon@snowflake.com>
2022-04-09 03:33:19 +08:00
init ( MAX_VERSION_RATE_MODIFIER , 0.1 ) ;
init ( MAX_VERSION_RATE_OFFSET , VERSIONS_PER_SECOND ) ; // If the calculated version is more than this amount away from the expected version, it will be clamped to this value. This prevents huge version jumps.
2022-06-04 03:29:54 +08:00
init ( ENABLE_VERSION_VECTOR_HA_OPTIMIZATION , false ) ;
2018-07-13 03:09:48 +08:00
2017-05-26 04:48:44 +08:00
// TLogs
init ( TLOG_TIMEOUT , 0.4 ) ; //cannot buggify because of availability
2021-03-10 10:57:46 +08:00
init ( TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS , 60 ) ; if ( randomize & & BUGGIFY ) TLOG_SLOW_REJOIN_WARN_TIMEOUT_SECS = deterministicRandom ( ) - > randomInt ( 5 , 10 ) ;
2017-05-26 04:48:44 +08:00
init ( TLOG_STORAGE_MIN_UPDATE_INTERVAL , 0.5 ) ;
init ( BUGGIFY_TLOG_STORAGE_MIN_UPDATE_INTERVAL , 30 ) ;
init ( DESIRED_TOTAL_BYTES , 150000 ) ; if ( randomize & & BUGGIFY ) DESIRED_TOTAL_BYTES = 10000 ;
2018-10-19 04:37:31 +08:00
init ( DESIRED_UPDATE_BYTES , 2 * DESIRED_TOTAL_BYTES ) ;
init ( UPDATE_DELAY , 0.001 ) ;
2017-05-26 04:48:44 +08:00
init ( MAXIMUM_PEEK_BYTES , 10e6 ) ;
init ( APPLY_MUTATION_BYTES , 1e6 ) ;
init ( BUGGIFY_RECOVER_MEMORY_LIMIT , 1e6 ) ;
init ( BUGGIFY_WORKER_REMOVED_MAX_LAG , 30 ) ;
init ( UPDATE_STORAGE_BYTE_LIMIT , 1e6 ) ;
2021-10-20 23:45:13 +08:00
init ( TLOG_PEEK_DELAY , 0.0005 ) ;
2017-05-26 04:48:44 +08:00
init ( LEGACY_TLOG_UPGRADE_ENTRIES_PER_VERSION , 100 ) ;
init ( VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS , 1072 ) ; // Based on a naive interpretation of the gcc version of std::deque, we would expect this to be 16 bytes overhead per 512 bytes data. In practice, it seems to be 24 bytes overhead per 512.
2018-03-17 07:47:05 +08:00
init ( VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD , std : : ceil ( 16.0 * VERSION_MESSAGES_OVERHEAD_FACTOR_1024THS / 1024 ) ) ;
2017-05-26 04:48:44 +08:00
init ( LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE , 1e5 ) ;
init ( MAX_MESSAGE_SIZE , std : : max < int > ( LOG_SYSTEM_PUSHED_DATA_BLOCK_SIZE , 1e5 + 2e4 + 1 ) + 8 ) ; // VALUE_SIZE_LIMIT + SYSTEM_KEY_SIZE_LIMIT + 9 bytes (4 bytes for length, 4 bytes for sequence number, and 1 byte for mutation type)
init ( TLOG_MESSAGE_BLOCK_BYTES , 10e6 ) ;
2017-06-29 07:52:45 +08:00
init ( TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR , double ( TLOG_MESSAGE_BLOCK_BYTES ) / ( TLOG_MESSAGE_BLOCK_BYTES - MAX_MESSAGE_SIZE ) ) ; //1.0121466709838096006362758832473
2023-02-06 23:06:16 +08:00
init ( PEEK_TRACKER_EXPIRATION_TIME , 600 ) ; if ( randomize & & BUGGIFY ) PEEK_TRACKER_EXPIRATION_TIME = 120 ; // Cannot be buggified lower without changing the following assert in LogSystemPeekCursor.actor.cpp: ASSERT_WE_THINK(e.code() == error_code_operation_obsolete || SERVER_KNOBS->PEEK_TRACKER_EXPIRATION_TIME < 10);
2022-06-01 07:04:28 +08:00
init ( PEEK_USING_STREAMING , false ) ; if ( randomize & & isSimulated & & BUGGIFY ) PEEK_USING_STREAMING = true ;
2021-08-03 05:24:20 +08:00
init ( PARALLEL_GET_MORE_REQUESTS , 32 ) ; if ( randomize & & BUGGIFY ) PARALLEL_GET_MORE_REQUESTS = 2 ;
2018-10-04 04:57:45 +08:00
init ( MULTI_CURSOR_PRE_FETCH_LIMIT , 10 ) ;
2017-05-26 04:48:44 +08:00
init ( MAX_QUEUE_COMMIT_BYTES , 15e6 ) ; if ( randomize & & BUGGIFY ) MAX_QUEUE_COMMIT_BYTES = 5000 ;
2019-11-05 12:21:38 +08:00
init ( DESIRED_OUTSTANDING_MESSAGES , 5000 ) ; if ( randomize & & BUGGIFY ) DESIRED_OUTSTANDING_MESSAGES = deterministicRandom ( ) - > randomInt ( 0 , 100 ) ;
init ( DESIRED_GET_MORE_DELAY , 0.005 ) ;
2019-11-09 10:34:05 +08:00
init ( CONCURRENT_LOG_ROUTER_READS , 5 ) ; if ( randomize & & BUGGIFY ) CONCURRENT_LOG_ROUTER_READS = 1 ;
Log Routers will prefer to peek from satellite logs.
Formerly, they would prefer to peek from the primary's logs. Testing of
a failed region rejoining the cluster revealed that this becomes quite a
strain on the primary logs when extremely large volumes of peek requests
are coming from the Log Routers. It happens that we have satellites
that contain the same mutations with Log Router tags, that have no other
peeking load, so we can prefer to use the satellite to peek rather than
the primary to distribute load across TLogs better.
Unfortunately, this revealed a latent bug in how tagged mutations in the
KnownCommittedVersion->RecoveryVersion gap were copied across
generations when the number of log router tags were decreased.
Satellite TLogs would be assigned log router tags using the
team-building based logic in getPushLocations(), whereas TLogs would
internally re-index tags according to tag.id%logRouterTags. This
mismatch would mean that we could have:
Log0 -2:0 ----- -2:0 Log 0
Log1 -2:1 \
>--- -2:1,-2:0 (-2:2 mod 2 becomes -2:0) Log 1
Log2 -2:2 /
And now we have data that's tagged as -2:0 on a TLog that's not the
preferred location for -2:0, and therefore a BestLocationOnly cursor
would miss the mutations.
This was never noticed before, as we never
used a satellite as a preferred location to peek from. Merge cursors
always peek from all locations, and thus a peek for -2:0 that needed
data from the satellites would have gone to both TLogs and merged the
results.
We now take this mod-based re-indexing into account when assigning which
TLogs need to recover which tags from the previous generation, to make
sure that tag.id%logRouterTags always results in the assigned TLog being
the preferred location.
Unfortunately, previously existing will potentially have existing
satellites with log router tags indexed incorrectly, so this transition
needs to be gated on a `log_version` transition. Old LogSets will have
an old LogVersion, and we won't prefer the sattelite for peeking. Log
Sets post-6.2 (opt-in) or post-6.3 (default) will be indexed correctly,
and therefore we can safely offload peeking onto the satellites.
2019-07-09 13:25:01 +08:00
init ( LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED , 1 ) ; if ( randomize & & BUGGIFY ) LOG_ROUTER_PEEK_FROM_SATELLITES_PREFERRED = 0 ;
2018-10-04 04:58:55 +08:00
init ( DISK_QUEUE_ADAPTER_MIN_SWITCH_TIME , 1.0 ) ;
init ( DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME , 5.0 ) ;
2019-03-04 04:57:38 +08:00
init ( TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES , 2e9 ) ; if ( randomize & & BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES = 2e6 ;
2019-05-02 08:40:31 +08:00
init ( TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK , 100 ) ; if ( randomize & & BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK = 1 ;
2019-05-04 03:48:28 +08:00
init ( TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH , 16 < < 10 ) ; if ( randomize & & BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH = 500 ;
2019-03-04 04:57:44 +08:00
init ( DISK_QUEUE_FILE_EXTENSION_BYTES , 10 < < 20 ) ; // BUGGIFYd per file within the DiskQueue
2019-03-09 00:46:34 +08:00
init ( DISK_QUEUE_FILE_SHRINK_BYTES , 100 < < 20 ) ; // BUGGIFYd per file within the DiskQueue
2021-04-14 12:25:27 +08:00
init ( DISK_QUEUE_MAX_TRUNCATE_BYTES , 2LL < < 30 ) ; if ( randomize & & BUGGIFY ) DISK_QUEUE_MAX_TRUNCATE_BYTES = 0 ;
2019-03-09 00:46:34 +08:00
init ( TLOG_DEGRADED_DURATION , 5.0 ) ;
2019-11-13 05:01:29 +08:00
init ( MAX_CACHE_VERSIONS , 10e6 ) ;
2019-05-14 06:43:27 +08:00
init ( TLOG_IGNORE_POP_AUTO_ENABLE_DELAY , 300.0 ) ;
2019-08-07 07:32:28 +08:00
init ( TXS_POPPED_MAX_DELAY , 1.0 ) ; if ( randomize & & BUGGIFY ) TXS_POPPED_MAX_DELAY = deterministicRandom ( ) - > random01 ( ) ;
2022-09-01 06:46:39 +08:00
// In some rare simulation tests, particularly with log_spill:=1 configured, the 10 second limit is exceeded, causing SevError trace events
// and simulation test failure. Increasing the knob value to 15.0 in simulation is a workaround to avoid these failures.
init ( TLOG_MAX_CREATE_DURATION , 10.0 ) ; if ( isSimulated ) TLOG_MAX_CREATE_DURATION = 15.0 ;
2020-04-23 05:24:45 +08:00
init ( PEEK_LOGGING_AMOUNT , 5 ) ;
init ( PEEK_LOGGING_DELAY , 5.0 ) ;
2020-07-10 13:50:47 +08:00
init ( PEEK_RESET_INTERVAL , 300.0 ) ; if ( randomize & & BUGGIFY ) PEEK_RESET_INTERVAL = 20.0 ;
init ( PEEK_MAX_LATENCY , 0.5 ) ; if ( randomize & & BUGGIFY ) PEEK_MAX_LATENCY = 0.0 ;
init ( PEEK_COUNT_SMALL_MESSAGES , false ) ; if ( randomize & & BUGGIFY ) PEEK_COUNT_SMALL_MESSAGES = true ;
2020-08-15 01:16:20 +08:00
init ( PEEK_STATS_INTERVAL , 10.0 ) ;
2020-08-18 23:43:14 +08:00
init ( PEEK_STATS_SLOW_AMOUNT , 2 ) ;
2020-07-10 13:50:47 +08:00
init ( PEEK_STATS_SLOW_RATIO , 0.5 ) ;
2021-05-25 14:31:32 +08:00
// Buggified value must be larger than the amount of simulated time taken by snapshots, to prevent repeatedly failing
// snapshots due to closed commit proxy connections
init ( PUSH_RESET_INTERVAL , 300.0 ) ; if ( randomize & & BUGGIFY ) PUSH_RESET_INTERVAL = 40.0 ;
2020-08-18 23:43:14 +08:00
init ( PUSH_MAX_LATENCY , 0.5 ) ; if ( randomize & & BUGGIFY ) PUSH_MAX_LATENCY = 0.0 ;
2020-10-10 00:53:05 +08:00
init ( PUSH_STATS_INTERVAL , 10.0 ) ;
2020-08-18 23:43:14 +08:00
init ( PUSH_STATS_SLOW_AMOUNT , 2 ) ;
init ( PUSH_STATS_SLOW_RATIO , 0.5 ) ;
2020-12-04 01:40:41 +08:00
init ( TLOG_POP_BATCH_SIZE , 1000 ) ; if ( randomize & & BUGGIFY ) TLOG_POP_BATCH_SIZE = 10 ;
2021-07-08 02:36:22 +08:00
init ( TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE , 250e6 ) ;
2022-04-03 03:26:35 +08:00
init ( BLOCKING_PEEK_TIMEOUT , 0.4 ) ;
2021-09-25 11:41:38 +08:00
init ( ENABLE_DETAILED_TLOG_POP_TRACE , false ) ; if ( randomize & & BUGGIFY ) ENABLE_DETAILED_TLOG_POP_TRACE = true ;
2022-02-16 06:08:45 +08:00
init ( PEEK_BATCHING_EMPTY_MSG , false ) ; if ( randomize & & BUGGIFY ) PEEK_BATCHING_EMPTY_MSG = true ;
init ( PEEK_BATCHING_EMPTY_MSG_INTERVAL , 0.001 ) ; if ( randomize & & BUGGIFY ) PEEK_BATCHING_EMPTY_MSG_INTERVAL = 0.01 ;
2022-11-02 07:02:42 +08:00
init ( POP_FROM_LOG_DELAY , 1 ) ; if ( randomize & & BUGGIFY ) POP_FROM_LOG_DELAY = 0 ;
2023-01-11 13:35:37 +08:00
init ( TLOG_PULL_ASYNC_DATA_WARNING_TIMEOUT_SECS , 120 ) ;
2017-05-26 04:48:44 +08:00
2019-06-20 02:21:36 +08:00
// disk snapshot max timeout, to be put in TLog, storage and coordinator nodes
2021-01-29 17:31:26 +08:00
init ( MAX_FORKED_PROCESS_OUTPUT , 1024 ) ;
2022-06-30 02:23:07 +08:00
init ( SNAP_CREATE_MAX_TIMEOUT , isSimulated ? 70.0 : 300.0 ) ;
init ( SNAP_MINIMUM_TIME_GAP , 5.0 ) ;
init ( SNAP_NETWORK_FAILURE_RETRY_LIMIT , 10 ) ;
2022-04-04 14:28:57 +08:00
init ( MAX_STORAGE_SNAPSHOT_FAULT_TOLERANCE , 1 ) ;
init ( MAX_COORDINATOR_SNAPSHOT_FAULT_TOLERANCE , 1 ) ;
2022-07-09 11:53:49 +08:00
init ( SNAPSHOT_ALL_STATEFUL_PROCESSES , false ) ; if ( randomize & & BUGGIFY ) SNAPSHOT_ALL_STATEFUL_PROCESSES = true ;
2017-05-26 04:48:44 +08:00
// Data distribution queue
init ( HEALTH_POLL_TIME , 1.0 ) ;
init ( BEST_TEAM_STUCK_DELAY , 1.0 ) ;
2021-12-14 02:13:34 +08:00
init ( DEST_OVERLOADED_DELAY , 0.2 ) ;
2020-02-11 13:49:31 +08:00
init ( BG_REBALANCE_POLLING_INTERVAL , 10.0 ) ;
init ( BG_REBALANCE_SWITCH_CHECK_INTERVAL , 5.0 ) ; if ( randomize & & BUGGIFY ) BG_REBALANCE_SWITCH_CHECK_INTERVAL = 1.0 ;
2017-05-26 04:48:44 +08:00
init ( DD_QUEUE_LOGGING_INTERVAL , 5.0 ) ;
2022-09-01 06:46:39 +08:00
init ( DD_QUEUE_COUNTER_REFRESH_INTERVAL , 60.0 ) ;
// 100 / 60 < 2 trace/sec ~ 2 * 200 = 400b/sec
init ( DD_QUEUE_COUNTER_MAX_LOG , 100 ) ; if ( randomize & & BUGGIFY ) DD_QUEUE_COUNTER_MAX_LOG = 1 ;
init ( DD_QUEUE_COUNTER_SUMMARIZE , true ) ;
2019-04-27 04:54:49 +08:00
init ( RELOCATION_PARALLELISM_PER_SOURCE_SERVER , 2 ) ; if ( randomize & & BUGGIFY ) RELOCATION_PARALLELISM_PER_SOURCE_SERVER = 1 ;
2021-12-14 02:13:34 +08:00
init ( RELOCATION_PARALLELISM_PER_DEST_SERVER , 10 ) ; if ( randomize & & BUGGIFY ) RELOCATION_PARALLELISM_PER_DEST_SERVER = 1 ; // Note: if this is smaller than FETCH_KEYS_PARALLELISM, this will artificially reduce performance. The current default of 10 is probably too high but is set conservatively for now.
2022-12-02 06:10:01 +08:00
init ( DD_QUEUE_MAX_KEY_SERVERS , 100 ) ; // Do not buggify
2017-05-26 04:48:44 +08:00
init ( DD_REBALANCE_PARALLELISM , 50 ) ;
init ( DD_REBALANCE_RESET_AMOUNT , 30 ) ;
init ( INFLIGHT_PENALTY_HEALTHY , 1.0 ) ;
2020-02-21 08:47:50 +08:00
init ( INFLIGHT_PENALTY_UNHEALTHY , 500.0 ) ;
2017-05-26 04:48:44 +08:00
init ( INFLIGHT_PENALTY_ONE_LEFT , 1000.0 ) ;
2020-03-05 08:23:49 +08:00
init ( USE_OLD_NEEDED_SERVERS , false ) ;
2020-01-24 22:58:33 +08:00
2019-10-12 08:50:43 +08:00
init ( PRIORITY_RECOVER_MOVE , 110 ) ;
init ( PRIORITY_REBALANCE_UNDERUTILIZED_TEAM , 120 ) ;
2022-05-27 06:55:14 +08:00
init ( PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM , 121 ) ;
2022-05-26 07:41:15 +08:00
init ( PRIORITY_REBALANCE_OVERUTILIZED_TEAM , 122 ) ;
2022-04-27 08:11:38 +08:00
init ( PRIORITY_REBALANCE_READ_OVERUTIL_TEAM , 123 ) ;
2021-06-12 06:58:05 +08:00
init ( PRIORITY_PERPETUAL_STORAGE_WIGGLE , 139 ) ;
2019-10-12 08:50:43 +08:00
init ( PRIORITY_TEAM_HEALTHY , 140 ) ;
init ( PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER , 150 ) ;
init ( PRIORITY_TEAM_REDUNDANT , 200 ) ;
2020-01-18 06:24:58 +08:00
init ( PRIORITY_MERGE_SHARD , 340 ) ;
2020-03-05 06:07:32 +08:00
init ( PRIORITY_POPULATE_REGION , 600 ) ;
2019-10-12 08:50:43 +08:00
init ( PRIORITY_TEAM_UNHEALTHY , 700 ) ;
init ( PRIORITY_TEAM_2_LEFT , 709 ) ;
init ( PRIORITY_TEAM_1_LEFT , 800 ) ;
2019-10-22 04:31:35 +08:00
init ( PRIORITY_TEAM_FAILED , 805 ) ;
2019-10-12 08:50:43 +08:00
init ( PRIORITY_TEAM_0_LEFT , 809 ) ;
2020-01-10 09:04:17 +08:00
init ( PRIORITY_SPLIT_SHARD , 950 ) ; if ( randomize & & BUGGIFY ) PRIORITY_SPLIT_SHARD = 350 ;
2022-09-01 06:46:39 +08:00
init ( PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD , 960 ) ; if ( randomize & & BUGGIFY ) PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD = 360 ; // Set as the lowest priority
2017-05-26 04:48:44 +08:00
// Data distribution
2022-08-04 04:51:40 +08:00
init ( SHARD_ENCODE_LOCATION_METADATA , false ) ; if ( randomize & & BUGGIFY ) SHARD_ENCODE_LOCATION_METADATA = true ;
2022-09-01 06:46:39 +08:00
init ( ENABLE_DD_PHYSICAL_SHARD , false ) ; // EXPERIMENTAL; If true, SHARD_ENCODE_LOCATION_METADATA must be true; When true, optimization of data move between DCs is disabled
init ( MAX_PHYSICAL_SHARD_BYTES , 500000000 ) ; // 500 MB; for ENABLE_DD_PHYSICAL_SHARD; smaller leads to larger number of physicalShard per storage server
init ( PHYSICAL_SHARD_METRICS_DELAY , 300.0 ) ; // 300 seconds; for ENABLE_DD_PHYSICAL_SHARD
init ( ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME , 600.0 ) ; if ( randomize & & BUGGIFY ) ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME = 0.0 ; // 600 seconds; for ENABLE_DD_PHYSICAL_SHARD
2022-05-13 07:30:21 +08:00
init ( READ_REBALANCE_CPU_THRESHOLD , 15.0 ) ;
2022-05-20 01:50:24 +08:00
init ( READ_REBALANCE_SRC_PARALLELISM , 20 ) ;
2022-05-17 12:25:56 +08:00
init ( READ_REBALANCE_SHARD_TOPK , READ_REBALANCE_SRC_PARALLELISM * 2 ) ;
2022-05-27 06:55:14 +08:00
init ( READ_REBALANCE_DIFF_FRAC , 0.3 ) ;
init ( READ_REBALANCE_MAX_SHARD_FRAC , 0.2 ) ; // FIXME: add buggify here when we have DD test, seems DD is pretty sensitive to this parameter
2017-05-26 04:48:44 +08:00
init ( RETRY_RELOCATESHARD_DELAY , 0.1 ) ;
2019-06-20 04:35:27 +08:00
init ( DATA_DISTRIBUTION_FAILURE_REACTION_TIME , 60.0 ) ; if ( randomize & & BUGGIFY ) DATA_DISTRIBUTION_FAILURE_REACTION_TIME = 1.0 ;
2022-01-12 22:10:53 +08:00
bool buggifySmallShards = randomize & & BUGGIFY ;
2022-04-02 06:57:30 +08:00
bool simulationMediumShards = ! buggifySmallShards & & isSimulated & & randomize & & ! BUGGIFY ; // prefer smaller shards in simulation
2022-04-21 07:45:20 +08:00
// FIXME: increase this even more eventually
init ( MIN_SHARD_BYTES , 10000000 ) ; if ( buggifySmallShards ) MIN_SHARD_BYTES = 40000 ; if ( simulationMediumShards ) MIN_SHARD_BYTES = 200000 ; //FIXME: data distribution tracker (specifically StorageMetrics) relies on this number being larger than the maximum size of a key value pair
2017-05-26 04:48:44 +08:00
init ( SHARD_BYTES_RATIO , 4 ) ;
2022-01-12 22:10:53 +08:00
init ( SHARD_BYTES_PER_SQRT_BYTES , 45 ) ; if ( buggifySmallShards ) SHARD_BYTES_PER_SQRT_BYTES = 0 ; //Approximately 10000 bytes per shard
2017-05-26 04:48:44 +08:00
init ( MAX_SHARD_BYTES , 500000000 ) ;
2017-10-28 02:52:11 +08:00
init ( KEY_SERVER_SHARD_BYTES , 500000000 ) ;
2020-08-13 06:15:25 +08:00
init ( SHARD_MAX_READ_DENSITY_RATIO , 8.0 ) ; if ( randomize & & BUGGIFY ) SHARD_MAX_READ_DENSITY_RATIO = 2.0 ;
2019-12-19 09:11:34 +08:00
/*
2020-08-13 06:15:25 +08:00
The bytesRead / byteSize radio . Will be declared as read hot when larger than this . 8.0 was chosen to avoid reporting table scan as read hot .
2019-12-19 09:11:34 +08:00
*/
2022-06-28 00:56:02 +08:00
init ( SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS , 1666667 * 1000 ) ;
2020-01-13 07:30:36 +08:00
/*
2020-08-13 05:27:34 +08:00
The read bandwidth of a given shard needs to be larger than this value in order to be evaluated if it ' s read hot . The roughly 1.67 MB per second is calculated as following :
- Heuristic data suggests that each storage process can do max 500 K read operations per second
2020-01-13 07:30:36 +08:00
- Each read has a minimum cost of EMPTY_READ_PENALTY , which is 20 bytes
2020-08-13 05:27:34 +08:00
- Thus that gives a minimum 10 MB per second
- But to be conservative , set that number to be 1 / 6 of 10 MB , which is roughly 1 , 666 , 667 bytes per second
2020-01-13 07:30:36 +08:00
Shard with a read bandwidth smaller than this value will never be too busy to handle the reads .
2019-10-10 07:30:22 +08:00
*/
init ( SHARD_MAX_BYTES_READ_PER_KSEC_JITTER , 0.1 ) ;
2017-05-26 04:48:44 +08:00
bool buggifySmallBandwidthSplit = randomize & & BUGGIFY ;
init ( SHARD_MAX_BYTES_PER_KSEC , 1LL * 1000000 * 1000 ) ; if ( buggifySmallBandwidthSplit ) SHARD_MAX_BYTES_PER_KSEC = 10LL * 1000 * 1000 ;
2019-10-10 07:30:22 +08:00
/* 1*1MB/sec * 1000sec/ksec
2017-05-26 04:48:44 +08:00
Shards with more than this bandwidth will be split immediately .
2020-02-04 01:49:26 +08:00
For a large shard ( 100 MB ) , it will be split into multiple shards with sizes < SHARD_SPLIT_BYTES_PER_KSEC ;
all but one split shard will be moved ; so splitting may cost ~ 100 MB of work or about 10 MB / sec over a 10 sec sampling window .
2017-05-26 04:48:44 +08:00
If the sampling window is too much longer , the MVCC window will fill up while we wait .
If SHARD_MAX_BYTES_PER_KSEC is too much lower , we could do a lot of data movement work in response to a small impulse of bandwidth .
If SHARD_MAX_BYTES_PER_KSEC is too high relative to the I / O bandwidth of a given server , a workload can remain concentrated on a single
team indefinitely , limiting performance .
*/
init ( SHARD_MIN_BYTES_PER_KSEC , 100 * 1000 * 1000 ) ; if ( buggifySmallBandwidthSplit ) SHARD_MIN_BYTES_PER_KSEC = 200 * 1 * 1000 ;
2019-10-10 07:30:22 +08:00
/* 100*1KB/sec * 1000sec/ksec
2017-05-26 04:48:44 +08:00
Shards with more than this bandwidth will not be merged .
Obviously this needs to be significantly less than SHARD_MAX_BYTES_PER_KSEC , else we will repeatedly merge and split .
It should probably be significantly less than SHARD_SPLIT_BYTES_PER_KSEC , else we will merge right after splitting .
The number of extra shards in the database because of bandwidth splitting can ' t be more than about W / SHARD_MIN_BYTES_PER_KSEC , where
W is the maximum bandwidth of the entire database in bytes / ksec . For 250 MB / sec write bandwidth , ( 250 MB / sec ) / ( 200 KB / sec ) = 1250 extra
shards .
The bandwidth sample maintained by the storage server needs to be accurate enough to reliably measure this minimum bandwidth . See
2022-11-15 04:31:13 +08:00
BYTES_WRITTEN_UNITS_PER_SAMPLE . If this number is too low , the storage server needs to spend more memory and time on sampling .
2017-05-26 04:48:44 +08:00
*/
init ( SHARD_SPLIT_BYTES_PER_KSEC , 250 * 1000 * 1000 ) ; if ( buggifySmallBandwidthSplit ) SHARD_SPLIT_BYTES_PER_KSEC = 50 * 1000 * 1000 ;
2019-10-10 07:30:22 +08:00
/* 250*1KB/sec * 1000sec/ksec
2017-05-26 04:48:44 +08:00
When splitting a shard , it is split into pieces with less than this bandwidth .
Obviously this should be less than half of SHARD_MAX_BYTES_PER_KSEC .
Smaller values mean that high bandwidth shards are split into more pieces , more quickly utilizing large numbers of servers to handle the
bandwidth .
Too many pieces ( too small a value ) may stress data movement mechanisms ( see e . g . RELOCATION_PARALLELISM_PER_SOURCE_SERVER ) .
If this value is too small relative to SHARD_MIN_BYTES_PER_KSEC immediate merging work will be generated .
*/
2020-02-20 07:20:38 +08:00
init ( STORAGE_METRIC_TIMEOUT , isSimulated ? 60.0 : 600.0 ) ; if ( randomize & & BUGGIFY ) STORAGE_METRIC_TIMEOUT = deterministicRandom ( ) - > coinflip ( ) ? 10.0 : 30.0 ;
2017-05-26 04:48:44 +08:00
init ( METRIC_DELAY , 0.1 ) ; if ( randomize & & BUGGIFY ) METRIC_DELAY = 1.0 ;
init ( ALL_DATA_REMOVED_DELAY , 1.0 ) ;
init ( INITIAL_FAILURE_REACTION_DELAY , 30.0 ) ; if ( randomize & & BUGGIFY ) INITIAL_FAILURE_REACTION_DELAY = 0.0 ;
2018-06-11 11:21:39 +08:00
init ( CHECK_TEAM_DELAY , 30.0 ) ;
2022-12-16 04:14:40 +08:00
// This is a safety knob to avoid busy spinning and the case a small cluster don't have enough space when excluding and including too fast. The basic idea is let PW wait for the re-included storage to take on data before wiggling the next one.
// This knob's ideal value would vary by cluster based on its size and disk type. In the meanwhile, the wiggle will also wait until the storage load is almost (85%) balanced.
2022-12-08 00:45:49 +08:00
init ( PERPETUAL_WIGGLE_DELAY , 60 ) ;
init ( PERPETUAL_WIGGLE_SMALL_LOAD_RATIO , 10 ) ;
2022-11-17 04:52:25 +08:00
init ( PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO , 0.85 ) ;
2021-10-22 13:48:21 +08:00
init ( PERPETUAL_WIGGLE_DISABLE_REMOVER , true ) ;
2017-05-26 04:48:44 +08:00
init ( LOG_ON_COMPLETION_DELAY , DD_QUEUE_LOGGING_INTERVAL ) ;
init ( BEST_TEAM_MAX_TEAM_TRIES , 10 ) ;
init ( BEST_TEAM_OPTION_COUNT , 4 ) ;
init ( BEST_OF_AMT , 4 ) ;
init ( SERVER_LIST_DELAY , 1.0 ) ;
init ( RECRUITMENT_IDLE_DELAY , 1.0 ) ;
2018-10-03 08:31:07 +08:00
init ( STORAGE_RECRUITMENT_DELAY , 10.0 ) ;
2021-05-13 02:53:20 +08:00
init ( TSS_HACK_IDENTITY_MAPPING , false ) ; // THIS SHOULD NEVER BE SET IN PROD. Only for performance testing
init ( TSS_RECRUITMENT_TIMEOUT , 3 * STORAGE_RECRUITMENT_DELAY ) ; if ( randomize & & BUGGIFY ) TSS_RECRUITMENT_TIMEOUT = 1.0 ; // Super low timeout should cause tss recruitments to fail
2021-05-29 02:15:52 +08:00
init ( TSS_DD_CHECK_INTERVAL , 60.0 ) ; if ( randomize & & BUGGIFY ) TSS_DD_CHECK_INTERVAL = 1.0 ; // May kill all TSS quickly
2017-05-26 04:48:44 +08:00
init ( DATA_DISTRIBUTION_LOGGING_INTERVAL , 5.0 ) ;
init ( DD_ENABLED_CHECK_DELAY , 1.0 ) ;
2019-02-21 06:18:36 +08:00
init ( DD_STALL_CHECK_DELAY , 0.4 ) ; //Must be larger than 2*MAX_BUGGIFIED_DELAY
2020-02-20 07:20:38 +08:00
init ( DD_LOW_BANDWIDTH_DELAY , isSimulated ? 15.0 : 240.0 ) ; if ( randomize & & BUGGIFY ) DD_LOW_BANDWIDTH_DELAY = 0 ; //Because of delayJitter, this should be less than 0.9 * DD_MERGE_COALESCE_DELAY
init ( DD_MERGE_COALESCE_DELAY , isSimulated ? 30.0 : 300.0 ) ; if ( randomize & & BUGGIFY ) DD_MERGE_COALESCE_DELAY = 0.001 ;
2017-05-26 04:48:44 +08:00
init ( STORAGE_METRICS_POLLING_DELAY , 2.0 ) ; if ( randomize & & BUGGIFY ) STORAGE_METRICS_POLLING_DELAY = 15.0 ;
init ( STORAGE_METRICS_RANDOM_DELAY , 0.2 ) ;
2020-02-26 07:47:10 +08:00
init ( AVAILABLE_SPACE_RATIO_CUTOFF , 0.05 ) ;
2020-04-25 03:54:28 +08:00
init ( DESIRED_TEAMS_PER_SERVER , 5 ) ; if ( randomize & & BUGGIFY ) DESIRED_TEAMS_PER_SERVER = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
2019-08-01 07:08:18 +08:00
init ( MAX_TEAMS_PER_SERVER , 5 * DESIRED_TEAMS_PER_SERVER ) ;
2017-05-26 04:48:44 +08:00
init ( DD_SHARD_SIZE_GRANULARITY , 5000000 ) ;
init ( DD_SHARD_SIZE_GRANULARITY_SIM , 500000 ) ; if ( randomize & & BUGGIFY ) DD_SHARD_SIZE_GRANULARITY_SIM = 0 ;
2019-08-22 05:55:21 +08:00
init ( DD_MOVE_KEYS_PARALLELISM , 15 ) ; if ( randomize & & BUGGIFY ) DD_MOVE_KEYS_PARALLELISM = 1 ;
2020-07-10 01:38:19 +08:00
init ( DD_FETCH_SOURCE_PARALLELISM , 1000 ) ; if ( randomize & & BUGGIFY ) DD_FETCH_SOURCE_PARALLELISM = 1 ;
2017-05-26 04:48:44 +08:00
init ( DD_MERGE_LIMIT , 2000 ) ; if ( randomize & & BUGGIFY ) DD_MERGE_LIMIT = 2 ;
init ( DD_SHARD_METRICS_TIMEOUT , 60.0 ) ; if ( randomize & & BUGGIFY ) DD_SHARD_METRICS_TIMEOUT = 0.1 ;
init ( DD_LOCATION_CACHE_SIZE , 2000000 ) ; if ( randomize & & BUGGIFY ) DD_LOCATION_CACHE_SIZE = 3 ;
init ( MOVEKEYS_LOCK_POLLING_DELAY , 5.0 ) ;
init ( DEBOUNCE_RECRUITING_DELAY , 5.0 ) ;
2021-09-16 01:41:59 +08:00
init ( DD_FAILURE_TIME , 1.0 ) ; if ( randomize & & BUGGIFY ) DD_FAILURE_TIME = 10.0 ;
2019-02-13 06:02:21 +08:00
init ( DD_ZERO_HEALTHY_TEAM_DELAY , 1.0 ) ;
2022-07-14 06:00:19 +08:00
init ( REMOTE_KV_STORE , false ) ;
2022-09-01 06:46:39 +08:00
init ( REBOOT_KV_STORE_DELAY , 0.1 ) ;
Remote ikvs debugging (#6465)
* initial structure for remote IKVS server
* moved struct to .h file, added new files to CMakeList
* happy path implementation, connection error when testing
* saved minor local change
* changed tracing to debug
* fixed onClosed and getError being called before init is finished
* fix spawn process bug, now use absolute path
* added server knob to set ikvs process port number
* added server knob for remote/local kv store
* implement simulator remote process spawning
* fixed bug for simulator timeout
* commit all changes
* removed print lines in trace
* added FlowProcess implementation by Markus
* initial debug of FlowProcess, stuck at parent sending OpenKVStoreRequest to child
* temporary fix for process factory throwing segfault on create
* specify public address in command
* change remote kv store knob to false for jenkins build
* made port 0 open random unused port
* change remote store knob to true for benchmark
* set listening port to randomly opened port
* added print lines for jenkins run open kv store timeout debug
* removed most tracing and print lines
* removed tutorial changes
* update handleIOErrors error handling to handle remote-ikvs cases
* Push all debugging changes
* A version where worker bug exists
* A version where restarting tests fail
* Use both the name and the port to determine the child process
* Remove unnecessary update on local address
* Disable remote-kvs for DiskFailureCycle test
* A version where restarting stuck
* A version where most restarting tests green
* Reset connection with child process explicitly
* Remove change on unnecessary files
* Unify flags from _ to -
* fix merging unexpected changes
* fix trac.error to .errorUnsuppressed
* Add license header
* Remove unnecessary header in FlowProcess.actor.cpp
* Fix Windows build
* Fix Windows build, add missing ;
* Fix a stupid bug caused by code dropped by code merging
* Disable remote kvs by default
* Pass the conn_file path to the flow process, though not needed, but the buildNetwork is difficult to tune
* serialization change on readrange
* Update traces
* Refactor the RemoteIKVS interface
* Format files
* Update sim2 interface to not clog connections between parent and child processes in simulation
* Update comments; remove debugging symbols; Add error handling for remote_kvs_cancelled
* Add comments, format files
* Change method name from isBuggifyDisabled to isStableConnection; Decrease(0.1x) latency for stable connections
* Commit the IConnection interface change, forgot in previous commit
* Fix the issue that onClosed request is cancelled by ActorCollection
* Enable the remote kv store knob
* Remove FlowProcess.actor.cpp and move functions to RemoteIKeyValueStore.actor.cpp; Add remote kv store delay to avoid race; Bind the child process to die with parent process
* Fix the bug where one process starts storage server more than once
* Add a please_reboot_remote_kv_store error to restart the storage server worker if remote kvs died abnormally
* Remove unreachable code path and add comments
* Clang format the code
* Fix a simple wait error
* Clang format after merging the main branch
* Testing mixed mode in simulation if remote_kvs knob is enabled, setting the default to false
* Disable remote kvs for PhysicalShardMove which is for RocksDB
* Cleanup #include orders, remove debugging traces
* Revert the reorder in fdbserver.actor.cpp, which fails the gcc build
Co-authored-by: “Lincoln <“lincoln.xiao@snowflake.com”>
2022-04-01 08:08:59 +08:00
init ( REMOTE_KV_STORE_MAX_INIT_DURATION , 10.0 ) ;
2019-07-31 08:04:41 +08:00
init ( REBALANCE_MAX_RETRIES , 100 ) ;
init ( DD_OVERLAP_PENALTY , 10000 ) ;
2019-09-28 04:12:41 +08:00
init ( DD_EXCLUDE_MIN_REPLICAS , 1 ) ;
2019-09-14 06:34:41 +08:00
init ( DD_VALIDATE_LOCALITY , true ) ; if ( randomize & & BUGGIFY ) DD_VALIDATE_LOCALITY = false ;
2019-09-18 04:03:57 +08:00
init ( DD_CHECK_INVALID_LOCALITY_DELAY , 60 ) ; if ( randomize & & BUGGIFY ) DD_CHECK_INVALID_LOCALITY_DELAY = 1 + deterministicRandom ( ) - > random01 ( ) * 600 ;
2020-03-01 05:45:00 +08:00
init ( DD_ENABLE_VERBOSE_TRACING , false ) ; if ( randomize & & BUGGIFY ) DD_ENABLE_VERBOSE_TRACING = true ;
2020-08-15 01:16:20 +08:00
init ( DD_SS_FAILURE_VERSIONLAG , 250000000 ) ;
2020-02-20 06:13:27 +08:00
init ( DD_SS_ALLOWED_VERSIONLAG , 200000000 ) ; if ( randomize & & BUGGIFY ) { DD_SS_FAILURE_VERSIONLAG = deterministicRandom ( ) - > randomInt ( 15000000 , 500000000 ) ; DD_SS_ALLOWED_VERSIONLAG = 0.75 * DD_SS_FAILURE_VERSIONLAG ; }
2020-04-07 14:45:49 +08:00
init ( DD_SS_STUCK_TIME_LIMIT , 300.0 ) ; if ( randomize & & BUGGIFY ) { DD_SS_STUCK_TIME_LIMIT = 200.0 + deterministicRandom ( ) - > random01 ( ) * 100.0 ; }
2020-10-24 05:01:53 +08:00
init ( DD_TEAMS_INFO_PRINT_INTERVAL , 60 ) ; if ( randomize & & BUGGIFY ) DD_TEAMS_INFO_PRINT_INTERVAL = 10 ;
2020-10-17 07:11:14 +08:00
init ( DD_TEAMS_INFO_PRINT_YIELD_COUNT , 100 ) ; if ( randomize & & BUGGIFY ) DD_TEAMS_INFO_PRINT_YIELD_COUNT = deterministicRandom ( ) - > random01 ( ) * 1000 + 1 ;
2020-10-22 02:24:18 +08:00
init ( DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY , 120 ) ; if ( randomize & & BUGGIFY ) DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY = 5 ;
2021-10-12 11:52:21 +08:00
init ( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD , 10 ) ; if ( randomize & & BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000 ;
2021-10-14 00:18:01 +08:00
init ( DD_STORAGE_WIGGLE_STUCK_THRESHOLD , 20 ) ;
2022-09-01 06:46:39 +08:00
init ( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC , isSimulated ? 2 : 21 * 60 * 60 * 24 ) ; if ( randomize & & BUGGIFY ) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0 : 120 ;
2022-11-11 11:35:21 +08:00
init ( DD_TENANT_AWARENESS_ENABLED , false ) ;
2023-02-18 05:08:56 +08:00
init ( STORAGE_QUOTA_ENABLED , true ) ; if ( isSimulated ) STORAGE_QUOTA_ENABLED = deterministicRandom ( ) - > coinflip ( ) ;
2022-09-01 06:46:39 +08:00
init ( TENANT_CACHE_LIST_REFRESH_INTERVAL , 2 ) ; if ( randomize & & BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
2022-10-22 06:44:13 +08:00
init ( TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL , 2 ) ; if ( randomize & & BUGGIFY ) TENANT_CACHE_STORAGE_USAGE_REFRESH_INTERVAL = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
init ( TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL , 10 ) ; if ( randomize & & BUGGIFY ) TENANT_CACHE_STORAGE_QUOTA_REFRESH_INTERVAL = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
2022-11-16 07:27:14 +08:00
init ( TENANT_CACHE_STORAGE_USAGE_TRACE_INTERVAL , 300 ) ;
2022-11-04 04:47:29 +08:00
init ( CP_FETCH_TENANTS_OVER_STORAGE_QUOTA_INTERVAL , 5 ) ; if ( randomize & & BUGGIFY ) CP_FETCH_TENANTS_OVER_STORAGE_QUOTA_INTERVAL = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
2022-12-06 08:20:04 +08:00
init ( DD_BUILD_EXTRA_TEAMS_OVERRIDE , 10 ) ; if ( randomize & & BUGGIFY ) DD_BUILD_EXTRA_TEAMS_OVERRIDE = 2 ;
2017-05-26 04:48:44 +08:00
2019-02-14 07:11:53 +08:00
// TeamRemover
2020-02-11 13:49:31 +08:00
init ( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER , false ) ; if ( randomize & & BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom ( ) - > random01 ( ) < 0.1 ? true : false ; // false by default. disable the consistency check when it's true
2019-05-11 05:01:52 +08:00
init ( TR_REMOVE_MACHINE_TEAM_DELAY , 60.0 ) ; if ( randomize & & BUGGIFY ) TR_REMOVE_MACHINE_TEAM_DELAY = deterministicRandom ( ) - > random01 ( ) * 60.0 ;
2020-02-11 13:49:31 +08:00
init ( TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS , true ) ; if ( randomize & & BUGGIFY ) TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS = deterministicRandom ( ) - > random01 ( ) < 0.1 ? true : false ;
init ( TR_FLAG_DISABLE_SERVER_TEAM_REMOVER , false ) ; if ( randomize & & BUGGIFY ) TR_FLAG_DISABLE_SERVER_TEAM_REMOVER = deterministicRandom ( ) - > random01 ( ) < 0.1 ? true : false ; // false by default. disable the consistency check when it's true
2019-07-09 09:17:19 +08:00
init ( TR_REMOVE_SERVER_TEAM_DELAY , 60.0 ) ; if ( randomize & & BUGGIFY ) TR_REMOVE_SERVER_TEAM_DELAY = deterministicRandom ( ) - > random01 ( ) * 60.0 ;
2019-07-20 07:48:50 +08:00
init ( TR_REMOVE_SERVER_TEAM_EXTRA_DELAY , 5.0 ) ; if ( randomize & & BUGGIFY ) TR_REMOVE_SERVER_TEAM_EXTRA_DELAY = deterministicRandom ( ) - > random01 ( ) * 10.0 ;
2019-02-14 07:11:53 +08:00
2019-08-17 07:46:54 +08:00
init ( DD_REMOVE_STORE_ENGINE_DELAY , 60.0 ) ; if ( randomize & & BUGGIFY ) DD_REMOVE_STORE_ENGINE_DELAY = deterministicRandom ( ) - > random01 ( ) * 60.0 ;
2019-08-13 01:08:12 +08:00
2017-05-26 04:48:44 +08:00
// KeyValueStore SQLITE
init ( CLEAR_BUFFER_SIZE , 20000 ) ;
init ( READ_VALUE_TIME_ESTIMATE , .00005 ) ;
init ( READ_RANGE_TIME_ESTIMATE , .00005 ) ;
init ( SET_TIME_ESTIMATE , .00005 ) ;
init ( CLEAR_TIME_ESTIMATE , .00005 ) ;
init ( COMMIT_TIME_ESTIMATE , .005 ) ;
init ( CHECK_FREE_PAGE_AMOUNT , 100 ) ; if ( randomize & & BUGGIFY ) CHECK_FREE_PAGE_AMOUNT = 5 ;
init ( SOFT_HEAP_LIMIT , 300e6 ) ;
init ( SQLITE_PAGE_SCAN_ERROR_LIMIT , 10000 ) ;
init ( SQLITE_BTREE_PAGE_USABLE , 4096 - 8 ) ; // pageSize - reserveSize for page checksum
2019-04-27 04:53:56 +08:00
init ( SQLITE_CHUNK_SIZE_PAGES , 25600 ) ; // 100MB
init ( SQLITE_CHUNK_SIZE_PAGES_SIM , 1024 ) ; // 4MB
2020-12-10 04:09:07 +08:00
init ( SQLITE_READER_THREADS , 64 ) ; // number of read threads
2021-02-03 06:36:49 +08:00
init ( SQLITE_WRITE_WINDOW_SECONDS , - 1 ) ;
2022-03-15 00:48:15 +08:00
init ( SQLITE_CURSOR_MAX_LIFETIME_BYTES , 1e6 ) ; if ( buggifySmallShards | | simulationMediumShards ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = MIN_SHARD_BYTES ; if ( randomize & & BUGGIFY ) SQLITE_CURSOR_MAX_LIFETIME_BYTES = 0 ;
2021-02-03 06:36:49 +08:00
init ( SQLITE_WRITE_WINDOW_LIMIT , - 1 ) ;
2021-02-03 04:39:05 +08:00
if ( randomize & & BUGGIFY ) {
// Choose an window between .01 and 1.01 seconds.
2021-02-03 05:31:00 +08:00
SQLITE_WRITE_WINDOW_SECONDS = 0.01 + deterministicRandom ( ) - > random01 ( ) ;
2021-02-03 06:26:39 +08:00
// Choose random operations per second
2021-02-03 04:39:05 +08:00
int opsPerSecond = deterministicRandom ( ) - > randomInt ( 1000 , 5000 ) ;
// Set window limit to opsPerSecond scaled down to window size
2021-02-03 05:31:00 +08:00
SQLITE_WRITE_WINDOW_LIMIT = opsPerSecond * SQLITE_WRITE_WINDOW_SECONDS ;
2021-02-03 04:39:05 +08:00
}
2017-05-26 04:48:44 +08:00
// Maximum and minimum cell payload bytes allowed on primary page as calculated in SQLite.
// These formulas are copied from SQLite, using its hardcoded constants, so if you are
// changing this you should also be changing SQLite.
init ( SQLITE_BTREE_CELL_MAX_LOCAL , ( SQLITE_BTREE_PAGE_USABLE - 12 ) * 64 / 255 - 23 ) ;
init ( SQLITE_BTREE_CELL_MIN_LOCAL , ( SQLITE_BTREE_PAGE_USABLE - 12 ) * 32 / 255 - 23 ) ;
// Maximum FDB fragment key and value bytes that can fit in a primary btree page
init ( SQLITE_FRAGMENT_PRIMARY_PAGE_USABLE ,
SQLITE_BTREE_CELL_MAX_LOCAL
- 1 // vdbeRecord header length size
- 2 // max key length size
- 4 // max index length size
- 2 // max value fragment length size
) ;
// Maximum FDB fragment value bytes in an overflow page
init ( SQLITE_FRAGMENT_OVERFLOW_PAGE_USABLE ,
SQLITE_BTREE_PAGE_USABLE
- 4 // next pageNumber size
) ;
init ( SQLITE_FRAGMENT_MIN_SAVINGS , 0.20 ) ;
// KeyValueStoreSqlite spring cleaning
2019-05-11 05:01:52 +08:00
init ( SPRING_CLEANING_NO_ACTION_INTERVAL , 1.0 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_NO_ACTION_INTERVAL = deterministicRandom ( ) - > coinflip ( ) ? 0.1 : deterministicRandom ( ) - > random01 ( ) * 5 ;
init ( SPRING_CLEANING_LAZY_DELETE_INTERVAL , 0.1 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_LAZY_DELETE_INTERVAL = deterministicRandom ( ) - > coinflip ( ) ? 1.0 : deterministicRandom ( ) - > random01 ( ) * 5 ;
init ( SPRING_CLEANING_VACUUM_INTERVAL , 1.0 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_VACUUM_INTERVAL = deterministicRandom ( ) - > coinflip ( ) ? 0.1 : deterministicRandom ( ) - > random01 ( ) * 5 ;
init ( SPRING_CLEANING_LAZY_DELETE_TIME_ESTIMATE , .010 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_LAZY_DELETE_TIME_ESTIMATE = deterministicRandom ( ) - > random01 ( ) * 5 ;
init ( SPRING_CLEANING_VACUUM_TIME_ESTIMATE , .010 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_VACUUM_TIME_ESTIMATE = deterministicRandom ( ) - > random01 ( ) * 5 ;
init ( SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE , 0.0 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE = deterministicRandom ( ) - > coinflip ( ) ? 1e9 : deterministicRandom ( ) - > random01 ( ) * 5 ;
init ( SPRING_CLEANING_MIN_LAZY_DELETE_PAGES , 0 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_MIN_LAZY_DELETE_PAGES = deterministicRandom ( ) - > randomInt ( 1 , 100 ) ;
init ( SPRING_CLEANING_MAX_LAZY_DELETE_PAGES , 1e9 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_MAX_LAZY_DELETE_PAGES = deterministicRandom ( ) - > coinflip ( ) ? 0 : deterministicRandom ( ) - > randomInt ( 1 , 1e4 ) ;
init ( SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE , 100 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE = deterministicRandom ( ) - > randomInt ( 1 , 1000 ) ;
init ( SPRING_CLEANING_MIN_VACUUM_PAGES , 1 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_MIN_VACUUM_PAGES = deterministicRandom ( ) - > randomInt ( 0 , 100 ) ;
init ( SPRING_CLEANING_MAX_VACUUM_PAGES , 1e9 ) ; if ( randomize & & BUGGIFY ) SPRING_CLEANING_MAX_VACUUM_PAGES = deterministicRandom ( ) - > coinflip ( ) ? 0 : deterministicRandom ( ) - > randomInt ( 1 , 1e4 ) ;
2017-05-26 04:48:44 +08:00
2018-05-02 10:43:35 +08:00
// KeyValueStoreMemory
2019-07-29 10:34:17 +08:00
init ( REPLACE_CONTENTS_BYTES , 1e5 ) ;
2018-05-02 10:43:35 +08:00
2020-08-15 01:16:20 +08:00
// KeyValueStoreRocksDB
2022-11-08 07:59:52 +08:00
init ( ROCKSDB_SET_READ_TIMEOUT , ! isSimulated ) ;
2022-09-30 06:40:29 +08:00
init ( ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES , true ) ; if ( randomize & & BUGGIFY ) ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES = false ;
init ( ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE , true ) ; if ( randomize & & BUGGIFY ) ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE = false ;
2022-09-01 06:46:39 +08:00
init ( ROCKSDB_READ_RANGE_ROW_LIMIT , 65535 ) ; if ( randomize & & BUGGIFY ) ROCKSDB_READ_RANGE_ROW_LIMIT = deterministicRandom ( ) - > randomInt ( 2 , 10 ) ;
2022-09-20 02:10:47 +08:00
init ( ROCKSDB_READER_THREAD_PRIORITY , 0 ) ;
init ( ROCKSDB_WRITER_THREAD_PRIORITY , 0 ) ;
2022-03-04 07:14:37 +08:00
init ( ROCKSDB_BACKGROUND_PARALLELISM , 4 ) ;
2020-11-05 06:06:30 +08:00
init ( ROCKSDB_READ_PARALLELISM , 4 ) ;
2022-12-15 09:44:47 +08:00
init ( ROCKSDB_CHECKPOINT_READER_PARALLELISM , 4 ) ;
2022-11-01 05:01:38 +08:00
// If true, do not process and store RocksDB logs
2022-11-11 06:35:44 +08:00
init ( ROCKSDB_MUTE_LOGS , true ) ;
2021-07-17 00:01:01 +08:00
// Use a smaller memtable in simulation to avoid OOMs.
2022-11-15 06:25:54 +08:00
int64_t memtableBytes = isSimulated ? 1024 * 1024 : 512 * 1024 * 1024 ;
2021-07-17 00:01:01 +08:00
init ( ROCKSDB_MEMTABLE_BYTES , memtableBytes ) ;
2022-10-05 06:13:50 +08:00
init ( ROCKSDB_LEVEL_STYLE_COMPACTION , true ) ;
2020-07-31 06:31:18 +08:00
init ( ROCKSDB_UNSAFE_AUTO_FSYNC , false ) ;
2020-08-12 02:00:01 +08:00
init ( ROCKSDB_PERIODIC_COMPACTION_SECONDS , 0 ) ;
2021-01-12 05:38:34 +08:00
init ( ROCKSDB_PREFIX_LEN , 0 ) ;
2022-09-21 22:37:15 +08:00
// If rocksdb block cache size is 0, the default 8MB is used.
2022-11-15 06:25:54 +08:00
int64_t blockCacheSize = isSimulated ? 16 * 1024 * 1024 : 1024 * 1024 * 1024 /* 1GB */ ;
2022-09-21 22:37:15 +08:00
init ( ROCKSDB_BLOCK_CACHE_SIZE , blockCacheSize ) ;
2021-06-23 06:37:37 +08:00
init ( ROCKSDB_METRICS_DELAY , 60.0 ) ;
2022-11-15 08:17:49 +08:00
// ROCKSDB_READ_VALUE_TIMEOUT, ROCKSDB_READ_VALUE_PREFIX_TIMEOUT, ROCKSDB_READ_RANGE_TIMEOUT knobs:
// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
2022-11-15 06:25:54 +08:00
// very high load and single read thread cannot process all the load within the timeouts.
2022-11-18 11:56:26 +08:00
init ( ROCKSDB_READ_VALUE_TIMEOUT , 5.0 ) ; if ( isSimulated ) ROCKSDB_READ_VALUE_TIMEOUT = 5 * 60 ;
init ( ROCKSDB_READ_VALUE_PREFIX_TIMEOUT , 5.0 ) ; if ( isSimulated ) ROCKSDB_READ_VALUE_PREFIX_TIMEOUT = 5 * 60 ;
init ( ROCKSDB_READ_RANGE_TIMEOUT , 5.0 ) ; if ( isSimulated ) ROCKSDB_READ_RANGE_TIMEOUT = 5 * 60 ;
2022-12-15 09:44:47 +08:00
init ( ROCKSDB_READ_CHECKPOINT_TIMEOUT , 60.0 ) ; if ( isSimulated ) ROCKSDB_READ_CHECKPOINT_TIMEOUT = 5 * 60 ;
init ( ROCKSDB_CHECKPOINT_READ_AHEAD_SIZE , 2 < < 20 ) ; // 2M
2021-10-30 01:58:34 +08:00
init ( ROCKSDB_READ_QUEUE_WAIT , 1.0 ) ;
2021-10-20 05:03:05 +08:00
init ( ROCKSDB_READ_QUEUE_HARD_MAX , 1000 ) ;
init ( ROCKSDB_READ_QUEUE_SOFT_MAX , 500 ) ;
init ( ROCKSDB_FETCH_QUEUE_HARD_MAX , 100 ) ;
init ( ROCKSDB_FETCH_QUEUE_SOFT_MAX , 50 ) ;
2021-12-15 06:21:56 +08:00
init ( ROCKSDB_HISTOGRAMS_SAMPLE_RATE , 0.001 ) ; if ( randomize & & BUGGIFY ) ROCKSDB_HISTOGRAMS_SAMPLE_RATE = 0 ;
2022-01-06 23:34:42 +08:00
init ( ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME , 30.0 ) ; if ( randomize & & BUGGIFY ) ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME = 0.1 ;
2022-11-08 07:59:52 +08:00
init ( ROCKSDB_READ_RANGE_REUSE_ITERATORS , true ) ; if ( randomize & & BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom ( ) - > coinflip ( ) ;
init ( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS , false ) ; if ( randomize & & BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom ( ) - > coinflip ( ) ;
2022-11-04 01:26:47 +08:00
init ( ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT , 200 ) ;
2021-12-11 06:47:49 +08:00
// Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second.
init ( ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC , 0 ) ;
// If true, enables dynamic adjustment of ROCKSDB_WRITE_RATE_LIMITER_BYTES according to the recent demand of background IO.
init ( ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE , true ) ;
2022-03-16 04:03:23 +08:00
init ( DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY , " fdb " ) ;
2022-10-05 06:13:50 +08:00
init ( ROCKSDB_DISABLE_AUTO_COMPACTIONS , false ) ; // RocksDB default
2022-03-16 04:03:23 +08:00
2022-11-08 07:59:52 +08:00
init ( ROCKSDB_PERFCONTEXT_ENABLE , false ) ; if ( randomize & & BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom ( ) - > coinflip ( ) ;
2022-04-22 10:53:14 +08:00
init ( ROCKSDB_PERFCONTEXT_SAMPLE_RATE , 0.0001 ) ;
2022-09-01 06:46:39 +08:00
init ( ROCKSDB_METRICS_SAMPLE_INTERVAL , 0.0 ) ;
2022-03-04 07:14:37 +08:00
init ( ROCKSDB_MAX_SUBCOMPACTIONS , 2 ) ;
2022-03-05 01:57:07 +08:00
init ( ROCKSDB_SOFT_PENDING_COMPACT_BYTES_LIMIT , 64000000000 ) ; // 64GB, Rocksdb option, Writes will slow down.
init ( ROCKSDB_HARD_PENDING_COMPACT_BYTES_LIMIT , 100000000000 ) ; // 100GB, Rocksdb option, Writes will stall.
init ( ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT , 50000000000 ) ; // 50GB, Commit waits.
2022-10-11 03:32:05 +08:00
// Enable this knob only for experminatal purpose, never enable this in production.
// If enabled, all the committed in-memory memtable writes are lost on a crash.
init ( ROCKSDB_DISABLE_WAL_EXPERIMENTAL , false ) ;
2022-10-29 03:24:53 +08:00
// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
2022-10-22 06:47:19 +08:00
// These knobs have contrary functionality.
2022-11-08 14:32:48 +08:00
init ( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE , false ) ; if ( randomize & & BUGGIFY ) ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE = deterministicRandom ( ) - > coinflip ( ) ;
2022-10-22 06:47:19 +08:00
init ( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT , 200000 ) ; // 200KB
2022-11-08 14:32:48 +08:00
init ( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS , true ) ; if ( randomize & & BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom ( ) - > coinflip ( ) ;
// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
2022-11-17 07:08:39 +08:00
// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
2022-11-08 14:32:48 +08:00
init ( ROCKSDB_STATS_LEVEL , 1 ) ; if ( randomize & & BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom ( ) - > randomInt ( 0 , 6 ) ;
2023-01-19 07:40:34 +08:00
init ( ROCKSDB_ENABLE_COMPACT_ON_DELETION , true ) ;
// CDCF: CompactOnDeletionCollectorFactory. The below 3 are parameters of the CompactOnDeletionCollectorFactory
// which controls the compaction on deleted data.
2023-01-26 01:50:40 +08:00
init ( ROCKSDB_CDCF_SLIDING_WINDOW_SIZE , 128 ) ;
2023-01-19 07:40:34 +08:00
init ( ROCKSDB_CDCF_DELETION_TRIGGER , 1 ) ;
init ( ROCKSDB_CDCF_DELETION_RATIO , 0 ) ;
2022-03-05 01:57:07 +08:00
// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
// Set ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD to 0, to disable
init ( ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD , 1 ) ;
init ( ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD , 5 ) ;
2022-03-15 15:38:07 +08:00
init ( ROCKSDB_COMPACTION_READAHEAD_SIZE , 32768 ) ; // 32 KB, performs bigger reads when doing compaction.
2022-03-27 14:17:36 +08:00
init ( ROCKSDB_BLOCK_SIZE , 32768 ) ; // 32 KB, size of the block in rocksdb cache.
2022-04-22 10:53:14 +08:00
init ( ENABLE_SHARDED_ROCKSDB , false ) ;
2022-09-01 06:46:39 +08:00
init ( ROCKSDB_WRITE_BUFFER_SIZE , 1 < < 30 ) ; // 1G
init ( ROCKSDB_CF_WRITE_BUFFER_SIZE , 64 < < 20 ) ; // 64M, RocksDB default.
init ( ROCKSDB_MAX_TOTAL_WAL_SIZE , 0 ) ; // RocksDB default.
init ( ROCKSDB_MAX_BACKGROUND_JOBS , 2 ) ; // RocksDB default.
init ( ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD , 21600 ) ; // 6h, RocksDB default.
init ( ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY , isSimulated ? 10.0 : 300.0 ) ; // Delays shard clean up, must be larger than ROCKSDB_READ_VALUE_TIMEOUT to prevent reading deleted shard.
2020-08-15 01:16:20 +08:00
2017-05-26 04:48:44 +08:00
// Leader election
bool longLeaderElection = randomize & & BUGGIFY ;
2018-07-07 08:40:29 +08:00
init ( MAX_NOTIFICATIONS , 100000 ) ;
2018-08-09 08:29:32 +08:00
init ( MIN_NOTIFICATIONS , 100 ) ;
init ( NOTIFICATION_FULL_CLEAR_TIME , 10000.0 ) ;
2017-05-26 04:48:44 +08:00
init ( CANDIDATE_MIN_DELAY , 0.05 ) ;
init ( CANDIDATE_MAX_DELAY , 1.0 ) ;
init ( CANDIDATE_GROWTH_RATE , 1.2 ) ;
2019-08-01 08:19:41 +08:00
init ( POLLING_FREQUENCY , 2.0 ) ; if ( longLeaderElection ) POLLING_FREQUENCY = 8.0 ;
init ( HEARTBEAT_FREQUENCY , 0.5 ) ; if ( longLeaderElection ) HEARTBEAT_FREQUENCY = 1.0 ;
2017-05-26 04:48:44 +08:00
2020-09-11 08:44:15 +08:00
// Commit CommitProxy and GRV CommitProxy
2017-05-26 04:48:44 +08:00
init ( START_TRANSACTION_BATCH_INTERVAL_MIN , 1e-6 ) ;
init ( START_TRANSACTION_BATCH_INTERVAL_MAX , 0.010 ) ;
init ( START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION , 0.5 ) ;
init ( START_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA , 0.1 ) ;
init ( START_TRANSACTION_BATCH_QUEUE_CHECK_INTERVAL , 0.001 ) ;
2019-03-19 07:16:03 +08:00
init ( START_TRANSACTION_MAX_TRANSACTIONS_TO_START , 100000 ) ;
2019-03-17 07:18:58 +08:00
init ( START_TRANSACTION_MAX_REQUESTS_TO_START , 10000 ) ;
2020-02-25 01:52:31 +08:00
init ( START_TRANSACTION_RATE_WINDOW , 2.0 ) ;
init ( START_TRANSACTION_MAX_EMPTY_QUEUE_BUDGET , 10.0 ) ;
2020-03-14 01:17:49 +08:00
init ( START_TRANSACTION_MAX_QUEUE_SIZE , 1e6 ) ;
init ( KEY_LOCATION_MAX_QUEUE_SIZE , 1e6 ) ;
2022-12-21 08:30:59 +08:00
init ( TENANT_ID_REQUEST_MAX_QUEUE_SIZE , 1e6 ) ;
2023-02-22 03:21:44 +08:00
init ( BLOB_GRANULE_LOCATION_MAX_QUEUE_SIZE , 1e5 ) ; if ( randomize & & BUGGIFY ) BLOB_GRANULE_LOCATION_MAX_QUEUE_SIZE = 100 ;
2021-07-20 12:00:28 +08:00
init ( COMMIT_PROXY_LIVENESS_TIMEOUT , 20.0 ) ;
2017-05-26 04:48:44 +08:00
init ( COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE , 0.0005 ) ; if ( randomize & & BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE = 0.005 ;
init ( COMMIT_TRANSACTION_BATCH_INTERVAL_MIN , 0.001 ) ; if ( randomize & & BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_MIN = 0.1 ;
init ( COMMIT_TRANSACTION_BATCH_INTERVAL_MAX , 0.020 ) ;
init ( COMMIT_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION , 0.1 ) ;
init ( COMMIT_TRANSACTION_BATCH_INTERVAL_SMOOTHER_ALPHA , 0.1 ) ;
init ( COMMIT_TRANSACTION_BATCH_COUNT_MAX , 32768 ) ; if ( randomize & & BUGGIFY ) COMMIT_TRANSACTION_BATCH_COUNT_MAX = 1000 ; // Do NOT increase this number beyond 32768, as CommitIds only budget 2 bytes for storing transaction id within each batch
2019-05-11 05:01:52 +08:00
init ( COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT , 8LL < < 30 ) ; if ( randomize & & BUGGIFY ) COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT = deterministicRandom ( ) - > randomInt64 ( 100LL < < 20 , 8LL < < 30 ) ;
2018-09-18 09:32:39 +08:00
init ( COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL , 0.5 ) ;
2020-10-05 10:16:51 +08:00
init ( COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR , 5.0 ) ;
2023-01-13 01:11:30 +08:00
init ( COMMIT_TRIGGER_DELAY , 0.01 ) ; if ( randomize & & BUGGIFY ) COMMIT_TRIGGER_DELAY = deterministicRandom ( ) - > random01 ( ) * 4 ;
2017-05-26 04:48:44 +08:00
// these settings disable batch bytes scaling. Try COMMIT_TRANSACTION_BATCH_BYTES_MAX=1e6, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE=50000, COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER=0.5?
init ( COMMIT_TRANSACTION_BATCH_BYTES_MIN , 100000 ) ;
init ( COMMIT_TRANSACTION_BATCH_BYTES_MAX , 100000 ) ; if ( randomize & & BUGGIFY ) { COMMIT_TRANSACTION_BATCH_BYTES_MIN = COMMIT_TRANSACTION_BATCH_BYTES_MAX = 1000000 ; }
init ( COMMIT_TRANSACTION_BATCH_BYTES_SCALE_BASE , 100000 ) ;
init ( COMMIT_TRANSACTION_BATCH_BYTES_SCALE_POWER , 0.0 ) ;
init ( RESOLVER_COALESCE_TIME , 1.0 ) ;
2019-05-11 05:01:52 +08:00
init ( BUGGIFIED_ROW_LIMIT , APPLY_MUTATION_BYTES ) ; if ( randomize & & BUGGIFY ) BUGGIFIED_ROW_LIMIT = deterministicRandom ( ) - > randomInt ( 3 , 30 ) ;
2017-05-26 04:48:44 +08:00
init ( PROXY_SPIN_DELAY , 0.01 ) ;
2018-09-18 09:32:39 +08:00
init ( UPDATE_REMOTE_LOG_VERSION_INTERVAL , 2.0 ) ;
init ( MAX_TXS_POP_VERSION_HISTORY , 1e5 ) ;
2019-07-13 08:58:16 +08:00
init ( MIN_CONFIRM_INTERVAL , 0.05 ) ;
2017-05-26 04:48:44 +08:00
2019-07-13 04:10:21 +08:00
bool shortRecoveryDuration = randomize & & BUGGIFY ;
2019-07-13 08:58:16 +08:00
init ( ENFORCED_MIN_RECOVERY_DURATION , 0.085 ) ; if ( shortRecoveryDuration ) ENFORCED_MIN_RECOVERY_DURATION = 0.01 ;
init ( REQUIRED_MIN_RECOVERY_DURATION , 0.080 ) ; if ( shortRecoveryDuration ) REQUIRED_MIN_RECOVERY_DURATION = 0.01 ;
2019-07-31 09:21:46 +08:00
init ( ALWAYS_CAUSAL_READ_RISKY , false ) ;
2020-01-18 06:24:58 +08:00
init ( MAX_COMMIT_UPDATES , 2000 ) ; if ( randomize & & BUGGIFY ) MAX_COMMIT_UPDATES = 1 ;
2020-02-26 07:46:13 +08:00
init ( MAX_PROXY_COMPUTE , 2.0 ) ;
2020-10-05 10:19:05 +08:00
init ( MAX_COMPUTE_PER_OPERATION , 0.1 ) ;
2022-04-21 06:42:17 +08:00
init ( MAX_COMPUTE_DURATION_LOG_CUTOFF , 0.05 ) ;
2020-02-22 11:22:14 +08:00
init ( PROXY_COMPUTE_BUCKETS , 20000 ) ;
2020-01-27 11:47:12 +08:00
init ( PROXY_COMPUTE_GROWTH_RATE , 0.01 ) ;
2020-07-28 01:59:06 +08:00
init ( TXN_STATE_SEND_AMOUNT , 4 ) ;
2020-08-30 03:35:31 +08:00
init ( REPORT_TRANSACTION_COST_ESTIMATION_DELAY , 0.1 ) ;
2020-11-29 11:58:39 +08:00
init ( PROXY_REJECT_BATCH_QUEUED_TOO_LONG , true ) ;
2022-05-21 04:02:07 +08:00
2022-09-01 06:46:39 +08:00
bool buggfyUseResolverPrivateMutations = randomize & & BUGGIFY & & ! ENABLE_VERSION_VECTOR_TLOG_UNICAST ;
2022-05-21 04:02:07 +08:00
init ( PROXY_USE_RESOLVER_PRIVATE_MUTATIONS , false ) ; if ( buggfyUseResolverPrivateMutations ) PROXY_USE_RESOLVER_PRIVATE_MUTATIONS = deterministicRandom ( ) - > coinflip ( ) ;
2017-05-26 04:48:44 +08:00
2020-10-12 14:06:26 +08:00
init ( RESET_MASTER_BATCHES , 200 ) ;
init ( RESET_RESOLVER_BATCHES , 200 ) ;
init ( RESET_MASTER_DELAY , 300.0 ) ;
init ( RESET_RESOLVER_DELAY , 300.0 ) ;
2022-07-13 02:25:04 +08:00
init ( GLOBAL_CONFIG_MIGRATE_TIMEOUT , 5.0 ) ;
2022-07-09 08:51:45 +08:00
init ( GLOBAL_CONFIG_REFRESH_INTERVAL , 1.0 ) ; if ( randomize & & BUGGIFY ) GLOBAL_CONFIG_REFRESH_INTERVAL = 0.1 ;
2022-07-13 02:25:04 +08:00
init ( GLOBAL_CONFIG_REFRESH_TIMEOUT , 10.0 ) ; if ( randomize & & BUGGIFY ) GLOBAL_CONFIG_REFRESH_TIMEOUT = 1.0 ;
2022-07-08 09:20:18 +08:00
2017-05-26 04:48:44 +08:00
// Master Server
// masterCommitter() in the master server will allow lower priority tasks (e.g. DataDistibution)
// by delay()ing for this amount of time between accepted batches of TransactionRequests.
2018-11-12 16:03:07 +08:00
bool fastBalancing = randomize & & BUGGIFY ;
2017-05-26 04:48:44 +08:00
init ( COMMIT_SLEEP_TIME , 0.0001 ) ; if ( randomize & & BUGGIFY ) COMMIT_SLEEP_TIME = 0 ;
2018-11-13 11:11:28 +08:00
init ( KEY_BYTES_PER_SAMPLE , 2e4 ) ; if ( fastBalancing ) KEY_BYTES_PER_SAMPLE = 1e3 ;
2017-05-26 04:48:44 +08:00
init ( MIN_BALANCE_TIME , 0.2 ) ;
2018-11-13 11:11:28 +08:00
init ( MIN_BALANCE_DIFFERENCE , 1e6 ) ; if ( fastBalancing ) MIN_BALANCE_DIFFERENCE = 1e4 ;
2017-05-26 04:48:44 +08:00
init ( SECONDS_BEFORE_NO_FAILURE_DELAY , 8 * 3600 ) ;
init ( MAX_TXS_SEND_MEMORY , 1e7 ) ; if ( randomize & & BUGGIFY ) MAX_TXS_SEND_MEMORY = 1e5 ;
2018-07-08 08:41:20 +08:00
init ( MAX_RECOVERY_VERSIONS , 200 * VERSIONS_PER_SECOND ) ;
2018-06-14 09:14:14 +08:00
init ( MAX_RECOVERY_TIME , 20.0 ) ; if ( randomize & & BUGGIFY ) MAX_RECOVERY_TIME = 1.0 ;
2019-03-23 05:13:58 +08:00
init ( PROVISIONAL_START_DELAY , 1.0 ) ;
init ( PROVISIONAL_MAX_DELAY , 60.0 ) ;
init ( PROVISIONAL_DELAY_GROWTH , 1.5 ) ;
2020-07-21 02:29:37 +08:00
init ( SECONDS_BEFORE_RECRUIT_BACKUP_WORKER , 4.0 ) ; if ( randomize & & BUGGIFY ) SECONDS_BEFORE_RECRUIT_BACKUP_WORKER = deterministicRandom ( ) - > random01 ( ) * 8 ;
init ( CC_INTERFACE_TIMEOUT , 10.0 ) ; if ( randomize & & BUGGIFY ) CC_INTERFACE_TIMEOUT = 0.0 ;
2017-05-26 04:48:44 +08:00
// Resolver
init ( SAMPLE_OFFSET_PER_KEY , 100 ) ;
init ( SAMPLE_EXPIRATION_TIME , 1.0 ) ;
init ( SAMPLE_POLL_TIME , 0.1 ) ;
init ( RESOLVER_STATE_MEMORY_LIMIT , 1e6 ) ;
2020-01-10 09:05:22 +08:00
init ( LAST_LIMITED_RATIO , 2.0 ) ;
2017-05-26 04:48:44 +08:00
2019-04-25 06:12:37 +08:00
// Backup Worker
2019-05-21 05:22:31 +08:00
init ( BACKUP_TIMEOUT , 0.4 ) ;
2020-01-08 06:15:29 +08:00
init ( BACKUP_NOOP_POP_DELAY , 5.0 ) ;
init ( BACKUP_FILE_BLOCK_BYTES , 1024 * 1024 ) ;
2023-01-20 09:36:06 +08:00
init ( BACKUP_LOCK_BYTES , 3e9 ) ; if ( randomize & & BUGGIFY ) BACKUP_LOCK_BYTES = deterministicRandom ( ) - > randomInt ( 1024 , 4096 ) * 256 * 1024 ;
2020-05-15 03:05:16 +08:00
init ( BACKUP_UPLOAD_DELAY , 10.0 ) ; if ( randomize & & BUGGIFY ) BACKUP_UPLOAD_DELAY = deterministicRandom ( ) - > random01 ( ) * 60 ;
2019-04-25 06:12:37 +08:00
2017-05-26 04:48:44 +08:00
//Cluster Controller
2018-06-22 07:31:52 +08:00
init ( CLUSTER_CONTROLLER_LOGGING_DELAY , 5.0 ) ;
2017-05-26 04:48:44 +08:00
init ( MASTER_FAILURE_REACTION_TIME , 0.4 ) ; if ( randomize & & BUGGIFY ) MASTER_FAILURE_REACTION_TIME = 10.0 ;
init ( MASTER_FAILURE_SLOPE_DURING_RECOVERY , 0.1 ) ;
init ( WORKER_COORDINATION_PING_DELAY , 60 ) ;
init ( SIM_SHUTDOWN_TIMEOUT , 10 ) ;
2017-09-19 02:04:51 +08:00
init ( SHUTDOWN_TIMEOUT , 600 ) ; if ( randomize & & BUGGIFY ) SHUTDOWN_TIMEOUT = 60.0 ;
2017-05-26 04:48:44 +08:00
init ( MASTER_SPIN_DELAY , 1.0 ) ; if ( randomize & & BUGGIFY ) MASTER_SPIN_DELAY = 10.0 ;
2021-10-18 11:47:11 +08:00
init ( CC_PRUNE_CLIENTS_INTERVAL , 60.0 ) ;
2018-02-10 08:48:55 +08:00
init ( CC_CHANGE_DELAY , 0.1 ) ;
2018-06-29 16:11:59 +08:00
init ( CC_CLASS_DELAY , 0.01 ) ;
2018-06-23 01:15:24 +08:00
init ( WAIT_FOR_GOOD_RECRUITMENT_DELAY , 1.0 ) ;
init ( WAIT_FOR_GOOD_REMOTE_RECRUITMENT_DELAY , 5.0 ) ;
2018-02-07 10:11:45 +08:00
init ( ATTEMPT_RECRUITMENT_DELAY , 0.035 ) ;
2019-02-01 02:51:25 +08:00
init ( WAIT_FOR_DISTRIBUTOR_JOIN_DELAY , 1.0 ) ;
2019-02-15 08:24:46 +08:00
init ( WAIT_FOR_RATEKEEPER_JOIN_DELAY , 1.0 ) ;
2022-09-17 00:03:06 +08:00
init ( WAIT_FOR_CONSISTENCYSCAN_JOIN_DELAY , 1.0 ) ;
2021-09-15 23:35:58 +08:00
init ( WAIT_FOR_BLOB_MANAGER_JOIN_DELAY , 1.0 ) ;
2022-01-14 03:11:01 +08:00
init ( WAIT_FOR_ENCRYPT_KEY_PROXY_JOIN_DELAY , 1.0 ) ;
2017-05-26 04:48:44 +08:00
init ( WORKER_FAILURE_TIME , 1.0 ) ; if ( randomize & & BUGGIFY ) WORKER_FAILURE_TIME = 10.0 ;
2018-06-28 14:02:08 +08:00
init ( CHECK_OUTSTANDING_INTERVAL , 0.5 ) ; if ( randomize & & BUGGIFY ) CHECK_OUTSTANDING_INTERVAL = 0.001 ;
2018-06-14 09:14:14 +08:00
init ( VERSION_LAG_METRIC_INTERVAL , 0.5 ) ; if ( randomize & & BUGGIFY ) VERSION_LAG_METRIC_INTERVAL = 10.0 ;
init ( MAX_VERSION_DIFFERENCE , 20 * VERSIONS_PER_SECOND ) ;
2021-09-10 11:29:28 +08:00
init ( INITIAL_UPDATE_CROSS_DC_INFO_DELAY , 300 ) ;
init ( CHECK_REMOTE_HEALTH_INTERVAL , 60 ) ;
2019-02-20 08:05:20 +08:00
init ( FORCE_RECOVERY_CHECK_DELAY , 5.0 ) ;
2019-02-15 08:24:46 +08:00
init ( RATEKEEPER_FAILURE_TIME , 1.0 ) ;
2022-09-17 00:03:06 +08:00
init ( CONSISTENCYSCAN_FAILURE_TIME , 1.0 ) ;
2021-09-15 23:35:58 +08:00
init ( BLOB_MANAGER_FAILURE_TIME , 1.0 ) ;
2022-09-03 10:21:52 +08:00
init ( BLOB_MIGRATOR_FAILURE_TIME , 1.0 ) ;
2019-05-30 07:57:13 +08:00
init ( REPLACE_INTERFACE_DELAY , 60.0 ) ;
init ( REPLACE_INTERFACE_CHECK_DELAY , 5.0 ) ;
2019-08-22 06:02:09 +08:00
init ( COORDINATOR_REGISTER_INTERVAL , 5.0 ) ;
init ( CLIENT_REGISTER_INTERVAL , 600.0 ) ;
2021-06-29 12:45:02 +08:00
init ( CC_ENABLE_WORKER_HEALTH_MONITOR , false ) ;
init ( CC_WORKER_HEALTH_CHECKING_INTERVAL , 60.0 ) ;
init ( CC_DEGRADED_LINK_EXPIRATION_INTERVAL , 300.0 ) ;
init ( CC_MIN_DEGRADATION_INTERVAL , 120.0 ) ;
2022-01-14 03:11:01 +08:00
init ( ENCRYPT_KEY_PROXY_FAILURE_TIME , 0.1 ) ;
2021-06-29 12:45:02 +08:00
init ( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE , 3 ) ;
init ( CC_MAX_EXCLUSION_DUE_TO_HEALTH , 2 ) ;
init ( CC_HEALTH_TRIGGER_RECOVERY , false ) ;
init ( CC_TRACKING_HEALTH_RECOVERY_INTERVAL , 3600.0 ) ;
2021-09-10 11:29:28 +08:00
init ( CC_MAX_HEALTH_RECOVERY_COUNT , 5 ) ;
init ( CC_HEALTH_TRIGGER_FAILOVER , false ) ;
init ( CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION , 5 ) ;
init ( CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION , 10 ) ;
2022-04-07 14:57:34 +08:00
init ( CC_ENABLE_ENTIRE_SATELLITE_MONITORING , false ) ;
init ( CC_SATELLITE_DEGRADATION_MIN_COMPLAINER , 3 ) ;
init ( CC_SATELLITE_DEGRADATION_MIN_BAD_SERVER , 3 ) ;
2022-04-29 06:45:09 +08:00
init ( CC_THROTTLE_SINGLETON_RERECRUIT_INTERVAL , 0.5 ) ;
2018-06-14 09:14:14 +08:00
2017-09-19 02:04:51 +08:00
init ( INCOMPATIBLE_PEERS_LOGGING_INTERVAL , 600 ) ; if ( randomize & & BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0 ;
2020-02-11 13:49:31 +08:00
init ( EXPECTED_MASTER_FITNESS , ProcessClass : : UnsetFit ) ;
init ( EXPECTED_TLOG_FITNESS , ProcessClass : : UnsetFit ) ;
init ( EXPECTED_LOG_ROUTER_FITNESS , ProcessClass : : UnsetFit ) ;
2020-09-16 13:29:49 +08:00
init ( EXPECTED_COMMIT_PROXY_FITNESS , ProcessClass : : UnsetFit ) ;
2020-08-06 15:01:57 +08:00
init ( EXPECTED_GRV_PROXY_FITNESS , ProcessClass : : UnsetFit ) ;
2020-02-11 13:49:31 +08:00
init ( EXPECTED_RESOLVER_FITNESS , ProcessClass : : UnsetFit ) ;
2019-05-11 05:01:52 +08:00
init ( RECRUITMENT_TIMEOUT , 600 ) ; if ( randomize & & BUGGIFY ) RECRUITMENT_TIMEOUT = deterministicRandom ( ) - > coinflip ( ) ? 60.0 : 1.0 ;
2017-05-26 04:48:44 +08:00
init ( POLICY_RATING_TESTS , 200 ) ; if ( randomize & & BUGGIFY ) POLICY_RATING_TESTS = 20 ;
init ( POLICY_GENERATIONS , 100 ) ; if ( randomize & & BUGGIFY ) POLICY_GENERATIONS = 10 ;
2020-05-02 04:20:26 +08:00
init ( DBINFO_SEND_AMOUNT , 5 ) ;
2020-04-11 08:02:11 +08:00
init ( DBINFO_BATCH_DELAY , 0.1 ) ;
2021-12-07 05:12:27 +08:00
init ( SINGLETON_RECRUIT_BME_DELAY , 10.0 ) ;
2017-05-26 04:48:44 +08:00
//Move Keys
init ( SHARD_READY_DELAY , 0.25 ) ;
init ( SERVER_READY_QUORUM_INTERVAL , std : : min ( 1.0 , std : : min ( MAX_READ_TRANSACTION_LIFE_VERSIONS , MAX_WRITE_TRANSACTION_LIFE_VERSIONS ) / ( 5.0 * VERSIONS_PER_SECOND ) ) ) ;
init ( SERVER_READY_QUORUM_TIMEOUT , 15.0 ) ; if ( randomize & & BUGGIFY ) SERVER_READY_QUORUM_TIMEOUT = 1.0 ;
init ( REMOVE_RETRY_DELAY , 1.0 ) ;
init ( MOVE_KEYS_KRM_LIMIT , 2000 ) ; if ( randomize & & BUGGIFY ) MOVE_KEYS_KRM_LIMIT = 2 ;
init ( MOVE_KEYS_KRM_LIMIT_BYTES , 1e5 ) ; if ( randomize & & BUGGIFY ) MOVE_KEYS_KRM_LIMIT_BYTES = 5e4 ; //This must be sufficiently larger than CLIENT_KNOBS->KEY_SIZE_LIMIT (fdbclient/Knobs.h) to ensure that at least two entries will be returned from an attempt to read a key range map
2022-07-08 11:49:16 +08:00
init ( MOVE_SHARD_KRM_ROW_LIMIT , 20000 ) ;
init ( MOVE_SHARD_KRM_BYTE_LIMIT , 1e6 ) ;
2018-03-18 01:36:19 +08:00
init ( MAX_SKIP_TAGS , 1 ) ; //The TLogs require tags to be densely packed to be memory efficient, so be careful increasing this knob
2018-09-01 03:43:14 +08:00
init ( MAX_ADDED_SOURCES_MULTIPLIER , 2.0 ) ;
2017-05-26 04:48:44 +08:00
//FdbServer
bool longReboots = randomize & & BUGGIFY ;
init ( MIN_REBOOT_TIME , 4.0 ) ; if ( longReboots ) MIN_REBOOT_TIME = 10.0 ;
init ( MAX_REBOOT_TIME , 5.0 ) ; if ( longReboots ) MAX_REBOOT_TIME = 20.0 ;
2017-10-13 08:49:41 +08:00
init ( LOG_DIRECTORY , " . " ) ; // Will be set to the command line flag.
Remote ikvs debugging (#6465)
* initial structure for remote IKVS server
* moved struct to .h file, added new files to CMakeList
* happy path implementation, connection error when testing
* saved minor local change
* changed tracing to debug
* fixed onClosed and getError being called before init is finished
* fix spawn process bug, now use absolute path
* added server knob to set ikvs process port number
* added server knob for remote/local kv store
* implement simulator remote process spawning
* fixed bug for simulator timeout
* commit all changes
* removed print lines in trace
* added FlowProcess implementation by Markus
* initial debug of FlowProcess, stuck at parent sending OpenKVStoreRequest to child
* temporary fix for process factory throwing segfault on create
* specify public address in command
* change remote kv store knob to false for jenkins build
* made port 0 open random unused port
* change remote store knob to true for benchmark
* set listening port to randomly opened port
* added print lines for jenkins run open kv store timeout debug
* removed most tracing and print lines
* removed tutorial changes
* update handleIOErrors error handling to handle remote-ikvs cases
* Push all debugging changes
* A version where worker bug exists
* A version where restarting tests fail
* Use both the name and the port to determine the child process
* Remove unnecessary update on local address
* Disable remote-kvs for DiskFailureCycle test
* A version where restarting stuck
* A version where most restarting tests green
* Reset connection with child process explicitly
* Remove change on unnecessary files
* Unify flags from _ to -
* fix merging unexpected changes
* fix trac.error to .errorUnsuppressed
* Add license header
* Remove unnecessary header in FlowProcess.actor.cpp
* Fix Windows build
* Fix Windows build, add missing ;
* Fix a stupid bug caused by code dropped by code merging
* Disable remote kvs by default
* Pass the conn_file path to the flow process, though not needed, but the buildNetwork is difficult to tune
* serialization change on readrange
* Update traces
* Refactor the RemoteIKVS interface
* Format files
* Update sim2 interface to not clog connections between parent and child processes in simulation
* Update comments; remove debugging symbols; Add error handling for remote_kvs_cancelled
* Add comments, format files
* Change method name from isBuggifyDisabled to isStableConnection; Decrease(0.1x) latency for stable connections
* Commit the IConnection interface change, forgot in previous commit
* Fix the issue that onClosed request is cancelled by ActorCollection
* Enable the remote kv store knob
* Remove FlowProcess.actor.cpp and move functions to RemoteIKeyValueStore.actor.cpp; Add remote kv store delay to avoid race; Bind the child process to die with parent process
* Fix the bug where one process starts storage server more than once
* Add a please_reboot_remote_kv_store error to restart the storage server worker if remote kvs died abnormally
* Remove unreachable code path and add comments
* Clang format the code
* Fix a simple wait error
* Clang format after merging the main branch
* Testing mixed mode in simulation if remote_kvs knob is enabled, setting the default to false
* Disable remote kvs for PhysicalShardMove which is for RocksDB
* Cleanup #include orders, remove debugging traces
* Revert the reorder in fdbserver.actor.cpp, which fails the gcc build
Co-authored-by: “Lincoln <“lincoln.xiao@snowflake.com”>
2022-04-01 08:08:59 +08:00
init ( CONN_FILE , " " ) ; // Will be set to the command line flag.
2020-02-11 13:49:31 +08:00
init ( SERVER_MEM_LIMIT , 8LL < < 30 ) ;
2021-03-24 02:15:37 +08:00
init ( SYSTEM_MONITOR_FREQUENCY , 5.0 ) ;
2017-05-26 04:48:44 +08:00
//Ratekeeper
2019-03-27 23:24:25 +08:00
bool slowRatekeeper = randomize & & BUGGIFY ;
init ( SMOOTHING_AMOUNT , 1.0 ) ; if ( slowRatekeeper ) SMOOTHING_AMOUNT = 5.0 ;
init ( SLOW_SMOOTHING_AMOUNT , 10.0 ) ; if ( slowRatekeeper ) SLOW_SMOOTHING_AMOUNT = 50.0 ;
init ( METRIC_UPDATE_RATE , .1 ) ; if ( slowRatekeeper ) METRIC_UPDATE_RATE = 0.5 ;
2019-02-01 04:56:58 +08:00
init ( DETAILED_METRIC_UPDATE_RATE , 5.0 ) ;
2022-09-01 06:46:39 +08:00
init ( RATEKEEPER_DEFAULT_LIMIT , 1e6 ) ; if ( randomize & & BUGGIFY ) RATEKEEPER_DEFAULT_LIMIT = 0 ;
2022-02-09 05:58:49 +08:00
init ( RATEKEEPER_LIMIT_REASON_SAMPLE_RATE , 0.1 ) ;
init ( RATEKEEPER_PRINT_LIMIT_REASON , false ) ; if ( randomize & & BUGGIFY ) RATEKEEPER_PRINT_LIMIT_REASON = true ;
2022-09-01 06:46:39 +08:00
init ( RATEKEEPER_MIN_RATE , 0.0 ) ;
init ( RATEKEEPER_MAX_RATE , 1e9 ) ;
init ( RATEKEEPER_BATCH_MIN_RATE , 0.0 ) ;
init ( RATEKEEPER_BATCH_MAX_RATE , 1e9 ) ;
2017-05-26 04:48:44 +08:00
bool smallStorageTarget = randomize & & BUGGIFY ;
2018-05-08 08:26:44 +08:00
init ( TARGET_BYTES_PER_STORAGE_SERVER , 1000e6 ) ; if ( smallStorageTarget ) TARGET_BYTES_PER_STORAGE_SERVER = 3000e3 ;
init ( SPRING_BYTES_STORAGE_SERVER , 100e6 ) ; if ( smallStorageTarget ) SPRING_BYTES_STORAGE_SERVER = 300e3 ;
2020-04-25 02:31:16 +08:00
init ( AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES , 800e6 ) ; if ( smallStorageTarget ) AUTO_TAG_THROTTLE_STORAGE_QUEUE_BYTES = 2500e3 ;
2019-08-01 08:19:41 +08:00
init ( TARGET_BYTES_PER_STORAGE_SERVER_BATCH , 750e6 ) ; if ( smallStorageTarget ) TARGET_BYTES_PER_STORAGE_SERVER_BATCH = 1500e3 ;
init ( SPRING_BYTES_STORAGE_SERVER_BATCH , 100e6 ) ; if ( smallStorageTarget ) SPRING_BYTES_STORAGE_SERVER_BATCH = 150e3 ;
2018-05-08 08:26:44 +08:00
init ( STORAGE_HARD_LIMIT_BYTES , 1500e6 ) ; if ( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES = 4500e3 ;
2021-10-20 23:45:13 +08:00
init ( STORAGE_HARD_LIMIT_BYTES_OVERAGE , 5000e3 ) ; if ( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_OVERAGE = 100e3 ; // byte+version overage ensures storage server makes enough progress on freeing up storage queue memory at hard limit by ensuring it advances desiredOldestVersion enough per commit cycle.
2022-09-01 01:59:46 +08:00
init ( STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM , STORAGE_HARD_LIMIT_BYTES ) ; if ( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_SPEED_UP_SIM * = 10 ;
init ( STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM , STORAGE_HARD_LIMIT_BYTES_OVERAGE ) ; if ( smallStorageTarget ) STORAGE_HARD_LIMIT_BYTES_OVERAGE_SPEED_UP_SIM * = 10 ;
2021-10-20 23:45:13 +08:00
init ( STORAGE_HARD_LIMIT_VERSION_OVERAGE , VERSIONS_PER_SECOND / 4.0 ) ;
2019-06-29 04:05:43 +08:00
init ( STORAGE_DURABILITY_LAG_HARD_MAX , 2000e6 ) ; if ( smallStorageTarget ) STORAGE_DURABILITY_LAG_HARD_MAX = 100e6 ;
2021-02-08 14:58:11 +08:00
init ( STORAGE_DURABILITY_LAG_SOFT_MAX , 250e6 ) ; if ( smallStorageTarget ) STORAGE_DURABILITY_LAG_SOFT_MAX = 10e6 ;
2022-09-09 07:37:44 +08:00
init ( STORAGE_INCLUDE_FEED_STORAGE_QUEUE , true ) ; if ( randomize & & BUGGIFY ) STORAGE_INCLUDE_FEED_STORAGE_QUEUE = false ;
2017-05-26 04:48:44 +08:00
2021-01-21 08:16:50 +08:00
//FIXME: Low priority reads are disabled by assigning very high knob values, reduce knobs for 7.0
2021-02-03 09:53:07 +08:00
init ( LOW_PRIORITY_STORAGE_QUEUE_BYTES , 775e8 ) ; if ( smallStorageTarget ) LOW_PRIORITY_STORAGE_QUEUE_BYTES = 1750e3 ;
2021-02-08 14:58:11 +08:00
init ( LOW_PRIORITY_DURABILITY_LAG , 200e6 ) ; if ( smallStorageTarget ) LOW_PRIORITY_DURABILITY_LAG = 15e6 ;
2021-01-21 08:16:50 +08:00
2017-05-26 04:48:44 +08:00
bool smallTlogTarget = randomize & & BUGGIFY ;
2018-08-05 01:31:30 +08:00
init ( TARGET_BYTES_PER_TLOG , 2400e6 ) ; if ( smallTlogTarget ) TARGET_BYTES_PER_TLOG = 2000e3 ;
2019-03-16 12:01:20 +08:00
init ( SPRING_BYTES_TLOG , 400e6 ) ; if ( smallTlogTarget ) SPRING_BYTES_TLOG = 200e3 ;
2019-08-01 08:19:41 +08:00
init ( TARGET_BYTES_PER_TLOG_BATCH , 1400e6 ) ; if ( smallTlogTarget ) TARGET_BYTES_PER_TLOG_BATCH = 1400e3 ;
init ( SPRING_BYTES_TLOG_BATCH , 300e6 ) ; if ( smallTlogTarget ) SPRING_BYTES_TLOG_BATCH = 150e3 ;
2017-05-26 04:48:44 +08:00
init ( TLOG_SPILL_THRESHOLD , 1500e6 ) ; if ( smallTlogTarget ) TLOG_SPILL_THRESHOLD = 1500e3 ; if ( randomize & & BUGGIFY ) TLOG_SPILL_THRESHOLD = 0 ;
2019-03-16 12:01:20 +08:00
init ( REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT , 20e6 ) ; if ( ( randomize & & BUGGIFY ) | | smallTlogTarget ) REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT = 1e6 ;
2020-04-30 04:50:13 +08:00
init ( TLOG_HARD_LIMIT_BYTES , 3000e6 ) ; if ( smallTlogTarget ) TLOG_HARD_LIMIT_BYTES = 30e6 ;
2018-08-22 12:11:23 +08:00
init ( TLOG_RECOVER_MEMORY_LIMIT , TARGET_BYTES_PER_TLOG + SPRING_BYTES_TLOG ) ;
2017-05-26 04:48:44 +08:00
init ( MAX_TRANSACTIONS_PER_BYTE , 1000 ) ;
2020-02-22 07:14:32 +08:00
init ( MIN_AVAILABLE_SPACE , 1e8 ) ;
init ( MIN_AVAILABLE_SPACE_RATIO , 0.05 ) ;
2022-02-17 11:26:40 +08:00
init ( MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER , 0.01 ) ;
2020-02-22 07:14:32 +08:00
init ( TARGET_AVAILABLE_SPACE_RATIO , 0.30 ) ;
init ( AVAILABLE_SPACE_UPDATE_DELAY , 5.0 ) ;
2017-05-26 04:48:44 +08:00
init ( MAX_TL_SS_VERSION_DIFFERENCE , 1e99 ) ; // if( randomize && BUGGIFY ) MAX_TL_SS_VERSION_DIFFERENCE = std::max(1.0, 0.25 * VERSIONS_PER_SECOND); // spring starts at half this value //FIXME: this knob causes ratekeeper to clamp on idle cluster in simulation that have a large number of logs
2019-02-28 02:31:56 +08:00
init ( MAX_TL_SS_VERSION_DIFFERENCE_BATCH , 1e99 ) ;
2017-05-26 04:48:44 +08:00
init ( MAX_MACHINES_FALLING_BEHIND , 1 ) ;
2019-07-12 09:34:19 +08:00
init ( MAX_TPS_HISTORY_SAMPLES , 600 ) ;
init ( NEEDED_TPS_HISTORY_SAMPLES , 200 ) ;
2019-08-22 05:55:21 +08:00
init ( TARGET_DURABILITY_LAG_VERSIONS , 350e6 ) ; // Should be larger than STORAGE_DURABILITY_LAG_SOFT_MAX
2020-04-25 02:31:16 +08:00
init ( AUTO_TAG_THROTTLE_DURABILITY_LAG_VERSIONS , 250e6 ) ;
2021-02-08 14:58:11 +08:00
init ( TARGET_DURABILITY_LAG_VERSIONS_BATCH , 150e6 ) ; // Should be larger than STORAGE_DURABILITY_LAG_SOFT_MAX
2019-07-13 04:40:18 +08:00
init ( DURABILITY_LAG_UNLIMITED_THRESHOLD , 50e6 ) ;
2019-07-12 09:34:19 +08:00
init ( INITIAL_DURABILITY_LAG_MULTIPLIER , 1.02 ) ;
init ( DURABILITY_LAG_REDUCTION_RATE , 0.9999 ) ;
2019-07-13 04:40:18 +08:00
init ( DURABILITY_LAG_INCREASE_RATE , 1.001 ) ;
2019-07-18 08:36:09 +08:00
init ( STORAGE_SERVER_LIST_FETCH_TIMEOUT , 20.0 ) ;
2022-09-01 06:46:39 +08:00
init ( BW_THROTTLING_ENABLED , true ) ;
bool buggifySmallBWLag = randomize & & BUGGIFY ;
2023-01-27 06:56:45 +08:00
init ( TARGET_BW_LAG , 90.0 ) ; if ( buggifySmallBWLag ) TARGET_BW_LAG = 10.0 ;
init ( TARGET_BW_LAG_BATCH , 60.0 ) ; if ( buggifySmallBWLag ) TARGET_BW_LAG_BATCH = 4.0 ;
2022-09-01 06:46:39 +08:00
init ( TARGET_BW_LAG_UPDATE , 9.0 ) ; if ( buggifySmallBWLag ) TARGET_BW_LAG_UPDATE = 1.0 ;
init ( MIN_BW_HISTORY , 10 ) ;
init ( BW_ESTIMATION_INTERVAL , 10.0 ) ; if ( buggifySmallBWLag ) BW_ESTIMATION_INTERVAL = 2.0 ;
init ( BW_LAG_INCREASE_AMOUNT , 1.1 ) ;
init ( BW_LAG_DECREASE_AMOUNT , 0.9 ) ;
init ( BW_FETCH_WORKERS_INTERVAL , 5.0 ) ;
init ( BW_RW_LOGGING_INTERVAL , 5.0 ) ;
init ( BW_MAX_BLOCKED_INTERVAL , 10.0 ) ; if ( buggifySmallBWLag ) BW_MAX_BLOCKED_INTERVAL = 2.0 ;
2023-01-27 06:56:45 +08:00
init ( BW_RK_SIM_QUIESCE_DELAY , 300.0 ) ;
2019-07-31 13:35:34 +08:00
2020-04-18 02:48:02 +08:00
init ( MAX_AUTO_THROTTLED_TRANSACTION_TAGS , 5 ) ; if ( randomize & & BUGGIFY ) MAX_AUTO_THROTTLED_TRANSACTION_TAGS = 1 ;
2020-05-05 01:37:53 +08:00
init ( MAX_MANUAL_THROTTLED_TRANSACTION_TAGS , 40 ) ; if ( randomize & & BUGGIFY ) MAX_MANUAL_THROTTLED_TRANSACTION_TAGS = 1 ;
2020-05-08 00:15:33 +08:00
init ( MIN_TAG_COST , 200 ) ; if ( randomize & & BUGGIFY ) MIN_TAG_COST = 0.0 ;
2020-04-25 02:31:16 +08:00
init ( AUTO_THROTTLE_TARGET_TAG_BUSYNESS , 0.1 ) ; if ( randomize & & BUGGIFY ) AUTO_THROTTLE_TARGET_TAG_BUSYNESS = 0.0 ;
2020-04-29 06:50:45 +08:00
init ( AUTO_TAG_THROTTLE_RAMP_UP_TIME , 120.0 ) ; if ( randomize & & BUGGIFY ) AUTO_TAG_THROTTLE_RAMP_UP_TIME = 5.0 ;
init ( AUTO_TAG_THROTTLE_DURATION , 240.0 ) ; if ( randomize & & BUGGIFY ) AUTO_TAG_THROTTLE_DURATION = 20.0 ;
2020-04-23 03:28:51 +08:00
init ( TAG_THROTTLE_PUSH_INTERVAL , 1.0 ) ; if ( randomize & & BUGGIFY ) TAG_THROTTLE_PUSH_INTERVAL = 0.0 ;
2020-04-29 05:30:37 +08:00
init ( AUTO_TAG_THROTTLE_START_AGGREGATION_TIME , 5.0 ) ; if ( randomize & & BUGGIFY ) AUTO_TAG_THROTTLE_START_AGGREGATION_TIME = 0.5 ;
2020-04-29 06:50:45 +08:00
init ( AUTO_TAG_THROTTLE_UPDATE_FREQUENCY , 10.0 ) ; if ( randomize & & BUGGIFY ) AUTO_TAG_THROTTLE_UPDATE_FREQUENCY = 0.5 ;
2020-05-02 12:36:28 +08:00
init ( TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL , 30.0 ) ; if ( randomize & & BUGGIFY ) TAG_THROTTLE_EXPIRED_CLEANUP_INTERVAL = 1.0 ;
2020-04-04 06:24:14 +08:00
init ( AUTO_TAG_THROTTLING_ENABLED , true ) ; if ( randomize & & BUGGIFY ) AUTO_TAG_THROTTLING_ENABLED = false ;
2022-04-03 09:42:27 +08:00
init ( SS_THROTTLE_TAGS_TRACKED , 1 ) ; if ( randomize & & BUGGIFY ) SS_THROTTLE_TAGS_TRACKED = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
2023-02-09 03:33:32 +08:00
init ( GLOBAL_TAG_THROTTLING , true ) ; if ( isSimulated ) GLOBAL_TAG_THROTTLING = deterministicRandom ( ) - > coinflip ( ) ;
2022-10-15 08:08:49 +08:00
init ( ENFORCE_TAG_THROTTLING_ON_PROXIES , GLOBAL_TAG_THROTTLING ) ;
2022-03-29 00:32:18 +08:00
init ( GLOBAL_TAG_THROTTLING_MIN_RATE , 1.0 ) ;
2022-11-09 06:46:03 +08:00
// 60 seconds was chosen as a default value to ensure that
// the global tag throttler does not react too drastically to
// changes in workload. To make the global tag throttler more reactive,
// lower this knob. To make global tag throttler more smooth, raise this knob.
// Setting this knob lower than TAG_MEASUREMENT_INTERVAL can cause erratic
// behaviour and is not recommended.
2022-11-09 02:02:24 +08:00
init ( GLOBAL_TAG_THROTTLING_FOLDING_TIME , 60.0 ) ;
2022-10-11 03:28:26 +08:00
init ( GLOBAL_TAG_THROTTLING_MAX_TAGS_TRACKED , 10 ) ;
init ( GLOBAL_TAG_THROTTLING_TAG_EXPIRE_AFTER , 240.0 ) ;
2022-10-26 08:37:26 +08:00
init ( GLOBAL_TAG_THROTTLING_PROXY_LOGGING_INTERVAL , 60.0 ) ;
2023-01-25 02:56:48 +08:00
init ( GLOBAL_TAG_THROTTLING_MIN_TPS , 1.0 ) ;
2020-04-04 06:24:14 +08:00
2017-05-26 04:48:44 +08:00
//Storage Metrics
init ( STORAGE_METRICS_AVERAGE_INTERVAL , 120.0 ) ;
init ( STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS , 1000.0 / STORAGE_METRICS_AVERAGE_INTERVAL ) ; // milliHz!
init ( SPLIT_JITTER_AMOUNT , 0.05 ) ; if ( randomize & & BUGGIFY ) SPLIT_JITTER_AMOUNT = 0.2 ;
init ( IOPS_UNITS_PER_SAMPLE , 10000 * 1000 / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 100 ) ;
2022-11-15 04:31:13 +08:00
init ( BYTES_WRITTEN_UNITS_PER_SAMPLE , SHARD_MIN_BYTES_PER_KSEC / STORAGE_METRICS_AVERAGE_INTERVAL_PER_KSECONDS / 25 ) ;
2019-10-31 01:04:19 +08:00
init ( BYTES_READ_UNITS_PER_SAMPLE , 100000 ) ; // 100K bytes
2019-12-20 08:02:42 +08:00
init ( READ_HOT_SUB_RANGE_CHUNK_SIZE , 10000000 ) ; // 10MB
2019-11-20 13:00:57 +08:00
init ( EMPTY_READ_PENALTY , 20 ) ; // 20 bytes
2022-04-28 14:37:35 +08:00
init ( DD_SHARD_COMPARE_LIMIT , 1000 ) ;
2020-09-01 00:30:51 +08:00
init ( READ_SAMPLING_ENABLED , false ) ; if ( randomize & & BUGGIFY ) READ_SAMPLING_ENABLED = true ; // enable/disable read sampling
2017-05-26 04:48:44 +08:00
//Storage Server
init ( STORAGE_LOGGING_DELAY , 5.0 ) ;
init ( STORAGE_SERVER_POLL_METRICS_DELAY , 1.0 ) ;
2018-06-30 02:29:22 +08:00
init ( FUTURE_VERSION_DELAY , 1.0 ) ;
2017-05-26 04:48:44 +08:00
init ( STORAGE_LIMIT_BYTES , 500000 ) ;
init ( BUGGIFY_LIMIT_BYTES , 1000 ) ;
2022-06-01 07:04:28 +08:00
init ( FETCH_USING_STREAMING , false ) ; if ( randomize & & isSimulated & & BUGGIFY ) FETCH_USING_STREAMING = true ; //Determines if fetch keys uses streaming reads
2022-09-01 06:46:39 +08:00
init ( FETCH_USING_BLOB , false ) ;
2017-05-26 04:48:44 +08:00
init ( FETCH_BLOCK_BYTES , 2e6 ) ;
2019-04-27 04:54:49 +08:00
init ( FETCH_KEYS_PARALLELISM_BYTES , 4e6 ) ; if ( randomize & & BUGGIFY ) FETCH_KEYS_PARALLELISM_BYTES = 3e6 ;
2021-06-20 00:47:13 +08:00
init ( FETCH_KEYS_PARALLELISM , 2 ) ;
2022-09-01 06:46:39 +08:00
init ( FETCH_KEYS_PARALLELISM_FULL , 6 ) ;
2019-07-09 05:28:19 +08:00
init ( FETCH_KEYS_LOWER_PRIORITY , 0 ) ;
2022-04-09 04:30:00 +08:00
init ( SERVE_FETCH_CHECKPOINT_PARALLELISM , 4 ) ;
2022-11-08 02:02:35 +08:00
init ( SERVE_AUDIT_STORAGE_PARALLELISM , 1 ) ;
2017-05-26 04:48:44 +08:00
init ( BUGGIFY_BLOCK_BYTES , 10000 ) ;
2022-03-28 03:37:38 +08:00
init ( STORAGE_RECOVERY_VERSION_LAG_LIMIT , 2 * MAX_READ_TRANSACTION_LIFE_VERSIONS ) ;
2017-05-26 04:48:44 +08:00
init ( STORAGE_COMMIT_BYTES , 10000000 ) ; if ( randomize & & BUGGIFY ) STORAGE_COMMIT_BYTES = 2000000 ;
2021-06-20 00:47:13 +08:00
init ( STORAGE_FETCH_BYTES , 2500000 ) ; if ( randomize & & BUGGIFY ) STORAGE_FETCH_BYTES = 500000 ;
2019-06-29 04:05:43 +08:00
init ( STORAGE_DURABILITY_LAG_REJECT_THRESHOLD , 0.25 ) ;
2019-07-06 06:46:16 +08:00
init ( STORAGE_DURABILITY_LAG_MIN_RATE , 0.1 ) ;
2017-05-26 04:48:44 +08:00
init ( STORAGE_COMMIT_INTERVAL , 0.5 ) ; if ( randomize & & BUGGIFY ) STORAGE_COMMIT_INTERVAL = 2.0 ;
init ( BYTE_SAMPLING_FACTOR , 250 ) ; //cannot buggify because of differences in restarting tests
init ( BYTE_SAMPLING_OVERHEAD , 100 ) ;
init ( MAX_STORAGE_SERVER_WATCH_BYTES , 100e6 ) ; if ( randomize & & BUGGIFY ) MAX_STORAGE_SERVER_WATCH_BYTES = 10e3 ;
init ( MAX_BYTE_SAMPLE_CLEAR_MAP_SIZE , 1e9 ) ; if ( randomize & & BUGGIFY ) MAX_BYTE_SAMPLE_CLEAR_MAP_SIZE = 1e3 ;
init ( LONG_BYTE_SAMPLE_RECOVERY_DELAY , 60.0 ) ;
2019-06-14 06:55:25 +08:00
init ( BYTE_SAMPLE_LOAD_PARALLELISM , 8 ) ; if ( randomize & & BUGGIFY ) BYTE_SAMPLE_LOAD_PARALLELISM = 1 ;
2019-01-05 02:32:31 +08:00
init ( BYTE_SAMPLE_LOAD_DELAY , 0.0 ) ; if ( randomize & & BUGGIFY ) BYTE_SAMPLE_LOAD_DELAY = 0.1 ;
2019-06-26 00:30:11 +08:00
init ( BYTE_SAMPLE_START_DELAY , 1.0 ) ; if ( randomize & & BUGGIFY ) BYTE_SAMPLE_START_DELAY = 0.0 ;
2020-01-11 04:23:59 +08:00
init ( BEHIND_CHECK_DELAY , 2.0 ) ;
init ( BEHIND_CHECK_COUNT , 2 ) ;
init ( BEHIND_CHECK_VERSIONS , 5 * VERSIONS_PER_SECOND ) ;
2020-02-20 07:20:38 +08:00
init ( WAIT_METRICS_WRONG_SHARD_CHANCE , isSimulated ? 1.0 : 0.1 ) ;
2020-09-12 04:50:19 +08:00
init ( MIN_TAG_READ_PAGES_RATE , 1.0e4 ) ; if ( randomize & & BUGGIFY ) MIN_TAG_READ_PAGES_RATE = 0 ;
init ( MIN_TAG_WRITE_PAGES_RATE , 3200 ) ; if ( randomize & & BUGGIFY ) MIN_TAG_WRITE_PAGES_RATE = 0 ;
2020-08-03 02:02:36 +08:00
init ( TAG_MEASUREMENT_INTERVAL , 30.0 ) ; if ( randomize & & BUGGIFY ) TAG_MEASUREMENT_INTERVAL = 1.0 ;
2020-07-28 01:59:06 +08:00
init ( PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS , true ) ; if ( randomize & & BUGGIFY ) PREFIX_COMPRESS_KVS_MEM_SNAPSHOTS = false ;
2020-11-08 12:22:29 +08:00
init ( REPORT_DD_METRICS , true ) ;
init ( DD_METRICS_REPORT_INTERVAL , 30.0 ) ;
2020-11-13 05:01:41 +08:00
init ( FETCH_KEYS_TOO_LONG_TIME_CRITERIA , 300.0 ) ;
2021-02-04 06:24:39 +08:00
init ( MAX_STORAGE_COMMIT_TIME , 120.0 ) ; //The max fsync stall time on the storage server and tlog before marking a disk as failed
2021-06-20 00:47:13 +08:00
init ( RANGESTREAM_LIMIT_BYTES , 2e6 ) ; if ( randomize & & BUGGIFY ) RANGESTREAM_LIMIT_BYTES = 1 ;
2022-02-05 06:41:25 +08:00
init ( CHANGEFEEDSTREAM_LIMIT_BYTES , 1e6 ) ; if ( randomize & & BUGGIFY ) CHANGEFEEDSTREAM_LIMIT_BYTES = 1 ;
init ( BLOBWORKERSTATUSSTREAM_LIMIT_BYTES , 1e4 ) ; if ( randomize & & BUGGIFY ) BLOBWORKERSTATUSSTREAM_LIMIT_BYTES = 1 ;
2022-11-12 07:20:26 +08:00
init ( ENABLE_CLEAR_RANGE_EAGER_READS , true ) ; if ( randomize & & BUGGIFY ) ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom ( ) - > coinflip ( ) ;
2022-03-16 04:03:23 +08:00
init ( CHECKPOINT_TRANSFER_BLOCK_BYTES , 40e6 ) ;
2022-03-11 02:05:44 +08:00
init ( QUICK_GET_VALUE_FALLBACK , true ) ;
init ( QUICK_GET_KEY_VALUES_FALLBACK , true ) ;
2022-11-12 07:20:26 +08:00
init ( STRICTLY_ENFORCE_BYTE_LIMIT , false ) ; if ( randomize & & BUGGIFY ) STRICTLY_ENFORCE_BYTE_LIMIT = deterministicRandom ( ) - > coinflip ( ) ;
init ( FRACTION_INDEX_BYTELIMIT_PREFETCH , 0.2 ) ; if ( randomize & & BUGGIFY ) FRACTION_INDEX_BYTELIMIT_PREFETCH = 0.01 + deterministicRandom ( ) - > random01 ( ) ;
2022-11-12 05:36:06 +08:00
init ( MAX_PARALLEL_QUICK_GET_VALUE , 10 ) ; if ( randomize & & BUGGIFY ) MAX_PARALLEL_QUICK_GET_VALUE = deterministicRandom ( ) - > randomInt ( 1 , 100 ) ;
2022-03-11 02:05:44 +08:00
init ( QUICK_GET_KEY_VALUES_LIMIT , 2000 ) ;
init ( QUICK_GET_KEY_VALUES_LIMIT_BYTES , 1e7 ) ;
2022-09-14 22:07:28 +08:00
init ( STORAGE_FEED_QUERY_HARD_LIMIT , 100000 ) ;
2022-11-11 16:02:47 +08:00
// Read priority definitions in the form of a list of their relative concurrency share weights
init ( STORAGESERVER_READ_PRIORITIES , " 120,10,20,40,60 " ) ;
// The total concurrency which will be shared by active priorities according to their relative weights
2022-10-07 14:41:28 +08:00
init ( STORAGE_SERVER_READ_CONCURRENCY , 70 ) ;
2022-11-11 16:02:47 +08:00
// The priority number which each ReadType maps to in enumeration order
// This exists for flexibility but assigning each ReadType to its own unique priority number makes the most sense
// The enumeration is currently: eager, fetch, low, normal, high
init ( STORAGESERVER_READTYPE_PRIORITY_MAP , " 0,1,2,3,4 " ) ;
2023-02-16 07:40:55 +08:00
init ( SPLIT_METRICS_MAX_ROWS , 10000 ) ; if ( randomize & & BUGGIFY ) SPLIT_METRICS_MAX_ROWS = 10 ;
2017-05-26 04:48:44 +08:00
//Wait Failure
init ( MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS , 250 ) ; if ( randomize & & BUGGIFY ) MAX_OUTSTANDING_WAIT_FAILURE_REQUESTS = 2 ;
init ( WAIT_FAILURE_DELAY_LIMIT , 1.0 ) ; if ( randomize & & BUGGIFY ) WAIT_FAILURE_DELAY_LIMIT = 5.0 ;
//Worker
init ( WORKER_LOGGING_INTERVAL , 5.0 ) ;
2019-04-06 05:45:58 +08:00
init ( HEAP_PROFILER_INTERVAL , 30.0 ) ;
2021-05-13 07:21:44 +08:00
init ( UNKNOWN_CC_TIMEOUT , 600.0 ) ;
2022-06-07 04:14:49 +08:00
init ( DEGRADED_RESET_INTERVAL , 24 * 60 * 60 ) ; // FIXME: short interval causes false positive degraded state to flap, e.g. when everyone tries and fails to connect to dead coordinator: if ( randomize && BUGGIFY ) DEGRADED_RESET_INTERVAL = 10;
2019-04-05 05:11:12 +08:00
init ( DEGRADED_WARNING_LIMIT , 1 ) ;
2019-04-08 14:00:58 +08:00
init ( DEGRADED_WARNING_RESET_DELAY , 7 * 24 * 60 * 60 ) ;
2020-03-13 05:34:19 +08:00
init ( TRACE_LOG_FLUSH_FAILURE_CHECK_INTERVAL_SECONDS , 10 ) ;
init ( TRACE_LOG_PING_TIMEOUT_SECONDS , 5.0 ) ;
2020-06-11 00:59:56 +08:00
init ( MIN_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS , 10.0 ) ;
2020-08-15 01:16:20 +08:00
init ( MAX_DELAY_CC_WORST_FIT_CANDIDACY_SECONDS , 30.0 ) ;
2020-04-12 11:54:17 +08:00
init ( DBINFO_FAILED_DELAY , 1.0 ) ;
2021-06-16 08:36:56 +08:00
init ( ENABLE_WORKER_HEALTH_MONITOR , false ) ;
init ( WORKER_HEALTH_MONITOR_INTERVAL , 60.0 ) ;
2021-07-26 13:37:08 +08:00
init ( PEER_LATENCY_CHECK_MIN_POPULATION , 30 ) ;
2022-09-30 05:18:26 +08:00
init ( PEER_LATENCY_DEGRADATION_PERCENTILE , 0.50 ) ;
2021-06-16 08:36:56 +08:00
init ( PEER_LATENCY_DEGRADATION_THRESHOLD , 0.05 ) ;
2022-09-30 05:18:26 +08:00
init ( PEER_LATENCY_DEGRADATION_PERCENTILE_SATELLITE , 0.50 ) ;
2022-04-07 14:57:34 +08:00
init ( PEER_LATENCY_DEGRADATION_THRESHOLD_SATELLITE , 0.1 ) ;
2021-06-16 08:36:56 +08:00
init ( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD , 0.1 ) ;
2022-09-27 05:45:47 +08:00
init ( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT , 5 ) ;
2022-04-06 13:22:06 +08:00
init ( WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER , true ) ;
2022-09-07 05:35:01 +08:00
init ( STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT , false ) ; if ( randomize & & BUGGIFY ) STORAGE_SERVER_REBOOT_ON_IO_TIMEOUT = true ;
2017-05-26 04:48:44 +08:00
// Test harness
init ( WORKER_POLL_DELAY , 1.0 ) ;
// Coordination
init ( COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL , 1.0 ) ; if ( randomize & & BUGGIFY ) COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL = 10.0 ;
2021-06-10 02:24:00 +08:00
init ( FORWARD_REQUEST_TOO_OLD , 4 * 24 * 60 * 60 ) ; if ( randomize & & BUGGIFY ) FORWARD_REQUEST_TOO_OLD = 60.0 ;
2021-03-31 00:31:10 +08:00
init ( ENABLE_CROSS_CLUSTER_SUPPORT , true ) ; if ( randomize & & BUGGIFY ) ENABLE_CROSS_CLUSTER_SUPPORT = false ;
2021-07-20 11:17:46 +08:00
init ( COORDINATOR_LEADER_CONNECTION_TIMEOUT , 20.0 ) ;
2017-05-26 04:48:44 +08:00
2021-10-22 14:47:05 +08:00
// Dynamic Knobs (implementation)
2022-02-23 02:40:36 +08:00
init ( COMPACTION_INTERVAL , isSimulated ? 5.0 : 300.0 ) ;
2022-08-03 14:51:24 +08:00
init ( BROADCASTER_SELF_UPDATE_DELAY , 1.0 ) ;
2022-02-02 14:27:12 +08:00
init ( GET_COMMITTED_VERSION_TIMEOUT , 3.0 ) ;
init ( GET_SNAPSHOT_AND_CHANGES_TIMEOUT , 3.0 ) ;
2022-02-09 04:35:47 +08:00
init ( FETCH_CHANGES_TIMEOUT , 3.0 ) ;
2021-10-22 14:47:05 +08:00
2017-05-26 04:48:44 +08:00
// Buggification
init ( BUGGIFIED_EVENTUAL_CONSISTENCY , 1.0 ) ;
2020-02-11 13:49:31 +08:00
init ( BUGGIFY_ALL_COORDINATION , false ) ; if ( randomize & & BUGGIFY ) BUGGIFY_ALL_COORDINATION = true ;
2017-05-26 04:48:44 +08:00
// Status
init ( STATUS_MIN_TIME_BETWEEN_REQUESTS , 0.0 ) ;
2019-01-29 07:37:30 +08:00
init ( MAX_STATUS_REQUESTS_PER_SECOND , 256.0 ) ;
2017-05-26 04:48:44 +08:00
init ( CONFIGURATION_ROWS_TO_FETCH , 20000 ) ;
2019-10-03 08:06:19 +08:00
init ( DISABLE_DUPLICATE_LOG_WARNING , false ) ;
2020-11-13 09:03:41 +08:00
init ( HISTOGRAM_REPORT_INTERVAL , 300.0 ) ;
2017-05-26 04:48:44 +08:00
2017-09-26 03:40:24 +08:00
// Timekeeper
2017-09-28 07:31:38 +08:00
init ( TIME_KEEPER_DELAY , 10 ) ;
2020-02-11 13:49:31 +08:00
init ( TIME_KEEPER_MAX_ENTRIES , 3600 * 24 * 30 * 6 ) ; if ( randomize & & BUGGIFY ) { TIME_KEEPER_MAX_ENTRIES = 2 ; }
2017-09-26 03:40:24 +08:00
2019-06-01 02:09:31 +08:00
// Fast Restore
init ( FASTRESTORE_FAILURE_TIMEOUT , 3600 ) ;
init ( FASTRESTORE_HEARTBEAT_INTERVAL , 60 ) ;
2020-06-30 01:13:18 +08:00
init ( FASTRESTORE_SAMPLING_PERCENT , 100 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SAMPLING_PERCENT = deterministicRandom ( ) - > random01 ( ) * 100 ; }
2020-08-27 00:54:33 +08:00
init ( FASTRESTORE_NUM_LOADERS , 3 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_NUM_LOADERS = deterministicRandom ( ) - > random01 ( ) * 10 + 1 ; }
2020-04-29 07:05:35 +08:00
init ( FASTRESTORE_NUM_APPLIERS , 3 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_NUM_APPLIERS = deterministicRandom ( ) - > random01 ( ) * 10 + 1 ; }
2020-08-27 00:54:33 +08:00
init ( FASTRESTORE_TXN_BATCH_MAX_BYTES , 1024.0 * 1024.0 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_TXN_BATCH_MAX_BYTES = deterministicRandom ( ) - > random01 ( ) * 1024.0 * 1024.0 + 1.0 ; }
2022-06-28 05:57:03 +08:00
init ( FASTRESTORE_VERSIONBATCH_MAX_BYTES , 10.0 * 1024.0 * 1024.0 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_VERSIONBATCH_MAX_BYTES = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 50 * 1024 : deterministicRandom ( ) - > random01 ( ) < 0.4 ? 100 * 1024 * 1024 : deterministicRandom ( ) - > random01 ( ) * 1000.0 * 1024.0 * 1024.0 ; } // too small value may increase chance of TooManyFile error
2020-08-27 00:54:33 +08:00
init ( FASTRESTORE_VB_PARALLELISM , 5 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_VB_PARALLELISM = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 2 : deterministicRandom ( ) - > random01 ( ) * 10 + 1 ; }
2020-06-30 01:13:18 +08:00
init ( FASTRESTORE_VB_MONITOR_DELAY , 30 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_VB_MONITOR_DELAY = deterministicRandom ( ) - > random01 ( ) * 20 + 1 ; }
2020-08-27 00:54:33 +08:00
init ( FASTRESTORE_VB_LAUNCH_DELAY , 1.0 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_VB_LAUNCH_DELAY = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 0.1 : deterministicRandom ( ) - > random01 ( ) * 10.0 + 1 ; }
2020-06-30 01:13:18 +08:00
init ( FASTRESTORE_ROLE_LOGGING_DELAY , 5 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_ROLE_LOGGING_DELAY = deterministicRandom ( ) - > random01 ( ) * 60 + 1 ; }
2020-04-29 07:05:35 +08:00
init ( FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL , 5 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL = deterministicRandom ( ) - > random01 ( ) * 60 + 1 ; }
2020-06-30 01:13:18 +08:00
init ( FASTRESTORE_ATOMICOP_WEIGHT , 1 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_ATOMICOP_WEIGHT = deterministicRandom ( ) - > random01 ( ) * 200 + 1 ; }
2020-04-29 07:05:35 +08:00
init ( FASTRESTORE_MONITOR_LEADER_DELAY , 5 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_MONITOR_LEADER_DELAY = deterministicRandom ( ) - > random01 ( ) * 100 ; }
2020-03-03 02:52:44 +08:00
init ( FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS , 60 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_STRAGGLER_THRESHOLD_SECONDS = deterministicRandom ( ) - > random01 ( ) * 240 + 10 ; }
2020-05-01 10:15:32 +08:00
init ( FASTRESTORE_TRACK_REQUEST_LATENCY , false ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_TRACK_REQUEST_LATENCY = false ; }
2020-02-27 05:43:30 +08:00
init ( FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT , 6144 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_MEMORY_THRESHOLD_MB_SOFT = 1 ; }
init ( FASTRESTORE_WAIT_FOR_MEMORY_LATENCY , 10 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_WAIT_FOR_MEMORY_LATENCY = 60 ; }
2020-02-29 12:31:59 +08:00
init ( FASTRESTORE_HEARTBEAT_DELAY , 10 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_HEARTBEAT_DELAY = deterministicRandom ( ) - > random01 ( ) * 120 + 2 ; }
2020-02-27 07:45:21 +08:00
init ( FASTRESTORE_HEARTBEAT_MAX_DELAY , 10 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_HEARTBEAT_MAX_DELAY = FASTRESTORE_HEARTBEAT_DELAY * 10 ; }
2020-04-18 01:02:53 +08:00
init ( FASTRESTORE_APPLIER_FETCH_KEYS_SIZE , 100 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_APPLIER_FETCH_KEYS_SIZE = deterministicRandom ( ) - > random01 ( ) * 10240 + 1 ; }
2020-08-25 08:27:40 +08:00
init ( FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES , 1.0 * 1024.0 * 1024.0 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_LOADER_SEND_MUTATION_MSG_BYTES = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1024 : deterministicRandom ( ) - > random01 ( ) * 5.0 * 1024.0 * 1024.0 + 1 ; }
2020-04-18 02:20:57 +08:00
init ( FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE , false ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_GET_RANGE_VERSIONS_EXPENSIVE = deterministicRandom ( ) - > random01 ( ) < 0.5 ? true : false ; }
2020-05-02 01:47:44 +08:00
init ( FASTRESTORE_REQBATCH_PARALLEL , 50 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_REQBATCH_PARALLEL = deterministicRandom ( ) - > random01 ( ) * 100 + 1 ; }
2020-05-05 11:12:59 +08:00
init ( FASTRESTORE_REQBATCH_LOG , false ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_REQBATCH_LOG = deterministicRandom ( ) - > random01 ( ) < 0.2 ? true : false ; }
2020-06-30 01:13:18 +08:00
init ( FASTRESTORE_TXN_CLEAR_MAX , 100 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_TXN_CLEAR_MAX = deterministicRandom ( ) - > random01 ( ) * 100 + 1 ; }
2020-05-08 06:06:59 +08:00
init ( FASTRESTORE_TXN_RETRY_MAX , 10 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_TXN_RETRY_MAX = deterministicRandom ( ) - > random01 ( ) * 100 + 1 ; }
2020-09-23 10:57:19 +08:00
init ( FASTRESTORE_TXN_EXTRA_DELAY , 0.0 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_TXN_EXTRA_DELAY = deterministicRandom ( ) - > random01 ( ) * 1 + 0.001 ; }
2020-08-01 01:40:29 +08:00
init ( FASTRESTORE_NOT_WRITE_DB , false ) ; // Perf test only: set it to true will cause simulation failure
init ( FASTRESTORE_USE_RANGE_FILE , true ) ; // Perf test only: set it to false will cause simulation failure
init ( FASTRESTORE_USE_LOG_FILE , true ) ; // Perf test only: set it to false will cause simulation failure
2020-08-05 04:35:36 +08:00
init ( FASTRESTORE_SAMPLE_MSG_BYTES , 1048576 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SAMPLE_MSG_BYTES = deterministicRandom ( ) - > random01 ( ) * 2048 ; }
2020-08-17 00:43:04 +08:00
init ( FASTRESTORE_SCHED_UPDATE_DELAY , 0.1 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_UPDATE_DELAY = deterministicRandom ( ) - > random01 ( ) * 2 ; }
2020-08-16 21:29:09 +08:00
init ( FASTRESTORE_SCHED_TARGET_CPU_PERCENT , 70 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_TARGET_CPU_PERCENT = deterministicRandom ( ) - > random01 ( ) * 100 + 50 ; } // simulate cpu usage can be larger than 100
2020-08-16 12:40:24 +08:00
init ( FASTRESTORE_SCHED_MAX_CPU_PERCENT , 90 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_MAX_CPU_PERCENT = FASTRESTORE_SCHED_TARGET_CPU_PERCENT + deterministicRandom ( ) - > random01 ( ) * 100 ; }
2020-08-27 06:21:33 +08:00
init ( FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS , 50 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_LOAD_REQS = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1 : deterministicRandom ( ) - > random01 ( ) * 30 + 1 ; }
init ( FASTRESTORE_SCHED_INFLIGHT_SEND_REQS , 3 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_SEND_REQS = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1 : deterministicRandom ( ) - > random01 ( ) * 10 + 1 ; }
init ( FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE , 5 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_LOAD_REQ_BATCHSIZE = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1 : deterministicRandom ( ) - > random01 ( ) * 10 + 1 ; }
2020-08-25 08:27:40 +08:00
init ( FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD , 10 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_INFLIGHT_SENDPARAM_THRESHOLD = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1 : deterministicRandom ( ) - > random01 ( ) * 15 + 1 ; }
init ( FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH , 2 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_SCHED_SEND_FUTURE_VB_REQS_BATCH = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1 : deterministicRandom ( ) - > random01 ( ) * 15 + 1 ; }
2020-08-18 11:34:33 +08:00
init ( FASTRESTORE_NUM_TRACE_EVENTS , 100 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_NUM_TRACE_EVENTS = deterministicRandom ( ) - > random01 ( ) < 0.2 ? 1 : deterministicRandom ( ) - > random01 ( ) * 500 + 1 ; }
2020-08-25 01:45:46 +08:00
init ( FASTRESTORE_EXPENSIVE_VALIDATION , false ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_EXPENSIVE_VALIDATION = deterministicRandom ( ) - > random01 ( ) < 0.5 ? true : false ; }
2020-09-22 23:14:23 +08:00
init ( FASTRESTORE_WRITE_BW_MB , 70 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_WRITE_BW_MB = deterministicRandom ( ) - > random01 ( ) < 0.5 ? 2 : 100 ; }
init ( FASTRESTORE_RATE_UPDATE_SECONDS , 1.0 ) ; if ( randomize & & BUGGIFY ) { FASTRESTORE_RATE_UPDATE_SECONDS = deterministicRandom ( ) - > random01 ( ) < 0.5 ? 0.1 : 2 ; }
2021-10-20 01:27:55 +08:00
init ( FASTRESTORE_DUMP_INSERT_RANGE_VERSION , false ) ;
2019-12-05 03:45:05 +08:00
2022-01-20 14:13:42 +08:00
init ( REDWOOD_DEFAULT_PAGE_SIZE , 8192 ) ;
2021-06-08 07:28:13 +08:00
init ( REDWOOD_DEFAULT_EXTENT_SIZE , 32 * 1024 * 1024 ) ;
init ( REDWOOD_DEFAULT_EXTENT_READ_SIZE , 1024 * 1024 ) ;
2021-05-27 14:20:31 +08:00
init ( REDWOOD_EXTENT_CONCURRENT_READS , 4 ) ;
2021-07-04 16:45:58 +08:00
init ( REDWOOD_KVSTORE_RANGE_PREFETCH , true ) ;
2021-04-04 10:54:49 +08:00
init ( REDWOOD_PAGE_REBUILD_MAX_SLACK , 0.33 ) ;
2022-12-20 22:22:54 +08:00
init ( REDWOOD_PAGE_REBUILD_SLACK_DISTRIBUTION , 0.50 ) ;
2020-05-13 17:27:03 +08:00
init ( REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES , 10 ) ;
init ( REDWOOD_LAZY_CLEAR_MIN_PAGES , 0 ) ;
init ( REDWOOD_LAZY_CLEAR_MAX_PAGES , 1e6 ) ;
2022-02-25 14:31:34 +08:00
init ( REDWOOD_REMAP_CLEANUP_WINDOW_BYTES , 4LL * 1024 * 1024 * 1024 ) ;
2022-03-01 07:40:40 +08:00
init ( REDWOOD_REMAP_CLEANUP_TOLERANCE_RATIO , 0.05 ) ;
2021-11-13 05:47:07 +08:00
init ( REDWOOD_PAGEFILE_GROWTH_SIZE_PAGES , 20000 ) ; if ( randomize & & BUGGIFY ) { REDWOOD_PAGEFILE_GROWTH_SIZE_PAGES = deterministicRandom ( ) - > randomInt ( 200 , 1000 ) ; }
2021-08-25 00:57:39 +08:00
init ( REDWOOD_METRICS_INTERVAL , 5.0 ) ;
2021-08-25 06:47:04 +08:00
init ( REDWOOD_HISTOGRAM_INTERVAL , 30.0 ) ;
2021-11-29 13:03:44 +08:00
init ( REDWOOD_EVICT_UPDATED_PAGES , true ) ; if ( randomize & & BUGGIFY ) { REDWOOD_EVICT_UPDATED_PAGES = false ; }
2022-04-06 06:11:07 +08:00
init ( REDWOOD_DECODECACHE_REUSE_MIN_HEIGHT , 2 ) ; if ( randomize & & BUGGIFY ) { REDWOOD_DECODECACHE_REUSE_MIN_HEIGHT = deterministicRandom ( ) - > randomInt ( 1 , 7 ) ; }
2023-02-17 14:46:11 +08:00
init ( REDWOOD_NODE_MAX_UNBALANCE , 2 ) ;
2022-11-12 12:07:48 +08:00
init ( REDWOOD_IO_PRIORITIES , " 32,32,32,32 " ) ;
2020-08-15 01:16:20 +08:00
2020-07-10 07:39:15 +08:00
// Server request latency measurement
2022-10-01 01:39:43 +08:00
init ( LATENCY_SKETCH_ACCURACY , 0.01 ) ;
init ( FILE_LATENCY_SKETCH_ACCURACY , 0.01 ) ;
2020-07-10 07:39:15 +08:00
init ( LATENCY_METRICS_LOGGING_INTERVAL , 60.0 ) ;
2020-05-15 17:10:51 +08:00
2022-01-07 04:15:51 +08:00
// Cluster recovery
2022-06-08 12:00:13 +08:00
init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX , " Master " ) ;
2022-01-07 04:15:51 +08:00
2022-07-14 04:53:20 +08:00
// Encryption
2022-06-08 12:00:13 +08:00
init ( SIM_KMS_MAX_KEYS , 4096 ) ;
init ( ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH , 100000 ) ;
2022-07-14 04:53:20 +08:00
// encrypt key proxy
2022-07-15 07:36:27 +08:00
init ( ENABLE_BLOB_GRANULE_COMPRESSION , false ) ; if ( randomize & & BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom ( ) - > coinflip ( ) ; }
2022-09-23 05:31:49 +08:00
init ( BLOB_GRANULE_COMPRESSION_FILTER , " NONE " ) ; if ( randomize & & BUGGIFY ) { BLOB_GRANULE_COMPRESSION_FILTER = CompressionUtils : : toString ( CompressionUtils : : getRandomFilter ( ) ) ; }
2022-07-15 07:36:27 +08:00
2022-09-01 06:46:39 +08:00
// KMS connector type
2022-06-08 12:00:13 +08:00
init ( KMS_CONNECTOR_TYPE , " RESTKmsConnector " ) ;
2022-04-22 23:53:39 +08:00
2021-07-08 06:04:49 +08:00
// Blob granlues
2022-11-05 02:26:44 +08:00
init ( BG_URL , isSimulated ? " file://simfdb/fdbblob/ " : " " ) ; // TODO: store in system key space or something, eventually
2022-07-23 04:13:32 +08:00
bool buggifyMediumGranules = simulationMediumShards | | ( randomize & & BUGGIFY ) ;
2022-09-01 06:46:39 +08:00
// BlobGranuleVerify* simulation tests use "knobs", BlobGranuleCorrectness* use "tenant", default in real clusters is "knobs"
2022-05-28 04:15:56 +08:00
init ( BG_METADATA_SOURCE , " knobs " ) ;
2023-01-27 06:56:45 +08:00
init ( BG_SNAPSHOT_FILE_TARGET_BYTES , 20000000 ) ; if ( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 50000 * deterministicRandom ( ) - > randomInt ( 1 , 4 ) ; else if ( buggifyMediumGranules ) BG_SNAPSHOT_FILE_TARGET_BYTES = 50000 * deterministicRandom ( ) - > randomInt ( 1 , 20 ) ;
2022-07-27 03:20:35 +08:00
init ( BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES , 64 * 1024 ) ; if ( randomize & & BUGGIFY ) BG_SNAPSHOT_FILE_TARGET_CHUNK_BYTES = BG_SNAPSHOT_FILE_TARGET_BYTES / ( 1 < < deterministicRandom ( ) - > randomInt ( 0 , 8 ) ) ;
2023-01-27 06:56:45 +08:00
init ( BG_DELTA_BYTES_BEFORE_COMPACT , BG_SNAPSHOT_FILE_TARGET_BYTES / 2 ) ; if ( randomize & & BUGGIFY ) BG_DELTA_BYTES_BEFORE_COMPACT * = ( 1.0 + deterministicRandom ( ) - > random01 ( ) * 3.0 ) / 2.0 ;
2021-07-08 06:04:49 +08:00
init ( BG_DELTA_FILE_TARGET_BYTES , BG_DELTA_BYTES_BEFORE_COMPACT / 10 ) ;
2022-08-03 00:36:44 +08:00
init ( BG_DELTA_FILE_TARGET_CHUNK_BYTES , 32 * 1024 ) ; if ( randomize & & BUGGIFY ) BG_DELTA_FILE_TARGET_CHUNK_BYTES = BG_DELTA_FILE_TARGET_BYTES / ( 1 < < deterministicRandom ( ) - > randomInt ( 0 , 7 ) ) ;
2022-03-10 23:22:53 +08:00
init ( BG_MAX_SPLIT_FANOUT , 10 ) ; if ( randomize & & BUGGIFY ) BG_MAX_SPLIT_FANOUT = deterministicRandom ( ) - > randomInt ( 5 , 15 ) ;
2022-04-26 02:41:00 +08:00
init ( BG_MAX_MERGE_FANIN , 10 ) ; if ( randomize & & BUGGIFY ) BG_MAX_MERGE_FANIN = deterministicRandom ( ) - > randomInt ( 2 , 15 ) ;
2022-03-10 23:22:53 +08:00
init ( BG_HOT_SNAPSHOT_VERSIONS , 5000000 ) ;
2021-07-08 06:04:49 +08:00
2022-04-01 01:36:01 +08:00
init ( BG_CONSISTENCY_CHECK_ENABLED , true ) ; if ( randomize & & BUGGIFY ) BG_CONSISTENCY_CHECK_ENABLED = false ;
init ( BG_CONSISTENCY_CHECK_TARGET_SPEED_KB , 1000 ) ; if ( randomize & & BUGGIFY ) BG_CONSISTENCY_CHECK_TARGET_SPEED_KB * = ( deterministicRandom ( ) - > randomInt ( 2 , 50 ) / 10 ) ;
blob: allow for alignment of granules to tuple boundaries (#7746)
* blob: read TenantMap during recovery
Future functionality in the blob subsystem will rely on the tenant data
being loaded. This fixes this issue by loading the tenant data before
completing recovery such that continued actions on existing blob
granules will have access to the tenant data.
Example scenario with failover, splits are restarted before loading the
tenant data:
BM - BlobManager
epoch 3: epoch 4:
BM record intent to split.
Epoch fails.
BM recovery begins.
BM fails to persist split.
BM recovery finishes.
BM.checkBlobWorkerList()
maybeSplitRange().
BM.monitorClientRanges().
loads tenant data.
bin/fdbserver -r simulation -f tests/slow/BlobGranuleCorrectness.toml \
-s 223570924 -b on --crash --trace_format json
* blob: add tuple key truncation for blob granule alignment
FDB has a backup system available using the blob manager and blob
granule subsystem. If we want to audit the data in the blobs, it's a lot
easier if we can align them to something meaningful.
When a blob granule is being split, we ask the storage metrics system
for split points as it holds approximate data distribution metrics.
These keys are then processed to determine if they are a tuple and
should be truncated according to the new knob,
BG_KEY_TUPLE_TRUNCATE_OFFSET.
Here we keep all aligned keys together in the same granule even if it is
larger than the allowed granule size. The following commit will address
this by adding merge boundaries.
* blob: minor clean ups in merging code
1. Rename mergeNow -> seen. This is more inline with clocksweep naming
and removes the confusion between mergeNow and canMergeNow.
2. Make clearMergeCandidate() reset to MergeCandidateCannotMerge to make
a clear distinction what we're accomplishing.
3. Rename canMergeNow() -> mergeEligble().
* blob: add explicit (hard) boundaries
Blob ranges can be specified either through explicit ranges or at the
tenant level. Right now this is managed implicitly. This commit aims to
make it a little more explicit.
Blobification begins in monitorClientRanges() which parses either the
explicit blob ranges or the tenant map. As we do this and add new
ranges, let's explicitly track what is a hard boundary and what isn't.
When blob merging occurs, we respect this boundary. When a hard boundary
is encountered, we submit the found eligible ranges and start looking
for a new range beginning with this hard boundary.
* blob: create BlobGranuleSplitPoints struct
This is a setup for the following commit. Our goal here is to provide a
structure for split points to be passed around. The need is for us to be
able to carry uncommitted state until it is committed and we can apply
these mutations to the in-memory data structures.
* blob: implement soft boundaries
An earlier commit establishes the need to create data boundaries within
a tenant. The reality is we may encounter a set of keys that degnerate
to the same key prefix. We'll need to be able to split those across
granules, but we want to ensure we merge the split granules together
before merging with other granules.
This adds to the BlobGranuleSplitPoints state of new
BlobGranuleMergeBoundary items. BlobGranuleMergeBoundary contains state
saying if it is a left or right boundary. This information is used to,
like hard boundaries, force merging of like granules first.
We read the BlobGranuleMergeBoundary map into memory at recovery.
2022-08-03 05:06:25 +08:00
init ( BG_KEY_TUPLE_TRUNCATE_OFFSET , 0 ) ;
2022-10-27 00:02:50 +08:00
init ( BG_ENABLE_READ_DRIVEN_COMPACTION , true ) ; if ( randomize & & BUGGIFY ) BG_ENABLE_READ_DRIVEN_COMPACTION = false ;
init ( BG_RDC_BYTES_FACTOR , 2 ) ; if ( randomize & & BUGGIFY ) BG_RDC_BYTES_FACTOR = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
init ( BG_RDC_READ_FACTOR , 3 ) ; if ( randomize & & BUGGIFY ) BG_RDC_READ_FACTOR = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
2022-11-09 07:30:01 +08:00
init ( BG_WRITE_MULTIPART , false ) ; if ( randomize & & BUGGIFY ) BG_WRITE_MULTIPART = true ;
2023-01-27 06:56:45 +08:00
init ( BG_ENABLE_DYNAMIC_WRITE_AMP , true ) ; if ( randomize & & BUGGIFY ) BG_ENABLE_DYNAMIC_WRITE_AMP = false ;
init ( BG_DYNAMIC_WRITE_AMP_MIN_FACTOR , 0.5 ) ;
init ( BG_DYNAMIC_WRITE_AMP_DECREASE_FACTOR , 0.8 ) ;
2022-04-01 01:36:01 +08:00
2022-04-26 02:41:00 +08:00
init ( BG_ENABLE_MERGING , true ) ; if ( randomize & & BUGGIFY ) BG_ENABLE_MERGING = false ;
init ( BG_MERGE_CANDIDATE_THRESHOLD_SECONDS , isSimulated ? 20.0 : 30 * 60 ) ; if ( randomize & & BUGGIFY ) BG_MERGE_CANDIDATE_THRESHOLD_SECONDS = 5.0 ;
2022-07-16 04:52:10 +08:00
init ( BG_MERGE_CANDIDATE_DELAY_SECONDS , BG_MERGE_CANDIDATE_THRESHOLD_SECONDS / 10.0 ) ;
2022-04-01 01:36:01 +08:00
2022-03-10 23:22:53 +08:00
init ( BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM , 8 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_INITIAL_SNAPSHOT_PARALLELISM = 1 ;
2023-01-27 06:56:45 +08:00
// The resnapshot/delta parallelism knobs are deprecated and replaced by the budget_bytes knobs! FIXME: remove after next release
2022-09-01 06:46:39 +08:00
init ( BLOB_WORKER_RESNAPSHOT_PARALLELISM , 40 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_RESNAPSHOT_PARALLELISM = deterministicRandom ( ) - > randomInt ( 1 , 10 ) ;
init ( BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM , 2000 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_DELTA_FILE_WRITE_PARALLELISM = deterministicRandom ( ) - > randomInt ( 10 , 100 ) ;
2022-10-27 00:02:50 +08:00
init ( BLOB_WORKER_RDC_PARALLELISM , 2 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_RDC_PARALLELISM = deterministicRandom ( ) - > randomInt ( 1 , 6 ) ;
2023-01-27 06:56:45 +08:00
init ( BLOB_WORKER_RESNAPSHOT_BUDGET_BYTES , 1024 * 1024 * 1024 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_RESNAPSHOT_BUDGET_BYTES = deterministicRandom ( ) - > random01 ( ) * 10 * BG_SNAPSHOT_FILE_TARGET_BYTES ;
init ( BLOB_WORKER_DELTA_WRITE_BUDGET_BYTES , 1024 * 1024 * 1024 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_DELTA_WRITE_BUDGET_BYTES = ( 5 + 45 * deterministicRandom ( ) - > random01 ( ) ) * BG_DELTA_FILE_TARGET_BYTES ;
2021-10-13 21:46:38 +08:00
init ( BLOB_WORKER_TIMEOUT , 10.0 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_TIMEOUT = 1.0 ;
2022-02-10 05:35:49 +08:00
init ( BLOB_WORKER_REQUEST_TIMEOUT , 5.0 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_REQUEST_TIMEOUT = 1.0 ;
2021-12-04 02:31:51 +08:00
init ( BLOB_WORKERLIST_FETCH_INTERVAL , 1.0 ) ;
2022-03-10 23:22:53 +08:00
init ( BLOB_WORKER_BATCH_GRV_INTERVAL , 0.1 ) ;
2022-09-01 06:46:39 +08:00
init ( BLOB_WORKER_DO_REJECT_WHEN_FULL , true ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_DO_REJECT_WHEN_FULL = false ;
init ( BLOB_WORKER_REJECT_WHEN_FULL_THRESHOLD , 0.9 ) ;
2022-09-22 03:36:02 +08:00
init ( BLOB_WORKER_FORCE_FLUSH_CLEANUP_DELAY , 30.0 ) ; if ( randomize & & BUGGIFY ) BLOB_WORKER_FORCE_FLUSH_CLEANUP_DELAY = deterministicRandom ( ) - > randomInt ( 0 , 10 ) - 1 ;
2022-03-10 01:44:45 +08:00
2022-03-10 23:22:53 +08:00
init ( BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN , 0.1 ) ;
init ( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX , 5.0 ) ;
init ( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT , 1.5 ) ;
2022-07-16 04:52:10 +08:00
init ( BLOB_MANAGER_CONCURRENT_MERGE_CHECKS , 64 ) ; if ( randomize & & BUGGIFY ) BLOB_MANAGER_CONCURRENT_MERGE_CHECKS = 1 < < deterministicRandom ( ) - > randomInt ( 0 , 7 ) ;
2022-08-20 04:29:58 +08:00
init ( BLOB_MANIFEST_BACKUP , false ) ;
2023-02-01 07:22:29 +08:00
init ( BLOB_MANIFEST_BACKUP_INTERVAL , isSimulated ? 5.0 : 300.0 ) ;
2022-08-20 04:29:58 +08:00
init ( BLOB_FULL_RESTORE_MODE , false ) ;
2023-02-01 07:22:29 +08:00
init ( BLOB_MIGRATOR_CHECK_INTERVAL , isSimulated ? 1.0 : 5.0 ) ;
init ( BLOB_MANIFEST_RW_ROWS , isSimulated ? 10 : 1000 ) ;
init ( BLOB_RESTORE_MLOGS_URL , isSimulated ? " file://simfdb/backups/ " : " " ) ;
init ( BLOB_MIGRATOR_ERROR_RETRIES , 20 ) ;
init ( BLOB_RESTORE_MANIFEST_URL , isSimulated ? " file://simfdb/fdbblob/manifest " : " " ) ;
init ( BLOB_RESTORE_MANIFEST_FILE_MAX_SIZE , isSimulated ? 10000 : 10000000 ) ;
init ( BLOB_RESTORE_MANIFEST_RETENTION_MAX , 10 ) ;
2023-02-02 05:56:07 +08:00
init ( BLOB_RESTORE_MLOGS_RETENTION_SECS , isSimulated ? 120 : 3600 * 24 * 14 ) ;
2021-09-04 04:13:26 +08:00
2022-04-05 23:23:30 +08:00
init ( BGCC_TIMEOUT , isSimulated ? 10.0 : 120.0 ) ;
init ( BGCC_MIN_INTERVAL , isSimulated ? 1.0 : 10.0 ) ;
2022-05-28 04:15:56 +08:00
// Blob Metadata
init ( BLOB_METADATA_CACHE_TTL , isSimulated ? 120 : 24 * 60 * 60 ) ;
if ( randomize & & BUGGIFY ) { BLOB_METADATA_CACHE_TTL = deterministicRandom ( ) - > randomInt ( 50 , 100 ) ; }
2022-05-08 04:18:35 +08:00
// HTTP KMS Connector
init ( REST_KMS_CONNECTOR_KMS_DISCOVERY_URL_MODE , " file " ) ;
init ( REST_KMS_CONNECTOR_VALIDATION_TOKEN_MODE , " file " ) ;
init ( REST_KMS_CONNECTOR_VALIDATION_TOKEN_MAX_SIZE , 1024 ) ;
init ( REST_KMS_CONNECTOR_VALIDATION_TOKENS_MAX_PAYLOAD_SIZE , 10 * 1024 ) ;
init ( REST_KMS_CONNECTOR_REFRESH_KMS_URLS , true ) ;
init ( REST_KMS_CONNECTOR_REFRESH_KMS_URLS_INTERVAL_SEC , 600 ) ;
// Below KMS configurations are responsible for:
// Discovering KMS URLs, fetch encryption keys endpoint and validation token details.
// Configurations are expected to be passed as command-line arguments.
// NOTE: Care must be taken when attempting to update below configurations for a up/running FDB cluster.
init ( REST_KMS_CONNECTOR_DISCOVER_KMS_URL_FILE , " " ) ;
init ( REST_KMS_CONNECTOR_GET_ENCRYPTION_KEYS_ENDPOINT , " " ) ;
2023-01-10 02:55:53 +08:00
init ( REST_KMS_CONNECTOR_GET_LATEST_ENCRYPTION_KEYS_ENDPOINT , " " ) ;
2022-10-15 06:49:00 +08:00
init ( REST_KMS_CONNECTOR_GET_BLOB_METADATA_ENDPOINT , " " ) ;
2022-05-08 04:18:35 +08:00
// Details to fetch validation token from a localhost file
// acceptable format: "<token_name1>#<absolute_file_path1>,<token_name2>#<absolute_file_path2>,.."
// NOTE: 'token-name" can NOT contain '#' character
init ( REST_KMS_CONNECTOR_VALIDATION_TOKEN_DETAILS , " " ) ;
2023-01-17 04:18:25 +08:00
init ( REST_KMS_CURRENT_BLOB_METADATA_REQUEST_VERSION , 1 ) ;
init ( REST_KMS_CURRENT_CIPHER_REQUEST_VERSION , 1 ) ;
2022-05-08 04:18:35 +08:00
2022-10-29 00:07:54 +08:00
// Drop in-memory state associated with an idempotency id after this many seconds. Once dropped, this id cannot be
// expired proactively, but will eventually get cleaned up by the idempotency id cleaner.
init ( IDEMPOTENCY_ID_IN_MEMORY_LIFETIME , 10 ) ;
2022-12-15 06:24:24 +08:00
// Attempt to clean old idempotency ids automatically this often
init ( IDEMPOTENCY_IDS_CLEANER_POLLING_INTERVAL , 10 ) ;
// Don't clean idempotency ids younger than this
init ( IDEMPOTENCY_IDS_MIN_AGE_SECONDS , 3600 * 24 * 7 ) ;
2022-10-29 00:07:54 +08:00
2019-07-31 13:35:34 +08:00
// clang-format on
2019-06-01 02:09:31 +08:00
2020-08-30 03:35:31 +08:00
if ( clientKnobs ) {
2020-08-11 06:29:59 +08:00
clientKnobs - > IS_ACCEPTABLE_DELAY =
clientKnobs - > IS_ACCEPTABLE_DELAY *
std : : min ( MAX_READ_TRANSACTION_LIFE_VERSIONS , MAX_WRITE_TRANSACTION_LIFE_VERSIONS ) /
( 5.0 * VERSIONS_PER_SECOND ) ;
clientKnobs - > INIT_MID_SHARD_BYTES = MIN_SHARD_BYTES ;
}
2017-05-26 04:48:44 +08:00
}
