Add transaction size option

This commit is contained in:
Jingyu Zhou 2019-06-19 07:40:54 -07:00
parent 4c62458172
commit 9c2257a0e5
11 changed files with 55 additions and 11 deletions

View File

@ -1550,6 +1550,7 @@ struct UnitTestsFunc : InstructionFunc {
const uint64_t retryLimit = 50;
const uint64_t noRetryLimit = -1;
const uint64_t maxRetryDelay = 100;
const uint64_t sizeLimit = 100000;
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, Optional<StringRef>(StringRef((const uint8_t*)&locationCacheSize, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, Optional<StringRef>(StringRef((const uint8_t*)&maxWatches, 8)));
@ -1558,6 +1559,7 @@ struct UnitTestsFunc : InstructionFunc {
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&timeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, Optional<StringRef>(StringRef((const uint8_t*)&noTimeout, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, Optional<StringRef>(StringRef((const uint8_t*)&maxRetryDelay, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&sizeLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, Optional<StringRef>(StringRef((const uint8_t*)&noRetryLimit, 8)));
data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE);

View File

@ -277,6 +277,13 @@ func (o DatabaseOptions) SetTransactionMaxRetryDelay(param int64) error {
return o.setOpt(502, int64ToBytes(param))
}
// Set the maximum transaction size which, if exceeded, will cause the transaction to be cancelled. Default to 10,000,000 bytes.
//
// Parameter: value in bytes
func (o DatabaseOptions) SetTransactionSizeLimit(param int64) error {
return o.setOpt(503, int64ToBytes(param))
}
// Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior.
func (o DatabaseOptions) SetSnapshotRywEnable() error {
return o.setOpt(26, nil)
@ -402,6 +409,13 @@ func (o TransactionOptions) SetMaxRetryDelay(param int64) error {
return o.setOpt(502, int64ToBytes(param))
}
// Set the maximum transaction size which, if exceeded, will cause the transaction to be cancelled. Valid parameter values are ``[32, 10,000,000]```.
//
// Parameter: value in bytes
func (o TransactionOptions) SetMaxTransactionSize(param int64) error {
return o.setOpt(503, int64ToBytes(param))
}
// Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior.
func (o TransactionOptions) SetSnapshotRywEnable() error {
return o.setOpt(600, nil)
@ -451,7 +465,7 @@ const (
// Infrequently used. The client has passed a specific row limit and wants
// that many rows delivered in a single batch. Because of iterator operation
// in client drivers make request batches transparent to the user, consider
// “WANT_ALL“ StreamingMode instead. A row limit must be specified if this
// ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this
// mode is used.
StreamingModeExact StreamingMode = 1
@ -568,15 +582,15 @@ type ErrorPredicate int
const (
// Returns “true“ if the error indicates the operations in the transactions
// should be retried because of transient error.
// Returns ``true`` if the error indicates the operations in the
// transactions should be retried because of transient error.
ErrorPredicateRetryable ErrorPredicate = 50000
// Returns “true“ if the error indicates the transaction may have succeeded,
// though not in a way the system can verify.
// Returns ``true`` if the error indicates the transaction may have
// succeeded, though not in a way the system can verify.
ErrorPredicateMaybeCommitted ErrorPredicate = 50001
// Returns “true“ if the error indicates the transaction has not committed,
// though in a way that can be retried.
// Returns ``true`` if the error indicates the transaction has not
// committed, though in a way that can be retried.
ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002
)

View File

@ -133,6 +133,7 @@ def test_db_options(db):
db.options.set_transaction_timeout(0)
db.options.set_transaction_timeout(0)
db.options.set_transaction_max_retry_delay(100)
db.options.set_transaction_size_limit(100000)
db.options.set_transaction_retry_limit(10)
db.options.set_transaction_retry_limit(-1)
db.options.set_snapshot_ryw_enable()

View File

@ -459,6 +459,7 @@ class Tester
@db.options.set_transaction_timeout(100000)
@db.options.set_transaction_timeout(0)
@db.options.set_transaction_max_retry_delay(100)
@db.options.set_transaction_size_limit(100000)
@db.options.set_transaction_retry_limit(10)
@db.options.set_transaction_retry_limit(-1)
@db.options.set_snapshot_ryw_enable()

View File

@ -24,6 +24,7 @@
.. |retry-limit-database-option| replace:: :func:`Database.options.set_transaction_retry_limit`
.. |timeout-database-option| replace:: :func:`Database.options.set_transaction_timeout`
.. |max-retry-delay-database-option| replace:: :func:`Database.options.set_transaction_max_retry_delay`
.. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit`
.. |snapshot-ryw-enable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_enable`
.. |snapshot-ryw-disable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_disable`
.. |future-type-string| replace:: a :ref:`future <api-python-future>`
@ -378,6 +379,10 @@ Database options
|option-db-tr-max-retry-delay-blurb|
.. method:: Database.options.set_transaction_size_limit(size_limit)
|option-db-tr-size-limit-blurb|
.. method:: Database.options.set_snapshot_ryw_enable()
|option-db-snapshot-ryw-enable-blurb|

View File

@ -29,6 +29,7 @@
.. |retry-limit-transaction-option| replace:: :meth:`Transaction.options.set_retry_limit`
.. |timeout-transaction-option| replace:: :meth:`Transaction.options.set_timeout`
.. |max-retry-delay-transaction-option| replace:: :meth:`Transaction.options.set_max_retry_delay`
.. |size-limit-transaction-option| replace:: :meth:`Transaction.options.set_transaction_size_limit`
.. |snapshot-ryw-enable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_enable`
.. |snapshot-ryw-disable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_disable`
.. |lazy-iterator-object| replace:: :class:`Enumerator`
@ -374,6 +375,10 @@ Database options
|option-db-tr-max-retry-delay-blurb|
.. method:: Database.options.set_transaction_size_limit(size_limit) -> nil
|option-db-tr-size-limit-blurb|
.. method:: Database.options.set_snapshot_ryw_enable() -> nil
|option-db-snapshot-ryw-enable-blurb|

View File

@ -157,6 +157,7 @@ public:
double transactionTimeout;
int transactionMaxRetries;
double transactionMaxBackoff;
int transactionMaxBytes;
int snapshotRywEnabled;
Future<Void> logger;

View File

@ -51,7 +51,7 @@ public:
double RESOURCE_CONSTRAINED_MAX_BACKOFF;
int PROXY_COMMIT_OVERHEAD_BYTES;
int64_t TRANSACTION_SIZE_LIMIT;
int TRANSACTION_SIZE_LIMIT;
int64_t KEY_SIZE_LIMIT;
int64_t SYSTEM_KEY_SIZE_LIMIT;
int64_t VALUE_SIZE_LIMIT;

View File

@ -522,6 +522,7 @@ DatabaseContext::DatabaseContext(
maxOutstandingWatches = CLIENT_KNOBS->DEFAULT_MAX_OUTSTANDING_WATCHES;
transactionMaxBackoff = CLIENT_KNOBS->FAILURE_MAX_DELAY;
transactionMaxBytes = CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
snapshotRywEnabled = apiVersionAtLeast(300) ? 1 : 0;
logger = databaseLogger( this );
@ -747,6 +748,10 @@ void DatabaseContext::setOption( FDBDatabaseOptions::Option option, Optional<Str
validateOptionValue(value, true);
transactionMaxBackoff = extractIntOption(value, 0, std::numeric_limits<int32_t>::max()) / 1000.0;
break;
case FDBDatabaseOptions::TRANSACTION_SIZE_LIMIT:
validateOptionValue(value, true);
transactionMaxBytes = extractIntOption(value, 32, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
break;
case FDBDatabaseOptions::SNAPSHOT_RYW_ENABLE:
validateOptionValue(value, false);
snapshotRywEnabled++;
@ -2398,6 +2403,7 @@ double Transaction::getBackoff(int errCode) {
TransactionOptions::TransactionOptions(Database const& cx) {
maxBackoff = cx->transactionMaxBackoff;
customTransactionSizeLimit = cx->transactionMaxBytes;
reset(cx);
if (BUGGIFY) {
commitOnFirstProxy = true;
@ -2411,6 +2417,7 @@ TransactionOptions::TransactionOptions(Database const& cx) {
TransactionOptions::TransactionOptions() {
memset(this, 0, sizeof(*this));
maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF;
customTransactionSizeLimit = CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
}
void TransactionOptions::reset(Database const& cx) {
@ -2418,6 +2425,7 @@ void TransactionOptions::reset(Database const& cx) {
double oldMaxRetries = maxRetries;
memset(this, 0, sizeof(*this));
maxBackoff = cx->apiVersionAtLeast(610) ? oldMaxBackoff : cx->transactionMaxBackoff;
customTransactionSizeLimit = cx->transactionMaxBytes;
maxRetries = oldMaxRetries;
lockAware = cx->lockAware;
}
@ -2753,8 +2761,9 @@ Future<Void> Transaction::commitMutations() {
transactionSize = tr.transaction.mutations.expectedSize(); // Old API versions didn't account for conflict ranges when determining whether to throw transaction_too_large
}
if (transactionSize > (options.customTransactionSizeLimit == 0 ? (uint64_t)CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT : (uint64_t)options.customTransactionSizeLimit))
if (transactionSize > options.customTransactionSizeLimit) {
return transaction_too_large();
}
if( !readVersion.isValid() )
getReadVersion( GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY ); // sets up readVersion field. We had no reads, so no need for (expensive) full causal consistency.

View File

@ -126,7 +126,7 @@ public:
void getWriteConflicts( KeyRangeMap<bool> *result );
Database getDatabase() {
Database getDatabase() const {
return tr.getDatabase();
}
private:

View File

@ -145,6 +145,9 @@ description is not currently required but encouraged.
<Option name="transaction_max_retry_delay" code="502"
paramType="Int" paramDescription="value in milliseconds of maximum delay"
description="Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. This sets the ``max_retry_delay`` option of each transaction created by this database. See the transaction option description for more information." />
<Option name="transaction_size_limit" code="503"
paramType="Int" paramDescription="value in bytes"
description="Set the maximum transaction size which, if exceeded, will cause the transaction to be cancelled. Default to 10,000,000 bytes." />
<Option name="snapshot_ryw_enable" code="26"
description="Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior." />
<Option name="snapshot_ryw_disable" code="27"
@ -200,7 +203,10 @@ description is not currently required but encouraged.
<Option name="max_retry_delay" code="502"
paramType="Int" paramDescription="value in milliseconds of maximum delay"
description="Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. Defaults to 1000 ms. Valid parameter values are ``[0, INT_MAX]``. If the maximum retry delay is less than the current retry delay of the transaction, then the current retry delay will be clamped to the maximum retry delay. Prior to API version 610, like all other transaction options, the maximum retry delay must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the maximum retry delay each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option."/>
<Option name="snapshot_ryw_enable" code="600"
<Option name="max_transaction_size" code="503"
paramType="Int" paramDescription="value in bytes"
description="Set the maximum transaction size which, if exceeded, will cause the transaction to be cancelled. Valid parameter values are ``[32, 10,000,000]```." />
<Option name="snapshot_ryw_enable" code="600"
description="Snapshot read operations will see the results of writes done in the same transaction. This is the default behavior." />
<Option name="snapshot_ryw_disable" code="601"
description="Snapshot read operations will not see the results of writes done in the same transaction. This was the default behavior prior to API version 300." />