Merge pull request #7487 from sfc-gh-ajbeamon/buggified-commit-helper

Add a helper to simulate a commit that can fail with unknown result or take a long time
This commit is contained in:
A.J. Beamon 2022-07-07 16:17:19 -07:00 committed by GitHub
commit 346d3c3265
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 43 additions and 19 deletions

View File

@ -160,15 +160,7 @@ Future<TenantMapEntry> createTenant(Reference<DB> db, TenantName name) {
tr->set(tenantLastIdKey, TenantMapEntry::idToPrefix(tenantId));
state std::pair<TenantMapEntry, bool> newTenant = wait(createTenantTransaction(tr, name, tenantId));
if (BUGGIFY) {
throw commit_unknown_result();
}
wait(safeThreadFutureToFuture(tr->commit()));
if (BUGGIFY) {
throw commit_unknown_result();
}
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
TraceEvent("CreatedTenant")
.detail("Tenant", name)
@ -226,16 +218,7 @@ Future<Void> deleteTenant(Reference<DB> db, TenantName name) {
}
wait(deleteTenantTransaction(tr, name));
if (BUGGIFY) {
throw commit_unknown_result();
}
wait(safeThreadFutureToFuture(tr->commit()));
if (BUGGIFY) {
throw commit_unknown_result();
}
wait(buggifiedCommit(tr, BUGGIFY_WITH_PROB(0.1)));
TraceEvent("DeletedTenant").detail("Tenant", name).detail("Version", tr->getCommittedVersion());
return Void();

View File

@ -1921,6 +1921,47 @@ Future<T> forward(Future<T> from, Promise<T> to) {
}
}
ACTOR template <class Transaction>
Future<Void> buggifiedCommit(Transaction tr, bool buggify, int maxDelayDuration = 60.0) {
state int buggifyUnknownResultPoint = 0;
state int buggifyDelayPoint = 0;
if (buggify) {
int choice = deterministicRandom()->randomInt(1, 9);
buggifyUnknownResultPoint = choice / 3;
buggifyDelayPoint = choice % 3;
}
// Simulate a delay before commit that could potentially trigger a timeout
if (buggifyDelayPoint == 1) {
wait(delay(deterministicRandom()->random01() * maxDelayDuration));
}
// Simulate an unknown result that didn't commit
if (buggifyUnknownResultPoint == 1) {
// The delay avoids a no-wait commit.
if (!BUGGIFY) {
wait(delay(0));
}
throw commit_unknown_result();
}
wait(safeThreadFutureToFuture(tr->commit()));
// Simulate a long delay after commit that could potentially trigger a timeout
if (buggifyDelayPoint == 2) {
wait(delay(deterministicRandom()->random01() * maxDelayDuration));
}
// Simulate an unknown result that did commit
if (buggifyUnknownResultPoint == 2) {
throw commit_unknown_result();
}
return Void();
}
// Monad
ACTOR template <class Fun, class T>