From 6e7d2ff7dd682472da03520a0793ff78e8b3d169 Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Tue, 25 Feb 2020 15:46:13 -0800 Subject: [PATCH] prevent the proxy from delaying too long based on an incorrect estimate of the compute time --- fdbrpc/sim2.actor.cpp | 2 +- fdbserver/Knobs.cpp | 1 + fdbserver/Knobs.h | 1 + fdbserver/MasterProxyServer.actor.cpp | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index f9e8b482a3..74e06f8310 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -756,7 +756,7 @@ public: // timer() can be up to one second ahead of now() virtual double timer() { - timerTime += deterministicRandom()->random01()*(time+1.0-timerTime)/2.0; + timerTime += deterministicRandom()->random01()*(time+0.1-timerTime)/2.0; return timerTime; } diff --git a/fdbserver/Knobs.cpp b/fdbserver/Knobs.cpp index 992f84625c..7b8e25247b 100644 --- a/fdbserver/Knobs.cpp +++ b/fdbserver/Knobs.cpp @@ -318,6 +318,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs, bool isSimula init( ALWAYS_CAUSAL_READ_RISKY, false ); init( MAX_COMMIT_UPDATES, 2000 ); if( randomize && BUGGIFY ) MAX_COMMIT_UPDATES = 1; init( MIN_PROXY_COMPUTE, 0.001 ); + init( MAX_PROXY_COMPUTE, 2.0 ); init( PROXY_COMPUTE_BUCKETS, 20000 ); init( PROXY_COMPUTE_GROWTH_RATE, 0.01 ); diff --git a/fdbserver/Knobs.h b/fdbserver/Knobs.h index 42aee44ef5..70ab0be45f 100644 --- a/fdbserver/Knobs.h +++ b/fdbserver/Knobs.h @@ -264,6 +264,7 @@ public: bool ALWAYS_CAUSAL_READ_RISKY; int MAX_COMMIT_UPDATES; double MIN_PROXY_COMPUTE; + double MAX_PROXY_COMPUTE; int PROXY_COMPUTE_BUCKETS; double PROXY_COMPUTE_GROWTH_RATE; diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index 3ef38de80a..7859b4a4e5 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -534,7 +534,7 @@ ACTOR Future commitBatch( /////// Phase 1: Pre-resolution processing (CPU bound except waiting for a version # which is separately pipelined and *should* be available by now (unless empty commit); ordered; currently atomic but could yield) TEST(self->latestLocalCommitBatchResolving.get() < localBatchNumber-1); // Queuing pre-resolution commit processing wait(self->latestLocalCommitBatchResolving.whenAtLeast(localBatchNumber-1)); - state Future releaseDelay = delay(batchOperations*self->commitComputePerOperation[latencyBucket], TaskPriority::ProxyMasterVersionReply); + state Future releaseDelay = delay(std::min(SERVER_KNOBS->MAX_PROXY_COMPUTE, batchOperations*self->commitComputePerOperation[latencyBucket]), TaskPriority::ProxyMasterVersionReply); if (debugID.present()) g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "MasterProxyServer.commitBatch.GettingCommitVersion");