Merge pull request #4152 from sears/merge-6.2-to-6.3

Merge 6.2 to 6.3

build/Dockerfile

@@ -1,17 +1,27 @@
 FROM centos:6
 
+# Clean yum cache, disable default Base repo and enable Vault
+RUN yum clean all &&\
+  sed -i -e 's/gpgcheck=1/enabled=0/g' /etc/yum.repos.d/CentOS-Base.repo &&\
+  sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo &&\
+  sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo &&\
+  sed -i -e "s/6\.0/$(cut -d\  -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo &&\
+  yum install -y yum-utils &&\
+  yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
+  yum -y install centos-release-scl-rh epel-release  \
+    http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
+  sed -i -e 's/#baseurl=/baseurl=/g' -e 's/mirror.centos.org/vault.centos.org/g' \
+     -e 's/mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo &&\
+  yum clean all
+
 # Install dependencies for developer tools, bindings,\
 # documentation, actorcompiler, and packaging tools\
-RUN yum install -y yum-utils &&\
+RUN yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
-  yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
+      devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
-  yum -y install centos-release-scl epel-release \
+      devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
-    http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
+      rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
-  yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
+      mono-core debbuild python-pip dos2unix valgrind-devel ccache \
-    devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
+      distcc wget libxslt git lz4 lz4-devel lz4-static &&\
-    devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
-    rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
-    mono-core debbuild python-pip dos2unix valgrind-devel ccache \
-    distcc wget git lz4 lz4-devel lz4-static &&\
   pip install boto3==1.1.1
 
 USER root
@@ -41,6 +51,8 @@ RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.1
 
 # install Ninja
 RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\
+    echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef  ninja.zip" > /tmp/ninja-sha.txt &&\
+    sha256sum -c /tmp/ninja-sha.txt &&\
     unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\
     cd .. && rm -rf ninja-1.9.0 ninja.zip
 
@@ -57,13 +69,48 @@ RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.
     echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee  rocksdb.tar.gz" > rocksdb-sha.txt &&\
     sha256sum -c rocksdb-sha.txt && tar xf rocksdb.tar.gz && rm -rf rocksdb.tar.gz rocksdb-sha.txt
 
+RUN cd /opt/ && curl -L https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o ctest2junit.xsl
+
+# Setting this environment variable switches from OpenSSL to BoringSSL
+#ENV OPENSSL_ROOT_DIR=/opt/boringssl
+
+# install BoringSSL:  TODO: They don't seem to have releases(?)  I picked today's master SHA.
+RUN cd /opt &&\
+    git clone https://boringssl.googlesource.com/boringssl &&\
+    cd boringssl &&\
+    git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\
+    mkdir build
+
+# ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS
+# also, enable -fPIC.
+# this is moderately uglier than creating a patchfile, but easier to maintain.
+RUN cd /opt/boringssl &&\
+	for f in crypto/fipsmodule/rand/fork_detect_test.cc \
+	         include/openssl/bn.h \
+		 ssl/test/bssl_shim.cc ; do \
+		perl -p -i -e 's/#include <inttypes.h>/#define __STDC_FORMAT_MACROS 1\n#include <inttypes.h>/g;' $f ; \
+	done &&\
+	perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\
+	git diff
+
+RUN cd /opt/boringssl/build &&\
+    scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\
+    scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\
+	./ssl/ssl_test &&\
+	mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib
+
 # Localize time zone
 ARG TIMEZONEINFO=America/Los_Angeles
 RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime
 
-LABEL version=0.1.19
+LABEL version=0.1.21
-ENV DOCKER_IMAGEVER=0.1.19
+ENV DOCKER_IMAGEVER=0.1.21
 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++
+
+ENV CCACHE_NOHASHDIR=true
+ENV CCACHE_UMASK=0000
+ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches"
+
 CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash

build/Dockerfile.devel

@@ -1,4 +1,5 @@
-FROM foundationdb/foundationdb-build:0.1.19
+ARG IMAGE_TAG=0.1.21
+FROM foundationdb/foundationdb-build:${IMAGE_TAG}
 
 USER root
 
@@ -50,8 +51,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\
 	ldconfig &&\
 	rm -rf /mnt/artifacts
 
-LABEL version=0.11.10
+LABEL version=0.11.12
-ENV DOCKER_IMAGEVER=0.11.10
+ENV DOCKER_IMAGEVER=0.11.12
 
 ENV CLANGCC=/usr/local/bin/clang.de8a65ef
 ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef
@@ -63,8 +64,5 @@ ENV CC=/usr/local/bin/clang.de8a65ef
 ENV CXX=/usr/local/bin/clang++.de8a65ef
 ENV USE_LD=LLD
 ENV USE_LIBCXX=1
-ENV CCACHE_NOHASHDIR=true
-ENV CCACHE_UMASK=0000
-ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches"
 
 CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash

build/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   common: &common
-    image: foundationdb/foundationdb-build:0.1.19
+    image: foundationdb/foundationdb-build:0.1.21
 
   build-setup: &build-setup
     <<: *common

documentation/sphinx/source/release-notes/release-notes-620.rst

@@ -6,6 +6,7 @@ Release Notes
 ======
 * Fix invalid memory access on data distributor when snapshotting large clusters. `(PR #4076) <https://github.com/apple/foundationdb/pull/4076>`_
 * Add human-readable DateTime to trace events `(PR #4087) <https://github.com/apple/foundationdb/pull/4087>`_
+* Proxy rejects transaction batch that exceeds MVCC window `(PR #4113) <https://github.com/apple/foundationdb/pull/4113>`_
 
 6.2.28
 ======

fdbclient/NativeAPI.actor.cpp

@@ -2197,7 +2197,6 @@ ACTOR Future<Standalone<RangeResultRef>> getRange( Database cx, Reference<Transa
 				}
 
 				++cx->transactionPhysicalReads;
-				++cx->transactionGetRangeRequests;
 				state GetKeyValuesReply rep;
 				try {
 					if (CLIENT_BUGGIFY) {

fdbserver/Knobs.cpp

@@ -381,6 +381,7 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi
 	init( PROXY_COMPUTE_BUCKETS,                                20000 );
 	init( PROXY_COMPUTE_GROWTH_RATE,                             0.01 );
 	init( TXN_STATE_SEND_AMOUNT,                                    4 );
+	init( PROXY_REJECT_BATCH_QUEUED_TOO_LONG,                    true );
 
 	init( RESET_MASTER_BATCHES,                                   200 );
 	init( RESET_RESOLVER_BATCHES,                                 200 );

fdbserver/Knobs.h

@@ -311,6 +311,7 @@ public:
 	int PROXY_COMPUTE_BUCKETS;
 	double PROXY_COMPUTE_GROWTH_RATE;
 	int TXN_STATE_SEND_AMOUNT;
+	bool PROXY_REJECT_BATCH_QUEUED_TOO_LONG;
 
 	int RESET_MASTER_BATCHES;
 	int RESET_RESOLVER_BATCHES;

fdbserver/MasterProxyServer.actor.cpp

@@ -854,6 +854,20 @@ ACTOR Future<Void> releaseResolvingAfter(ProxyCommitData* self, Future<Void> rel
 	return Void();
 }
 
+// Try to identify recovery transaction and backup's apply mutations (blind writes).
+// Both cannot be rejected and are approximated by looking at first mutation
+// starting with 0xff.
+bool canReject(const std::vector<CommitTransactionRequest>& trs) {
+	for (const auto& tr : trs) {
+		if (tr.transaction.mutations.empty()) continue;
+		if (tr.transaction.mutations[0].param1.startsWith(LiteralStringRef("\xff")) ||
+		    tr.transaction.read_conflict_ranges.empty()) {
+			return false;
+		}
+	}
+	return true;
+}
+
 // Commit one batch of transactions trs
 ACTOR Future<Void> commitBatch(
 	ProxyCommitData* self,
@@ -898,6 +912,7 @@ ACTOR Future<Void> commitBatch(
 
 	if (debugID.present())
 		g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "MasterProxyServer.commitBatch.Before");
+	state double timeStart = now();
 
 	if(localBatchNumber-self->latestLocalCommitBatchResolving.get()>SERVER_KNOBS->RESET_MASTER_BATCHES && now()-self->lastMasterReset>SERVER_KNOBS->RESET_MASTER_DELAY) {
 		TraceEvent(SevWarnAlways, "ConnectionResetMaster", self->dbgid)
@@ -913,6 +928,32 @@ ACTOR Future<Void> commitBatch(
 	// Queuing pre-resolution commit processing
 	TEST(self->latestLocalCommitBatchResolving.get() < localBatchNumber - 1);
 	wait(self->latestLocalCommitBatchResolving.whenAtLeast(localBatchNumber-1));
+	double queuingDelay = g_network->now() - timeStart;
+	if ((queuingDelay > (double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS / SERVER_KNOBS->VERSIONS_PER_SECOND ||
+	     (g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01))) &&
+	    SERVER_KNOBS->PROXY_REJECT_BATCH_QUEUED_TOO_LONG && canReject(trs)) {
+		// Disabled for the recovery transaction. otherwise, recovery can't finish and keeps doing more recoveries.
+		TEST(true); // Reject transactions in the batch
+		TraceEvent(SevWarnAlways, "ProxyReject", self->dbgid)
+		    .suppressFor(0.1)
+		    .detail("QDelay", queuingDelay)
+		    .detail("Transactions", trs.size())
+		    .detail("BatchNumber", localBatchNumber);
+		ASSERT(self->latestLocalCommitBatchResolving.get() == localBatchNumber - 1);
+		self->latestLocalCommitBatchResolving.set(localBatchNumber);
+
+		wait(self->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber-1));
+		ASSERT(self->latestLocalCommitBatchLogging.get() == localBatchNumber - 1);
+		self->latestLocalCommitBatchLogging.set(localBatchNumber);
+		for (const auto& tr : trs) {
+			tr.reply.sendError(transaction_too_old());
+		}
+		++self->stats.commitBatchOut;
+		self->stats.txnCommitOut += trs.size();
+		self->stats.txnConflicts += trs.size();
+		return Void();
+	}
+
 	state Future<Void> releaseDelay = delay(std::min(SERVER_KNOBS->MAX_PROXY_COMPUTE, batchOperations*self->commitComputePerOperation[latencyBucket]), TaskPriority::ProxyMasterVersionReply);
 
 	if (debugID.present())