Jingyu Zhou
GitHub
commit
5f06c20d08
|
|
@ -1,17 +1,27 @@
|
|||
FROM centos:6
|
||||
|
||||
# Clean yum cache, disable default Base repo and enable Vault
|
||||
RUN yum clean all &&\
|
||||
sed -i -e 's/gpgcheck=1/enabled=0/g' /etc/yum.repos.d/CentOS-Base.repo &&\
|
||||
sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo &&\
|
||||
sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo &&\
|
||||
sed -i -e "s/6\.0/$(cut -d\ -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo &&\
|
||||
yum install -y yum-utils &&\
|
||||
yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
|
||||
yum -y install centos-release-scl-rh epel-release \
|
||||
http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
|
||||
sed -i -e 's/#baseurl=/baseurl=/g' -e 's/mirror.centos.org/vault.centos.org/g' \
|
||||
-e 's/mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo &&\
|
||||
yum clean all
|
||||
|
||||
# Install dependencies for developer tools, bindings,\
|
||||
# documentation, actorcompiler, and packaging tools\
|
||||
RUN yum install -y yum-utils &&\
|
||||
yum-config-manager --enable rhel-server-rhscl-7-rpms &&\
|
||||
yum -y install centos-release-scl epel-release \
|
||||
http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\
|
||||
yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
|
||||
devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
|
||||
devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
|
||||
rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
|
||||
mono-core debbuild python-pip dos2unix valgrind-devel ccache \
|
||||
distcc wget git lz4 lz4-devel lz4-static &&\
|
||||
RUN yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \
|
||||
devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \
|
||||
devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \
|
||||
rh-python36-python-devel rh-ruby24 golang python27 rpm-build \
|
||||
mono-core debbuild python-pip dos2unix valgrind-devel ccache \
|
||||
distcc wget libxslt git lz4 lz4-devel lz4-static &&\
|
||||
pip install boto3==1.1.1
|
||||
|
||||
USER root
|
||||
|
|
@ -41,6 +51,8 @@ RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.1
|
|||
|
||||
# install Ninja
|
||||
RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\
|
||||
echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > /tmp/ninja-sha.txt &&\
|
||||
sha256sum -c /tmp/ninja-sha.txt &&\
|
||||
unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\
|
||||
cd .. && rm -rf ninja-1.9.0 ninja.zip
|
||||
|
||||
|
|
@ -57,13 +69,48 @@ RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.
|
|||
echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt &&\
|
||||
sha256sum -c rocksdb-sha.txt && tar xf rocksdb.tar.gz && rm -rf rocksdb.tar.gz rocksdb-sha.txt
|
||||
|
||||
RUN cd /opt/ && curl -L https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o ctest2junit.xsl
|
||||
|
||||
# Setting this environment variable switches from OpenSSL to BoringSSL
|
||||
#ENV OPENSSL_ROOT_DIR=/opt/boringssl
|
||||
|
||||
# install BoringSSL: TODO: They don't seem to have releases(?) I picked today's master SHA.
|
||||
RUN cd /opt &&\
|
||||
git clone https://boringssl.googlesource.com/boringssl &&\
|
||||
cd boringssl &&\
|
||||
git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\
|
||||
mkdir build
|
||||
|
||||
# ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS
|
||||
# also, enable -fPIC.
|
||||
# this is moderately uglier than creating a patchfile, but easier to maintain.
|
||||
RUN cd /opt/boringssl &&\
|
||||
for f in crypto/fipsmodule/rand/fork_detect_test.cc \
|
||||
include/openssl/bn.h \
|
||||
ssl/test/bssl_shim.cc ; do \
|
||||
perl -p -i -e 's/#include <inttypes.h>/#define __STDC_FORMAT_MACROS 1\n#include <inttypes.h>/g;' $f ; \
|
||||
done &&\
|
||||
perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\
|
||||
git diff
|
||||
|
||||
RUN cd /opt/boringssl/build &&\
|
||||
scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\
|
||||
scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\
|
||||
./ssl/ssl_test &&\
|
||||
mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib
|
||||
|
||||
# Localize time zone
|
||||
ARG TIMEZONEINFO=America/Los_Angeles
|
||||
RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime
|
||||
|
||||
LABEL version=0.1.19
|
||||
ENV DOCKER_IMAGEVER=0.1.19
|
||||
LABEL version=0.1.21
|
||||
ENV DOCKER_IMAGEVER=0.1.21
|
||||
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
|
||||
ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc
|
||||
ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++
|
||||
|
||||
ENV CCACHE_NOHASHDIR=true
|
||||
ENV CCACHE_UMASK=0000
|
||||
ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches"
|
||||
|
||||
CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
FROM foundationdb/foundationdb-build:0.1.19
|
||||
ARG IMAGE_TAG=0.1.21
|
||||
FROM foundationdb/foundationdb-build:${IMAGE_TAG}
|
||||
|
||||
USER root
|
||||
|
||||
|
|
@ -50,8 +51,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\
|
|||
ldconfig &&\
|
||||
rm -rf /mnt/artifacts
|
||||
|
||||
LABEL version=0.11.10
|
||||
ENV DOCKER_IMAGEVER=0.11.10
|
||||
LABEL version=0.11.12
|
||||
ENV DOCKER_IMAGEVER=0.11.12
|
||||
|
||||
ENV CLANGCC=/usr/local/bin/clang.de8a65ef
|
||||
ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef
|
||||
|
|
@ -63,8 +64,5 @@ ENV CC=/usr/local/bin/clang.de8a65ef
|
|||
ENV CXX=/usr/local/bin/clang++.de8a65ef
|
||||
ENV USE_LD=LLD
|
||||
ENV USE_LIBCXX=1
|
||||
ENV CCACHE_NOHASHDIR=true
|
||||
ENV CCACHE_UMASK=0000
|
||||
ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches"
|
||||
|
||||
CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ version: "3"
|
|||
|
||||
services:
|
||||
common: &common
|
||||
image: foundationdb/foundationdb-build:0.1.19
|
||||
image: foundationdb/foundationdb-build:0.1.21
|
||||
|
||||
build-setup: &build-setup
|
||||
<<: *common
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Release Notes
|
|||
======
|
||||
* Fix invalid memory access on data distributor when snapshotting large clusters. `(PR #4076) <https://github.com/apple/foundationdb/pull/4076>`_
|
||||
* Add human-readable DateTime to trace events `(PR #4087) <https://github.com/apple/foundationdb/pull/4087>`_
|
||||
* Proxy rejects transaction batch that exceeds MVCC window `(PR #4113) <https://github.com/apple/foundationdb/pull/4113>`_
|
||||
|
||||
6.2.28
|
||||
======
|
||||
|
|
|
|||
|
|
@ -2197,7 +2197,6 @@ ACTOR Future<Standalone<RangeResultRef>> getRange( Database cx, Reference<Transa
|
|||
}
|
||||
|
||||
++cx->transactionPhysicalReads;
|
||||
++cx->transactionGetRangeRequests;
|
||||
state GetKeyValuesReply rep;
|
||||
try {
|
||||
if (CLIENT_BUGGIFY) {
|
||||
|
|
|
|||
|
|
@ -381,6 +381,7 @@ void ServerKnobs::initialize(bool randomize, ClientKnobs* clientKnobs, bool isSi
|
|||
init( PROXY_COMPUTE_BUCKETS, 20000 );
|
||||
init( PROXY_COMPUTE_GROWTH_RATE, 0.01 );
|
||||
init( TXN_STATE_SEND_AMOUNT, 4 );
|
||||
init( PROXY_REJECT_BATCH_QUEUED_TOO_LONG, true );
|
||||
|
||||
init( RESET_MASTER_BATCHES, 200 );
|
||||
init( RESET_RESOLVER_BATCHES, 200 );
|
||||
|
|
|
|||
|
|
@ -311,6 +311,7 @@ public:
|
|||
int PROXY_COMPUTE_BUCKETS;
|
||||
double PROXY_COMPUTE_GROWTH_RATE;
|
||||
int TXN_STATE_SEND_AMOUNT;
|
||||
bool PROXY_REJECT_BATCH_QUEUED_TOO_LONG;
|
||||
|
||||
int RESET_MASTER_BATCHES;
|
||||
int RESET_RESOLVER_BATCHES;
|
||||
|
|
|
|||
|
|
@ -854,6 +854,20 @@ ACTOR Future<Void> releaseResolvingAfter(ProxyCommitData* self, Future<Void> rel
|
|||
return Void();
|
||||
}
|
||||
|
||||
// Try to identify recovery transaction and backup's apply mutations (blind writes).
|
||||
// Both cannot be rejected and are approximated by looking at first mutation
|
||||
// starting with 0xff.
|
||||
bool canReject(const std::vector<CommitTransactionRequest>& trs) {
|
||||
for (const auto& tr : trs) {
|
||||
if (tr.transaction.mutations.empty()) continue;
|
||||
if (tr.transaction.mutations[0].param1.startsWith(LiteralStringRef("\xff")) ||
|
||||
tr.transaction.read_conflict_ranges.empty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Commit one batch of transactions trs
|
||||
ACTOR Future<Void> commitBatch(
|
||||
ProxyCommitData* self,
|
||||
|
|
@ -898,6 +912,7 @@ ACTOR Future<Void> commitBatch(
|
|||
|
||||
if (debugID.present())
|
||||
g_traceBatch.addEvent("CommitDebug", debugID.get().first(), "MasterProxyServer.commitBatch.Before");
|
||||
state double timeStart = now();
|
||||
|
||||
if(localBatchNumber-self->latestLocalCommitBatchResolving.get()>SERVER_KNOBS->RESET_MASTER_BATCHES && now()-self->lastMasterReset>SERVER_KNOBS->RESET_MASTER_DELAY) {
|
||||
TraceEvent(SevWarnAlways, "ConnectionResetMaster", self->dbgid)
|
||||
|
|
@ -913,6 +928,32 @@ ACTOR Future<Void> commitBatch(
|
|||
// Queuing pre-resolution commit processing
|
||||
TEST(self->latestLocalCommitBatchResolving.get() < localBatchNumber - 1);
|
||||
wait(self->latestLocalCommitBatchResolving.whenAtLeast(localBatchNumber-1));
|
||||
double queuingDelay = g_network->now() - timeStart;
|
||||
if ((queuingDelay > (double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS / SERVER_KNOBS->VERSIONS_PER_SECOND ||
|
||||
(g_network->isSimulated() && BUGGIFY_WITH_PROB(0.01))) &&
|
||||
SERVER_KNOBS->PROXY_REJECT_BATCH_QUEUED_TOO_LONG && canReject(trs)) {
|
||||
// Disabled for the recovery transaction. otherwise, recovery can't finish and keeps doing more recoveries.
|
||||
TEST(true); // Reject transactions in the batch
|
||||
TraceEvent(SevWarnAlways, "ProxyReject", self->dbgid)
|
||||
.suppressFor(0.1)
|
||||
.detail("QDelay", queuingDelay)
|
||||
.detail("Transactions", trs.size())
|
||||
.detail("BatchNumber", localBatchNumber);
|
||||
ASSERT(self->latestLocalCommitBatchResolving.get() == localBatchNumber - 1);
|
||||
self->latestLocalCommitBatchResolving.set(localBatchNumber);
|
||||
|
||||
wait(self->latestLocalCommitBatchLogging.whenAtLeast(localBatchNumber-1));
|
||||
ASSERT(self->latestLocalCommitBatchLogging.get() == localBatchNumber - 1);
|
||||
self->latestLocalCommitBatchLogging.set(localBatchNumber);
|
||||
for (const auto& tr : trs) {
|
||||
tr.reply.sendError(transaction_too_old());
|
||||
}
|
||||
++self->stats.commitBatchOut;
|
||||
self->stats.txnCommitOut += trs.size();
|
||||
self->stats.txnConflicts += trs.size();
|
||||
return Void();
|
||||
}
|
||||
|
||||
state Future<Void> releaseDelay = delay(std::min(SERVER_KNOBS->MAX_PROXY_COMPUTE, batchOperations*self->commitComputePerOperation[latencyBucket]), TaskPriority::ProxyMasterVersionReply);
|
||||
|
||||
if (debugID.present())
|
||||
|
|
|
|||
Loading…
Reference in New Issue