From bf5acc7296f2a837d7b3ced9525a95b287dbc8c9 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 31 Aug 2020 20:07:35 +0000 Subject: [PATCH 1/4] Use lz4 --- build/Dockerfile | 2 +- cmake/CompileRocksDB.cmake | 4 ++-- fdbserver/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index e99c45357d..baecae0c9c 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -11,7 +11,7 @@ RUN yum install -y yum-utils &&\ devtoolset-8-libubsan-devel devtoolset-8-valgrind-devel \ rh-python36-python-devel rh-ruby24 golang python27 rpm-build \ mono-core debbuild python-pip dos2unix valgrind-devel ccache \ - distcc wget git &&\ + distcc wget git lz4 &&\ pip install boto3==1.1.1 USER root diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake index 3de2696beb..aeceaba998 100644 --- a/cmake/CompileRocksDB.cmake +++ b/cmake/CompileRocksDB.cmake @@ -18,7 +18,7 @@ if (RocksDB_FOUND) -DWITH_CORE_TOOLS=OFF -DWITH_BENCHMARK_TOOLS=OFF -DWITH_BZ2=OFF - -DWITH_LZ4=OFF + -DWITH_LZ4=ON -DWITH_SNAPPY=OFF -DWITH_ZLIB=OFF -DWITH_ZSTD=OFF @@ -45,7 +45,7 @@ else() -DWITH_CORE_TOOLS=OFF -DWITH_BENCHMARK_TOOLS=OFF -DWITH_BZ2=OFF - -DWITH_LZ4=OFF + -DWITH_LZ4=ON -DWITH_SNAPPY=OFF -DWITH_ZLIB=OFF -DWITH_ZSTD=OFF diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 3236687d65..dbc9d9e42e 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -243,7 +243,7 @@ target_include_directories(fdbserver PRIVATE if (WITH_ROCKSDB_EXPERIMENTAL) add_dependencies(fdbserver rocksdb) target_include_directories(fdbserver PRIVATE ${ROCKSDB_INCLUDE_DIR}) - target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite ${ROCKSDB_LIBRARIES}) + target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite ${ROCKSDB_LIBRARIES} lz4) else() target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite) endif() From a8194b4ae309bb2a8b5fa5378b8592974c8fb73b Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Mon, 31 Aug 2020 20:20:46 +0000 Subject: [PATCH 2/4] Bump Dockerfile versions --- build/Dockerfile | 4 ++-- build/Dockerfile.devel | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index baecae0c9c..0aa092533b 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -61,8 +61,8 @@ RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar. ARG TIMEZONEINFO=America/Los_Angeles RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime -LABEL version=0.1.15 -ENV DOCKER_IMAGEVER=0.1.15 +LABEL version=0.1.16 +ENV DOCKER_IMAGEVER=0.1.16 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ diff --git a/build/Dockerfile.devel b/build/Dockerfile.devel index 9552a064dc..95869bdd35 100644 --- a/build/Dockerfile.devel +++ b/build/Dockerfile.devel @@ -1,4 +1,4 @@ -FROM foundationdb/foundationdb-build:0.1.15 +FROM foundationdb/foundationdb-build:0.1.16 USER root @@ -50,8 +50,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\ ldconfig &&\ rm -rf /mnt/artifacts -LABEL version=0.11.8 -ENV DOCKER_IMAGEVER=0.11.8 +LABEL version=0.11.9 +ENV DOCKER_IMAGEVER=0.11.9 ENV CLANGCC=/usr/local/bin/clang.de8a65ef ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef From 59ebdfb38868356af7da0c7f1a654dd5ae7dba1c Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Tue, 8 Sep 2020 12:25:11 -0400 Subject: [PATCH 3/4] Update with newest docker container version --- build/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml index 8241c48cb3..0044ad481a 100644 --- a/build/docker-compose.yaml +++ b/build/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: common: &common - image: foundationdb/foundationdb-build:0.1.15 + image: foundationdb/foundationdb-build:0.1.16 build-setup: &build-setup <<: *common From 62fb44e3b9d9196e4998de49b753db9ba0b117f8 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Wed, 9 Sep 2020 19:30:13 +0000 Subject: [PATCH 4/4] Use the live data estimate instead of total data --- fdbserver/KeyValueStoreRocksDB.actor.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp index 39b71a98fe..62138d25fa 100644 --- a/fdbserver/KeyValueStoreRocksDB.actor.cpp +++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp @@ -372,16 +372,14 @@ struct RocksDBKeyValueStore : IKeyValueStore { } StorageBytes getStorageBytes() override { + uint64_t live = 0; + ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kEstimateLiveDataSize, &live)); + int64_t free; int64_t total; - - uint64_t sstBytes = 0; - ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kTotalSstFilesSize, &sstBytes)); - uint64_t memtableBytes = 0; - ASSERT(db->GetIntProperty(rocksdb::DB::Properties::kSizeAllMemTables, &memtableBytes)); g_network->getDiskBytes(path, free, total); - return StorageBytes(free, total, sstBytes + memtableBytes, free); + return StorageBytes(free, total, live, free); } };