From ace2750ae30ca0f87d8664d5a1c3dba2ab4dee37 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Tue, 1 Mar 2022 17:11:12 -0600 Subject: [PATCH] Build AWS SDK as part of FDB, and use it to acquire blob credentials. This supports IAM role-based authentication, in addition to all existing credential types. --- cmake/FDBComponents.cmake | 12 ++++ cmake/awssdk.cmake | 98 +++++++++++++++++++++++++++ fdbclient/CMakeLists.txt | 16 +++++ fdbclient/FDBAWSCredentialsProvider.h | 47 +++++++++++++ fdbclient/S3BlobStore.actor.cpp | 37 +++++++++- fdbclient/S3BlobStore.h | 6 +- 6 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 cmake/awssdk.cmake create mode 100644 fdbclient/FDBAWSCredentialsProvider.h diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake index a31d386688..60fbe44a0d 100644 --- a/cmake/FDBComponents.cmake +++ b/cmake/FDBComponents.cmake @@ -212,6 +212,17 @@ endif() set(COROUTINE_IMPL ${DEFAULT_COROUTINE_IMPL} CACHE STRING "Which coroutine implementation to use. Options are boost and libcoro") +################################################################################ +# AWS SDK +################################################################################ + +set(BUILD_AWS_BACKUP OFF CACHE BOOL "Build AWS S3 SDK backup client") +if (BUILD_AWS_BACKUP) + set(WITH_AWS_BACKUP ON) +else() + set(WITH_AWS_BACKUP OFF) +endif() + ################################################################################ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/packages) @@ -232,6 +243,7 @@ function(print_components) message(STATUS "Build Python sdist (make package): ${WITH_PYTHON_BINDING}") message(STATUS "Configure CTest (depends on Python): ${WITH_PYTHON}") message(STATUS "Build with RocksDB: ${WITH_ROCKSDB_EXPERIMENTAL}") + message(STATUS "Build with AWS SDK: ${WITH_AWS_BACKUP}") message(STATUS "=========================================") endfunction() diff --git a/cmake/awssdk.cmake b/cmake/awssdk.cmake new file mode 100644 index 0000000000..c3b9e95a4e --- /dev/null +++ b/cmake/awssdk.cmake @@ -0,0 +1,98 @@ +project(awssdk-download NONE) + +# Compile the sdk with clang and libc++, since otherwise we get libc++ vs libstdc++ link errors when compiling fdb with clang +set(AWSSDK_COMPILER_FLAGS "") +set(AWSSDK_LINK_FLAGS "") +if(APPLE OR CLANG OR USE_LIBCXX) + set(AWSSDK_COMPILER_FLAGS -stdlib=libc++ -nostdlib++) + set(AWSSDK_LINK_FLAGS -stdlib=libc++ -lc++abi) +endif() + +include(ExternalProject) +ExternalProject_Add(awssdk_project + GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git + GIT_TAG 2af3ce543c322cb259471b3b090829464f825972 # v1.9.200 + SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/awssdk-src" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build" + GIT_CONFIG advice.detachedHead=false + CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF # SDK builds shared libs by default, we want static libs + -DENABLE_TESTING=OFF + -DBUILD_ONLY=core # git repo contains SDK for every AWS product, we only want the core auth libraries + -DSIMPLE_INSTALL=ON + -DCMAKE_INSTALL_PREFIX=install # need to specify an install prefix so it doesn't install in /usr/lib - FIXME: use absolute path + -DBYO_CRYPTO=ON # we have our own crypto libraries that conflict if we let aws sdk build and link its own + + + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_EXE_LINKER_FLAGS=${AWSSDK_COMPILER_FLAGS} + -DCMAKE_CXX_FLAGS=${AWSSDK_LINK_FLAGS} + TEST_COMMAND "" + BUILD_ALWAYS TRUE + # the sdk build produces a ton of artifacts, with their own dependency tree, so there is a very specific dependency order they must be linked in + BUILD_BYPRODUCTS "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-s3.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-auth.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a" + "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a" +) + +add_library(awssdk_core STATIC IMPORTED) +add_dependencies(awssdk_core awssdk_project) +set_target_properties(awssdk_core PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a") + +add_library(awssdk_crt STATIC IMPORTED) +add_dependencies(awssdk_crt awssdk_project) +set_target_properties(awssdk_crt PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a") + +# TODO: can we remove c_s3? It seems to be a dependency of libaws-crt +add_library(awssdk_c_s3 STATIC IMPORTED) +add_dependencies(awssdk_c_s3 awssdk_project) +set_target_properties(awssdk_c_s3 PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-s3.a") + +add_library(awssdk_c_auth STATIC IMPORTED) +add_dependencies(awssdk_c_auth awssdk_project) +set_target_properties(awssdk_c_auth PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-auth.a") + +add_library(awssdk_c_eventstream STATIC IMPORTED) +add_dependencies(awssdk_c_eventstream awssdk_project) +set_target_properties(awssdk_c_eventstream PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a") + +add_library(awssdk_c_http STATIC IMPORTED) +add_dependencies(awssdk_c_http awssdk_project) +set_target_properties(awssdk_c_http PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a") + +add_library(awssdk_c_mqtt STATIC IMPORTED) +add_dependencies(awssdk_c_mqtt awssdk_project) +set_target_properties(awssdk_c_mqtt PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a") + +add_library(awssdk_c_io STATIC IMPORTED) +add_dependencies(awssdk_c_io awssdk_project) +set_target_properties(awssdk_c_io PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a") + +add_library(awssdk_checksums STATIC IMPORTED) +add_dependencies(awssdk_checksums awssdk_project) +set_target_properties(awssdk_checksums PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a") + +add_library(awssdk_c_compression STATIC IMPORTED) +add_dependencies(awssdk_c_compression awssdk_project) +set_target_properties(awssdk_c_compression PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a") + +add_library(awssdk_c_cal STATIC IMPORTED) +add_dependencies(awssdk_c_cal awssdk_project) +set_target_properties(awssdk_c_cal PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a") + +add_library(awssdk_c_common STATIC IMPORTED) +add_dependencies(awssdk_c_common awssdk_project) +set_target_properties(awssdk_c_common PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a") + +# link them all together in one interface target +add_library(awssdk_target INTERFACE) +target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include) +target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl) \ No newline at end of file diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 81b41314d1..3cb841dbd7 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -205,6 +205,17 @@ if(BUILD_AZURE_BACKUP) ) endif() + +if(WITH_AWS_BACKUP) + add_compile_definitions(BUILD_AWS_BACKUP) + + set(FDBCLIENT_SRCS + ${FDBCLIENT_SRCS} + FDBAWSCredentialsProvider.h) + + include(awssdk) +endif() + add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs}) add_dependencies(fdbclient fdboptions) target_link_libraries(fdbclient PUBLIC fdbrpc msgpack) @@ -224,3 +235,8 @@ if(BUILD_AZURE_BACKUP) target_link_libraries(fdbclient PRIVATE curl uuid azure-storage-lite) target_link_libraries(fdbclient_sampling PRIVATE curl uuid azure-storage-lite) endif() + +if(BUILD_AWS_BACKUP) + target_link_libraries(fdbclient PUBLIC awssdk_target) + target_link_libraries(fdbclient_sampling PUBLIC awssdk_target) +endif() diff --git a/fdbclient/FDBAWSCredentialsProvider.h b/fdbclient/FDBAWSCredentialsProvider.h new file mode 100644 index 0000000000..f09e2f858e --- /dev/null +++ b/fdbclient/FDBAWSCredentialsProvider.h @@ -0,0 +1,47 @@ +/* + * FDBAWSCredentialsProvider.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if (!defined FDB_AWS_CREDENTIALS_PROVIDER_H) && (defined BUILD_AWS_BACKUP) +#define FDB_AWS_CREDENTIALS_PROVIDER_H +#pragma once + +#include "aws/core/Aws.h" +#include "aws/core/auth/AWSCredentialsProviderChain.h" + +// Singleton +namespace FDBAWSCredentialsProvider { +bool doneInit = false; + +// You're supposed to call AWS::ShutdownAPI(options); once done +// But we want this to live for the lifetime of the process, so we don't do that +static Aws::Auth::AWSCredentials getAwsCredentials() { + if (!doneInit) { + doneInit = true; + Aws::SDKOptions options; + Aws::InitAPI(options); + TraceEvent("AWSSDKInitSuccessful"); + } + Aws::Auth::DefaultAWSCredentialsProviderChain credProvider; + Aws::Auth::AWSCredentials creds = credProvider.GetAWSCredentials(); + return creds; +} +} // namespace FDBAWSCredentialsProvider + +#endif diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp index a9bd360cde..b63b930ee8 100644 --- a/fdbclient/S3BlobStore.actor.cpp +++ b/fdbclient/S3BlobStore.actor.cpp @@ -34,6 +34,8 @@ #include "fdbrpc/IAsyncFile.h" #include "flow/UnitTest.h" #include "fdbclient/rapidxml/rapidxml.hpp" +#include "fdbclient/FDBAWSCredentialsProvider.h" + #include "flow/actorcompiler.h" // has to be last include using namespace rapidxml; @@ -82,6 +84,7 @@ S3BlobStoreEndpoint::BlobKnobs::BlobKnobs() { read_cache_blocks_per_file = CLIENT_KNOBS->BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE; max_send_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_SEND_BYTES_PER_SECOND; max_recv_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_RECV_BYTES_PER_SECOND; + sdk_auth = false; } bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) { @@ -118,6 +121,7 @@ bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) { TRY_PARAM(read_cache_blocks_per_file, rcb); TRY_PARAM(max_send_bytes_per_second, sbps); TRY_PARAM(max_recv_bytes_per_second, rbps); + TRY_PARAM(sdk_auth, sa); #undef TRY_PARAM return false; } @@ -506,7 +510,38 @@ ACTOR Future> tryReadJSONFile(std::string path) { return Optional(); } +// If the credentials expire, the connection will eventually fail and be discarded from the pool, and then a new +// connection will be constructed, which will call this again to get updated credentials +static S3BlobStoreEndpoint::Credentials getSecretSdk() { +#ifdef BUILD_AWS_BACKUP + double elapsed = -timer_monotonic(); + Aws::Auth::AWSCredentials awsCreds = FDBAWSCredentialsProvider::getAwsCredentials(); + elapsed += timer_monotonic(); + + if (awsCreds.IsEmpty()) { + TraceEvent(SevWarn, "S3BlobStoreAWSCredsEmpty"); + throw backup_auth_missing(); + } + + S3BlobStoreEndpoint::Credentials fdbCreds; + fdbCreds.key = awsCreds.GetAWSAccessKeyId(); + fdbCreds.secret = awsCreds.GetAWSSecretKey(); + fdbCreds.securityToken = awsCreds.GetSessionToken(); + + TraceEvent("S3BlobStoreGotSdkCredentials").suppressFor(60).detail("Duration", elapsed); + + return fdbCreds; +#else + TraceEvent(SevError, "S3BlobStoreNoSDK"); + throw backup_auth_missing(); +#endif +} + ACTOR Future updateSecret_impl(Reference b) { + if (b->knobs.sdk_auth) { + b->credentials = getSecretSdk(); + return Void(); + } std::vector* pFiles = (std::vector*)g_network->global(INetwork::enBlobCredentialFiles); if (pFiles == nullptr) return Void(); @@ -601,7 +636,7 @@ ACTOR Future connect_impl(ReferencegetPeerAddress()) .detail("ExpiresIn", b->knobs.max_connection_life); - if (b->lookupKey || b->lookupSecret) + if (b->lookupKey || b->lookupSecret || b->knobs.sdk_auth) wait(b->updateSecret()); return S3BlobStoreEndpoint::ReusableConnection({ conn, now() + b->knobs.max_connection_life }); diff --git a/fdbclient/S3BlobStore.h b/fdbclient/S3BlobStore.h index c259d6a4da..21f39e1d0e 100644 --- a/fdbclient/S3BlobStore.h +++ b/fdbclient/S3BlobStore.h @@ -59,7 +59,7 @@ public: delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests, concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file, read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second, - max_recv_bytes_per_second; + max_recv_bytes_per_second, sdk_auth; bool set(StringRef name, int value); std::string getURLParameters() const; static std::vector getKnobDescriptions() { @@ -91,7 +91,9 @@ public: "read_cache_blocks_per_file (or rcb) Size of the read cache for a file in blocks.", "max_send_bytes_per_second (or sbps) Max send bytes per second for all requests combined.", "max_recv_bytes_per_second (or rbps) Max receive bytes per second for all requests combined (NOT YET " - "USED)." + "USED).", + "sdk_auth (or sa) Use AWS SDK to resolve credentials. Only valid if " + "BUILD_AWS_BACKUP is enabled." }; } };