Build AWS SDK as part of FDB, and use it to acquire blob credentials. This supports IAM role-based authentication, in addition to all existing credential types.
This commit is contained in:
Josh Slocum
parent
017709aec6
commit
ace2750ae3
|
|
@ -212,6 +212,17 @@ endif()
|
|||
|
||||
set(COROUTINE_IMPL ${DEFAULT_COROUTINE_IMPL} CACHE STRING "Which coroutine implementation to use. Options are boost and libcoro")
|
||||
|
||||
################################################################################
|
||||
# AWS SDK
|
||||
################################################################################
|
||||
|
||||
set(BUILD_AWS_BACKUP OFF CACHE BOOL "Build AWS S3 SDK backup client")
|
||||
if (BUILD_AWS_BACKUP)
|
||||
set(WITH_AWS_BACKUP ON)
|
||||
else()
|
||||
set(WITH_AWS_BACKUP OFF)
|
||||
endif()
|
||||
|
||||
################################################################################
|
||||
|
||||
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/packages)
|
||||
|
|
@ -232,6 +243,7 @@ function(print_components)
|
|||
message(STATUS "Build Python sdist (make package): ${WITH_PYTHON_BINDING}")
|
||||
message(STATUS "Configure CTest (depends on Python): ${WITH_PYTHON}")
|
||||
message(STATUS "Build with RocksDB: ${WITH_ROCKSDB_EXPERIMENTAL}")
|
||||
message(STATUS "Build with AWS SDK: ${WITH_AWS_BACKUP}")
|
||||
message(STATUS "=========================================")
|
||||
endfunction()
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
project(awssdk-download NONE)
|
||||
|
||||
# Compile the sdk with clang and libc++, since otherwise we get libc++ vs libstdc++ link errors when compiling fdb with clang
|
||||
set(AWSSDK_COMPILER_FLAGS "")
|
||||
set(AWSSDK_LINK_FLAGS "")
|
||||
if(APPLE OR CLANG OR USE_LIBCXX)
|
||||
set(AWSSDK_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
|
||||
set(AWSSDK_LINK_FLAGS -stdlib=libc++ -lc++abi)
|
||||
endif()
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(awssdk_project
|
||||
GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git
|
||||
GIT_TAG 2af3ce543c322cb259471b3b090829464f825972 # v1.9.200
|
||||
SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/awssdk-src"
|
||||
BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build"
|
||||
GIT_CONFIG advice.detachedHead=false
|
||||
CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF # SDK builds shared libs by default, we want static libs
|
||||
-DENABLE_TESTING=OFF
|
||||
-DBUILD_ONLY=core # git repo contains SDK for every AWS product, we only want the core auth libraries
|
||||
-DSIMPLE_INSTALL=ON
|
||||
-DCMAKE_INSTALL_PREFIX=install # need to specify an install prefix so it doesn't install in /usr/lib - FIXME: use absolute path
|
||||
-DBYO_CRYPTO=ON # we have our own crypto libraries that conflict if we let aws sdk build and link its own
|
||||
|
||||
|
||||
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
|
||||
-DCMAKE_EXE_LINKER_FLAGS=${AWSSDK_COMPILER_FLAGS}
|
||||
-DCMAKE_CXX_FLAGS=${AWSSDK_LINK_FLAGS}
|
||||
TEST_COMMAND ""
|
||||
BUILD_ALWAYS TRUE
|
||||
# the sdk build produces a ton of artifacts, with their own dependency tree, so there is a very specific dependency order they must be linked in
|
||||
BUILD_BYPRODUCTS "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-s3.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-auth.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a"
|
||||
)
|
||||
|
||||
add_library(awssdk_core STATIC IMPORTED)
|
||||
add_dependencies(awssdk_core awssdk_project)
|
||||
set_target_properties(awssdk_core PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a")
|
||||
|
||||
add_library(awssdk_crt STATIC IMPORTED)
|
||||
add_dependencies(awssdk_crt awssdk_project)
|
||||
set_target_properties(awssdk_crt PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a")
|
||||
|
||||
# TODO: can we remove c_s3? It seems to be a dependency of libaws-crt
|
||||
add_library(awssdk_c_s3 STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_s3 awssdk_project)
|
||||
set_target_properties(awssdk_c_s3 PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-s3.a")
|
||||
|
||||
add_library(awssdk_c_auth STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_auth awssdk_project)
|
||||
set_target_properties(awssdk_c_auth PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-auth.a")
|
||||
|
||||
add_library(awssdk_c_eventstream STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_eventstream awssdk_project)
|
||||
set_target_properties(awssdk_c_eventstream PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a")
|
||||
|
||||
add_library(awssdk_c_http STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_http awssdk_project)
|
||||
set_target_properties(awssdk_c_http PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a")
|
||||
|
||||
add_library(awssdk_c_mqtt STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_mqtt awssdk_project)
|
||||
set_target_properties(awssdk_c_mqtt PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a")
|
||||
|
||||
add_library(awssdk_c_io STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_io awssdk_project)
|
||||
set_target_properties(awssdk_c_io PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a")
|
||||
|
||||
add_library(awssdk_checksums STATIC IMPORTED)
|
||||
add_dependencies(awssdk_checksums awssdk_project)
|
||||
set_target_properties(awssdk_checksums PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a")
|
||||
|
||||
add_library(awssdk_c_compression STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_compression awssdk_project)
|
||||
set_target_properties(awssdk_c_compression PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a")
|
||||
|
||||
add_library(awssdk_c_cal STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_cal awssdk_project)
|
||||
set_target_properties(awssdk_c_cal PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a")
|
||||
|
||||
add_library(awssdk_c_common STATIC IMPORTED)
|
||||
add_dependencies(awssdk_c_common awssdk_project)
|
||||
set_target_properties(awssdk_c_common PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a")
|
||||
|
||||
# link them all together in one interface target
|
||||
add_library(awssdk_target INTERFACE)
|
||||
target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
|
||||
target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl)
|
||||
|
|
@ -205,6 +205,17 @@ if(BUILD_AZURE_BACKUP)
|
|||
)
|
||||
endif()
|
||||
|
||||
|
||||
if(WITH_AWS_BACKUP)
|
||||
add_compile_definitions(BUILD_AWS_BACKUP)
|
||||
|
||||
set(FDBCLIENT_SRCS
|
||||
${FDBCLIENT_SRCS}
|
||||
FDBAWSCredentialsProvider.h)
|
||||
|
||||
include(awssdk)
|
||||
endif()
|
||||
|
||||
add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
|
||||
add_dependencies(fdbclient fdboptions)
|
||||
target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
|
||||
|
|
@ -224,3 +235,8 @@ if(BUILD_AZURE_BACKUP)
|
|||
target_link_libraries(fdbclient PRIVATE curl uuid azure-storage-lite)
|
||||
target_link_libraries(fdbclient_sampling PRIVATE curl uuid azure-storage-lite)
|
||||
endif()
|
||||
|
||||
if(BUILD_AWS_BACKUP)
|
||||
target_link_libraries(fdbclient PUBLIC awssdk_target)
|
||||
target_link_libraries(fdbclient_sampling PUBLIC awssdk_target)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
/*
|
||||
* FDBAWSCredentialsProvider.h
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#if (!defined FDB_AWS_CREDENTIALS_PROVIDER_H) && (defined BUILD_AWS_BACKUP)
|
||||
#define FDB_AWS_CREDENTIALS_PROVIDER_H
|
||||
#pragma once
|
||||
|
||||
#include "aws/core/Aws.h"
|
||||
#include "aws/core/auth/AWSCredentialsProviderChain.h"
|
||||
|
||||
// Singleton
|
||||
namespace FDBAWSCredentialsProvider {
|
||||
bool doneInit = false;
|
||||
|
||||
// You're supposed to call AWS::ShutdownAPI(options); once done
|
||||
// But we want this to live for the lifetime of the process, so we don't do that
|
||||
static Aws::Auth::AWSCredentials getAwsCredentials() {
|
||||
if (!doneInit) {
|
||||
doneInit = true;
|
||||
Aws::SDKOptions options;
|
||||
Aws::InitAPI(options);
|
||||
TraceEvent("AWSSDKInitSuccessful");
|
||||
}
|
||||
Aws::Auth::DefaultAWSCredentialsProviderChain credProvider;
|
||||
Aws::Auth::AWSCredentials creds = credProvider.GetAWSCredentials();
|
||||
return creds;
|
||||
}
|
||||
} // namespace FDBAWSCredentialsProvider
|
||||
|
||||
#endif
|
||||
|
|
@ -34,6 +34,8 @@
|
|||
#include "fdbrpc/IAsyncFile.h"
|
||||
#include "flow/UnitTest.h"
|
||||
#include "fdbclient/rapidxml/rapidxml.hpp"
|
||||
#include "fdbclient/FDBAWSCredentialsProvider.h"
|
||||
|
||||
#include "flow/actorcompiler.h" // has to be last include
|
||||
|
||||
using namespace rapidxml;
|
||||
|
|
@ -82,6 +84,7 @@ S3BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
|
|||
read_cache_blocks_per_file = CLIENT_KNOBS->BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
|
||||
max_send_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
|
||||
max_recv_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
|
||||
sdk_auth = false;
|
||||
}
|
||||
|
||||
bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
|
||||
|
|
@ -118,6 +121,7 @@ bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
|
|||
TRY_PARAM(read_cache_blocks_per_file, rcb);
|
||||
TRY_PARAM(max_send_bytes_per_second, sbps);
|
||||
TRY_PARAM(max_recv_bytes_per_second, rbps);
|
||||
TRY_PARAM(sdk_auth, sa);
|
||||
#undef TRY_PARAM
|
||||
return false;
|
||||
}
|
||||
|
|
@ -506,7 +510,38 @@ ACTOR Future<Optional<json_spirit::mObject>> tryReadJSONFile(std::string path) {
|
|||
return Optional<json_spirit::mObject>();
|
||||
}
|
||||
|
||||
// If the credentials expire, the connection will eventually fail and be discarded from the pool, and then a new
|
||||
// connection will be constructed, which will call this again to get updated credentials
|
||||
static S3BlobStoreEndpoint::Credentials getSecretSdk() {
|
||||
#ifdef BUILD_AWS_BACKUP
|
||||
double elapsed = -timer_monotonic();
|
||||
Aws::Auth::AWSCredentials awsCreds = FDBAWSCredentialsProvider::getAwsCredentials();
|
||||
elapsed += timer_monotonic();
|
||||
|
||||
if (awsCreds.IsEmpty()) {
|
||||
TraceEvent(SevWarn, "S3BlobStoreAWSCredsEmpty");
|
||||
throw backup_auth_missing();
|
||||
}
|
||||
|
||||
S3BlobStoreEndpoint::Credentials fdbCreds;
|
||||
fdbCreds.key = awsCreds.GetAWSAccessKeyId();
|
||||
fdbCreds.secret = awsCreds.GetAWSSecretKey();
|
||||
fdbCreds.securityToken = awsCreds.GetSessionToken();
|
||||
|
||||
TraceEvent("S3BlobStoreGotSdkCredentials").suppressFor(60).detail("Duration", elapsed);
|
||||
|
||||
return fdbCreds;
|
||||
#else
|
||||
TraceEvent(SevError, "S3BlobStoreNoSDK");
|
||||
throw backup_auth_missing();
|
||||
#endif
|
||||
}
|
||||
|
||||
ACTOR Future<Void> updateSecret_impl(Reference<S3BlobStoreEndpoint> b) {
|
||||
if (b->knobs.sdk_auth) {
|
||||
b->credentials = getSecretSdk();
|
||||
return Void();
|
||||
}
|
||||
std::vector<std::string>* pFiles = (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
|
||||
if (pFiles == nullptr)
|
||||
return Void();
|
||||
|
|
@ -601,7 +636,7 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
|
|||
.detail("RemoteEndpoint", conn->getPeerAddress())
|
||||
.detail("ExpiresIn", b->knobs.max_connection_life);
|
||||
|
||||
if (b->lookupKey || b->lookupSecret)
|
||||
if (b->lookupKey || b->lookupSecret || b->knobs.sdk_auth)
|
||||
wait(b->updateSecret());
|
||||
|
||||
return S3BlobStoreEndpoint::ReusableConnection({ conn, now() + b->knobs.max_connection_life });
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public:
|
|||
delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
|
||||
concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file,
|
||||
read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second,
|
||||
max_recv_bytes_per_second;
|
||||
max_recv_bytes_per_second, sdk_auth;
|
||||
bool set(StringRef name, int value);
|
||||
std::string getURLParameters() const;
|
||||
static std::vector<std::string> getKnobDescriptions() {
|
||||
|
|
@ -91,7 +91,9 @@ public:
|
|||
"read_cache_blocks_per_file (or rcb) Size of the read cache for a file in blocks.",
|
||||
"max_send_bytes_per_second (or sbps) Max send bytes per second for all requests combined.",
|
||||
"max_recv_bytes_per_second (or rbps) Max receive bytes per second for all requests combined (NOT YET "
|
||||
"USED)."
|
||||
"USED).",
|
||||
"sdk_auth (or sa) Use AWS SDK to resolve credentials. Only valid if "
|
||||
"BUILD_AWS_BACKUP is enabled."
|
||||
};
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue