wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[fdbbackup] make credentials optional 

when blob storage is configured on a private network (such as an AWS
VPC), there isn't a need to specify http auth
This commit is contained in:
Kevin King 2020-09-27 17:58:07 -07:00 committed by sfc-gh-tclinkenbeard
parent ccfc48d89a
commit e3114a4656
2 changed files with 61 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
fdbclient/S3BlobStore.actor.cpp
View File

@ -164,7 +164,11 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(std::string const
StringRef prefix = t.eat("://"); StringRef prefix = t.eat("://");
if (prefix != LiteralStringRef("blobstore")) if (prefix != LiteralStringRef("blobstore"))
throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str()); throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str());
StringRef cred = t.eat("@");
Optional<StringRef> cred;
if (url.find("@") != std::string::npos) {
cred = t.eat("@");
}
uint8_t foundSeparator = 0; uint8_t foundSeparator = 0;
StringRef hostPort = t.eatAny("/?", &foundSeparator); StringRef hostPort = t.eatAny("/?", &foundSeparator);
StringRef resource; StringRef resource;
@ -238,12 +242,16 @@ Reference<S3BlobStoreEndpoint> S3BlobStoreEndpoint::fromString(std::string const
if (resourceFromURL != nullptr) if (resourceFromURL != nullptr)
*resourceFromURL = resource.toString(); *resourceFromURL = resource.toString();
StringRef c(cred); Optional<S3BlobStoreEndpoint::Credentials> creds;
if (cred.present()) {
StringRef c(cred.get());
StringRef key = c.eat(":"); StringRef key = c.eat(":");
StringRef secret = c.eat(); StringRef secret = c.eat();
creds = S3BlobStoreEndpoint::Credentials{key.toString(), secret.toString()};
}
return makeReference<S3BlobStoreEndpoint>( return makeReference<S3BlobStoreEndpoint>(
host.toString(), service.toString(), key.toString(), secret.toString(), knobs, extraHeaders); host.toString(), service.toString(), creds, knobs, extraHeaders);
} catch (std::string& err) { } catch (std::string& err) {
if (error != nullptr) if (error != nullptr)
@ -264,12 +272,17 @@ std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::strin
hostPort.append(service); hostPort.append(service);
} }
// If secret isn't being looked up from credentials files then it was passed explicitly in th URL so show it here. // If secret isn't being looked up from credentials files then it was passed explicitly in the URL so show it here.
std::string s; std::string credsString;
if (!lookupSecret) if (credentials.present()) {
s = std::string(":") + secret; credsString = credentials.get().key;
if (!lookupSecret) {
credsString +=":" + credentials.get().secret;
}
credsString += "@";
}
std::string r = format("blobstore://%s%s@%s/%s", key.c_str(), s.c_str(), hostPort.c_str(), resource.c_str()); std::string r = format("blobstore://%s%s/%s", credsString.c_str(), hostPort.c_str(), resource.c_str());
// Get params that are deviations from knob defaults // Get params that are deviations from knob defaults
std::string knobParams = knobs.getURLParameters(); std::string knobParams = knobs.getURLParameters();
@ -484,13 +497,18 @@ ACTOR Future<Void> updateSecret_impl(Reference<S3BlobStoreEndpoint> b) {
if (pFiles == nullptr) if (pFiles == nullptr)
return Void(); return Void();
if (!b->credentials.present()) {
return Void();
}
state std::vector<Future<Optional<json_spirit::mObject>>> reads; state std::vector<Future<Optional<json_spirit::mObject>>> reads;
for (auto& f : *pFiles) for (auto& f : *pFiles)
reads.push_back(tryReadJSONFile(f)); reads.push_back(tryReadJSONFile(f));
wait(waitForAll(reads)); wait(waitForAll(reads));
std::string key = b->key + "@" + b->host; std::string accessKey = b->credentials.get().key;
std::string credentialsFileKey = accessKey + "@" + b->host;
int invalid = 0; int invalid = 0;
@ -504,12 +522,12 @@ ACTOR Future<Void> updateSecret_impl(Reference<S3BlobStoreEndpoint> b) {
JSONDoc doc(f.get().get()); JSONDoc doc(f.get().get());
if (doc.has("accounts") && doc.last().type() == json_spirit::obj_type) { if (doc.has("accounts") && doc.last().type() == json_spirit::obj_type) {
JSONDoc accounts(doc.last().get_obj()); JSONDoc accounts(doc.last().get_obj());
if (accounts.has(key, false) && accounts.last().type() == json_spirit::obj_type) { if (accounts.has(credentialsFileKey, false) && accounts.last().type() == json_spirit::obj_type) {
JSONDoc account(accounts.last()); JSONDoc account(accounts.last());
std::string secret; std::string secret;
// Once we find a matching account, use it. // Once we find a matching account, use it.
if (account.tryGet("secret", secret)) { if (account.tryGet("secret", secret)) {
b->secret = secret; b->credentials = S3BlobStoreEndpoint::Credentials{accessKey, secret};
return Void(); return Void();
} }
} }
@ -643,7 +661,9 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<S3BlobStoreEndp
// Finish/update the request headers (which includes Date header) // Finish/update the request headers (which includes Date header)
// This must be done AFTER the connection is ready because if credentials are coming from disk they are // This must be done AFTER the connection is ready because if credentials are coming from disk they are
// refreshed when a new connection is established and setAuthHeaders() would need the updated secret. // refreshed when a new connection is established and setAuthHeaders() would need the updated secret.
bstore->setAuthHeaders(verb, resource, headers); if (bstore->credentials.present()) {
bstore->setAuthHeaders(bstore->credentials.get(), verb, resource, headers);
}
remoteAddress = rconn.conn->getPeerAddress(); remoteAddress = rconn.conn->getPeerAddress();
wait(bstore->requestRate->getAllowance(1)); wait(bstore->requestRate->getAllowance(1));
Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn, Reference<HTTP::Response> _r = wait(timeoutError(HTTP::doRequest(rconn.conn,
@ -1041,8 +1061,8 @@ Future<std::vector<std::string>> S3BlobStoreEndpoint::listBuckets() {
return listBuckets_impl(Reference<S3BlobStoreEndpoint>::addRef(this)); return listBuckets_impl(Reference<S3BlobStoreEndpoint>::addRef(this));
} }
std::string S3BlobStoreEndpoint::hmac_sha1(std::string const& msg) { std::string S3BlobStoreEndpoint::hmac_sha1(Credentials const &creds, std::string const& msg) {
std::string key = secret; std::string key = creds.secret;
// Hash key to shorten it if it is longer than SHA1 block size // Hash key to shorten it if it is longer than SHA1 block size
if (key.size() > 64) { if (key.size() > 64) {
@ -1066,7 +1086,7 @@ std::string S3BlobStoreEndpoint::hmac_sha1(std::string const& msg) {
return SHA1::from_string(kopad); return SHA1::from_string(kopad);
} }
void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) { void S3BlobStoreEndpoint::setAuthHeaders(Credentials const &creds, std::string const& verb, std::string const& resource, HTTP::Headers& headers) {
std::string& date = headers["Date"]; std::string& date = headers["Date"];
char dateBuf[20]; char dateBuf[20];
@ -1106,11 +1126,11 @@ void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string co
msg.resize(msg.size() - (resource.size() - q)); msg.resize(msg.size() - (resource.size() - q));
} }
std::string sig = base64::encoder::from_string(hmac_sha1(msg)); std::string sig = base64::encoder::from_string(hmac_sha1(creds, msg));
// base64 encoded blocks end in \n so remove it. // base64 encoded blocks end in \n so remove it.
sig.resize(sig.size() - 1); sig.resize(sig.size() - 1);
std::string auth = "AWS "; std::string auth = "AWS ";
auth.append(key); auth.append(creds.key);
auth.append(":"); auth.append(":");
auth.append(sig); auth.append(sig);
headers["Authorization"] = auth; headers["Authorization"] = auth;

34
fdbclient/S3BlobStore.h
View File

@ -46,6 +46,11 @@ public:
static Stats s_stats; static Stats s_stats;
struct Credentials {
std::string key;
std::string secret;
};
struct BlobKnobs { struct BlobKnobs {
BlobKnobs(); BlobKnobs();
int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min, int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min,
@ -90,23 +95,20 @@ public:
} }
}; };
S3BlobStoreEndpoint(std::string const& host, S3BlobStoreEndpoint(std::string const &host, std::string service, Optional<Credentials> const& creds, BlobKnobs const &knobs = BlobKnobs(), HTTP::Headers extraHeaders = HTTP::Headers())
std::string service, : host(host), service(service), credentials(creds), lookupSecret(creds.present() && creds.get().secret.empty()), knobs(knobs), extraHeaders(extraHeaders),
std::string const& key, requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
std::string const& secret,
BlobKnobs const& knobs = BlobKnobs(),
HTTP::Headers extraHeaders = HTTP::Headers())
: host(host), service(service), key(key), secret(secret), lookupSecret(secret.empty()), knobs(knobs),
extraHeaders(extraHeaders), requestRate(new SpeedLimit(knobs.requests_per_second, 1)),
requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)), requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)),
requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)), requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)),
requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)), requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)),
requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)), requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)),
sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)), sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests), recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)),
concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) { concurrentRequests(knobs.concurrent_requests),
concurrentUploads(knobs.concurrent_uploads),
concurrentLists(knobs.concurrent_lists) {
if (host.empty()) if(host.empty())
throw connection_string_invalid(); throw connection_string_invalid();
} }
@ -114,8 +116,7 @@ public:
const char* resource = ""; const char* resource = "";
if (withResource) if (withResource)
resource = "<name>"; resource = "<name>";
return format("blobstore://<api_key>:<secret>@<host>[:<port>]/%s[?<param>=<value>[&<param>=<value>]...]", return format("blobstore://[<api_key>:<secret>]@<host>[:<port>]/%s[?<param>=<value>[&<param>=<value>]...]", resource);
resource);
} }
typedef std::map<std::string, std::string> ParametersT; typedef std::map<std::string, std::string> ParametersT;
@ -142,8 +143,7 @@ public:
std::string host; std::string host;
std::string service; std::string service;
std::string key; Optional<Credentials> credentials;
std::string secret;
bool lookupSecret; bool lookupSecret;
BlobKnobs knobs; BlobKnobs knobs;
HTTP::Headers extraHeaders; HTTP::Headers extraHeaders;
@ -163,10 +163,10 @@ public:
Future<Void> updateSecret(); Future<Void> updateSecret();
// Calculates the authentication string from the secret key // Calculates the authentication string from the secret key
std::string hmac_sha1(std::string const& msg); std::string hmac_sha1(Credentials const &creds, std::string const &msg);
// Sets headers needed for Authorization (including Date which will be overwritten if present) // Sets headers needed for Authorization (including Date which will be overwritten if present)
void setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers); void setAuthHeaders(Credentials const &creds, std::string const &verb, std::string const &resource, HTTP::Headers &headers);
// Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain // Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain
static PacketBuffer* writeRequestHeader(std::string const& request, static PacketBuffer* writeRequestHeader(std::string const& request,