From e3114a465672f50ae038eb297231542bd0449535 Mon Sep 17 00:00:00 2001 From: Kevin King <4kevinking@gmail.com> Date: Sun, 27 Sep 2020 17:58:07 -0700 Subject: [PATCH] [fdbbackup] make credentials optional when blob storage is configured on a private network (such as an AWS VPC), there isn't a need to specify http auth --- fdbclient/S3BlobStore.actor.cpp | 58 ++++++++++++++++++++++----------- fdbclient/S3BlobStore.h | 44 ++++++++++++------------- 2 files changed, 61 insertions(+), 41 deletions(-) diff --git a/fdbclient/S3BlobStore.actor.cpp b/fdbclient/S3BlobStore.actor.cpp index 2e328698cd..4a9c375be6 100644 --- a/fdbclient/S3BlobStore.actor.cpp +++ b/fdbclient/S3BlobStore.actor.cpp @@ -164,7 +164,11 @@ Reference S3BlobStoreEndpoint::fromString(std::string const StringRef prefix = t.eat("://"); if (prefix != LiteralStringRef("blobstore")) throw format("Invalid blobstore URL prefix '%s'", prefix.toString().c_str()); - StringRef cred = t.eat("@"); + + Optional cred; + if (url.find("@") != std::string::npos) { + cred = t.eat("@"); + } uint8_t foundSeparator = 0; StringRef hostPort = t.eatAny("/?", &foundSeparator); StringRef resource; @@ -238,12 +242,16 @@ Reference S3BlobStoreEndpoint::fromString(std::string const if (resourceFromURL != nullptr) *resourceFromURL = resource.toString(); - StringRef c(cred); - StringRef key = c.eat(":"); - StringRef secret = c.eat(); + Optional creds; + if (cred.present()) { + StringRef c(cred.get()); + StringRef key = c.eat(":"); + StringRef secret = c.eat(); + creds = S3BlobStoreEndpoint::Credentials{key.toString(), secret.toString()}; + } return makeReference( - host.toString(), service.toString(), key.toString(), secret.toString(), knobs, extraHeaders); + host.toString(), service.toString(), creds, knobs, extraHeaders); } catch (std::string& err) { if (error != nullptr) @@ -264,12 +272,17 @@ std::string S3BlobStoreEndpoint::getResourceURL(std::string resource, std::strin hostPort.append(service); } - // If secret isn't being looked up from credentials files then it was passed explicitly in th URL so show it here. - std::string s; - if (!lookupSecret) - s = std::string(":") + secret; + // If secret isn't being looked up from credentials files then it was passed explicitly in the URL so show it here. + std::string credsString; + if (credentials.present()) { + credsString = credentials.get().key; + if (!lookupSecret) { + credsString +=":" + credentials.get().secret; + } + credsString += "@"; + } - std::string r = format("blobstore://%s%s@%s/%s", key.c_str(), s.c_str(), hostPort.c_str(), resource.c_str()); + std::string r = format("blobstore://%s%s/%s", credsString.c_str(), hostPort.c_str(), resource.c_str()); // Get params that are deviations from knob defaults std::string knobParams = knobs.getURLParameters(); @@ -484,13 +497,18 @@ ACTOR Future updateSecret_impl(Reference b) { if (pFiles == nullptr) return Void(); + if (!b->credentials.present()) { + return Void(); + } + state std::vector>> reads; for (auto& f : *pFiles) reads.push_back(tryReadJSONFile(f)); wait(waitForAll(reads)); - std::string key = b->key + "@" + b->host; + std::string accessKey = b->credentials.get().key; + std::string credentialsFileKey = accessKey + "@" + b->host; int invalid = 0; @@ -504,12 +522,12 @@ ACTOR Future updateSecret_impl(Reference b) { JSONDoc doc(f.get().get()); if (doc.has("accounts") && doc.last().type() == json_spirit::obj_type) { JSONDoc accounts(doc.last().get_obj()); - if (accounts.has(key, false) && accounts.last().type() == json_spirit::obj_type) { + if (accounts.has(credentialsFileKey, false) && accounts.last().type() == json_spirit::obj_type) { JSONDoc account(accounts.last()); std::string secret; // Once we find a matching account, use it. if (account.tryGet("secret", secret)) { - b->secret = secret; + b->credentials = S3BlobStoreEndpoint::Credentials{accessKey, secret}; return Void(); } } @@ -643,7 +661,9 @@ ACTOR Future> doRequest_impl(ReferencesetAuthHeaders(verb, resource, headers); + if (bstore->credentials.present()) { + bstore->setAuthHeaders(bstore->credentials.get(), verb, resource, headers); + } remoteAddress = rconn.conn->getPeerAddress(); wait(bstore->requestRate->getAllowance(1)); Reference _r = wait(timeoutError(HTTP::doRequest(rconn.conn, @@ -1041,8 +1061,8 @@ Future> S3BlobStoreEndpoint::listBuckets() { return listBuckets_impl(Reference::addRef(this)); } -std::string S3BlobStoreEndpoint::hmac_sha1(std::string const& msg) { - std::string key = secret; +std::string S3BlobStoreEndpoint::hmac_sha1(Credentials const &creds, std::string const& msg) { + std::string key = creds.secret; // Hash key to shorten it if it is longer than SHA1 block size if (key.size() > 64) { @@ -1066,7 +1086,7 @@ std::string S3BlobStoreEndpoint::hmac_sha1(std::string const& msg) { return SHA1::from_string(kopad); } -void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers) { +void S3BlobStoreEndpoint::setAuthHeaders(Credentials const &creds, std::string const& verb, std::string const& resource, HTTP::Headers& headers) { std::string& date = headers["Date"]; char dateBuf[20]; @@ -1106,11 +1126,11 @@ void S3BlobStoreEndpoint::setAuthHeaders(std::string const& verb, std::string co msg.resize(msg.size() - (resource.size() - q)); } - std::string sig = base64::encoder::from_string(hmac_sha1(msg)); + std::string sig = base64::encoder::from_string(hmac_sha1(creds, msg)); // base64 encoded blocks end in \n so remove it. sig.resize(sig.size() - 1); std::string auth = "AWS "; - auth.append(key); + auth.append(creds.key); auth.append(":"); auth.append(sig); headers["Authorization"] = auth; diff --git a/fdbclient/S3BlobStore.h b/fdbclient/S3BlobStore.h index 87ee25a620..02cf47d13b 100644 --- a/fdbclient/S3BlobStore.h +++ b/fdbclient/S3BlobStore.h @@ -46,6 +46,11 @@ public: static Stats s_stats; + struct Credentials { + std::string key; + std::string secret; + }; + struct BlobKnobs { BlobKnobs(); int secure_connection, connect_tries, connect_timeout, max_connection_life, request_tries, request_timeout_min, @@ -90,23 +95,20 @@ public: } }; - S3BlobStoreEndpoint(std::string const& host, - std::string service, - std::string const& key, - std::string const& secret, - BlobKnobs const& knobs = BlobKnobs(), - HTTP::Headers extraHeaders = HTTP::Headers()) - : host(host), service(service), key(key), secret(secret), lookupSecret(secret.empty()), knobs(knobs), - extraHeaders(extraHeaders), requestRate(new SpeedLimit(knobs.requests_per_second, 1)), - requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)), - requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)), - requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)), - requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)), - sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)), - recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), concurrentRequests(knobs.concurrent_requests), - concurrentUploads(knobs.concurrent_uploads), concurrentLists(knobs.concurrent_lists) { + S3BlobStoreEndpoint(std::string const &host, std::string service, Optional const& creds, BlobKnobs const &knobs = BlobKnobs(), HTTP::Headers extraHeaders = HTTP::Headers()) + : host(host), service(service), credentials(creds), lookupSecret(creds.present() && creds.get().secret.empty()), knobs(knobs), extraHeaders(extraHeaders), + requestRate(new SpeedLimit(knobs.requests_per_second, 1)), + requestRateList(new SpeedLimit(knobs.list_requests_per_second, 1)), + requestRateWrite(new SpeedLimit(knobs.write_requests_per_second, 1)), + requestRateRead(new SpeedLimit(knobs.read_requests_per_second, 1)), + requestRateDelete(new SpeedLimit(knobs.delete_requests_per_second, 1)), + sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)), + recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)), + concurrentRequests(knobs.concurrent_requests), + concurrentUploads(knobs.concurrent_uploads), + concurrentLists(knobs.concurrent_lists) { - if (host.empty()) + if(host.empty()) throw connection_string_invalid(); } @@ -114,8 +116,7 @@ public: const char* resource = ""; if (withResource) resource = ""; - return format("blobstore://:@[:]/%s[?=[&=]...]", - resource); + return format("blobstore://[:]@[:]/%s[?=[&=]...]", resource); } typedef std::map ParametersT; @@ -142,8 +143,7 @@ public: std::string host; std::string service; - std::string key; - std::string secret; + Optional credentials; bool lookupSecret; BlobKnobs knobs; HTTP::Headers extraHeaders; @@ -163,10 +163,10 @@ public: Future updateSecret(); // Calculates the authentication string from the secret key - std::string hmac_sha1(std::string const& msg); + std::string hmac_sha1(Credentials const &creds, std::string const &msg); // Sets headers needed for Authorization (including Date which will be overwritten if present) - void setAuthHeaders(std::string const& verb, std::string const& resource, HTTP::Headers& headers); + void setAuthHeaders(Credentials const &creds, std::string const &verb, std::string const &resource, HTTP::Headers &headers); // Prepend the HTTP request header to the given PacketBuffer, returning the new head of the buffer chain static PacketBuffer* writeRequestHeader(std::string const& request,