wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug fix, in recursive blob folder listings the recent removal of common prefixes from the result stream caused the list marker to not be set correctly when a folder level requires multiple requests due to folder size.

This commit is contained in:
Stephen Atherton 2018-01-06 20:58:48 -08:00
parent 96cb06cbc7
commit 0e7d538c94
1 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
fdbrpc/BlobStore.actor.cpp
View File

@ -534,7 +534,9 @@ ACTOR Future<Void> listBucketStream_impl(Reference<BlobStoreEndpoint> bstore, st
state FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
HTTP::Headers headers;
Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource + HTTP::urlEncode(lastFile), headers, NULL, 0, {200}));
state std::string fullResource = resource + HTTP::urlEncode(lastFile);
lastFile.clear();
Reference<HTTP::Response> r = wait(bstore->doRequest("GET", fullResource, headers, NULL, 0, {200}));
listReleaser.release();
try {
@ -590,23 +592,32 @@ ACTOR Future<Void> listBucketStream_impl(Reference<BlobStoreEndpoint> bstore, st
std::string p;
objectDoc.get("Prefix", p);
// If recursing, queue a sub-request, otherwise add the common prefix to the result.
if(maxDepth > 0)
if(maxDepth > 0) {
subLists.push_back(bstore->listBucketStream(bucket, results, p, delimiter, maxDepth - 1));
if(more)
lastFile = std::move(p);
}
else
result.commonPrefixes.push_back(p);
result.commonPrefixes.push_back(std::move(p));
}
}
results.send(result);
if(more) {
if(!result.objects.empty())
// lastFile will be the last commonprefix for which a sublist was started, if any
if(!result.objects.empty() && lastFile < result.objects.back().name)
lastFile = result.objects.back().name;
if(!result.commonPrefixes.empty() && lastFile < result.commonPrefixes.back())
lastFile = result.commonPrefixes.back();
}
results.send(result);
if(lastFile.empty()) {
TraceEvent(SevWarn, "BlobStoreEndpointListNoNextMarker").detail("Resource", fullResource).suppressFor(60, true);
throw backup_error();
}
}
} catch(Error &e) {
TraceEvent(SevWarn, "BlobStoreEndpointListResultParseError").detail("Resource", resource + HTTP::urlEncode(lastFile)).suppressFor(60, true);
TraceEvent(SevWarn, "BlobStoreEndpointListResultParseError").detail("Resource", fullResource).error(e).suppressFor(60, true);
throw http_bad_response();
}
}