Finished backup container discovery / listing via base URL.

fdbclient/BackupContainer.actor.cpp

@@ -512,6 +512,32 @@ public:
 		m_path = path;
 	}
 
+	static Future<std::vector<std::string>> listURLs(std::string url) {
+		std::string path;
+		if(url.find("file://") != 0) {
+			TraceEvent(SevWarn, "BackupContainerLocalDirectory").detail("Description", "Invalid URL for BackupContainerLocalDirectory").detail("URL", url);
+		}
+
+		path = url.substr(7);
+		// Remove trailing slashes on path
+		path.erase(path.find_last_not_of("\\/") + 1);
+
+		if(!g_network->isSimulated() && path != abspath(path)) {
+			TraceEvent(SevWarn, "BackupContainerLocalDirectory").detail("Description", "Backup path must be absolute (e.g. file:///some/path)").detail("URL", url).detail("Path", path);
+			throw io_error();
+		}
+		std::vector<std::string> dirs = platform::listDirectories(path);
+		std::vector<std::string> results;
+
+		for(auto &r : dirs) {
+			if(r == "." || r == "..")
+				continue;
+			results.push_back(std::string("file://") + joinPath(path, r));
+		}
+
+		return results;
+	}
+
 	Future<Void> create() {
 		// Nothing should be done here because create() can be called by any process working with the container URL, such as fdbbackup.
 		// Since "local directory" containers are by definition local to the machine they are accessed from,
@@ -653,6 +679,15 @@ public:
 			);
 	}
 
+	ACTOR static Future<std::vector<std::string>> listURLs(Reference<BlobStoreEndpoint> bstore) {
+		BlobStoreEndpoint::ListResult contents = wait(bstore->listBucket(BACKUP_BUCKET, {}, '/'));
+		std::vector<std::string> results;
+		for(auto &f : contents.commonPrefixes) {
+			results.push_back(bstore->getResourceURL(f));
+		}
+		return results;
+	}
+
 	class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {
 	public:
 		BackupFile(std::string fileName, Reference<IAsyncFile> file) : IBackupFile(fileName), m_file(file) {}
@@ -800,6 +835,44 @@ Reference<IBackupContainer> IBackupContainer::openContainer(std::string url)
 	}
 }
 
+// Get a list of URLS to backup containers based on some a shorter URL.  This function knows about some set of supported
+// URL types which support this sort of backup discovery.
+ACTOR Future<std::vector<std::string>> IBackupContainer::listContainers(std::string baseURL) {
+	try {
+		StringRef u(baseURL);
+		if(u.startsWith(LiteralStringRef("file://"))) {
+			std::vector<std::string> results = wait(BackupContainerLocalDirectory::listURLs(baseURL));
+			return results;
+		}
+		else if(u.startsWith(LiteralStringRef("blobstore://"))) {
+			std::string resource;
+			Reference<BlobStoreEndpoint> bstore = BlobStoreEndpoint::fromString(baseURL, &resource, &IBackupContainer::lastOpenError);
+			if(!resource.empty()) {
+				TraceEvent(SevWarn, "BackupContainer").detail("Description", "Invalid backup container base URL, resource aka path should be blank.").detail("URL", baseURL);
+				throw backup_invalid_url();
+			}
+
+			std::vector<std::string> results = wait(BackupContainerBlobStore::listURLs(bstore));
+			return results;
+		}
+		else {
+			IBackupContainer::lastOpenError = "invalid URL prefix";
+			throw backup_invalid_url();
+		}
+
+	} catch(Error &e) {
+		if(e.code() == error_code_actor_cancelled)
+			throw;
+
+		TraceEvent m(SevWarn, "BackupContainer");
+		m.detail("Description", "Invalid backup container URL prefix.  See help.").detail("URL", baseURL);
+
+		if(e.code() == error_code_backup_invalid_url)
+			m.detail("lastOpenError", IBackupContainer::lastOpenError);
+		throw;
+	}
+}
+
 ACTOR Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c, Reference<IBackupFile> f) {
 	state Standalone<StringRef> content = makeString(g_random->randomInt(0, 10000000));
 	for(int i = 0; i < content.size(); ++i)
@@ -820,6 +893,8 @@ ACTOR Future<Void> testBackupContainer(std::string url) {
 	printf("BackupContainerTest URL %s\n", url.c_str());
 
 	state Reference<IBackupContainer> c = IBackupContainer::openContainer(url);
+	// Make sure container doesn't exist, then create it.
+	Void _ = wait(c->deleteContainer());
 	Void _ = wait(c->create());
 
 	state Reference<IBackupFile> log1 = wait(c->writeLogFile(100, 150, 10));
@@ -914,3 +989,14 @@ TEST_CASE("backup/containers/url") {
 	}
 	return Void();
 };
+
+TEST_CASE("backup/containers/list") {
+	state const char *url = getenv("FDB_TEST_BACKUP_URL");
+	ASSERT(url != nullptr);
+	printf("Listing %s\n", url);
+	std::vector<std::string> urls = wait(IBackupContainer::listContainers(url));
+	for(auto &u : urls) {
+		printf("%s\n", u.c_str());
+	}
+	return Void();
+};

fdbclient/BackupContainer.h

@@ -169,7 +169,7 @@ public:
 	// Get an IBackupContainer based on a container spec string
 	static Reference<IBackupContainer> openContainer(std::string url);
 	static std::vector<std::string> getURLFormats();
-	static std::vector<std::string> listContainers(std::string baseURL);
+	static Future<std::vector<std::string>> listContainers(std::string const &baseURL);
 
 	std::string getURL() const {
 		return URL;

tests/BackupContainers.txt

@@ -3,5 +3,5 @@ testName=UnitTests
 startDelay=0
 useDB=false
 maxTestCases=0
-testsMatching=backup/containers/
+testsMatching=backup/containers/url