Merge pull request #1209 from satherton/master-merge-release-6.0

Merge release 6.0 into master

fdbclient/BackupContainer.actor.cpp

@@ -1180,7 +1180,31 @@ public:
 		// can't be used because backup files are read-only.  Cached mode can only help during restore task retries handled
 		// by the same process that failed the first task execution anyway, which is a very rare case.
 		#endif
-		return IAsyncFileSystem::filesystem()->open(fullPath, flags, 0644);
+		Future<Reference<IAsyncFile>> f = IAsyncFileSystem::filesystem()->open(fullPath, flags, 0644);
+
+		if(g_network->isSimulated()) {
+			int blockSize = 0;
+			// Extract block size from the filename, if present
+			size_t lastComma = path.find_last_of(',');
+			if(lastComma != path.npos) {
+				blockSize = atoi(path.substr(lastComma + 1).c_str());
+			}
+			if(blockSize <= 0) {
+				blockSize = g_random->randomInt(1e4, 1e6);
+			}
+			if(g_random->random01() < .01) {
+				blockSize /= g_random->randomInt(1, 3);
+			}
+
+			return map(f, [=](Reference<IAsyncFile> fr) {
+				int readAhead = g_random->randomInt(0, 3);
+				int reads = g_random->randomInt(1, 3);
+				int cacheSize = g_random->randomInt(0, 3);
+				return Reference<IAsyncFile>(new AsyncFileReadAheadCache(fr, blockSize, readAhead, reads, cacheSize));
+			});
+		}
+
+		return f;
 	}
 
 	class BackupFile : public IBackupFile, ReferenceCounted<BackupFile> {

fdbclient/FileBackupAgent.actor.cpp

@@ -400,10 +400,16 @@ FileBackupAgent::FileBackupAgent()
 
 namespace fileBackup {
 
-	// Padding bytes for backup files.  The largest padded area that could ever have to be written is
-	// the size of two 32 bit ints and the largest key size and largest value size.  Since CLIENT_KNOBS
-	// may not be initialized yet a conservative constant is being used.
-	std::string paddingFFs(128 * 1024, 0xFF);
+	// Return a block of contiguous padding bytes, growing if needed.
+	Value makePadding(int size) {
+		static Value pad;
+		if(pad.size() < size) {
+			pad = makeString(size);
+			memset(mutateString(pad), '\xff', pad.size());
+		}
+
+		return pad.substr(0, size);
+	}
 
 	// File Format handlers.
 	// Both Range and Log formats are designed to be readable starting at any 1MB boundary
@@ -441,11 +447,18 @@ namespace fileBackup {
 		RangeFileWriter(Reference<IBackupFile> file = Reference<IBackupFile>(), int blockSize = 0) : file(file), blockSize(blockSize), blockEnd(0), fileVersion(1001) {}
 
 		// Handles the first block and internal blocks.  Ends current block if needed.
-		ACTOR static Future<Void> newBlock(RangeFileWriter *self, int bytesNeeded) {
+		// The final flag is used in simulation to pad the file's final block to a whole block size
+		ACTOR static Future<Void> newBlock(RangeFileWriter *self, int bytesNeeded, bool final = false) {
 			// Write padding to finish current block if needed
 			int bytesLeft = self->blockEnd - self->file->size();
 			if(bytesLeft > 0) {
-				wait(self->file->append((uint8_t *)paddingFFs.data(), bytesLeft));
+				state Value paddingFFs = makePadding(bytesLeft);
+				wait(self->file->append(paddingFFs.begin(), bytesLeft));
+			}
+
+			if(final) {
+				ASSERT(g_network->isSimulated());
+				return Void();
 			}
 
 			// Set new blockEnd
@@ -468,6 +481,15 @@ namespace fileBackup {
 			return Void();
 		}
 
+		// Used in simulation only to create backup file sizes which are an integer multiple of the block size
+		Future<Void> padEnd() {
+			ASSERT(g_network->isSimulated());
+			if(file->size() > 0) {
+				return newBlock(this, 0, true);
+			}
+			return Void();
+		}
+
 		// Ends the current block if necessary based on bytesNeeded.
 		Future<Void> newBlockIfNeeded(int bytesNeeded) {
 			if(file->size() + bytesNeeded > blockEnd)
@@ -620,7 +642,8 @@ namespace fileBackup {
 				// Write padding if needed
 				int bytesLeft = self->blockEnd - self->file->size();
 				if(bytesLeft > 0) {
-					wait(self->file->append((uint8_t *)paddingFFs.data(), bytesLeft));
+					state Value paddingFFs = makePadding(bytesLeft);
+					wait(self->file->append(paddingFFs.begin(), bytesLeft));
 				}
 
 				// Set new blockEnd
@@ -1108,6 +1131,10 @@ namespace fileBackup {
 						state Key nextKey = done ? endKey : keyAfter(lastKey);
 						wait(rangeFile.writeKey(nextKey));
 
+						if(BUGGIFY) {
+							rangeFile.padEnd();
+						}
+
 						bool usedFile = wait(finishRangeFile(outFile, cx, task, taskBucket, KeyRangeRef(beginKey, nextKey), outVersion));
 						TraceEvent("FileBackupWroteRangeFile")
 							.suppressFor(60)

fdbrpc/AsyncFileReadAhead.actor.h

@@ -67,14 +67,18 @@ public:
 		if(offset >= fileSize)
 			return 0;  // TODO:  Should this throw since the input isn't really valid?
 
+		if(length == 0) {
+			return 0;
+		}
+
 		// If reading past the end then clip length to just read to the end
 		if(offset + length > fileSize)
 			length = fileSize - offset;   // Length is at least 1 since offset < fileSize
 
 		// Calculate block range for the blocks that contain this data
 		state int firstBlockNum = offset / f->m_block_size;
-		state int lastBlockNum = (offset + length) / f->m_block_size;
-		state int blockNum;
+		ASSERT(f->m_block_size > 0);
+		state int lastBlockNum = (offset + length - 1) / f->m_block_size;
 
 		// Start reads (if needed) of the block range required for this read, plus the read ahead blocks
 		// The futures for the read started will be stored in the cache but since things can be evicted from
@@ -84,8 +88,10 @@ public:
 
 		// Start blocks up to the read ahead size beyond the last needed block but don't go past the end of the file
 		state int lastBlockNumInFile = ((fileSize + f->m_block_size - 1) / f->m_block_size) - 1;
+		ASSERT(lastBlockNum <= lastBlockNumInFile);
 		int lastBlockToStart = std::min<int>(lastBlockNum + f->m_read_ahead_blocks, lastBlockNumInFile);
 
+		state int blockNum;
 		for(blockNum = firstBlockNum; blockNum <= lastBlockToStart; ++blockNum) {
 			Future<Reference<CacheBlock>> fblock;