Major refactor primarily to change BTree page ids from a single LogicalPageID to multiple, but also refactored write path data structures and memory lifetimes to use Refs and Arenas and carefully avoid unnecessary copying as it involved much of the same code. Pager reads can now explicitly avoid cache pollution. Refactored toString() helpers for easier debug output using common container types.

fdbserver/IPager.h

@@ -33,10 +33,15 @@
 
 #define debug_printf_noop(...)
 
-#if REDWOOD_DEBUG
-  #define debug_printf debug_printf_always
+#if defined(NO_INTELLISENSE)
+	#if REDWOOD_DEBUG
+		#define debug_printf debug_printf_always
+	#else
+		#define debug_printf debug_printf_noop
+	#endif
 #else
-#define debug_printf debug_printf_noop
+	// To get error-checking on debug_printf statements in IDE
+	#define debug_printf printf
 #endif
 
 #define BEACON fprintf(stderr, "%s: %s line %d \n", __FUNCTION__, __FILE__, __LINE__)
@@ -79,7 +84,7 @@ public:
 
 class IPagerSnapshot {
 public:
-	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID) = 0;
+	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID, bool cacheable) = 0;
 	virtual Version getVersion() const = 0;
 
 	virtual Key getMetaKey() const {
@@ -165,25 +170,28 @@ public:
 	// regardless of whether or not it was written to.
 	virtual Future<LogicalPageID> newPageID() = 0;
 
-	// Replace the contents of a page with new data.  Existing holders of a page reference for pageID
-	// will see the effects of this write.
+	// Replace the contents of a page with new data across *all* versions.
+	// Existing holders of a page reference for pageID, read from any version,
+	// may see the effects of this write.
 	virtual void updatePage(LogicalPageID pageID, Reference<IPage> data) = 0;
 
-	// Try to atomically update the contents of a page as of the next successful commit()
-	// If the pager is unable to do this at this time, it may choose to write the data to a new page,
-	// call freePage(pageID), and return the new page id.  Otherwise the pageID argument will be returned.
-	virtual Future<LogicalPageID> atomicUpdatePage(LogicalPageID pageID, Reference<IPage> data) = 0;
+	// Try to atomically update the contents of a page as of version v in the next commit.
+	// If the pager is unable to do this at this time, it may choose to write the data to a new page ID
+	// instead and return the new page ID to the caller.  Otherwise the original pageID argument will be returned.
+	// If a new page ID is returned, the old page ID will be freed as of version v
+	virtual Future<LogicalPageID> atomicUpdatePage(LogicalPageID pageID, Reference<IPage> data, Version v) = 0;
 
-	// Free pageID to be used again after version v is durable
+	// Free pageID to be used again after the commit that moves oldestVersion past v
 	virtual void freePage(LogicalPageID pageID, Version v) = 0;
 
-	// Returns the data for a page by LogicalPageID
+	// Returns the latest data (regardless of version) for a page by LogicalPageID
 	// The data returned will be the later of
-	//   - the most recent committed atomic write
+	//   - the most recent committed atomic
 	//   - the most recent non-atomic write
-	virtual Future<Reference<IPage>> readPage(LogicalPageID pageID) = 0;
+	virtual Future<Reference<IPage>> readPage(LogicalPageID pageID, bool cacheable) = 0;
 
 	// Get a snapshot of the metakey and all pages as of the version v which must be >= getOldestVersion()
+	// Note that snapshots at any version may still see the results of updatePage() calls.
 	// The snapshot shall be usable until setOldVersion() is called with a version > v.
 	virtual Reference<IPagerSnapshot> getReadSnapshot(Version v) = 0;
 

fdbserver/IndirectShadowPager.actor.cpp

@@ -108,7 +108,7 @@ IndirectShadowPagerSnapshot::IndirectShadowPagerSnapshot(IndirectShadowPager *pa
 {
 }
 
-Future<Reference<const IPage>> IndirectShadowPagerSnapshot::getPhysicalPage(LogicalPageID pageID) {
+Future<Reference<const IPage>> IndirectShadowPagerSnapshot::getPhysicalPage(LogicalPageID pageID, bool cacheable) {
 	if(pagerError.isReady())
 		pagerError.get();
 	return pager->getPage(Reference<IndirectShadowPagerSnapshot>::addRef(this), pageID, version);

fdbserver/IndirectShadowPager.h

@@ -70,7 +70,7 @@ class IndirectShadowPagerSnapshot : public IPagerSnapshot, ReferenceCounted<Indi
 public:
 	IndirectShadowPagerSnapshot(IndirectShadowPager *pager, Version version);
 
-	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID);
+	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID, bool cacheable);
 
 	virtual Version getVersion() const {
 		return version;

fdbserver/MemoryPager.actor.cpp

@@ -61,7 +61,7 @@ private:
 class MemoryPagerSnapshot : public IPagerSnapshot, ReferenceCounted<MemoryPagerSnapshot> {
 public:
 	MemoryPagerSnapshot(MemoryPager *pager, Version version) : pager(pager), version(version) {}
-	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID);
+	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID, bool cacheable);
 	virtual Version getVersion() const {
 		return version;
 	}
@@ -155,7 +155,7 @@ int MemoryPage::size() const {
 
 const int MemoryPage::PAGE_BYTES = 4096;
 
-Future<Reference<const IPage>> MemoryPagerSnapshot::getPhysicalPage(LogicalPageID pageID) {
+Future<Reference<const IPage>> MemoryPagerSnapshot::getPhysicalPage(LogicalPageID pageID, bool cacheable) {
 	return pager->getPage(pageID, version);
 }
 
@@ -367,7 +367,7 @@ ACTOR Future<Void> read(IPager *pager, LogicalPageID pageID, Version version, Ve
 	state int myRead = readNum++;
 	state Reference<IPagerSnapshot> readSnapshot = pager->getReadSnapshot(version);
 	debug_printf("Read%d\n", myRead);
-	Reference<const IPage> readPage = wait(readSnapshot->getPhysicalPage(pageID));
+	Reference<const IPage> readPage = wait(readSnapshot->getPhysicalPage(pageID, true));
 	debug_printf("FinishedRead%d\n", myRead);
 	ASSERT(validatePage(readPage, pageID, expectedVersion >= 0 ? expectedVersion : version));
 	return Void();