Correctness test now has additional random reader which doesn't do verification but isn't stopped when the btree is closed. Fixed bug exposed by this where pager snapshots will still try to read pages after the pager has been shut down or even destroyed. Added new error type, shutdown_in_progress.

fdbserver/IndirectShadowPager.actor.cpp

@@ -96,8 +96,15 @@ int IndirectShadowPage::size() const {
 const int IndirectShadowPage::PAGE_BYTES = 4096;
 const int IndirectShadowPage::PAGE_OVERHEAD_BYTES = sizeof(SumType);
 
+IndirectShadowPagerSnapshot::IndirectShadowPagerSnapshot(IndirectShadowPager *pager, Version version)
+	: pager(pager), version(version), pagerError(pager->getError())
+{
+}
+
 Future<Reference<const IPage>> IndirectShadowPagerSnapshot::getPhysicalPage(LogicalPageID pageID) {
-	return pager->getPage(Reference<IndirectShadowPagerSnapshot>::addRef(this), pageID, version);
+	if(pagerError.isError())
+		throw pagerError.getError();
+	return waitOrError(pager->getPage(Reference<IndirectShadowPagerSnapshot>::addRef(this), pageID, version), pagerError);
 }
 
 template <class T>
@@ -557,8 +564,9 @@ Future<Void> IndirectShadowPager::onClosed() {
 }
 
 ACTOR void shutdown(IndirectShadowPager *pager, bool dispose) {
-	pager->recovery = Never(); // TODO: this is a hacky way to prevent users from performing operations after calling shutdown. Implement a better mechanism 
 
+	if(pager->errorPromise.canBeSet())
+		pager->errorPromise.sendError(shutdown_in_progress());
 	wait(pager->writeActors.signal());
 	wait(pager->operations.signal());
 	wait(pager->committing);

fdbserver/IndirectShadowPager.h

@@ -68,7 +68,7 @@ private:
 
 class IndirectShadowPagerSnapshot : public IPagerSnapshot, ReferenceCounted<IndirectShadowPagerSnapshot> {
 public:
-	IndirectShadowPagerSnapshot(IndirectShadowPager *pager, Version version) : pager(pager), version(version) {}
+	IndirectShadowPagerSnapshot(IndirectShadowPager *pager, Version version);
 
 	virtual Future<Reference<const IPage>> getPhysicalPage(LogicalPageID pageID);
 
@@ -90,6 +90,7 @@ public:
 private:
 	IndirectShadowPager *pager;
 	Version version;
+	Future<Void> pagerError;
 };
 
 class PagerFile {

fdbserver/VersionedBTree.actor.cpp

@@ -1937,7 +1937,7 @@ int randomSize(int max) {
 
 KeyValue randomKV(int keySize = 10, int valueSize = 5) {
 	int kLen = randomSize(1 + keySize);
-	int vLen = randomSize(valueSize);
+	int vLen = valueSize > 0 ? randomSize(valueSize) : 0;
 	KeyValue kv;
 	kv.key = makeString(kLen, kv.arena());
 	kv.value = makeString(vLen, kv.arena());
@@ -2143,7 +2143,24 @@ ACTOR Future<Void> verify(VersionedBTree *btree, FutureStream<Version> vStream,
 	return Void();
 }
 
-static void nullWaitHandler( const boost::system::error_code& ) {}
+// Does a random range read, doesn't trap/report errors
+ACTOR Future<Void> randomReader(VersionedBTree *btree) {
+	state Reference<IStoreCursor> cur;
+	loop {
+		wait(yield());
+		if(!cur || g_random->random01() > .1) {
+			Version v = g_random->randomInt(1, btree->getLastCommittedVersion() + 1);
+			cur = btree->readAtVersion(v);
+		}
+
+		wait(cur->findFirstEqualOrGreater(randomKV(10, 0).key, true, 0));
+		state int c = g_random->randomInt(0, 100);
+		while(cur->isValid() && c-- > 0) {
+			wait(success(cur->next(true)));
+			wait(yield());
+		}
+	}
+}
 
 TEST_CASE("/redwood/correctness") {
 	state bool useDisk = true;  // MemoryPager is not being maintained currently.
@@ -2188,6 +2205,7 @@ TEST_CASE("/redwood/correctness") {
 
 	state PromiseStream<Version> committedVersions;
 	state Future<Void> verifyTask = verify(btree, committedVersions.getFuture(), &written, &errorCount);
+	state Future<Void> randomTask = randomReader(btree);
 
 	state Future<Void> commit = Void();
 
@@ -2297,7 +2315,7 @@ TEST_CASE("/redwood/correctness") {
 				pager->close();
 				wait(closedFuture);
 
-				debug_printf_always("Opening btree\n");
+				debug_printf_always("Reopening btree\n");
 				pager = new IndirectShadowPager(pagerFile);
 				btree = new VersionedBTree(pager, pagerFile, pageSize);
 				wait(btree->init());
@@ -2309,6 +2327,7 @@ TEST_CASE("/redwood/correctness") {
 				// Create new promise stream and start the verifier again
 				committedVersions = PromiseStream<Version>();
 				verifyTask = verify(btree, committedVersions.getFuture(), &written, &errorCount);
+				randomTask = randomReader(btree);
 			}
 
 			// Check for errors

flow/error_definitions.h

@@ -63,6 +63,7 @@ ERROR( cluster_version_changed, 1039, "The protocol version of the cluster has c
 ERROR( external_client_already_loaded, 1040, "External client has already been loaded" )
 ERROR( lookup_failed, 1041, "DNS lookup failed" )
 ERROR( proxy_memory_limit_exceeded, 1042, "Proxy commit memory limit exceeded")
+ERROR( shutdown_in_progress, 1043, "Operation no longer supported due to shutdown")
 
 ERROR( broken_promise, 1100, "Broken promise" )
 ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )