Merge pull request #3374 from ajbeamon/merge-release-6.2-into-release-6.3

Merge release 6.2 into release 6.3

documentation/sphinx/source/mr-status-json-schemas.rst.inc

@@ -514,6 +514,7 @@
       "data_distribution_disabled_for_ss_failures":true,
       "data_distribution_disabled_for_rebalance":true,
       "data_distribution_disabled":true,
+      "active_primary_dc":"pv",
       "configuration":{
          "log_anti_quorum":0,
          "log_replicas":2,

documentation/sphinx/source/old-release-notes/release-notes-620.rst

@@ -2,6 +2,14 @@
 Release Notes
 #############
 
+6.2.23
+======
+
+Status
+------
+
+* Added ``cluster.active_primary_dc`` that indicates which datacenter is serving as the primary datacenter in multi-region setups. `(PR #3320) <https://github.com/apple/foundationdb/pull/3320>`_
+
 6.2.22
 ======
 
@@ -21,6 +29,7 @@ Fixes
 * ``fdbrestore`` prefix options required exactly a single hyphen instead of the standard two. `(PR #3056) <https://github.com/apple/foundationdb/pull/3056>`_
 * Commits could stall on a newly elected proxy because of inaccurate compute estimates. `(PR #3123) <https://github.com/apple/foundationdb/pull/3123>`_
 * A transaction class process with a bad disk could be repeatedly recruited as a transaction log. `(PR #3268) <https://github.com/apple/foundationdb/pull/3268>`_
+* Fix a potential race condition that could lead to undefined behavior when connecting to a database using the multi-version client API. `(PR #3265) <https://github.com/apple/foundationdb/pull/3265>`_
 
 Features
 --------

fdbserver/storageserver.actor.cpp

@@ -1013,19 +1013,25 @@ ACTOR Future<Void> watchValue_impl( StorageServer* data, WatchValueRequest req )
 		if( req.debugID.present() )
 			g_traceBatch.addEvent("WatchValueDebug", req.debugID.get().first(), "watchValueQ.AfterVersion"); //.detail("TaskID", g_network->getCurrentTask());
 
+		state Version minVersion = data->data().latestVersion;
+		state Future<Void> watchFuture = data->watches.onChange(req.key);
 		loop {
 			try {
-				state Version latest = data->data().latestVersion;
-				state Future<Void> watchFuture = data->watches.onChange(req.key);
+				state Version latest = data->version.get();
+				TEST(latest >= minVersion && latest < data->data().latestVersion); // Starting watch loop with latestVersion > data->version
 				GetValueRequest getReq( req.key, latest, req.tags, req.debugID );
 				state Future<Void> getValue = getValueQ( data, getReq ); //we are relying on the delay zero at the top of getValueQ, if removed we need one here
 				GetValueReply reply = wait( getReq.reply.getFuture() );
 				//TraceEvent("WatcherCheckValue").detail("Key",  req.key  ).detail("Value",  req.value  ).detail("CurrentValue",  v  ).detail("Ver", latest);
 
 				if(reply.error.present()) {
+					ASSERT(reply.error.get().code() != error_code_future_version);
 					throw reply.error.get();
 				}
-
+				if(BUGGIFY) {
+					throw transaction_too_old();
+				}
+				
 				debugMutation("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("<null>") ) );
 
 				if( req.debugID.present() )
@@ -1045,7 +1051,16 @@ ACTOR Future<Void> watchValue_impl( StorageServer* data, WatchValueRequest req )
 				++data->numWatches;
 				data->watchBytes += ( req.key.expectedSize() + req.value.expectedSize() + 1000 );
 				try {
-					wait( watchFuture );
+					if(latest < minVersion) {
+						// If the version we read is less than minVersion, then we may fail to be notified of any changes that occur up to or including minVersion
+						// To prevent that, we'll check the key again once the version reaches our minVersion
+						watchFuture = watchFuture || data->version.whenAtLeast(minVersion);
+					}
+					if(BUGGIFY) {
+						// Simulate a trigger on the watch that results in the loop going around without the value changing
+						watchFuture = watchFuture || delay(deterministicRandom()->random01());
+					}
+					wait(watchFuture);
 					--data->numWatches;
 					data->watchBytes -= ( req.key.expectedSize() + req.value.expectedSize() + 1000 );
 				} catch( Error &e ) {
@@ -1054,9 +1069,15 @@ ACTOR Future<Void> watchValue_impl( StorageServer* data, WatchValueRequest req )
 					throw;
 				}
 			} catch( Error &e ) {
-				if( e.code() != error_code_transaction_too_old )
+				if( e.code() != error_code_transaction_too_old ) {
 					throw;
+				}
+
+				TEST(true); // Reading a watched key failed with transaction_too_old
 			}
+
+			watchFuture = data->watches.onChange(req.key);
+			wait(data->version.whenAtLeast(data->data().latestVersion));
 		}
 	} catch (Error& e) {
 		if(!canReplyWith(e))