Add missing secondary queries tests for index prefetch (#7185)

* Add missing secondary queries tests for index prefetch

This change adds test for missing secondary queries for index prefetch,
in this case, MATCHED_ONLY mode would NOT return KV and UNMATCHED_ONLY
mode would return KV.

* remove default value for params

bindings/c/test/unit/unit_tests.cpp

@@ -958,7 +958,7 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
                                            int endId,
                                            fdb::Transaction& tr,
                                            std::string mapper,
-                                           int matchIndex = MATCH_INDEX_ALL) {
+                                           int matchIndex) {
 	std::string indexEntryKeyBegin = indexEntryKey(beginId);
 	std::string indexEntryKeyEnd = indexEntryKey(endId);
 
@@ -980,8 +980,15 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
 GetMappedRangeResult getMappedIndexEntries(int beginId,
                                            int endId,
                                            fdb::Transaction& tr,
-                                           int matchIndex = MATCH_INDEX_ALL) {
+                                           int matchIndex,
-	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).append("{...}"_sr).pack().toString();
+                                           bool allMissing) {
+	std::string mapper = Tuple()
+	                         .append(prefix)
+	                         .append(RECORD)
+	                         .append(allMissing ? "{K[2]}"_sr : "{K[3]}"_sr)
+	                         .append("{...}"_sr)
+	                         .pack()
+	                         .toString();
 	return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
 }
 
@@ -1003,7 +1010,7 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 		} else if (r < 0.75) {
 			matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
 		}
-		auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex);
+		auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, false);
 
 		if (result.err) {
 			fdb::EmptyFuture f1 = tr.on_error(result.err);
@@ -1023,22 +1030,14 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 			if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
 				CHECK(indexEntryKey(id).compare(key) == 0);
 			} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
-				// now we cannot generate a workload that only has partial results matched
-				// thus expecting everything matched
-				// TODO: create tests to generate workloads with partial secondary results present
 				CHECK(indexEntryKey(id).compare(key) == 0);
 			} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
-				// now we cannot generate a workload that only has partial results matched
-				// thus expecting everything NOT matched(except for the boundary asserted above)
-				// TODO: create tests to generate workloads with partial secondary results present
 				CHECK(EMPTY.compare(key) == 0);
 			} else {
 				CHECK(EMPTY.compare(key) == 0);
 			}
-
+			bool empty = range_results.empty();
-			// TODO: create tests to generate workloads with partial secondary results present
+			CHECK(boundaryAndExist == (boundary && !empty));
-			CHECK(boundaryAndExist == boundary);
-
 			CHECK(EMPTY.compare(value) == 0);
 			CHECK(range_results.size() == SPLIT_SIZE);
 			for (int split = 0; split < SPLIT_SIZE; split++) {
@@ -1051,6 +1050,58 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
 	}
 }
 
+TEST_CASE("fdb_transaction_get_mapped_range_missing_all_secondary") {
+	const int TOTAL_RECORDS = 20;
+	fillInRecords(TOTAL_RECORDS);
+
+	fdb::Transaction tr(db);
+	// RYW should be enabled.
+	while (1) {
+		int beginId = 1;
+		int endId = 19;
+		const double r = deterministicRandom()->random01();
+		int matchIndex = MATCH_INDEX_ALL;
+		if (r < 0.25) {
+			matchIndex = MATCH_INDEX_NONE;
+		} else if (r < 0.5) {
+			matchIndex = MATCH_INDEX_MATCHED_ONLY;
+		} else if (r < 0.75) {
+			matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
+		}
+		auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, true);
+
+		if (result.err) {
+			fdb::EmptyFuture f1 = tr.on_error(result.err);
+			fdb_check(wait_future(f1));
+			continue;
+		}
+
+		int expectSize = endId - beginId;
+		CHECK(result.mkvs.size() == expectSize);
+		CHECK(!result.more);
+
+		int id = beginId;
+		bool boundary;
+		for (int i = 0; i < expectSize; i++, id++) {
+			boundary = i == 0 || i == expectSize - 1;
+			const auto& [key, value, begin, end, range_results, boundaryAndExist] = result.mkvs[i];
+			if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
+				CHECK(indexEntryKey(id).compare(key) == 0);
+			} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
+				CHECK(EMPTY.compare(key) == 0);
+			} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
+				CHECK(indexEntryKey(id).compare(key) == 0);
+			} else {
+				CHECK(EMPTY.compare(key) == 0);
+			}
+			bool empty = range_results.empty();
+			CHECK(boundaryAndExist == (boundary && !empty));
+			CHECK(EMPTY.compare(value) == 0);
+		}
+		break;
+	}
+}
+
 TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
 	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
 	fdb::Transaction tr(db);
@@ -1111,7 +1162,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_fail_on_mapper_not_tuple") {
 	};
 	assertNotTuple(mapper);
 	fdb::Transaction tr(db);
-	auto result = getMappedIndexEntries(1, 3, tr, mapper);
+	auto result = getMappedIndexEntries(1, 3, tr, mapper, MATCH_INDEX_ALL);
 	ASSERT(result.err == error_code_mapper_not_tuple);
 }
 

fdbserver/workloads/GetMappedRange.actor.cpp

@@ -149,33 +149,26 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                           const MappedKeyValueRef* it,
 	                           GetMappedRangeWorkload* self,
 	                           int matchIndex,
-	                           bool isBoundary) {
+	                           bool isBoundary,
+	                           bool allMissing) {
 		//		std::cout << "validateRecord expectedId " << expectedId << " it->key " << printable(it->key) << "
 		// indexEntryKey(expectedId) " << printable(indexEntryKey(expectedId)) << std::endl;
 		if (matchIndex == MATCH_INDEX_ALL || isBoundary) {
 			ASSERT(it->key == indexEntryKey(expectedId));
 		} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
-			// now we cannot generate a workload that only has partial results matched
+			ASSERT(it->key == (allMissing ? EMPTY : indexEntryKey(expectedId)));
-			// thus expecting everything matched
-			// TODO: create tests to generate workloads with partial secondary results present
-			ASSERT(it->key == indexEntryKey(expectedId));
 		} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
-			// now we cannot generate a workload that only has partial results matched
+			ASSERT(it->key == (allMissing ? indexEntryKey(expectedId) : EMPTY));
-			// thus expecting everything NOT matched(except for the boundary asserted above)
-			// TODO: create tests to generate workloads with partial secondary results present
-			ASSERT(it->key == EMPTY);
 		} else {
 			ASSERT(it->key == EMPTY);
 		}
-		// TODO: create tests to generate workloads with partial secondary results present
-		ASSERT(it->boundaryAndExist == isBoundary);
-
 		ASSERT(it->value == EMPTY);
 
 		if (self->SPLIT_RECORDS) {
 			ASSERT(std::holds_alternative<GetRangeReqAndResultRef>(it->reqAndResult));
 			auto& getRange = std::get<GetRangeReqAndResultRef>(it->reqAndResult);
 			auto& rangeResult = getRange.result;
+			ASSERT(it->boundaryAndExist == (isBoundary && !rangeResult.empty()));
 			//					std::cout << "rangeResult.size()=" << rangeResult.size() << std::endl;
 			// In the future, we may be able to do the continuation more efficiently by combining partial results
 			// together and then validate.
@@ -183,17 +176,20 @@ struct GetMappedRangeWorkload : ApiWorkload {
 				// Retry if the underlying request is not fully completed.
 				return true;
 			}
-			ASSERT(rangeResult.size() == SPLIT_SIZE);
+			if (!allMissing) {
-			for (int split = 0; split < SPLIT_SIZE; split++) {
+				ASSERT(rangeResult.size() == SPLIT_SIZE);
-				auto& kv = rangeResult[split];
+				for (int split = 0; split < SPLIT_SIZE; split++) {
-				//						std::cout << "kv.key=" << printable(kv.key)
+					auto& kv = rangeResult[split];
-				//						          << ", recordKey(id, split)=" << printable(recordKey(id, split)) <<
+					//				std::cout << "kv.key=" << printable(kv.key)
-				// std::endl; std::cout << "kv.value=" << printable(kv.value)
+					//						   << ", recordKey(id, split)=" << printable(recordKey(id, split)) <<
-				//						          << ", recordValue(id, split)=" << printable(recordValue(id, split)) <<
+					// std::endl; std::cout << "kv.value=" << printable(kv.value)
-				// std::endl;
+					//						   << ", recordValue(id, split)=" << printable(recordValue(id,split)) <<
-				ASSERT(kv.key == recordKey(expectedId, split));
+					// std::endl;
-				ASSERT(kv.value == recordValue(expectedId, split));
+					ASSERT(kv.key == recordKey(expectedId, split));
+					ASSERT(kv.value == recordValue(expectedId, split));
+				}
 			}
+
 		} else {
 			ASSERT(std::holds_alternative<GetValueReqAndResultRef>(it->reqAndResult));
 			auto& getValue = std::get<GetValueReqAndResultRef>(it->reqAndResult);
@@ -211,7 +207,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                                                          int limit,
 	                                                          int expectedBeginId,
 	                                                          GetMappedRangeWorkload* self,
-	                                                          int matchIndex) {
+	                                                          int matchIndex,
+	                                                          bool allMissing) {
 
 		std::cout << "start scanMappedRangeWithLimits beginSelector:" << beginSelector.toString()
 		          << " endSelector:" << endSelector.toString() << " expectedBeginId:" << expectedBeginId
@@ -238,7 +235,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
 				int cnt = 0;
 				const MappedKeyValueRef* it = result.begin();
 				for (; cnt < result.size(); cnt++, it++) {
-					if (validateRecord(expectedId, it, self, matchIndex, cnt == 0 || cnt == result.size() - 1)) {
+					if (validateRecord(
+					        expectedId, it, self, matchIndex, cnt == 0 || cnt == result.size() - 1, allMissing)) {
 						needRetry = true;
 						break;
 					}
@@ -270,7 +268,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                                   int endId,
 	                                   Key mapper,
 	                                   GetMappedRangeWorkload* self,
-	                                   int matchIndex) {
+	                                   int matchIndex,
+	                                   bool allMissing = false) {
 		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
 		state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
 		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
@@ -279,7 +278,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		state int expectedBeginId = beginId;
 		while (true) {
 			MappedRangeResult result = wait(self->scanMappedRangeWithLimits(
-			    cx, beginSelector, endSelector, mapper, limit, expectedBeginId, self, matchIndex));
+			    cx, beginSelector, endSelector, mapper, limit, expectedBeginId, self, matchIndex, allMissing));
 			expectedBeginId += result.size();
 			if (result.more) {
 				if (result.empty()) {
@@ -322,7 +321,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
 	                                                   int endId,
 	                                                   Reference<TransactionWrapper>& tr,
 	                                                   GetMappedRangeWorkload* self) {
-		Key mapper = getMapper(self);
+		Key mapper = getMapper(self, false);
 		Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
 		KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
 		Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
@@ -427,7 +426,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
 		std::cout << "Test configuration: transactionType:" << self->transactionType << " snapshot:" << self->snapshot
 		          << "bad_mapper:" << self->BAD_MAPPER << std::endl;
 
-		Key mapper = getMapper(self);
+		Key mapper = getMapper(self, false);
 		// The scanned range cannot be too large to hit get_mapped_key_values_has_more. We have a unit validating the
 		// error is thrown when the range is large.
 		const double r = deterministicRandom()->random01();
@@ -440,15 +439,19 @@ struct GetMappedRangeWorkload : ApiWorkload {
 			matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
 		}
 		wait(self->scanMappedRange(cx, 10, 490, mapper, self, matchIndex));
+
+		Key mapper = getMapper(self, true);
+		wait(self->scanMappedRange(cx, 10, 490, mapper, self, MATCH_INDEX_UNMATCHED_ONLY, true));
+
 		return Void();
 	}
 
-	static Key getMapper(GetMappedRangeWorkload* self) {
+	static Key getMapper(GetMappedRangeWorkload* self, bool mapperForAllMissing) {
 		Tuple mapperTuple;
 		if (self->BAD_MAPPER) {
 			mapperTuple << prefix << RECORD << "{K[xxx]}"_sr;
 		} else {
-			mapperTuple << prefix << RECORD << "{K[3]}"_sr;
+			mapperTuple << prefix << RECORD << (mapperForAllMissing ? "{K[2]}"_sr : "{K[3]}"_sr);
 			if (self->SPLIT_RECORDS) {
 				mapperTuple << "{...}"_sr;
 			}