wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add missing secondary queries tests for index prefetch (#7185) 

* Add missing secondary queries tests for index prefetch

This change adds test for missing secondary queries for index prefetch,
in this case, MATCHED_ONLY mode would NOT return KV and UNMATCHED_ONLY
mode would return KV.

* remove default value for params
This commit is contained in:
Hao Fu 2022-05-18 12:13:19 -07:00 committed by GitHub
parent f00606fd71
commit dcacb30688
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 99 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
bindings/c/test/unit/unit_tests.cpp
View File

@ -958,7 +958,7 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
int endId,
fdb::Transaction& tr,
std::string mapper,
int matchIndex = MATCH_INDEX_ALL) {
int matchIndex) {
std::string indexEntryKeyBegin = indexEntryKey(beginId);
std::string indexEntryKeyEnd = indexEntryKey(endId);
@ -980,8 +980,15 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
GetMappedRangeResult getMappedIndexEntries(int beginId,
int endId,
fdb::Transaction& tr,
int matchIndex = MATCH_INDEX_ALL) {
std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).append("{...}"_sr).pack().toString();
int matchIndex,
bool allMissing) {
std::string mapper = Tuple()
.append(prefix)
.append(RECORD)
.append(allMissing ? "{K[2]}"_sr : "{K[3]}"_sr)
.append("{...}"_sr)
.pack()
.toString();
return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
}
@ -1003,7 +1010,7 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
} else if (r < 0.75) {
matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
}
auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex);
auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, false);
if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err);
@ -1023,22 +1030,14 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
// now we cannot generate a workload that only has partial results matched
// thus expecting everything matched
// TODO: create tests to generate workloads with partial secondary results present
CHECK(indexEntryKey(id).compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
// now we cannot generate a workload that only has partial results matched
// thus expecting everything NOT matched(except for the boundary asserted above)
// TODO: create tests to generate workloads with partial secondary results present
CHECK(EMPTY.compare(key) == 0);
} else {
CHECK(EMPTY.compare(key) == 0);
}
// TODO: create tests to generate workloads with partial secondary results present
CHECK(boundaryAndExist == boundary);
bool empty = range_results.empty();
CHECK(boundaryAndExist == (boundary && !empty));
CHECK(EMPTY.compare(value) == 0);
CHECK(range_results.size() == SPLIT_SIZE);
for (int split = 0; split < SPLIT_SIZE; split++) {
@ -1051,6 +1050,58 @@ TEST_CASE("fdb_transaction_get_mapped_range") {
}
}
TEST_CASE("fdb_transaction_get_mapped_range_missing_all_secondary") {
const int TOTAL_RECORDS = 20;
fillInRecords(TOTAL_RECORDS);
fdb::Transaction tr(db);
// RYW should be enabled.
while (1) {
int beginId = 1;
int endId = 19;
const double r = deterministicRandom()->random01();
int matchIndex = MATCH_INDEX_ALL;
if (r < 0.25) {
matchIndex = MATCH_INDEX_NONE;
} else if (r < 0.5) {
matchIndex = MATCH_INDEX_MATCHED_ONLY;
} else if (r < 0.75) {
matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
}
auto result = getMappedIndexEntries(beginId, endId, tr, matchIndex, true);
if (result.err) {
fdb::EmptyFuture f1 = tr.on_error(result.err);
fdb_check(wait_future(f1));
continue;
}
int expectSize = endId - beginId;
CHECK(result.mkvs.size() == expectSize);
CHECK(!result.more);
int id = beginId;
bool boundary;
for (int i = 0; i < expectSize; i++, id++) {
boundary = i == 0 || i == expectSize - 1;
const auto& [key, value, begin, end, range_results, boundaryAndExist] = result.mkvs[i];
if (matchIndex == MATCH_INDEX_ALL || i == 0 || i == expectSize - 1) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
CHECK(EMPTY.compare(key) == 0);
} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
CHECK(indexEntryKey(id).compare(key) == 0);
} else {
CHECK(EMPTY.compare(key) == 0);
}
bool empty = range_results.empty();
CHECK(boundaryAndExist == (boundary && !empty));
CHECK(EMPTY.compare(value) == 0);
}
break;
}
}
TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
fdb::Transaction tr(db);
@ -1111,7 +1162,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_fail_on_mapper_not_tuple") {
};
assertNotTuple(mapper);
fdb::Transaction tr(db);
auto result = getMappedIndexEntries(1, 3, tr, mapper);
auto result = getMappedIndexEntries(1, 3, tr, mapper, MATCH_INDEX_ALL);
ASSERT(result.err == error_code_mapper_not_tuple);
}

63
fdbserver/workloads/GetMappedRange.actor.cpp
View File

@ -149,33 +149,26 @@ struct GetMappedRangeWorkload : ApiWorkload {
const MappedKeyValueRef* it,
GetMappedRangeWorkload* self,
int matchIndex,
bool isBoundary) {
bool isBoundary,
bool allMissing) {
// std::cout << "validateRecord expectedId " << expectedId << " it->key " << printable(it->key) << "
// indexEntryKey(expectedId) " << printable(indexEntryKey(expectedId)) << std::endl;
if (matchIndex == MATCH_INDEX_ALL || isBoundary) {
ASSERT(it->key == indexEntryKey(expectedId));
} else if (matchIndex == MATCH_INDEX_MATCHED_ONLY) {
// now we cannot generate a workload that only has partial results matched
// thus expecting everything matched
// TODO: create tests to generate workloads with partial secondary results present
ASSERT(it->key == indexEntryKey(expectedId));
ASSERT(it->key == (allMissing ? EMPTY : indexEntryKey(expectedId)));
} else if (matchIndex == MATCH_INDEX_UNMATCHED_ONLY) {
// now we cannot generate a workload that only has partial results matched
// thus expecting everything NOT matched(except for the boundary asserted above)
// TODO: create tests to generate workloads with partial secondary results present
ASSERT(it->key == EMPTY);
ASSERT(it->key == (allMissing ? indexEntryKey(expectedId) : EMPTY));
} else {
ASSERT(it->key == EMPTY);
}
// TODO: create tests to generate workloads with partial secondary results present
ASSERT(it->boundaryAndExist == isBoundary);
ASSERT(it->value == EMPTY);
if (self->SPLIT_RECORDS) {
ASSERT(std::holds_alternative<GetRangeReqAndResultRef>(it->reqAndResult));
auto& getRange = std::get<GetRangeReqAndResultRef>(it->reqAndResult);
auto& rangeResult = getRange.result;
ASSERT(it->boundaryAndExist == (isBoundary && !rangeResult.empty()));
// std::cout << "rangeResult.size()=" << rangeResult.size() << std::endl;
// In the future, we may be able to do the continuation more efficiently by combining partial results
// together and then validate.
@ -183,17 +176,20 @@ struct GetMappedRangeWorkload : ApiWorkload {
// Retry if the underlying request is not fully completed.
return true;
}
ASSERT(rangeResult.size() == SPLIT_SIZE);
for (int split = 0; split < SPLIT_SIZE; split++) {
auto& kv = rangeResult[split];
// std::cout << "kv.key=" << printable(kv.key)
// << ", recordKey(id, split)=" << printable(recordKey(id, split)) <<
// std::endl; std::cout << "kv.value=" << printable(kv.value)
// << ", recordValue(id, split)=" << printable(recordValue(id, split)) <<
// std::endl;
ASSERT(kv.key == recordKey(expectedId, split));
ASSERT(kv.value == recordValue(expectedId, split));
if (!allMissing) {
ASSERT(rangeResult.size() == SPLIT_SIZE);
for (int split = 0; split < SPLIT_SIZE; split++) {
auto& kv = rangeResult[split];
// std::cout << "kv.key=" << printable(kv.key)
// << ", recordKey(id, split)=" << printable(recordKey(id, split)) <<
// std::endl; std::cout << "kv.value=" << printable(kv.value)
// << ", recordValue(id, split)=" << printable(recordValue(id,split)) <<
// std::endl;
ASSERT(kv.key == recordKey(expectedId, split));
ASSERT(kv.value == recordValue(expectedId, split));
}
}
} else {
ASSERT(std::holds_alternative<GetValueReqAndResultRef>(it->reqAndResult));
auto& getValue = std::get<GetValueReqAndResultRef>(it->reqAndResult);
@ -211,7 +207,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
int limit,
int expectedBeginId,
GetMappedRangeWorkload* self,
int matchIndex) {
int matchIndex,
bool allMissing) {
std::cout << "start scanMappedRangeWithLimits beginSelector:" << beginSelector.toString()
<< " endSelector:" << endSelector.toString() << " expectedBeginId:" << expectedBeginId
@ -238,7 +235,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
int cnt = 0;
const MappedKeyValueRef* it = result.begin();
for (; cnt < result.size(); cnt++, it++) {
if (validateRecord(expectedId, it, self, matchIndex, cnt == 0 || cnt == result.size() - 1)) {
if (validateRecord(
expectedId, it, self, matchIndex, cnt == 0 || cnt == result.size() - 1, allMissing)) {
needRetry = true;
break;
}
@ -270,7 +268,8 @@ struct GetMappedRangeWorkload : ApiWorkload {
int endId,
Key mapper,
GetMappedRangeWorkload* self,
int matchIndex) {
int matchIndex,
bool allMissing = false) {
Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
state KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
@ -279,7 +278,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
state int expectedBeginId = beginId;
while (true) {
MappedRangeResult result = wait(self->scanMappedRangeWithLimits(
cx, beginSelector, endSelector, mapper, limit, expectedBeginId, self, matchIndex));
cx, beginSelector, endSelector, mapper, limit, expectedBeginId, self, matchIndex, allMissing));
expectedBeginId += result.size();
if (result.more) {
if (result.empty()) {
@ -322,7 +321,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
int endId,
Reference<TransactionWrapper>& tr,
GetMappedRangeWorkload* self) {
Key mapper = getMapper(self);
Key mapper = getMapper(self, false);
Key beginTuple = Tuple().append(prefix).append(INDEX).append(indexKey(beginId)).getDataAsStandalone();
KeySelector beginSelector = KeySelector(firstGreaterOrEqual(beginTuple));
Key endTuple = Tuple().append(prefix).append(INDEX).append(indexKey(endId)).getDataAsStandalone();
@ -427,7 +426,7 @@ struct GetMappedRangeWorkload : ApiWorkload {
std::cout << "Test configuration: transactionType:" << self->transactionType << " snapshot:" << self->snapshot
<< "bad_mapper:" << self->BAD_MAPPER << std::endl;
Key mapper = getMapper(self);
Key mapper = getMapper(self, false);
// The scanned range cannot be too large to hit get_mapped_key_values_has_more. We have a unit validating the
// error is thrown when the range is large.
const double r = deterministicRandom()->random01();
@ -440,15 +439,19 @@ struct GetMappedRangeWorkload : ApiWorkload {
matchIndex = MATCH_INDEX_UNMATCHED_ONLY;
}
wait(self->scanMappedRange(cx, 10, 490, mapper, self, matchIndex));
Key mapper = getMapper(self, true);
wait(self->scanMappedRange(cx, 10, 490, mapper, self, MATCH_INDEX_UNMATCHED_ONLY, true));
return Void();
}
static Key getMapper(GetMappedRangeWorkload* self) {
static Key getMapper(GetMappedRangeWorkload* self, bool mapperForAllMissing) {
Tuple mapperTuple;
if (self->BAD_MAPPER) {
mapperTuple << prefix << RECORD << "{K[xxx]}"_sr;
} else {
mapperTuple << prefix << RECORD << "{K[3]}"_sr;
mapperTuple << prefix << RECORD << (mapperForAllMissing ? "{K[2]}"_sr : "{K[3]}"_sr);
if (self->SPLIT_RECORDS) {
mapperTuple << "{...}"_sr;
}