From 37ca0bf7e2259a70d7d24cdfc638cb384a0782d3 Mon Sep 17 00:00:00 2001 From: chaoguang <13974480+zjuLcg@users.noreply.github.com> Date: Mon, 30 Mar 2020 23:27:09 -0700 Subject: [PATCH] add code to handle the three flags in RangeResultRef --- fdbclient/SpecialKeySpace.actor.cpp | 94 ++++++++++++++----- fdbclient/SpecialKeySpace.actor.h | 1 + .../SpecialKeySpaceCorrectness.actor.cpp | 20 +++- tests/fast/SpecialKeySpaceCorrectness.txt | 2 +- 4 files changed, 89 insertions(+), 28 deletions(-) diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 04e042aa87..edf57dc21a 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -67,43 +67,75 @@ ACTOR Future> SpecialKeySpace::getRangeAggregationAct GetRangeLimits limits, bool reverse) { // This function handles ranges which cover more than one keyrange and aggregates all results // KeySelector, GetRangeLimits and reverse are all handled here - + state Standalone result; + state RangeMap::Iterator iter; + state int actualBeginOffset; + state int actualEndOffset; // make sure orEqual == false begin.removeOrEqual(begin.arena()); end.removeOrEqual(end.arena()); // make sure offset == 1 - state RangeMap::Iterator iter = + state RangeMap::Iterator beginIter = pks->impls.rangeContaining(begin.getKey()); - while ((begin.offset < 1 && iter != pks->impls.ranges().begin()) || ( begin.offset > 1 && iter != pks->impls.ranges().end())) { - if (iter->value() != nullptr) wait(iter->value()->normalizeKeySelectorActor(iter->value(), ryw, &begin)); - begin.offset < 1 ? --iter : ++iter; + while ((begin.offset < 1 && beginIter != pks->impls.ranges().begin()) || + (begin.offset > 1 && beginIter != pks->impls.ranges().end())) { + if (beginIter->value() != nullptr) + wait(beginIter->value()->normalizeKeySelectorActor(beginIter->value(), ryw, &begin)); + begin.offset < 1 ? --beginIter : ++beginIter; } + // TODO : change to is FirstGreaterOrEqual + actualBeginOffset = begin.offset; + if (beginIter == pks->impls.ranges().begin()) + begin.setKey(normalKeys.begin); + else if (beginIter == pks->impls.ranges().end()) + begin.setKey(normalKeys.end); + if (begin.offset != 1) { // The Key Selector points to key outside the whole special key space - TraceEvent(SevWarn, "IllegalBeginKeySelector") - .detail("TerminateKey", begin.getKey()) - .detail("TerminateOffset", begin.offset); + TraceEvent(SevInfo, "IllegalBeginKeySelector") + .detail("TerminateKey", begin.getKey()) + .detail("TerminateOffset", begin.offset); + if (begin.offset < 1 && beginIter == pks->impls.ranges().begin()) + result.readToBegin = true; + else + result.readThroughEnd = true; begin.offset = 1; } - iter = pks->impls.rangeContaining(end.getKey()); - while ((end.offset < 1 && iter != pks->impls.ranges().begin()) || (end.offset > 1 && iter != pks->impls.ranges().end())) { - if (iter->value() != nullptr) wait(iter->value()->normalizeKeySelectorActor(iter->value(), ryw, &end)); - end.offset < 1 ? --iter : ++iter; + state RangeMap::Iterator endIter = + pks->impls.rangeContaining(end.getKey()); + while ((end.offset < 1 && endIter != pks->impls.ranges().begin()) || + (end.offset > 1 && endIter != pks->impls.ranges().end())) { + if (endIter->value() != nullptr) wait(endIter->value()->normalizeKeySelectorActor(endIter->value(), ryw, &end)); + end.offset < 1 ? --endIter : ++endIter; } + // TODO : change to is FirstGreaterOrEqual + actualEndOffset = end.offset; + if (endIter == pks->impls.ranges().begin()) + end.setKey(normalKeys.begin); + else if (endIter == pks->impls.ranges().end()) + end.setKey(normalKeys.end); + if (end.offset != 1) { // The Key Selector points to key outside the whole special key space - TraceEvent(SevWarn, "IllegalEndKeySelector") + TraceEvent(SevInfo, "IllegalEndKeySelector") .detail("TerminateKey", end.getKey()) .detail("TerminateOffset", end.offset); + if (end.offset < 1 && endIter == pks->impls.ranges().begin()) + result.readToBegin = true; + else + result.readThroughEnd = true; end.offset = 1; } + // Handle all corner cases like what RYW does // return if range inverted - if (begin.offset >= end.offset && begin.getKey() >= end.getKey()) { + if (actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) { TEST(true); - return Standalone(); + return RangeResultRef(false, false); + } + if (beginIter == pks->impls.ranges().end() || endIter == pks->impls.ranges().begin()) { + return result; } - state Standalone result; state RangeMap::Ranges ranges = pks->impls.intersectingRanges(KeyRangeRef(begin.getKey(), end.getKey())); // TODO : workaround to write this two together to make the code compact @@ -120,10 +152,16 @@ ACTOR Future> SpecialKeySpace::getRangeAggregationAct // limits handler for (int i = pairs.size() - 1; i >= 0; --i) { result.push_back_deep(result.arena(), pairs[i]); - // Note : behavior here is even the last k-v pair makes total bytes larger than specified, it is still returned - // In other words, the total size of the returned value (less the last entry) will be less than byteLimit + // Note : behavior here is even the last k-v pair makes total bytes larger than specified, it is still + // returned In other words, the total size of the returned value (less the last entry) will be less than + // byteLimit limits.decrement(pairs[i]); - if (limits.isReached()) return result; + if (limits.isReached()) { + // result.more = (iter != ranges.begin()) || (iter == ranges.begin() && i != 0); + result.more = true; + result.readToBegin = false; + return result; + }; } } } else { @@ -134,12 +172,18 @@ ACTOR Future> SpecialKeySpace::getRangeAggregationAct KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end; Standalone pairs = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd))); // limits handler - for (const KeyValueRef& kv : pairs) { - result.push_back_deep(result.arena(), kv); - // Note : behavior here is even the last k-v pari makes total bytes larger than specified, it is still returned - // In other words, the total size of the returned value (less the last entry) will be less than byteLimit - limits.decrement(kv); - if (limits.isReached()) return result; + for (int i = 0; i < pairs.size(); ++i) { + result.push_back_deep(result.arena(), pairs[i]); + // Note : behavior here is even the last k-v pair makes total bytes larger than specified, it is still + // returned In other words, the total size of the returned value (less the last entry) will be less than + // byteLimit + limits.decrement(pairs[i]); + if (limits.isReached()) { + // result.more = (iter != ranges.end()) || (iter == ranges.end() && i != pairs.size() - 1); + result.more = true; + result.readThroughEnd = false; + return result; + }; } } } diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 8dc717aa9f..13e0a2c24c 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -47,6 +47,7 @@ public: SpecialKeySpace(KeyRef spaceStartKey = Key(), KeyRef spaceEndKey = allKeys.end) { // Default value is nullptr, begin of KeyRangeMap is Key() + // TODO : check do we need stop key here impls = KeyRangeMap(nullptr, spaceEndKey); range = KeyRangeRef(spaceStartKey, spaceEndKey); } diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index 371cbcd608..67af91da95 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -102,14 +102,30 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { if (!self->compareRangeResult(correctResult, testResult)) { // TODO : log here // TraceEvent("WrongGetRangeResult"). detail("KeySeleco") - // auto temp1 = cx->specialKeySpace->getRange(self->ryw, begin, end, limit, false, reverse); - // auto temp2 = self->ryw->getRange(begin, end, limit, false, reverse); + auto temp1 = self->ryw->getRange(begin, end, limit, false, reverse); + auto temp2 = cx->specialKeySpace->getRange(self->ryw, begin, end, limit, false, reverse); ++self->wrongResults; } } } bool compareRangeResult(Standalone& res1, Standalone& res2) { + if ((res1.more != res2.more) || (res1.readToBegin != res2.readToBegin) || + (res1.readThroughEnd != res2.readThroughEnd)) { + TraceEvent(SevError, "TestFailure") + .detail("Reason", "flags are different") + .detail("More", res1.more) + .detail("ReadToBegin", res1.readToBegin) + .detail("ReadThroughEnd", res1.readThroughEnd) + .detail("More2", res2.more) + .detail("ReadToBegin2", res2.readToBegin) + .detail("ReadThroughEnd2", res2.readThroughEnd); + return false; + } + // if (res1.more != res2.more) { + // TraceEvent(SevError, "TestFailure").detail("Reason", "More flags are different").detail("More", res1.more); + // return false; + // } if (res1.size() != res2.size()) return false; for (int i = 0; i < res1.size(); ++i) { if (res1[i] != res2[i]) return false; diff --git a/tests/fast/SpecialKeySpaceCorrectness.txt b/tests/fast/SpecialKeySpaceCorrectness.txt index 6de079e86e..6b98a73a19 100644 --- a/tests/fast/SpecialKeySpaceCorrectness.txt +++ b/tests/fast/SpecialKeySpaceCorrectness.txt @@ -1,5 +1,5 @@ testTitle=SpecialKeySpaceCorrectnessTest testName=SpecialKeySpaceCorrectness - testDuration=10.0 + testDuration=30.0 valueBytes=16 keyBytes=16