From c886a6efe4a522f85e522c3264b683d02a14a298 Mon Sep 17 00:00:00 2001 From: Xiaoge Su Date: Wed, 1 Jun 2022 21:29:59 -0700 Subject: [PATCH] Refactor the WriteMap.h Split the interface and implementation, so adding debugging information in the code will not trigger full rebuild. --- fdbclient/CMakeLists.txt | 1 + fdbclient/WriteMap.cpp | 596 ++++++++++++++++++++++++++++++++++++++ fdbclient/WriteMap.h | 610 ++------------------------------------- 3 files changed, 623 insertions(+), 584 deletions(-) create mode 100644 fdbclient/WriteMap.cpp diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index bdf008b37c..fc2840caa5 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -153,6 +153,7 @@ set(FDBCLIENT_SRCS VersionVector.cpp WellKnownEndpoints.h WriteMap.h + WriteMap.cpp json_spirit/json_spirit_error_position.h json_spirit/json_spirit_reader_template.h json_spirit/json_spirit_value.h diff --git a/fdbclient/WriteMap.cpp b/fdbclient/WriteMap.cpp new file mode 100644 index 0000000000..dcb6ee9581 --- /dev/null +++ b/fdbclient/WriteMap.cpp @@ -0,0 +1,596 @@ +/* + * WriteMap.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/WriteMap.h" + +void OperationStack::reset(RYWMutation initialEntry) { + defaultConstructed = false; + singletonOperation = initialEntry; + optionalOperations = Optional>(); +} + +void OperationStack::poppush(RYWMutation entry) { + if (hasVector()) { + optionalOperations.get().pop_back(); + optionalOperations.get().push_back(entry); + } else + singletonOperation = entry; +} + +void OperationStack::push(RYWMutation entry) { + if (defaultConstructed) { + singletonOperation = entry; + defaultConstructed = false; + } else if (hasVector()) + optionalOperations.get().push_back(entry); + else { + optionalOperations = std::vector(); + optionalOperations.get().push_back(entry); + } +} + +bool OperationStack::isDependent() const { + if (!size()) + return false; + return singletonOperation.type != MutationRef::SetValue && singletonOperation.type != MutationRef::ClearRange && + singletonOperation.type != MutationRef::SetVersionstampedValue && + singletonOperation.type != MutationRef::SetVersionstampedKey; +} + +bool OperationStack::operator==(const OperationStack& r) const { + if (size() != r.size()) + return false; + + if (size() == 0) + return true; + + if (singletonOperation != r.singletonOperation) + return false; + + if (size() == 1) + return true; + + for (int i = 0; i < optionalOperations.get().size(); i++) { + if (optionalOperations.get()[i] != r.optionalOperations.get()[i]) + return false; + } + + return true; +} + +WriteMap& WriteMap::operator=(WriteMap&& r) noexcept { + writeMapEmpty = r.writeMapEmpty; + writes = std::move(r.writes); + ver = r.ver; + scratch_iterator = std::move(r.scratch_iterator); + arena = r.arena; + return *this; +} + +void WriteMap::mutate(KeyRef key, MutationRef::Type operation, ValueRef param, bool addConflict) { + writeMapEmpty = false; + auto& it = scratch_iterator; + it.reset(writes, ver); + it.skip(key); + + bool is_cleared = it.entry().following_keys_cleared; + bool following_conflict = it.entry().following_keys_conflict; + bool is_conflict = addConflict || it.is_conflict_range(); + bool following_unreadable = it.entry().following_keys_unreadable; + bool is_unreadable = it.is_unreadable() || operation == MutationRef::SetVersionstampedValue || + operation == MutationRef::SetVersionstampedKey; + bool is_dependent = operation != MutationRef::SetValue && operation != MutationRef::SetVersionstampedValue && + operation != MutationRef::SetVersionstampedKey; + + if (it.entry().key != key) { + if (it.is_cleared_range() && is_dependent) { + it.tree.clear(); + OperationStack op(RYWMutation(Optional(), MutationRef::SetValue)); + coalesceOver(op, RYWMutation(param, operation), *arena); + PTreeImpl::insert( + writes, + ver, + WriteMapEntry( + key, std::move(op), true, following_conflict, is_conflict, following_unreadable, is_unreadable)); + } else { + it.tree.clear(); + PTreeImpl::insert(writes, + ver, + WriteMapEntry(key, + OperationStack(RYWMutation(param, operation)), + is_cleared, + following_conflict, + is_conflict, + following_unreadable, + is_unreadable)); + } + } else { + if (!it.is_unreadable() && + (operation == MutationRef::SetValue || operation == MutationRef::SetVersionstampedValue)) { + it.tree.clear(); + PTreeImpl::remove(writes, ver, key); + PTreeImpl::insert(writes, + ver, + WriteMapEntry(key, + OperationStack(RYWMutation(param, operation)), + is_cleared, + following_conflict, + is_conflict, + following_unreadable, + is_unreadable)); + } else { + WriteMapEntry e(it.entry()); + e.is_conflict = is_conflict; + e.is_unreadable = is_unreadable; + if (e.stack.size() == 0 && it.is_cleared_range() && is_dependent) { + e.stack.push(RYWMutation(Optional(), MutationRef::SetValue)); + coalesceOver(e.stack, RYWMutation(param, operation), *arena); + } else if (!is_unreadable && e.stack.size() > 0) + coalesceOver(e.stack, RYWMutation(param, operation), *arena); + else + e.stack.push(RYWMutation(param, operation)); + + it.tree.clear(); + PTreeImpl::remove( + writes, + ver, + e.key); // FIXME: Make PTreeImpl::insert do this automatically (see also VersionedMap.h FIXME) + PTreeImpl::insert(writes, ver, std::move(e)); + } + } +} + +void WriteMap::clear(KeyRangeRef keys, bool addConflict) { + writeMapEmpty = false; + if (!addConflict) { + clearNoConflict(keys); + return; + } + + auto& it = scratch_iterator; + it.reset(writes, ver); + it.skip(keys.begin); + + bool insert_begin = !it.is_cleared_range() || !it.is_conflict_range() || it.is_unreadable(); + + if (it.endKey() == keys.end) { + ++it; + } else if (it.endKey() < keys.end) { + it.skip(keys.end); + } + + bool insert_end = (it.is_unmodified_range() || !it.is_conflict_range() || it.is_unreadable()) && + (!it.keyAtBegin() || it.beginKey() != keys.end); + bool end_coalesce_clear = + it.is_cleared_range() && it.beginKey() == keys.end && it.is_conflict_range() && !it.is_unreadable(); + bool end_conflict = it.is_conflict_range(); + bool end_cleared = it.is_cleared_range(); + bool end_unreadable = it.is_unreadable(); + + it.tree.clear(); + + PTreeImpl::remove(writes, + ver, + ExtStringRef(keys.begin, !insert_begin ? 1 : 0), + ExtStringRef(keys.end, end_coalesce_clear ? 1 : 0)); + + if (insert_begin) + PTreeImpl::insert(writes, ver, WriteMapEntry(keys.begin, OperationStack(), true, true, true, false, false)); + + if (insert_end) + PTreeImpl::insert( + writes, + ver, + WriteMapEntry( + keys.end, OperationStack(), end_cleared, end_conflict, end_conflict, end_unreadable, end_unreadable)); +} + +void WriteMap::addUnmodifiedAndUnreadableRange(KeyRangeRef keys) { + auto& it = scratch_iterator; + it.reset(writes, ver); + it.skip(keys.begin); + + bool insert_begin = !it.is_unmodified_range() || it.is_conflict_range() || !it.is_unreadable(); + + if (it.endKey() == keys.end) { + ++it; + } else if (it.endKey() < keys.end) { + it.skip(keys.end); + } + + bool insert_end = (it.is_cleared_range() || it.is_conflict_range() || !it.is_unreadable()) && + (!it.keyAtBegin() || it.beginKey() != keys.end); + bool end_coalesce_unmodified = + it.is_unmodified_range() && it.beginKey() == keys.end && !it.is_conflict_range() && it.is_unreadable(); + bool end_conflict = it.is_conflict_range(); + bool end_cleared = it.is_cleared_range(); + bool end_unreadable = it.is_unreadable(); + + it.tree.clear(); + + PTreeImpl::remove(writes, + ver, + ExtStringRef(keys.begin, !insert_begin ? 1 : 0), + ExtStringRef(keys.end, end_coalesce_unmodified ? 1 : 0)); + + if (insert_begin) + PTreeImpl::insert(writes, ver, WriteMapEntry(keys.begin, OperationStack(), false, false, false, true, true)); + + if (insert_end) + PTreeImpl::insert( + writes, + ver, + WriteMapEntry( + keys.end, OperationStack(), end_cleared, end_conflict, end_conflict, end_unreadable, end_unreadable)); +} + +void WriteMap::addConflictRange(KeyRangeRef keys) { + writeMapEmpty = false; + auto& it = scratch_iterator; + it.reset(writes, ver); + it.skip(keys.begin); + + std::vector removals; + std::vector insertions; + + if (!it.entry().following_keys_conflict || !it.entry().is_conflict) { + if (it.keyAtBegin() && it.beginKey() == keys.begin) { + removals.push_back(keys.begin); + } + insertions.push_back(WriteMapEntry(keys.begin, + it.is_operation() ? OperationStack(it.op()) : OperationStack(), + it.entry().following_keys_cleared, + true, + true, + it.entry().following_keys_unreadable, + it.entry().is_unreadable)); + } + + while (it.endKey() < keys.end) { + ++it; + if (it.keyAtBegin() && (!it.entry().following_keys_conflict || !it.entry().is_conflict)) { + WriteMapEntry e(it.entry()); + e.following_keys_conflict = true; + e.is_conflict = true; + removals.push_back(e.key); + insertions.push_back(std::move(e)); + } + } + + ASSERT(it.beginKey() != keys.end); + if (!it.entry().following_keys_conflict || !it.entry().is_conflict) { + bool isCleared = it.entry().following_keys_cleared; + bool isUnreadable = it.entry().is_unreadable; + bool followingUnreadable = it.entry().following_keys_unreadable; + ++it; + + if (!it.keyAtBegin() || it.beginKey() != keys.end) { + insertions.push_back( + WriteMapEntry(keys.end, OperationStack(), isCleared, false, false, followingUnreadable, isUnreadable)); + } + } + + it.tree.clear(); + + // SOMEDAY: optimize this code by having a PTree removal/insertion that takes and returns an iterator + for (int i = 0; i < removals.size(); i++) { + PTreeImpl::remove( + writes, + ver, + removals[i]); // FIXME: Make PTreeImpl::insert do this automatically (see also VersionedMap.h FIXME) + } + + for (int i = 0; i < insertions.size(); i++) { + PTreeImpl::insert(writes, ver, std::move(insertions[i])); + } +} + +WriteMap::iterator::SEGMENT_TYPE WriteMap::iterator::type() const { + if (offset) + return entry().following_keys_cleared ? CLEARED_RANGE : UNMODIFIED_RANGE; + else + return entry().stack.isDependent() ? DEPENDENT_WRITE : INDEPENDENT_WRITE; +} + +WriteMap::iterator& WriteMap::iterator::operator++() { + if (!offset && !equalsKeyAfter(entry().key, nextEntry().key)) { + offset = true; + } else { + beginLen = endLen; + finger.resize(beginLen); + endLen = PTreeImpl::halfNext(at, finger); + offset = !entry().stack.size(); + } + return *this; +} +WriteMap::iterator& WriteMap::iterator::operator--() { + if (offset && entry().stack.size()) { + offset = false; + } else { + endLen = beginLen; + finger.resize(endLen); + beginLen = PTreeImpl::halfPrevious(at, finger); + offset = !entry().stack.size() || !equalsKeyAfter(entry().key, nextEntry().key); + } + return *this; +} + +void WriteMap::iterator::skip( + KeyRef key) { // Changes *this to the segment containing key (so that beginKey()<=key && key < endKey()) + finger.clear(); + + if (key == allKeys.end) + PTreeImpl::last(tree, at, finger); + else + PTreeImpl::upper_bound(tree, at, key, finger); + endLen = finger.size(); + beginLen = PTreeImpl::halfPrevious(at, finger); + + offset = !entry().stack.size() || (entry().key != key); +} + +void WriteMap::iterator::reset(Tree const& tree, Version ver) { + this->tree = tree; + this->at = ver; + this->finger.clear(); + beginLen = endLen = 0; + offset = false; +} + +RYWMutation WriteMap::coalesce(RYWMutation existingEntry, RYWMutation newEntry, Arena& arena) { + ASSERT(newEntry.value.present()); + + if (newEntry.type == MutationRef::SetValue || newEntry.type == MutationRef::SetVersionstampedValue) { + // independent mutations + return newEntry; + } else if (newEntry.type == MutationRef::AddValue) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doLittleEndianAdd(existingEntry.value, newEntry.value.get(), arena), + MutationRef::SetValue); + case MutationRef::AddValue: + return RYWMutation(doLittleEndianAdd(existingEntry.value, newEntry.value.get(), arena), + MutationRef::AddValue); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::CompareAndClear) { + switch (existingEntry.type) { + case MutationRef::SetValue: + if (doCompareAndClear(existingEntry.value, newEntry.value.get(), arena).present()) { + return existingEntry; + } else { + return RYWMutation(Optional(), MutationRef::SetValue); + } + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::AppendIfFits) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doAppendIfFits(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::AppendIfFits: + return RYWMutation(doAppendIfFits(existingEntry.value, newEntry.value.get(), arena), + MutationRef::AppendIfFits); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::And) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doAnd(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::And: + return RYWMutation(doAnd(existingEntry.value, newEntry.value.get(), arena), MutationRef::And); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::Or) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doOr(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::Or: + return RYWMutation(doOr(existingEntry.value, newEntry.value.get(), arena), MutationRef::Or); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::Xor) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doXor(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::Xor: + return RYWMutation(doXor(existingEntry.value, newEntry.value.get(), arena), MutationRef::Xor); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::Max) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::Max: + return RYWMutation(doMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::Max); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::Min) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::Min: + return RYWMutation(doMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::Min); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::ByteMin) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doByteMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::ByteMin: + return RYWMutation(doByteMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::ByteMin); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::ByteMax) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doByteMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::ByteMax: + return RYWMutation(doByteMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::ByteMax); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::MinV2) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doMinV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::MinV2: + return RYWMutation(doMinV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::MinV2); + default: + throw operation_failed(); + } + } else if (newEntry.type == MutationRef::AndV2) { + switch (existingEntry.type) { + case MutationRef::SetValue: + return RYWMutation(doAndV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); + case MutationRef::AndV2: + return RYWMutation(doAndV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::AndV2); + default: + throw operation_failed(); + } + } else + throw operation_failed(); +} + +void WriteMap::coalesceOver(OperationStack& stack, RYWMutation newEntry, Arena& arena) { + RYWMutation existingEntry = stack.top(); + if (existingEntry.type == newEntry.type && newEntry.type != MutationRef::CompareAndClear) { + if (isNonAssociativeOp(existingEntry.type) && existingEntry.value.present() && + existingEntry.value.get().size() != newEntry.value.get().size()) { + stack.push(newEntry); + } else { + stack.poppush(coalesce(existingEntry, newEntry, arena)); + } + } else { + if (isAtomicOp(newEntry.type) && isAtomicOp(existingEntry.type)) { + stack.push(newEntry); + } else { + stack.poppush(coalesce(existingEntry, newEntry, arena)); + } + } +} + +RYWMutation WriteMap::coalesceUnder(OperationStack const& stack, Optional const& value, Arena& arena) { + if (!stack.isDependent() && stack.size() == 1) + return stack.at(0); + + RYWMutation currentEntry = RYWMutation(value, MutationRef::SetValue); + for (int i = 0; i < stack.size(); ++i) { + currentEntry = coalesce(currentEntry, stack.at(i), arena); + } + + return currentEntry; +} + +void WriteMap::dump() { + iterator it(this); + it.skip(allKeys.begin); + while (it.beginKey() < allKeys.end) { + TraceEvent("WriteMapDump") + .detail("Begin", it.beginKey().toStandaloneStringRef()) + .detail("End", it.endKey()) + .detail("Cleared", it.is_cleared_range()) + .detail("Conflicted", it.is_conflict_range()) + .detail("Operation", it.is_operation()) + .detail("Unmodified", it.is_unmodified_range()) + .detail("Independent", it.is_operation() && it.is_independent()) + .detail("StackSize", it.is_operation() ? it.op().size() : 0); + ++it; + } +} + +void WriteMap::clearNoConflict(KeyRangeRef keys) { + auto& it = scratch_iterator; + it.reset(writes, ver); + + // Find all write conflict ranges within the cleared range + it.skip(keys.begin); + + bool insert_begin = !it.is_cleared_range() || it.is_unreadable(); + + bool lastConflicted = it.is_conflict_range(); + + bool conflicted = lastConflicted; + std::vector conflict_ranges; + + if (insert_begin) { + conflict_ranges.push_back(keys.begin); + } else { + conflicted = !conflicted; + } + + while (it.endKey() < keys.end) { + ++it; + if (lastConflicted != it.is_conflict_range()) { + conflict_ranges.push_back(it.beginKey()); + lastConflicted = it.is_conflict_range(); + } + } + + if (it.endKey() == keys.end) + ++it; + + ASSERT(it.beginKey() <= keys.end && keys.end < it.endKey()); + + bool insert_end = + ((it.is_unmodified_range() || it.is_unreadable()) && (!it.keyAtBegin() || it.beginKey() != keys.end)) || + (it.entry().is_conflict && !it.entry().following_keys_conflict && it.beginKey() == keys.end && + !it.keyAtBegin()); + bool end_cleared = it.is_cleared_range(); + bool end_coalesce_clear = it.is_cleared_range() && it.beginKey() == keys.end && + it.is_conflict_range() == lastConflicted && !it.is_unreadable(); + bool end_conflict = it.is_conflict_range(); + bool end_unreadable = it.is_unreadable(); + + TEST(it.is_conflict_range() != lastConflicted); // not last conflicted + + it.tree.clear(); + + PTreeImpl::remove(writes, + ver, + ExtStringRef(keys.begin, !insert_begin ? 1 : 0), + ExtStringRef(keys.end, end_coalesce_clear ? 1 : 0)); + + for (int i = 0; i < conflict_ranges.size(); i++) { + PTreeImpl::insert( + writes, + ver, + WriteMapEntry( + conflict_ranges[i].toArenaOrRef(*arena), OperationStack(), true, conflicted, conflicted, false, false)); + conflicted = !conflicted; + } + + ASSERT(conflicted != lastConflicted); + + if (insert_end) + PTreeImpl::insert( + writes, + ver, + WriteMapEntry( + keys.end, OperationStack(), end_cleared, end_conflict, end_conflict, end_unreadable, end_unreadable)); +} diff --git a/fdbclient/WriteMap.h b/fdbclient/WriteMap.h index 33fb6aee37..f4d2a43511 100644 --- a/fdbclient/WriteMap.h +++ b/fdbclient/WriteMap.h @@ -51,36 +51,11 @@ public: defaultConstructed = false; singletonOperation = initialEntry; } - void reset(RYWMutation initialEntry) { - defaultConstructed = false; - singletonOperation = initialEntry; - optionalOperations = Optional>(); - } - void poppush(RYWMutation entry) { - if (hasVector()) { - optionalOperations.get().pop_back(); - optionalOperations.get().push_back(entry); - } else - singletonOperation = entry; - } - void push(RYWMutation entry) { - if (defaultConstructed) { - singletonOperation = entry; - defaultConstructed = false; - } else if (hasVector()) - optionalOperations.get().push_back(entry); - else { - optionalOperations = std::vector(); - optionalOperations.get().push_back(entry); - } - } - bool isDependent() const { - if (!size()) - return false; - return singletonOperation.type != MutationRef::SetValue && singletonOperation.type != MutationRef::ClearRange && - singletonOperation.type != MutationRef::SetVersionstampedValue && - singletonOperation.type != MutationRef::SetVersionstampedKey; - } + void reset(RYWMutation initialEntry); + void poppush(RYWMutation entry); + void push(RYWMutation entry); + bool isDependent() const; + const RYWMutation& top() const { return hasVector() ? optionalOperations.get().back() : singletonOperation; } RYWMutation& operator[](int n) { return (n == 0) ? singletonOperation : optionalOperations.get()[n - 1]; } @@ -88,26 +63,7 @@ public: int size() const { return defaultConstructed ? 0 : hasVector() ? optionalOperations.get().size() + 1 : 1; } - bool operator==(const OperationStack& r) const { - if (size() != r.size()) - return false; - - if (size() == 0) - return true; - - if (singletonOperation != r.singletonOperation) - return false; - - if (size() == 1) - return true; - - for (int i = 0; i < optionalOperations.get().size(); i++) { - if (optionalOperations.get()[i] != r.optionalOperations.get()[i]) - return false; - } - - return true; - } + bool operator==(const OperationStack& r) const; }; struct WriteMapEntry { @@ -179,246 +135,17 @@ public: WriteMap(WriteMap&& r) noexcept : arena(r.arena), writeMapEmpty(r.writeMapEmpty), writes(std::move(r.writes)), ver(r.ver), scratch_iterator(std::move(r.scratch_iterator)) {} - WriteMap& operator=(WriteMap&& r) noexcept { - writeMapEmpty = r.writeMapEmpty; - writes = std::move(r.writes); - ver = r.ver; - scratch_iterator = std::move(r.scratch_iterator); - arena = r.arena; - return *this; - } + + WriteMap& operator=(WriteMap&& r) noexcept; // a write with addConflict false on top of an existing write with a conflict range will not remove the conflict - void mutate(KeyRef key, MutationRef::Type operation, ValueRef param, bool addConflict) { - writeMapEmpty = false; - auto& it = scratch_iterator; - it.reset(writes, ver); - it.skip(key); + void mutate(KeyRef key, MutationRef::Type operation, ValueRef param, bool addConflict); - bool is_cleared = it.entry().following_keys_cleared; - bool following_conflict = it.entry().following_keys_conflict; - bool is_conflict = addConflict || it.is_conflict_range(); - bool following_unreadable = it.entry().following_keys_unreadable; - bool is_unreadable = it.is_unreadable() || operation == MutationRef::SetVersionstampedValue || - operation == MutationRef::SetVersionstampedKey; - bool is_dependent = operation != MutationRef::SetValue && operation != MutationRef::SetVersionstampedValue && - operation != MutationRef::SetVersionstampedKey; + void clear(KeyRangeRef keys, bool addConflict); - if (it.entry().key != key) { - if (it.is_cleared_range() && is_dependent) { - it.tree.clear(); - OperationStack op(RYWMutation(Optional(), MutationRef::SetValue)); - coalesceOver(op, RYWMutation(param, operation), *arena); - PTreeImpl::insert(writes, - ver, - WriteMapEntry(key, - std::move(op), - true, - following_conflict, - is_conflict, - following_unreadable, - is_unreadable)); - } else { - it.tree.clear(); - PTreeImpl::insert(writes, - ver, - WriteMapEntry(key, - OperationStack(RYWMutation(param, operation)), - is_cleared, - following_conflict, - is_conflict, - following_unreadable, - is_unreadable)); - } - } else { - if (!it.is_unreadable() && - (operation == MutationRef::SetValue || operation == MutationRef::SetVersionstampedValue)) { - it.tree.clear(); - PTreeImpl::remove(writes, ver, key); - PTreeImpl::insert(writes, - ver, - WriteMapEntry(key, - OperationStack(RYWMutation(param, operation)), - is_cleared, - following_conflict, - is_conflict, - following_unreadable, - is_unreadable)); - } else { - WriteMapEntry e(it.entry()); - e.is_conflict = is_conflict; - e.is_unreadable = is_unreadable; - if (e.stack.size() == 0 && it.is_cleared_range() && is_dependent) { - e.stack.push(RYWMutation(Optional(), MutationRef::SetValue)); - coalesceOver(e.stack, RYWMutation(param, operation), *arena); - } else if (!is_unreadable && e.stack.size() > 0) - coalesceOver(e.stack, RYWMutation(param, operation), *arena); - else - e.stack.push(RYWMutation(param, operation)); + void addUnmodifiedAndUnreadableRange(KeyRangeRef keys); - it.tree.clear(); - PTreeImpl::remove( - writes, - ver, - e.key); // FIXME: Make PTreeImpl::insert do this automatically (see also VersionedMap.h FIXME) - PTreeImpl::insert(writes, ver, std::move(e)); - } - } - } - - void clear(KeyRangeRef keys, bool addConflict) { - writeMapEmpty = false; - if (!addConflict) { - clearNoConflict(keys); - return; - } - - auto& it = scratch_iterator; - it.reset(writes, ver); - it.skip(keys.begin); - - bool insert_begin = !it.is_cleared_range() || !it.is_conflict_range() || it.is_unreadable(); - - if (it.endKey() == keys.end) { - ++it; - } else if (it.endKey() < keys.end) { - it.skip(keys.end); - } - - bool insert_end = (it.is_unmodified_range() || !it.is_conflict_range() || it.is_unreadable()) && - (!it.keyAtBegin() || it.beginKey() != keys.end); - bool end_coalesce_clear = - it.is_cleared_range() && it.beginKey() == keys.end && it.is_conflict_range() && !it.is_unreadable(); - bool end_conflict = it.is_conflict_range(); - bool end_cleared = it.is_cleared_range(); - bool end_unreadable = it.is_unreadable(); - - it.tree.clear(); - - PTreeImpl::remove(writes, - ver, - ExtStringRef(keys.begin, !insert_begin ? 1 : 0), - ExtStringRef(keys.end, end_coalesce_clear ? 1 : 0)); - - if (insert_begin) - PTreeImpl::insert(writes, ver, WriteMapEntry(keys.begin, OperationStack(), true, true, true, false, false)); - - if (insert_end) - PTreeImpl::insert(writes, - ver, - WriteMapEntry(keys.end, - OperationStack(), - end_cleared, - end_conflict, - end_conflict, - end_unreadable, - end_unreadable)); - } - - void addUnmodifiedAndUnreadableRange(KeyRangeRef keys) { - auto& it = scratch_iterator; - it.reset(writes, ver); - it.skip(keys.begin); - - bool insert_begin = !it.is_unmodified_range() || it.is_conflict_range() || !it.is_unreadable(); - - if (it.endKey() == keys.end) { - ++it; - } else if (it.endKey() < keys.end) { - it.skip(keys.end); - } - - bool insert_end = (it.is_cleared_range() || it.is_conflict_range() || !it.is_unreadable()) && - (!it.keyAtBegin() || it.beginKey() != keys.end); - bool end_coalesce_unmodified = - it.is_unmodified_range() && it.beginKey() == keys.end && !it.is_conflict_range() && it.is_unreadable(); - bool end_conflict = it.is_conflict_range(); - bool end_cleared = it.is_cleared_range(); - bool end_unreadable = it.is_unreadable(); - - it.tree.clear(); - - PTreeImpl::remove(writes, - ver, - ExtStringRef(keys.begin, !insert_begin ? 1 : 0), - ExtStringRef(keys.end, end_coalesce_unmodified ? 1 : 0)); - - if (insert_begin) - PTreeImpl::insert( - writes, ver, WriteMapEntry(keys.begin, OperationStack(), false, false, false, true, true)); - - if (insert_end) - PTreeImpl::insert(writes, - ver, - WriteMapEntry(keys.end, - OperationStack(), - end_cleared, - end_conflict, - end_conflict, - end_unreadable, - end_unreadable)); - } - - void addConflictRange(KeyRangeRef keys) { - writeMapEmpty = false; - auto& it = scratch_iterator; - it.reset(writes, ver); - it.skip(keys.begin); - - std::vector removals; - std::vector insertions; - - if (!it.entry().following_keys_conflict || !it.entry().is_conflict) { - if (it.keyAtBegin() && it.beginKey() == keys.begin) { - removals.push_back(keys.begin); - } - insertions.push_back(WriteMapEntry(keys.begin, - it.is_operation() ? OperationStack(it.op()) : OperationStack(), - it.entry().following_keys_cleared, - true, - true, - it.entry().following_keys_unreadable, - it.entry().is_unreadable)); - } - - while (it.endKey() < keys.end) { - ++it; - if (it.keyAtBegin() && (!it.entry().following_keys_conflict || !it.entry().is_conflict)) { - WriteMapEntry e(it.entry()); - e.following_keys_conflict = true; - e.is_conflict = true; - removals.push_back(e.key); - insertions.push_back(std::move(e)); - } - } - - ASSERT(it.beginKey() != keys.end); - if (!it.entry().following_keys_conflict || !it.entry().is_conflict) { - bool isCleared = it.entry().following_keys_cleared; - bool isUnreadable = it.entry().is_unreadable; - bool followingUnreadable = it.entry().following_keys_unreadable; - ++it; - - if (!it.keyAtBegin() || it.beginKey() != keys.end) { - insertions.push_back(WriteMapEntry( - keys.end, OperationStack(), isCleared, false, false, followingUnreadable, isUnreadable)); - } - } - - it.tree.clear(); - - // SOMEDAY: optimize this code by having a PTree removal/insertion that takes and returns an iterator - for (int i = 0; i < removals.size(); i++) { - PTreeImpl::remove( - writes, - ver, - removals[i]); // FIXME: Make PTreeImpl::insert do this automatically (see also VersionedMap.h FIXME) - } - - for (int i = 0; i < insertions.size(); i++) { - PTreeImpl::insert(writes, ver, std::move(insertions[i])); - } - } + void addConflictRange(KeyRangeRef keys); struct iterator { // Iterates over three types of segments: unmodified ranges, cleared ranges, and modified keys. @@ -432,12 +159,7 @@ public: enum SEGMENT_TYPE { UNMODIFIED_RANGE, CLEARED_RANGE, INDEPENDENT_WRITE, DEPENDENT_WRITE }; - SEGMENT_TYPE type() const { - if (offset) - return entry().following_keys_cleared ? CLEARED_RANGE : UNMODIFIED_RANGE; - else - return entry().stack.isDependent() ? DEPENDENT_WRITE : INDEPENDENT_WRITE; - } + SEGMENT_TYPE type() const; bool is_cleared_range() const { return offset && entry().following_keys_cleared; } bool is_unmodified_range() const { return offset && !entry().following_keys_cleared; } bool is_operation() const { return !offset; } @@ -457,55 +179,16 @@ public: return entry().stack; } - iterator& operator++() { - if (!offset && !equalsKeyAfter(entry().key, nextEntry().key)) { - offset = true; - } else { - beginLen = endLen; - finger.resize(beginLen); - endLen = PTreeImpl::halfNext(at, finger); - offset = !entry().stack.size(); - } - return *this; - } - iterator& operator--() { - if (offset && entry().stack.size()) { - offset = false; - } else { - endLen = beginLen; - finger.resize(endLen); - beginLen = PTreeImpl::halfPrevious(at, finger); - offset = !entry().stack.size() || !equalsKeyAfter(entry().key, nextEntry().key); - } - return *this; - } + iterator& operator++(); + iterator& operator--(); bool operator==(const iterator& r) const { return offset == r.offset && beginLen == r.beginLen && finger[beginLen - 1] == r.finger[beginLen - 1]; } - - void skip( - KeyRef key) { // Changes *this to the segment containing key (so that beginKey()<=key && key < endKey()) - finger.clear(); - - if (key == allKeys.end) - PTreeImpl::last(tree, at, finger); - else - PTreeImpl::upper_bound(tree, at, key, finger); - endLen = finger.size(); - beginLen = PTreeImpl::halfPrevious(at, finger); - - offset = !entry().stack.size() || (entry().key != key); - } + void skip(KeyRef key); private: friend class WriteMap; - void reset(Tree const& tree, Version ver) { - this->tree = tree; - this->at = ver; - this->finger.clear(); - beginLen = endLen = 0; - offset = false; - } + void reset(Tree const& tree, Version ver); WriteMapEntry const& entry() const { return finger[beginLen - 1]->data; } WriteMapEntry const& nextEntry() const { return finger[endLen - 1]->data; } @@ -521,266 +204,25 @@ public: bool empty() const { return writeMapEmpty; } - static RYWMutation coalesce(RYWMutation existingEntry, RYWMutation newEntry, Arena& arena) { - ASSERT(newEntry.value.present()); - - if (newEntry.type == MutationRef::SetValue || newEntry.type == MutationRef::SetVersionstampedValue) { - // independent mutations - return newEntry; - } else if (newEntry.type == MutationRef::AddValue) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doLittleEndianAdd(existingEntry.value, newEntry.value.get(), arena), - MutationRef::SetValue); - case MutationRef::AddValue: - return RYWMutation(doLittleEndianAdd(existingEntry.value, newEntry.value.get(), arena), - MutationRef::AddValue); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::CompareAndClear) { - switch (existingEntry.type) { - case MutationRef::SetValue: - if (doCompareAndClear(existingEntry.value, newEntry.value.get(), arena).present()) { - return existingEntry; - } else { - return RYWMutation(Optional(), MutationRef::SetValue); - } - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::AppendIfFits) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doAppendIfFits(existingEntry.value, newEntry.value.get(), arena), - MutationRef::SetValue); - case MutationRef::AppendIfFits: - return RYWMutation(doAppendIfFits(existingEntry.value, newEntry.value.get(), arena), - MutationRef::AppendIfFits); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::And) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doAnd(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::And: - return RYWMutation(doAnd(existingEntry.value, newEntry.value.get(), arena), MutationRef::And); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::Or) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doOr(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::Or: - return RYWMutation(doOr(existingEntry.value, newEntry.value.get(), arena), MutationRef::Or); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::Xor) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doXor(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::Xor: - return RYWMutation(doXor(existingEntry.value, newEntry.value.get(), arena), MutationRef::Xor); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::Max) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::Max: - return RYWMutation(doMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::Max); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::Min) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::Min: - return RYWMutation(doMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::Min); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::ByteMin) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doByteMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::ByteMin: - return RYWMutation(doByteMin(existingEntry.value, newEntry.value.get(), arena), MutationRef::ByteMin); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::ByteMax) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doByteMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::ByteMax: - return RYWMutation(doByteMax(existingEntry.value, newEntry.value.get(), arena), MutationRef::ByteMax); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::MinV2) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doMinV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::MinV2: - return RYWMutation(doMinV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::MinV2); - default: - throw operation_failed(); - } - } else if (newEntry.type == MutationRef::AndV2) { - switch (existingEntry.type) { - case MutationRef::SetValue: - return RYWMutation(doAndV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::SetValue); - case MutationRef::AndV2: - return RYWMutation(doAndV2(existingEntry.value, newEntry.value.get(), arena), MutationRef::AndV2); - default: - throw operation_failed(); - } - } else - throw operation_failed(); - } - - static void coalesceOver(OperationStack& stack, RYWMutation newEntry, Arena& arena) { - RYWMutation existingEntry = stack.top(); - if (existingEntry.type == newEntry.type && newEntry.type != MutationRef::CompareAndClear) { - if (isNonAssociativeOp(existingEntry.type) && existingEntry.value.present() && - existingEntry.value.get().size() != newEntry.value.get().size()) { - stack.push(newEntry); - } else { - stack.poppush(coalesce(existingEntry, newEntry, arena)); - } - } else { - if (isAtomicOp(newEntry.type) && isAtomicOp(existingEntry.type)) { - stack.push(newEntry); - } else { - stack.poppush(coalesce(existingEntry, newEntry, arena)); - } - } - } - - static RYWMutation coalesceUnder(OperationStack const& stack, Optional const& value, Arena& arena) { - if (!stack.isDependent() && stack.size() == 1) - return stack.at(0); - - RYWMutation currentEntry = RYWMutation(value, MutationRef::SetValue); - for (int i = 0; i < stack.size(); ++i) { - currentEntry = coalesce(currentEntry, stack.at(i), arena); - } - - return currentEntry; - } + static RYWMutation coalesce(RYWMutation existingEntry, RYWMutation newEntry, Arena& arena); + static void coalesceOver(OperationStack& stack, RYWMutation newEntry, Arena& arena); + static RYWMutation coalesceUnder(OperationStack const& stack, Optional const& value, Arena& arena); private: friend class ReadYourWritesTransaction; Arena* arena; bool writeMapEmpty; Tree writes; - Version ver; // an internal version number for the tree - no connection to database versions! Currently this is - // incremented after reads, so that consecutive writes have the same version and those separated by - // reads have different versions. + // an internal version number for the tree - no connection to database versions! Currently this is + // incremented after reads, so that consecutive writes have the same version and those separated by + // reads have different versions. + Version ver; iterator scratch_iterator; // Avoid unnecessary memory allocation in write operations - void dump() { - iterator it(this); - it.skip(allKeys.begin); - while (it.beginKey() < allKeys.end) { - TraceEvent("WriteMapDump") - .detail("Begin", it.beginKey().toStandaloneStringRef()) - .detail("End", it.endKey()) - .detail("Cleared", it.is_cleared_range()) - .detail("Conflicted", it.is_conflict_range()) - .detail("Operation", it.is_operation()) - .detail("Unmodified", it.is_unmodified_range()) - .detail("Independent", it.is_operation() && it.is_independent()) - .detail("StackSize", it.is_operation() ? it.op().size() : 0); - ++it; - } - } + void dump(); // SOMEDAY: clearNoConflict replaces cleared sets with two map entries for everyone one item cleared - void clearNoConflict(KeyRangeRef keys) { - auto& it = scratch_iterator; - it.reset(writes, ver); - - // Find all write conflict ranges within the cleared range - it.skip(keys.begin); - - bool insert_begin = !it.is_cleared_range() || it.is_unreadable(); - - bool lastConflicted = it.is_conflict_range(); - - bool conflicted = lastConflicted; - std::vector conflict_ranges; - - if (insert_begin) { - conflict_ranges.push_back(keys.begin); - } else { - conflicted = !conflicted; - } - - while (it.endKey() < keys.end) { - ++it; - if (lastConflicted != it.is_conflict_range()) { - conflict_ranges.push_back(it.beginKey()); - lastConflicted = it.is_conflict_range(); - } - } - - if (it.endKey() == keys.end) - ++it; - - ASSERT(it.beginKey() <= keys.end && keys.end < it.endKey()); - - bool insert_end = - ((it.is_unmodified_range() || it.is_unreadable()) && (!it.keyAtBegin() || it.beginKey() != keys.end)) || - (it.entry().is_conflict && !it.entry().following_keys_conflict && it.beginKey() == keys.end && - !it.keyAtBegin()); - bool end_cleared = it.is_cleared_range(); - bool end_coalesce_clear = it.is_cleared_range() && it.beginKey() == keys.end && - it.is_conflict_range() == lastConflicted && !it.is_unreadable(); - bool end_conflict = it.is_conflict_range(); - bool end_unreadable = it.is_unreadable(); - - TEST(it.is_conflict_range() != lastConflicted); // not last conflicted - - it.tree.clear(); - - PTreeImpl::remove(writes, - ver, - ExtStringRef(keys.begin, !insert_begin ? 1 : 0), - ExtStringRef(keys.end, end_coalesce_clear ? 1 : 0)); - - for (int i = 0; i < conflict_ranges.size(); i++) { - PTreeImpl::insert(writes, - ver, - WriteMapEntry(conflict_ranges[i].toArenaOrRef(*arena), - OperationStack(), - true, - conflicted, - conflicted, - false, - false)); - conflicted = !conflicted; - } - - ASSERT(conflicted != lastConflicted); - - if (insert_end) - PTreeImpl::insert(writes, - ver, - WriteMapEntry(keys.end, - OperationStack(), - end_cleared, - end_conflict, - end_conflict, - end_unreadable, - end_unreadable)); - } + void clearNoConflict(KeyRangeRef keys); }; /*