Adding ART-based MutationBuffer

fdbserver/ArtMutationBuffer.h

@@ -0,0 +1,183 @@
+#ifndef ART_MUTATION_BUFFER
+#define ART_MUTATION_BUFFER
+
+#include "art.h"
+#include "flow/Arena.h"
+
+struct MutationBuffer {
+
+private:
+    Arena arena;
+    art_tree *mutations;
+
+public:
+    struct iterator : std::map<KeyRef, RangeMutation>::iterator {
+
+        art_iterator artIterator;
+
+        iterator() = default;
+
+        iterator(art_iterator i) {
+            artIterator = i;
+        }
+
+        iterator(const iterator &i) {
+            artIterator = i.artIterator;
+        }
+
+        const KeyRef &key() {
+            return artIterator.key();
+        }
+
+        RangeMutation &mutation() {
+            return (RangeMutation & )(*((RangeMutation *) artIterator.value()));
+        }
+
+        void **value_ptr() {
+            return artIterator.value_ptr();
+        }
+
+        bool operator==(const iterator &other) const {
+            return (artIterator) == (other.artIterator);
+        }
+
+        iterator &operator++() {
+            ++(artIterator);
+            return *this;
+        }
+
+        iterator &operator--() {
+            --(artIterator);
+            return *this;
+        }
+    };
+
+    struct const_iterator : std::map<KeyRef, RangeMutation>::const_iterator {
+        art_iterator artIterator;
+
+        const_iterator() = default;
+
+        const_iterator(art_iterator i) {
+            artIterator = i;
+        }
+
+        const_iterator(const const_iterator &i) {
+            artIterator = i.artIterator;
+        }
+
+        const KeyRef &key() {
+            return artIterator.key();
+        }
+
+        const RangeMutation &mutation() {
+            return (const RangeMutation &) (*((RangeMutation *) artIterator.value()));
+        }
+
+        bool operator==(const const_iterator &other) const {
+            return (artIterator) == (other.artIterator);
+        }
+
+        const_iterator &operator++() {
+            ++(artIterator);
+            return *this;
+        }
+
+        const_iterator &operator--() {
+            --(artIterator);
+            return *this;
+        }
+    };
+
+    MutationBuffer() {
+        printf("NEW MB\n");
+        mutations = new(arena) art_tree(arena);
+        // Create range representing the entire keyspace.  This reduces edge cases to applying mutations
+        // because now all existing keys are within some range in the mutation map.
+
+        mutations->insert(dbBegin.key, new(arena)RangeMutation());
+
+        // Setting the dbEnd key to be cleared prevents having to treat a range clear to dbEnd as a special
+        // case in order to avoid traversing down the rightmost edge of the tree.
+        RangeMutation *rm = new(arena) RangeMutation();
+        rm->clearBoundary();
+
+        mutations->insert(dbEnd.key, rm);
+    }
+
+    // Return a T constructed in arena
+    template<typename T>
+    T copyToArena(const T &object) {
+        return T(arena, object);
+    }
+
+    const_iterator upper_bound(const KeyRef &k) const {
+        return const_iterator(mutations->upper_bound(k));
+    }
+
+    const_iterator lower_bound(const KeyRef &k) const {
+        return const_iterator(mutations->lower_bound(k));
+    }
+
+    /*
+     * For now, let's not deal with conversions. Just create all combinations.
+     */
+    // erase [begin, end) from the mutation map
+    void erase(const iterator &begin, const iterator &end) {
+        art_iterator it = begin.artIterator;
+        art_iterator next = it;
+        ++next;
+        while (end.artIterator != it) {
+            mutations->erase(it);
+            it = next;
+            ++next;
+        }
+    }
+
+    void erase(const const_iterator &begin, const iterator &end) {
+        art_iterator it = begin.artIterator;
+        art_iterator next = it;
+        ++next;
+        while (end.artIterator != it) {
+            mutations->erase(it);
+            it = next;
+            ++next;
+        }
+    }
+
+    void erase(const iterator &begin, const const_iterator &end) {
+        art_iterator it = begin.artIterator;
+        art_iterator next = it;
+        ++next;
+        while (end.artIterator != it) {
+            mutations->erase(it);
+            it = next;
+            ++next;
+        }
+    }
+
+    // Find or create a mutation buffer boundary for bound and return an iterator to it
+    iterator insert(KeyRef boundary) {
+        int already_present = 0;
+        iterator ib = iterator(mutations->insert_if_absent(boundary, nullptr, &already_present));
+        if (already_present) {
+            return ib;
+        } else {
+            *(ib.value_ptr()) = new(arena) RangeMutation();
+        }
+        iterator iPrevious = ib;
+        --iPrevious;
+        if (iPrevious.mutation().clearAfterBoundary) {
+            ib.mutation().clearAll();
+        }
+        return ib;
+    }
+
+
+};
+
+#endif //ART_MUTATION_BUFFER
+
+
+
+
+

fdbserver/VersionedBTree.actor.cpp

@@ -3288,6 +3288,11 @@ private:
 	};
 
 public:
+#define USE_ART_MUTATION_BUFFER 1
+
+#if USE_ART_MUTATION_BUFFER
+#include "ArtMutationBuffer.h"
+#else
 	struct MutationBuffer {
 		MutationBuffer() {
 			// Create range representing the entire keyspace.  This reduces edge cases to applying mutations
@@ -3385,6 +3390,7 @@ public:
 		}
 
 	};
+#endif //USE_ART_MUTATION_BUFFER
 
 private:
 	/* Mutation Buffer Overview
@@ -4996,6 +5002,10 @@ private:
 
 };
 
+#if USE_ART_MUTATION_BUFFER
+#include "art_impl.h"
+#endif //USE_ART_MUTATION_BUFFER
+
 RedwoodRecordRef VersionedBTree::dbBegin(StringRef(), 0);
 RedwoodRecordRef VersionedBTree::dbEnd(LiteralStringRef("\xff\xff\xff\xff\xff"));
 VersionedBTree::Counts VersionedBTree::counts;

fdbserver/art.h

@@ -0,0 +1,426 @@
+/*
+ * art.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/*
+ *  Original copyright notice
+*/
+/*
+Copyright (c) 2012, Armon Dadgar
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+       modification, are permitted provided that the following conditions are met:
+* Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+* Neither the name of the organization nor the
+names of its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+       ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL ARMON DADGAR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+       ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+       (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include <stdint.h>
+#include "flow/Arena.h"
+
+#ifndef ART_H
+#define ART_H
+//#define DEBUG
+
+#ifdef DEBUG
+#define ARTRACE(format, arg...)\
+    do {                        \
+        fprintf(stdout,  "%s:%d:" format , __FUNCTION__, __LINE__,##arg); fflush(stdout);\
+    } while (0)
+
+#define ART_PRINT(format, arg...) fprintf(stdout,  "" format , ##arg); fflush(stdout);
+
+#else //NO DEBUG
+
+
+#define ART_PRINT(format, arg...)
+#endif //DEBUG
+
+
+struct art_iterator;
+
+struct art_tree {
+
+
+#define ART_LEAF 0
+#define ART_NODE4   1
+#define ART_NODE16  2
+#define ART_NODE48  3
+#define ART_NODE256 4
+#define ART_NODE4_KV   5
+#define ART_NODE16_KV  6
+#define ART_NODE48_KV  7
+#define ART_NODE256_KV 8
+#define ART_MIN_LEAF_UNSET 1UL
+#define ART_LEAF_MATCH_KEY 1
+#define ART_LEAF_SMALLER_KEY 2
+#define ART_LEAF_LARGER_KEY 3
+#define ART_I_FOUND 2
+#define ART_I_DEPTH 3
+#define ART_I_BACKTRACK 4
+
+#define ART_NEXT 1
+#define ART_PREV -1
+#define ART_NEITHER 0
+
+
+#define ART_IS_LEAF(x) ( (*((uint8_t*)x) == ART_LEAF))
+
+#define ART_LEAF_RAW(x) ((art_leaf*)(x))
+
+#define ART_LEAF_DISPL(x) fat_leaf_offset[(x)->type]
+#define ART_FAT_NODE_LEAF(node_ptr) (*((art_leaf**) (((char*)(node_ptr))+ART_LEAF_DISPL(node_ptr))))
+
+//In Bytes
+//This is needed so that we can pre-allocate a static stack to perform efficient backtracking in iterative_bound
+#define ART_MAX_KEY_LEN 10000
+
+#define _mm_cmpge_epu8(a, b) \
+        _mm_cmpeq_epi8(_mm_max_epu8(a, b), a)
+
+#define _mm_cmple_epu8(a, b) _mm_cmpge_epu8(b, a)
+
+#define _mm_cmpgt_epu8(a, b) \
+        _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1))
+
+#define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a)
+
+#define ART_MAX_PREFIX_LEN 10
+
+
+/**
+ * This struct is included as part
+ * of all the various node sizes
+ */
+    struct art_node {
+        uint8_t type;
+        uint8_t num_children;
+        uint32_t partial_len;
+        unsigned char partial[ART_MAX_PREFIX_LEN];
+    };
+
+/**
+ * Represents a leaf. These are
+ * of arbitrary size, as they include the key.
+ * Note that the first two fields must be the same as in art_node
+ */
+    struct art_leaf {
+        uint8_t type;
+        void *value;
+        art_leaf *prev = nullptr;
+        art_leaf *next = nullptr;
+        KeyRef key;
+
+        art_leaf(const KeyRef &k, void *v) : type(ART_LEAF), value(v), key(k.begin(), k.size()) {
+        }
+
+        art_leaf(const uint8_t *ptr, int len, void *v) : type(ART_LEAF), value(v), key(ptr, len) {}
+
+    };
+
+/**
+ * Small node with only 4 children
+ * @ddi: Each children is indexed by the (next) char in the prefix
+ */
+    struct art_node4 {
+        art_node n;
+        unsigned char keys[4];
+        art_node *children[4];
+    };
+
+/**
+ * Node with 16 children
+ * @ddi: Each children is indexed by the (next) char in the prefix
+ */
+    struct art_node16 {
+        art_node n;
+        unsigned char keys[16];
+        art_node *children[16];
+    };
+
+/**
+ * Node with 48 children, but
+ * a full 256 byte field.
+ * @ddi: keys[i] is the index in children such that children[i]'s partial key starts with (char)i (48 means no child)
+ */
+    struct art_node48 {
+        art_node n;
+        unsigned char keys[256];
+        art_node *children[48];
+    };
+
+/**
+ * Full node with 256 children
+ * @ddi: children[i] is the children whose partial key that starts with (char)i
+ */
+    struct art_node256 {
+        art_node n;
+        art_node *children[256];
+    };
+
+/**
+ * Small node with only 4 children
+ * @ddi: Each children is indexed by the (next) char in the prefix
+ */
+    struct art_node4_kv {
+        art_node4 n;
+        art_leaf *leaf;
+    };
+
+/**
+ * Node with 16 children
+ * @ddi: Each children is indexed by the (next) char in the prefix
+ */
+    struct art_node16_kv {
+        art_node16 n;
+        art_leaf *leaf;
+    };
+
+/**
+ * Node with 48 children, but
+ * a full 256 byte field.
+ * @ddi: keys[i] is the index in children such that children[i]'s partial key starts with (char)i (48 means no child)
+ */
+    struct art_node48_kv {
+        art_node48 n;
+        art_leaf *leaf;
+    };
+
+/**
+ * Full node with 256 children
+ * @ddi: children[i] is the children whose partial key that starts with (char)i
+ */
+    struct art_node256_kv {
+        art_node256 n;
+        art_leaf *leaf;
+    };
+
+    struct art_children_pair {
+        art_node *child = nullptr;
+        art_node *next = nullptr;
+        art_node *prev = nullptr;
+    };
+
+/**
+ * Main struct, points to root.
+ */
+
+
+private:
+    art_node *root;
+    uint64_t size;
+    Arena *arena;
+    static int fat_leaf_offset[9];
+    static int node_sizes[9];
+
+    static int check_bound_node(art_node *n, const KeyRef &k, int *depth_p, art_leaf **result, bool strict);
+
+    static int art_bound_leaf(art_node *n, const KeyRef &k, int depth, art_leaf **result);
+
+    static art_leaf *minimum(art_node *n);
+
+    static art_leaf *maximum(art_node *n);
+
+    static int leaf_matches_signed(art_leaf *n, const KeyRef &k, int depth);
+
+    static int leaf_matches(const art_leaf *n, const KeyRef &k, int depth);
+
+    static int signed_prefix_mismatch(art_node *n, const KeyRef &k, int depth, art_leaf **min_leaf, bool find_min);
+
+    static int prefix_mismatch(art_node *n, KeyRef &k, int depth, art_leaf **minout);
+
+    static art_leaf *minimum_kv(art_node *n);
+
+
+    static art_node **find_child(art_node *n, unsigned char c);
+
+    static void find_next(art_node *n, unsigned char c, art_node **out);
+
+    static void find_prev(art_node *n, unsigned char c, art_node **out);
+
+    static int art_next_prev(art_node *n, unsigned char c, art_node **out);
+
+    void art_bound_iterative(art_node *n, const KeyRef &k, int depth, art_leaf **result, bool strict);
+
+    static inline int min(int a, int b) {
+        return (a < b) ? a : b;
+    }
+
+    static inline void *SET_LEAF(void *x) {
+        (*((uint8_t *) x) = 0);
+        return x;
+    }
+
+    static int check_prefix(const art_node *n, const KeyRef &k, int depth);
+
+    void recursive_delete_binary(art_node *n, art_node **ref, const KeyRef &k, int depth);
+
+    void remove_child(art_node *n, art_node **ref, unsigned char c, art_node **l, int depth);
+
+    //needs this pointer to do allocation
+    art_node *alloc_node(uint8_t type);
+
+    art_node *alloc_kv_node(uint8_t type);
+
+    art_leaf *make_leaf(const KeyRef &k, void *value);
+
+    static void remove_fat_child(art_node *n, art_node **ref, int depth);
+
+    static void remove_fat_child4(art_node4_kv *n, art_node **ref, int depth);
+
+    static void remove_child4(art_node4 *n, art_node **ref, art_node **l, int depth);
+
+    static void remove_fat_child16(art_node16_kv *n, art_node **ref);
+
+    void remove_child16(art_node16 *n, art_node **ref, art_node **l);
+
+    static void remove_fat_child48(art_node48_kv *n, art_node **ref);
+
+    void remove_child48(art_node48 *n, art_node **ref, unsigned char c);
+
+    static void remove_fat_child256(art_node256_kv *n, art_node **ref);
+
+    void remove_child256(art_node256 *n, art_node **ref, unsigned char c);
+
+    static void copy_header(art_node *dest, art_node *src);
+
+    art_leaf *iterative_insert(art_node *root, art_node **root_ptr, KeyRef &k, void *value,
+                               int depth, int *old, int replace_existing);
+
+    art_leaf *insert_leaf(art_node *n, art_node **ref, const KeyRef &k, void *value, int depth,
+                          int *old, int replace_existing);
+
+    art_leaf *insert_fat_node(art_node *n, art_node **ref, const KeyRef &k, void *value, int depth,
+                              int *old, art_leaf *min_of_n, int replace_existing);
+
+    art_leaf *insert_internal_node(art_node *n, art_node **ref, const KeyRef &k, void *value, int depth,
+                                   int *old, art_leaf *min_of_n, int prefix_diff, int replace_existing);
+
+    art_leaf *insert_child(art_node *n, art_node **ref, const KeyRef &k, void *value, int depth,
+                           int *old, int replace_existing);
+
+    static inline int longest_common_prefix(const KeyRef &k1, const KeyRef &k2, int depth);
+
+    static void insert_after(art_leaf *l_new, art_leaf *l_existing);
+
+    static void insert_before(art_leaf *l_new, art_leaf *l_existing);
+
+    void add_child(art_node *n, art_node **ref, unsigned char c, void *child);
+
+    void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child);
+
+    void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *child);
+
+    void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *child);
+
+    void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *child);
+
+public:
+    art_tree(Arena &a) {
+        this->arena = &a;
+        this->root = alloc_node(ART_NODE4); //Sentinel node. Also needed for empty key
+        this->size = 0;
+    }
+
+    art_iterator lower_bound(const KeyRef &k);
+
+    art_iterator upper_bound(const KeyRef &k);
+
+    art_iterator insert(KeyRef &key, void *value);
+
+    art_iterator insert_if_absent(KeyRef &key, void *value, int *replaced);
+
+    void erase(const art_iterator &it);
+
+    uint64_t count() {
+        return size;
+    }
+
+
+}; //art_tree
+
+struct art_iterator {
+
+private:
+    art_tree::art_leaf *leaf;
+
+public:
+    art_iterator(art_tree::art_leaf *l) {
+        leaf = l;
+    }
+
+    art_iterator(const art_iterator &i) {
+        leaf = i.leaf;
+    }
+
+    art_iterator() {
+        leaf = nullptr;
+    }
+
+    art_iterator &operator++() {
+        leaf = leaf->next;
+        return (*this);
+    }
+
+    art_iterator &operator--() {
+        leaf = leaf->prev;
+        return (*this);
+    }
+
+    bool operator==(const art_iterator &other) const {
+        return leaf == other.leaf;
+    }
+
+    bool operator!=(const art_iterator &other) const {
+        return leaf != other.leaf;
+    }
+
+    const KeyRef &key() const {
+        return leaf->key;
+    }
+
+    void *value() const {
+        return leaf->value;
+    }
+
+    void **value_ptr() {
+        return &(leaf->value);
+    }
+};
+
+
+#endif  //ART_H