foundationdb/fdbserver/art_impl.h

/*
 * art_impl.h
 *
 * This source file is part of the FoundationDB open source project
 *
 * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ART_IMPL_H
#define ART_IMPL_H

using art_tree = VersionedBTree::art_tree;
using art_leaf = art_tree::art_leaf;
#define art_node art_tree::art_node

int art_tree::fat_leaf_offset[] = {
	0, 0, 0, 0, 0, sizeof(art_node4), sizeof(art_node16), sizeof(art_node48), sizeof(art_node256)
};
int art_tree::node_sizes[] = { 0,
	                           sizeof(art_node4),
	                           sizeof(art_node16),
	                           sizeof(art_node48),
	                           sizeof(art_node256),
	                           sizeof(art_node4_kv),
	                           sizeof(art_node16_kv),
	                           sizeof(art_node48_kv),
	                           sizeof(art_node256_kv) };

VersionedBTree::art_iterator art_tree::insert(KeyRef& k, void* value) {
#define INIT_DEPTH 0
#define REPLACE 1
	int old_val = 0;
	art_leaf* l = iterative_insert(this->root, &this->root, k, value, INIT_DEPTH, &old_val, REPLACE);

	if (!old_val)
		this->size++;
	return VersionedBTree::art_iterator(l);
}

VersionedBTree::art_iterator art_tree::insert_if_absent(KeyRef& k, void* value, int* existing) {
#define INIT_DEPTH 0
#define DONTREPLACE 0
	art_leaf* l = iterative_insert(this->root, &this->root, k, value, INIT_DEPTH, existing, DONTREPLACE);
	if (!existing)
		this->size++;
	return VersionedBTree::art_iterator(l);
}

VersionedBTree::art_iterator art_tree::lower_bound(const KeyRef& key) {
	if (!size)
		return art_iterator(nullptr);
	art_node* n = root;
	art_leaf* res = nullptr;
	int depth = 0;

	art_bound_iterative(n, key, depth, &res, false);

	return art_iterator(res);
}

VersionedBTree::art_iterator art_tree::upper_bound(const KeyRef& key) {
	if (!size)
		return art_iterator(nullptr);
	art_node* n = root;
	art_leaf* res = nullptr;
	int depth = 0;

	art_bound_iterative(n, key, depth, &res, true);

	return art_iterator(res);
}

struct stack_entry {
	art_node* node;
	unsigned char key;
	stack_entry* prev;

	stack_entry(art_node* n, unsigned char k, stack_entry* s) : node(n), key(k), prev(s) {}

	stack_entry() {}
};

art_leaf* art_tree::minimum(art_node* n) {
	// Handle base cases
	if (!n) {
		return nullptr;
	}
	if (ART_IS_LEAF(n)) {
		return ART_LEAF_RAW(n);
	}

	int idx;
	switch (n->type) {
	case ART_NODE4:
		return minimum(((art_node4*)n)->children[0]);
	case ART_NODE16:
		return minimum(((art_node16*)n)->children[0]);
	case ART_NODE48:
		idx = 0;
		while (!((art_node48*)n)->keys[idx])
			idx++;
		idx = ((art_node48*)n)->keys[idx] - 1;
		return minimum(((art_node48*)n)->children[idx]);
	case ART_NODE256:
		idx = 0;
		while (!((art_node256*)n)->children[idx])
			idx++;
		return minimum(((art_node256*)n)->children[idx]);
	case ART_NODE256_KV:
	case ART_NODE48_KV:
	case ART_NODE16_KV:
	case ART_NODE4_KV:
		return ART_FAT_NODE_LEAF(n);
	default:
		printf("%d\n", n->type);
		UNSTOPPABLE_ASSERT(false);
	}
}

art_leaf* art_tree::maximum(art_node* n) {
	// Handle base cases
	if (!n)
		return nullptr;
	if (ART_IS_LEAF(n))
		return ART_LEAF_RAW(n);

	int idx;
	switch (n->type) {
	case ART_NODE4:
	case ART_NODE4_KV:
		return maximum(((art_node4*)n)->children[n->num_children - 1]);
	case ART_NODE16:
	case ART_NODE16_KV:
		return maximum(((art_node16*)n)->children[n->num_children - 1]);
	case ART_NODE48:
	case ART_NODE48_KV:
		idx = 255;
		while (!((art_node48*)n)->keys[idx])
			idx--;
		idx = ((art_node48*)n)->keys[idx] - 1;
		return maximum(((art_node48*)n)->children[idx]);
	case ART_NODE256:
	case ART_NODE256_KV:
		idx = 255;
		while (!((art_node256*)n)->children[idx])
			idx--;
		return maximum(((art_node256*)n)->children[idx]);
	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

void art_tree::art_bound_iterative(art_node* n, const KeyRef& k, int depth, art_leaf** result, bool strict) {

	static stack_entry arena[ART_MAX_KEY_LEN]; // Single threaded implementation.

	stack_entry *head = nullptr, *curr_arena = arena;
	int ret;
	art_node** child;
	unsigned char* key = (unsigned char*)k.begin();
	while (n) {
		ret = check_bound_node(n, k, &depth, result, strict);
		if (ret == ART_I_FOUND) {
			return;
		}
		if (ret == ART_I_BACKTRACK)
			break;
		if (ret == ART_I_DEPTH) {
			head = new (curr_arena++) stack_entry(n, key[depth], head);
			// if the child is nullptr, then we have to look right; i.e., we start backtracking
			child = find_child(n, key[depth]);
			if (!child)
				break;
			n = *child;
			depth = depth + 1;
			// else go on
		} else {
			UNSTOPPABLE_ASSERT(false);
		}
	}
	art_node* next;
	while (head) {
		n = head->node;
		find_next(n, head->key, &next);
		if (next) {
			*result = minimum(next);
			break;
		} else {
			head = head->prev;
		}
	}
}

int art_tree::check_bound_node(art_node* n, const KeyRef& k, int* depth_p, art_leaf** result, bool strict) {
	if (ART_IS_LEAF(n)) {
		int ret = art_bound_leaf(n, k, *depth_p, result);
		*result = ART_LEAF_RAW(n);
		if (ret == ART_LEAF_SMALLER_KEY || (ret == ART_LEAF_MATCH_KEY && strict)) {
			*result = (*result)->next;
		}
		return ART_I_FOUND;
	}
	int depth = *depth_p;
	const int key_len = k.size();
	if (n->type < ART_NODE4_KV) {
		const uint32_t n_partial_len = n->partial_len;
		// Case 1: the search ends on this node
		if (key_len <= (depth + n_partial_len)) {
			// Easy case w/o checking prefix:
			if (0 == n_partial_len) {
				*result = minimum(n);
				return ART_I_FOUND;
			} else {
				// The Whole prefix is the set of bytes after the first depth bytes and up to partial_len
				art_leaf* min_leaf;
				int prefix_differs = signed_prefix_mismatch(n, k, depth, &min_leaf, true);

				if (prefix_differs < 0) { // our key is greater than or equal to the partial prefix. I.e., this subtree
					// only stores smaller keys.
					return ART_I_BACKTRACK;
				} else { // our key is less than or equal to the partial prefix. So this subtree stores our bound
					     // (modulo null leaf)
					if (min_leaf == (art_leaf*)ART_MIN_LEAF_UNSET) {
						min_leaf = minimum(n);
					}
					*result = min_leaf;
					return ART_I_FOUND;
				}
			}
		}
		// Case 2: the search has to go deeper
		// First, look if you have any prefix. If you do, you have to check this prefix before checking your children
		if (n_partial_len) {
			art_leaf* min_leaf;
			int prefix_differs = signed_prefix_mismatch(n, k, depth, &min_leaf, true);

			if (prefix_differs == 0) {
				// The partial prefix matches (0 ==> match)
				// The mismatch happens after the prefix covered by the node, so it is fine to check its children
				*depth_p += n->partial_len;
				return ART_I_DEPTH;
			} else {
				// The mismatch happens in the prefix of this node. Now there are two options
				// 1. The subtree stores smaller keys. Have to backtrack
				if (prefix_differs < 0) {
					return ART_I_BACKTRACK;
				} else {
					if (min_leaf == (art_leaf*)ART_MIN_LEAF_UNSET) {
						min_leaf = minimum(n);
					}
					*result = min_leaf;
					return ART_I_FOUND;
				}
			}
		}
		return ART_I_DEPTH;
	} else {
		if (key_len <= depth + n->partial_len) { // Case 1: the key ends here
			art_leaf* l = ART_FAT_NODE_LEAF(n);
			// Check if the expanded path matches
			int m = leaf_matches_signed(l, k, depth);
			if (!m) {
				// We have a match. If the bound is not strict, we save the result
				if (!strict) {
					*result = l;
					return ART_I_FOUND;
				} else { // Assuming --as we do-- that we have a child, the smallest child in the subtree is the bound
					*result = minimum_kv(n);
					return ART_I_FOUND; // We have found the bound. We use this return value so that the value
					// We set as result is propagated directly
				}
			}
			if (m < 0) {
				// The key in the fat node is smaller than the desired key
				// This means that the target key does not exist.
				// Also, it means that this subtree as a whole is smaller than the target key
				// So the bound has to be searched by my daddy
				return ART_I_BACKTRACK;
			} else {
				*result = l;
				return ART_I_FOUND;
			}
		}
		if (n->partial_len) {
			art_leaf* l = ART_FAT_NODE_LEAF(n);
			// Note we know target key *does not* end here, so we just need to check the prefix
			// To avoid distinguishing whether the prefix is larger or smaller than min, we compare with the leaf
			// That is readily available w/o issuing a "min"
			// The fat leaf key includes the prefix for sure
			int cmp =
			    memcmp(((unsigned char*)k.begin()) + depth, ((unsigned char*)l->key.begin()) + depth, n->partial_len);
			if (cmp < 0) { // Target key is smaller, fat key is then the smallest key larger than current key
				*result = l;
				return ART_I_FOUND;
			} else if (cmp > 0) { // Target key is larger than key and hence than subtree. Ask daddy to go right
				return ART_I_BACKTRACK;
			}
			// If prefix is the same, then go in depth with the updated depth value
			*depth_p += n->partial_len;
			return ART_I_DEPTH;
		}
		return ART_I_DEPTH;
	}
}

int art_tree::art_bound_leaf(art_node* n, const KeyRef& k, int depth, art_leaf** result) {
	n = (art_node*)ART_LEAF_RAW(n);
	// Check if the expanded path matches
	int m = leaf_matches_signed((art_leaf*)n, k, depth);
	if (0 == m)
		return ART_LEAF_MATCH_KEY;
	if (m < 0)
		return ART_LEAF_SMALLER_KEY;
	return ART_LEAF_LARGER_KEY;
}

int art_tree::leaf_matches_signed(art_leaf* n, const KeyRef& k, int depth) {

	const int key_len = k.size();
	const unsigned char* key = k.begin();
	int common_length = min(n->key.size() - depth, key_len - depth);

	// Compare the keys starting at the depth
	int cmp = memcmp((n->key.begin()) + depth, key + depth, common_length);
	if (cmp)
		return cmp;
	return n->key.size() - key_len;
}

int art_tree::leaf_matches(const art_leaf* n, const KeyRef& k, int depth) {
	const int key_len = k.size();
	// Fail if the key lengths are different
	if (n->key.size() != (uint32_t)key_len)
		return 1;

	const unsigned char* key = k.begin();

	// Compare the keys starting at the depth
	return memcmp(((unsigned char*)(n->key).begin()) + depth, key + depth, key_len - depth);
}

int art_tree::signed_prefix_mismatch(art_node* n, const KeyRef& k, int depth, art_leaf** min_leaf, bool find_min) {
	int all_to_check = min(n->partial_len, k.size() - depth);
	int max_cmp = min(ART_MAX_PREFIX_LEN, all_to_check);
	// Mark the leaf as unset right away, so that if you don't return in the first loop AND you don't enter the
	// Second loop, the leaf is marked correctly as unset
	*min_leaf = (art_leaf*)ART_MIN_LEAF_UNSET;
	int idx;
	for (idx = 0; idx < max_cmp; idx++) {
		if (n->partial[idx] != k.begin()[depth + idx]) {
			return ((int)n->partial[idx]) - ((int)(k.begin()[depth + idx]));
		}
	}
	// So far they are the same.
	// There are two cases now. They are the same BUT we have to check after ART_MAX_PREFIX_LEN, or they are the same
	// and there's nothing more to check.
	if (all_to_check <= ART_MAX_PREFIX_LEN)
		return 0;
	// Else, we need to go deeper and check the bytes after ART_MAX_PREFIX_LEN

	int remaining = all_to_check - ART_MAX_PREFIX_LEN;
	// If the prefix is short we can avoid finding a leaf
	if (remaining > 0) {
		// Prefix is longer than what we've checked, find a leaf
		art_leaf* l = find_min ? minimum(n) : maximum(n);
		*min_leaf = l;
		// We have to compare the last partial_len - ART_MAX_PREFIX_LEN bytes
		// If the minimum is below me, then it has at least my same prefix, so I can safely check
		// all its bytes from depth to depth+remaining
		for (; idx < ART_MAX_PREFIX_LEN + remaining; idx++) {
			if (l->key.begin()[idx + depth] != k.begin()[depth + idx]) {
				return ((int)l->key.begin()[idx + depth]) - ((int)k.begin()[depth + idx]);
			}
		}
	}
	return 0;
}

art_leaf* art_tree::minimum_kv(art_node* n) {
	// Handle base cases
	if (!n) {
		return nullptr;
	}
	if (ART_IS_LEAF(n)) {
		return ART_LEAF_RAW(n);
	}

	int idx;
	switch (n->type) {
	case ART_NODE4_KV:
		return minimum(((art_node4*)n)->children[0]);
	case ART_NODE16_KV:
		return minimum(((art_node16*)n)->children[0]);
	case ART_NODE48_KV:
		idx = 0;
		while (!((art_node48*)n)->keys[idx])
			idx++;
		idx = ((art_node48*)n)->keys[idx] - 1;
		return minimum(((art_node48*)n)->children[idx]);
	case ART_NODE256_KV:
		idx = 0;
		while (!((art_node256*)n)->children[idx])
			idx++;
		return minimum(((art_node256*)n)->children[idx]);
	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

art_node** art_tree::find_child(art_node* n, unsigned char c) {
	int i, mask, bitfield;
	union {
		art_node4* p1;
		art_node16* p2;
		art_node48* p3;
		art_node256* p4;
	} p;
	switch (n->type) {
	case ART_NODE4:
	case ART_NODE4_KV:
		p.p1 = (art_node4*)n;
		switch (n->num_children) {
		case 4:
			if (p.p1->keys[3] == c) {
				return &p.p1->children[3];
			}
		case 3:
			if (p.p1->keys[2] == c) {
				return &p.p1->children[2];
			}
		case 2:
			if (p.p1->keys[1] == c) {
				return &p.p1->children[1];
			}
			/*
			 * We need a case 1. Otherwise this could happen: we could be looking for char '0'
			 * On a node with 0 children (root node). Since the node has been zeroed on allocation,
			 * we get that key[0] is exactly '0', and we think we have found a child. Duh.
			 */
		case 1:
			if (p.p1->keys[0] == c) {
				return &p.p1->children[0];
			}
		default:
			break;
		}
		break;

		{
		case ART_NODE16:
		case ART_NODE16_KV:
			__m128i cmp;
			p.p2 = (art_node16*)n;

			// Compare the key to all 16 stored keys
			cmp = _mm_cmpeq_epi8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i*)p.p2->keys));

			// Use a mask to ignore children that don't exist
			mask = (1 << n->num_children) - 1;
			bitfield = _mm_movemask_epi8(cmp) & mask;

			/*
			 * If we have a match (any bit set) then we can
			 * return the pointer match using ctz to get
			 * the index.
			 */
			if (bitfield)
				return &p.p2->children[ctz(bitfield)];
			break;
		}

	case ART_NODE48:
	case ART_NODE48_KV:
		p.p3 = (art_node48*)n;
		i = p.p3->keys[c];
		if (i)
			return &p.p3->children[i - 1];
		break;

	case ART_NODE256:
	case ART_NODE256_KV:
		p.p4 = (art_node256*)n;
		if (p.p4->children[c])
			return &p.p4->children[c];
		break;

	default:
		UNSTOPPABLE_ASSERT(false);
	}
	return nullptr;
}

void art_tree::find_next(art_node* n, unsigned char c, art_node** out) {
	int i, mask, bitfield;
	union {
		art_node4* p1;
		art_node16* p2;
		art_node48* p3;
		art_node256* p4;
	} p;
	*out = nullptr;
	switch (n->type) {
	case ART_NODE4:
	case ART_NODE4_KV:
		p.p1 = (art_node4*)n;
		switch (n->num_children) { // unrolling loop
		case 4:
			if (p.p1->keys[0] > c) {
				*out = p.p1->children[0];
				break;
			}
			if (p.p1->keys[1] > c) {
				*out = p.p1->children[1];
				break;
			}
			if (p.p1->keys[2] > c) {
				*out = p.p1->children[2];
				break;
			}
			if (p.p1->keys[3] > c) {
				*out = p.p1->children[3];
				break;
			}
			break;
		case 3:
			if (p.p1->keys[0] > c) {
				*out = p.p1->children[0];
				break;
			}
			if (p.p1->keys[1] > c) {
				*out = p.p1->children[1];
				break;
			}
			if (p.p1->keys[2] > c) {
				*out = p.p1->children[2];
				break;
			}
			break;
		case 2:
			if (p.p1->keys[0] > c) {
				*out = p.p1->children[0];
				break;
			}
			if (p.p1->keys[1] > c) {
				*out = p.p1->children[1];
				break;
			}
			break;
			// We add a case 1 just in case... (see previous similar comment)
		case 1:
			if (p.p1->keys[0] > c) {
				*out = p.p1->children[0];
				break;
			}
		default:
			break;
		}
		break;

		{
		case ART_NODE16:
		case ART_NODE16_KV:
			__m128i cmp;
			p.p2 = (art_node16*)n;
			// We did not find the child corresponding to the key. Let's see if we have a next at least
			// Compare the key to all 16 stored keys for Greater than
			cmp = _mm_cmplt_epu8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i*)p.p2->keys));

			// Use a mask to ignore children that don't exist
			mask = (1 << n->num_children) - 1;
			bitfield = _mm_movemask_epi8(cmp) & mask;
			if (bitfield) {
				// ALL children greater than char have their bit set
				// We need the smallest one
				// let's get the least significant bit that is set in the bitfield
				int one = ctz(bitfield);
				*out = p.p2->children[one];
			}
			break;
		}
	case ART_NODE48:
	case ART_NODE48_KV: {
		p.p3 = (art_node48*)n;

		if (c == 255)
			break;
		unsigned char cc = c + 1;
		do {
			i = p.p3->keys[cc];
			if (i) {
				*out = p.p3->children[i - 1];
				break;
			}
			++cc;
		} while (cc > 0); // cc wraps around at 256
		break;
	}

	case ART_NODE256:
	case ART_NODE256_KV: {
		p.p4 = (art_node256*)n;
		unsigned char cc = c + 1;
		if (c == 255)
			break;
		do {
			if (p.p4->children[cc]) {
				*out = p.p4->children[cc];
				break;
			}
			++cc;
		} while (cc > 0); // cc wraps around at 256
		break;
	}

	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

void art_tree::recursive_delete_binary(art_node* n, art_node** ref, const KeyRef& k, int depth) {
	// Bail if the prefix does not match
	unsigned char* key = (unsigned char*)k.begin();
	unsigned int key_len = (unsigned int)k.size();
	if (n->partial_len) {
		int prefix_len = check_prefix(n, k, depth);
		if (prefix_len != min(ART_MAX_PREFIX_LEN, n->partial_len)) {
			return;
		}
		depth = depth + n->partial_len;
	}

	// Find child node
	art_node** child = find_child(n, key[depth]);
	if (!child)
		return;

	// The child contains the key to be deleted in two cases
	// 1. The child is a leaf (and the key matches)
	// 2. THe child is a fat node (and the key ands in the node and the key matches)

	// If the child is leaf, delete from this node
	if (ART_IS_LEAF(*child)) {
		art_leaf* l = ART_LEAF_RAW(*child);
		if (!leaf_matches(l, k, depth)) {
			remove_child(n, ref, key[depth], child, depth);
			if (l->prev) {
				l->prev->next = l->next;
			}
			if (l->next) {
				l->next->prev = l->prev;
			}
			return;
		}
		return;
	} else {
		// Fat node and key ends within the child
		// Note that the predicate is on the child node, so at depth+1
		if ((*child)->type >= ART_NODE4_KV && (key_len == depth + 1 + (*child)->partial_len)) {
			// Check if key matches
			art_leaf* l = ART_FAT_NODE_LEAF((*child));
			// Return nullptr if key does not match
			if (leaf_matches(l, k, depth + 1)) {
				return;
			}

			// Set prev and next, since the leaf is going away
			if (l->prev) {
				l->prev->next = l->next;
			}
			if (l->next) {
				l->next->prev = l->prev;
			}
			// Remove the fat child. This also triggers compaction
			// NOTE: we are removing the fat leaf on our child, which is at depth+1!!!
			remove_fat_child(*child, child, depth + 1);
			return;
		}
		// Recurse if key is not supposed to be in the child. The child is going to be an internal node
		recursive_delete_binary(*child, child, k, depth + 1);
	}
}

int art_tree::check_prefix(const art_node* n, const KeyRef& k, int depth) {
	int max_cmp = min(min(n->partial_len, ART_MAX_PREFIX_LEN), k.size() - depth);
	int idx;
	for (idx = 0; idx < max_cmp; idx++) {
		if (n->partial[idx] != ((unsigned char*)k.begin())[depth + idx])
			return idx;
	}
	return idx;
}

// Remove a LEAF from a node256/node256kv
void art_tree::remove_child256(art_node256* n, art_node** ref, unsigned char c) {
	n->children[c] = nullptr;
	n->n.num_children--;

	// Resize to a node48 on underflow, not immediately to prevent
	// trashing if we sit on the 48/49 boundary
	if (n->n.num_children == 37) {
		art_node48* new_node;

		if (n->n.type == ART_NODE256_KV) {
			new_node = (art_node48*)alloc_node(ART_NODE48_KV);
			ART_FAT_NODE_LEAF(&new_node->n) = ART_FAT_NODE_LEAF(&n->n);
		} else {
			new_node = (art_node48*)alloc_node(ART_NODE48);
		}
		*ref = (art_node*)new_node;
		copy_header((art_node*)new_node, (art_node*)n);

		int pos = 0;
		for (int i = 0; i < 256; i++) {
			if (n->children[i]) {
				new_node->children[pos] = n->children[i];
				new_node->keys[i] = pos + 1;
				pos++;
			}
		}
	}
}

void art_tree::remove_fat_child256(art_node256_kv* n, art_node** ref) {
	// Delete the child by turning the fat node to a normal internal node
	n->n.n.type = ART_NODE256;
	*ref = (art_node*)n;
}

// Remove a LEAF from a node48/node48kv
void art_tree::remove_child48(art_node48* n, art_node** ref, unsigned char c) {
	int pos = n->keys[c];
	n->keys[c] = 0;
	n->children[pos - 1] = nullptr;
	n->n.num_children--;

	if (n->n.num_children == 12) {
		art_node16* new_node;
		if (n->n.type == ART_NODE48_KV) {
			new_node = (art_node16*)alloc_node(ART_NODE16_KV);
			ART_FAT_NODE_LEAF(&new_node->n) = ART_FAT_NODE_LEAF(&n->n);
		} else {
			new_node = (art_node16*)alloc_node(ART_NODE16);
		}
		*ref = (art_node*)new_node;
		copy_header((art_node*)new_node, (art_node*)n);

		int child = 0;
		for (int i = 0; i < 256; i++) {
			pos = n->keys[i];
			if (pos) {
				new_node->keys[child] = i;
				new_node->children[child] = n->children[pos - 1];
				child++;
			}
		}
	}
}

void art_tree::remove_fat_child48(art_node48_kv* n, art_node** ref) {
	// Delete the child by turning the fat node to a normal internal node
	n->n.n.type = ART_NODE48;
	*ref = (art_node*)n;
}

// Remove a LEAF from a node16/node16kv
void art_tree::remove_child16(art_node16* n, art_node** ref, art_node** l) {
	int pos = l - n->children;
	memmove(n->keys + pos, n->keys + pos + 1, n->n.num_children - 1 - pos);
	memmove(n->children + pos, n->children + pos + 1, (n->n.num_children - 1 - pos) * sizeof(void*));
	n->n.num_children--;

	if (n->n.num_children == 3) {
		art_node4* new_node;
		if (n->n.type == ART_NODE16_KV) {
			new_node = (art_node4*)alloc_node(ART_NODE4_KV);
			ART_FAT_NODE_LEAF(&new_node->n) = ART_FAT_NODE_LEAF(&n->n);
		} else {
			new_node = (art_node4*)alloc_node(ART_NODE4);
		}

		*ref = (art_node*)new_node;
		copy_header((art_node*)new_node, (art_node*)n);
		memcpy(new_node->keys, n->keys, 4);
		memcpy(new_node->children, n->children, 4 * sizeof(void*));
	}
}

void art_tree::remove_fat_child16(art_node16_kv* n, art_node** ref) {
	// Delete the child by turning the fat node to a normal internal node
	n->n.n.type = ART_NODE16;
	*ref = (art_node*)n;
}

// Remove a LEAF from a node4/node4kv
void art_tree::remove_child4(art_node4* n, art_node** ref, art_node** l, int depth) {
	int pos = l - n->children;
	// We should do this only if then we do not remove the node altogether
	memmove(n->keys + pos, n->keys + pos + 1, n->n.num_children - 1 - pos);
	memmove(n->children + pos, n->children + pos + 1, (n->n.num_children - 1 - pos) * sizeof(void*));
	n->n.num_children--;

	// Remove nodes with only a single child
	// This can only be done if the node is not a fat node.
	// In that case, the key in the fat node is a prefix to the key in the only child
	// And hence has to be preserved in the fat node
	if (n->n.num_children == 1 && (n->n.type < ART_NODE4_KV) && depth) {
		art_node* child = n->children[0];
		if (!ART_IS_LEAF(child)) {
			// Concatenate the prefixes
			int prefix = n->n.partial_len;
			if (prefix < ART_MAX_PREFIX_LEN) {
				n->n.partial[prefix] = n->keys[0];
				prefix++;
			}
			if (prefix < ART_MAX_PREFIX_LEN) {
				int sub_prefix = min(child->partial_len, ART_MAX_PREFIX_LEN - prefix);
				memcpy(n->n.partial + prefix, child->partial, sub_prefix);
				prefix += sub_prefix;
			}

			// Store the prefix in the child
			memcpy(child->partial, n->n.partial, min(prefix, ART_MAX_PREFIX_LEN));
			child->partial_len += n->n.partial_len + 1;
		}
		// This is done also for the leaf. If the only son is a leaf, just delete the internal node
		// And have the father of the deleted internal node point directly to the leaf.
		*ref = child;
	}

	// What can happen now is that the node has ZERO children.
	// If you have a fat node, you cannot compress when you get to one child
	// So it can happen that the one child get removed
	// At that point, you transform the fat node into a leaf: the fat key is not prefix of any subtree!
	// This should only happen to a node4kv: if the node is normal, then already when there there is 1 child
	// the node gets compressed.
	// We still check for depth b/c we want to avoid that root becomes empty and becomes a leaf
	else if (n->n.num_children == 0 && depth) {
		*ref = (art_node*)SET_LEAF((art_node*)ART_FAT_NODE_LEAF(&n->n));
	} else if (!depth && n->n.num_children == 0) {
		n->children[0] = nullptr;
	}
}

void art_tree::remove_fat_child4(art_node4_kv* n, art_node** ref, int depth) {

	// Delete the child by turning the fat node to a normal internal node
	n->n.n.type = ART_NODE4;

	*ref = (art_node*)n; // Tell daddy that my address has changed
	art_node4* node = (art_node4*)n;

	// Remove nodes with only a single child
	if (node->n.num_children == 1 && depth) {
		art_node* child = node->children[0];
		if (!ART_IS_LEAF(child)) {
			// Concatenate the prefixes
			int prefix = node->n.partial_len;
			if (prefix < ART_MAX_PREFIX_LEN) {
				node->n.partial[prefix] = node->keys[0];
				prefix++;
			}
			if (prefix < ART_MAX_PREFIX_LEN) {
				int sub_prefix = min(child->partial_len, ART_MAX_PREFIX_LEN - prefix);
				memcpy(node->n.partial + prefix, child->partial, sub_prefix);
				prefix += sub_prefix;
			}

			// Store the prefix in the child
			memcpy(child->partial, node->n.partial, min(prefix, ART_MAX_PREFIX_LEN));
			child->partial_len += node->n.partial_len + 1;
		}
		*ref = child;

	} else if (!depth && node->n.num_children == 0) {
		// This happens if we have a fat root with the empty key.
		// If only the empty key remains, then the fat root has zero children
		node->children[0] = nullptr;
	}
}

void art_tree::remove_child(art_node* n, art_node** ref, unsigned char c, art_node** l, int depth) {
	switch (n->type) {
	case ART_NODE4_KV:
	case ART_NODE4:
		return remove_child4((art_node4*)n, ref, l, depth);
	case ART_NODE16_KV:
	case ART_NODE16:
		return remove_child16((art_node16*)n, ref, l);
	case ART_NODE48_KV:
	case ART_NODE48:
		return remove_child48((art_node48*)n, ref, c);
	case ART_NODE256_KV:
	case ART_NODE256:
		return remove_child256((art_node256*)n, ref, c);
	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

// NB: Here, n is the fat node from which we remove the kv/pair
void art_tree::remove_fat_child(art_node* n, art_node** ref, int depth) {
	switch (n->type) {
	case ART_NODE4_KV:
		return remove_fat_child4((art_node4_kv*)n, ref, depth);
	case ART_NODE16_KV:
		return remove_fat_child16((art_node16_kv*)n, ref);
	case ART_NODE48_KV:
		return remove_fat_child48((art_node48_kv*)n, ref);
	case ART_NODE256_KV:
		return remove_fat_child256((art_node256_kv*)n, ref);
	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

void art_tree::copy_header(art_node* dest, art_node* src) {
	dest->num_children = src->num_children;
	dest->partial_len = src->partial_len;
	memcpy(dest->partial, src->partial, min(ART_MAX_PREFIX_LEN, src->partial_len));
}

void art_tree::erase(const art_iterator& it) {
	recursive_delete_binary(this->root, &this->root, it.key(), 0);
}

art_leaf* art_tree::iterative_insert(art_node* root,
                                     art_node** root_ptr,
                                     KeyRef& k,
                                     void* value,
                                     int depth,
                                     int* old,
                                     int replace_existing) {

	art_node* n = root;
	art_node** ref = root_ptr;
	// Ref is the memory location of the father that stores the pointer to the child in which we add the new item
	// Of course, the root is not the child of anybody. We need the ptr to root bc it is possible that the root itself
	// Grows by adding nodes, and hence gets relocated. Then, we need to update the ptr to the root
	while (n) {
		if (ART_IS_LEAF(n)) {
			return insert_leaf(n, ref, k, value, depth, old, replace_existing);
		}
		art_leaf* min_of_n = nullptr;
		if (n->partial_len) {
			// Determine if the prefixes differ, since we need to split
			int prefix_diff = prefix_mismatch(n, k, depth, &min_of_n);
			if ((uint32_t)prefix_diff >= n->partial_len) {
				depth += n->partial_len;
				// handle fat key  or go in depth
			} else {
				return insert_internal_node(n, ref, k, value, depth, old, min_of_n, prefix_diff, replace_existing);
			}
		}
		if (k.size() == (depth)) {
			return insert_fat_node(n, ref, k, value, depth, old, min_of_n, replace_existing);
		}

		// Find a child to recurse to
		// Child is the pointer to the memory location within the node that contains the ptr to the child
		art_node** child = find_child(n, ((unsigned char*)k.begin())[depth]);
		if (!child) {
			break;
		}
		// Else go on. Update depth, the current node and its reference in the list of children in the parent
		depth++;
		ref = child;
		n = *ref;
	}
	return insert_child(n, ref, k, value, depth, old, replace_existing);
}

art_leaf* art_tree::insert_child(art_node* n,
                                 art_node** ref,
                                 const KeyRef& k,
                                 void* value,
                                 int depth,
                                 int* old,
                                 int replace_existing) {
	// No child, node goes within us
	art_leaf* l = make_leaf(k, value);
	const unsigned char* key = (const unsigned char*)k.begin();
	add_child(n, ref, key[depth], SET_LEAF(l));
	// After adding a child, the node might have grown (and old pointer n becomes invalid)
	art_node* new_node = *ref;

	art_node* pn = nullptr;
	art_leaf* lm = nullptr;
	int prev_next = art_next_prev(new_node, key[depth], &pn);
	if (prev_next == ART_NEXT) {
		// If the char of this key is NOT the maximum in the node, take the key's next node from curr_node
		// If the next node exists, then the MIN of the next node is the new_key's next
		// NB: The fat leaf cannot be the next, so we search for the min in the subtree
		lm = minimum(pn);
		// new key is LOWER than min
		insert_before(l, lm);
	} else if (prev_next == ART_PREV) {
		// If the next does not exist, it means there is a prev (the curr_node has at least one child).
		// The curr key is the next of the MAX of the prev node
		// Check if the prev is the fat_leaf
		if (pn == new_node) {
			lm = ART_FAT_NODE_LEAF(new_node);
		} else {
			lm = maximum(pn);
		}
		// new key is BIGGER than lm
		insert_after(l, lm);
	} // if it is ART_NEITHER, then there's no prev nor next. They have been set to nullptr already

	return l;
}

int art_tree::art_next_prev(art_node* n, unsigned char c, art_node** out) {
	find_next(n, c, out);
	if (*out) {
		return ART_NEXT;
	}
	find_prev(n, c, out);
	if (*out) {
		return ART_PREV;
	}
	return ART_NEITHER;
}

art_leaf* art_tree::insert_internal_node(art_node* n,
                                         art_node** ref,
                                         const KeyRef& k,
                                         void* value,
                                         int depth,
                                         int* old,
                                         art_leaf* min_of_n,
                                         int prefix_diff,
                                         int replace_existing) {

	art_node4* new_node;
	bool kv_creat = (prefix_diff == (k.size() - depth));
	if (!kv_creat) {
		new_node = (art_node4*)alloc_node(ART_NODE4);
		*ref = (art_node*)new_node;
		new_node->n.partial_len = prefix_diff;
		memcpy(new_node->n.partial, n->partial, min(ART_MAX_PREFIX_LEN, prefix_diff));

		// Adjust the prefix of the old node
		if (n->partial_len <= ART_MAX_PREFIX_LEN) {
			add_child4(new_node, ref, n->partial[prefix_diff], n);
			n->partial_len -= (prefix_diff + 1);
			memmove(n->partial, n->partial + prefix_diff + 1, min(ART_MAX_PREFIX_LEN, n->partial_len));
		} else {
			n->partial_len -= (prefix_diff + 1);
			art_leaf* l = min_of_n == nullptr ? minimum(n) : min_of_n;
			min_of_n = l;
			unsigned char* lkey = (unsigned char*)l->key.begin();
			add_child4(new_node, ref, lkey[depth + prefix_diff], n);
			memcpy(n->partial, lkey + depth + prefix_diff + 1, min(ART_MAX_PREFIX_LEN, n->partial_len));
		}

		// Insert the new leaf
		art_leaf* l = make_leaf(k, value);
		add_child4(new_node, ref, ((unsigned char*)k.begin())[depth + prefix_diff], SET_LEAF(l));

		art_node* pn;
		art_leaf* lm = nullptr;
		int prev_next = art_next_prev((art_node*)new_node, l->key[depth + prefix_diff], &pn);
		if (prev_next == ART_NEXT) {
			// If the char of this key is NOT the maximum in the node, take the key's next node from curr_node
			// If the next node exists, then the MIN of the next node is the new_key's next
			lm = minimum(pn);
			insert_before(l, lm);
		} else if (prev_next == ART_PREV) {
			// If the next does not exist, it means there is a prev (the curr_node has at least one child).
			// The curr key is the next of the MAX of the prev node
			// We do not check for fat node
			lm = maximum(pn);
			// new key is BIGGER than lm
			insert_after(l, lm);
		}
		return l;
	} else {
		// The new key becomes a fat node that has as child the current internal node.
		// The current internal node has to be indexed by the first char in the prefix, whose size has to be reduced
		// By one accordingly
		new_node = (art_node4*)alloc_kv_node(ART_NODE4_KV);
		ART_FAT_NODE_LEAF(&new_node->n) = make_leaf(k, value);
		// This is legacy code. I think it is valid, but probably it can be simplified in our case
		*ref = (art_node*)new_node;
		new_node->n.partial_len = prefix_diff;
		memcpy(new_node->n.partial, n->partial, min(ART_MAX_PREFIX_LEN, prefix_diff));

		// Adjust the prefix of the old node

		if (n->partial_len <= ART_MAX_PREFIX_LEN) {
			add_child4(new_node, ref, n->partial[prefix_diff], n);
			n->partial_len -= (prefix_diff + 1);
			memmove(n->partial, n->partial + prefix_diff + 1, min(ART_MAX_PREFIX_LEN, n->partial_len));
		} else {
			n->partial_len -= (prefix_diff + 1);
			art_leaf* l = min_of_n == nullptr ? minimum(n) : min_of_n; // minimum(n);
			min_of_n = l;
			add_child4(new_node, ref, ((unsigned char*)l->key.begin())[depth + prefix_diff], n);
			memcpy(n->partial, l->key.begin() + depth + prefix_diff + 1, min(ART_MAX_PREFIX_LEN, n->partial_len));
		}
		// Fat leaf is the smallest in this subtree. So the minimum in the subtree is its next
		art_leaf* lm =
		    min_of_n == nullptr ? minimum(n) : min_of_n; // minimum(n); //FIXME: reuse from before if already taken
		min_of_n = lm;
		// new key is LOWER than min
		art_leaf* l_new = ART_FAT_NODE_LEAF(&new_node->n);
		insert_before(l_new, lm);
		return l_new;
	}
}

art_leaf* art_tree::insert_fat_node(art_node* n,
                                    art_node** ref,
                                    const KeyRef& k,
                                    void* value,
                                    int depth,
                                    int* old,
                                    art_leaf* min_of_n,
                                    int replace_existing) {
	if (n->type >= ART_NODE4_KV) {
		// If the node is already fat, it means you already have the key
		*old = 1;
		art_leaf* l = ART_FAT_NODE_LEAF(n);
		if (replace_existing) {
			l->value = value;
		}
		return l;
	} else {
		// We have to transform a node into a fat node
		// The minimum function considers the leaf to be the minimum.
		// Let's grab the minimum BEFORE we create the fat node and insert the new leaf :)
		art_leaf* lm = min_of_n == nullptr ? minimum(n) : min_of_n;
		art_node* nkv = n;
		// change the type before deferencing the leaf
		nkv->type = static_cast<ART_NODE_TYPE>(n->type + 4);
		art_leaf* l_new = make_leaf(k, value);
		ART_FAT_NODE_LEAF(nkv) = l_new;
		*ref = (art_node*)nkv;

		// Fat leaf is the smallest in this subtree. So the minimum in the subtree is its next
		// new key is LOWER than min
		if (lm)
			insert_before(l_new, lm);
		return l_new;
	}
}

art_leaf* art_tree::insert_leaf(art_node* n,
                                art_node** ref,
                                const KeyRef& k,
                                void* value,
                                int depth,
                                int* old,
                                int replace_existing) {
	art_leaf* l = ART_LEAF_RAW(n);

	// Check if we are updating an existing value
	// Original case
	if (!leaf_matches(l, k, depth)) {
		*old = 1;
		if (replace_existing) {
			l->value = value;
		}
		return l;
	}
	const unsigned char* key = (const unsigned char*)k.begin();
	const int key_len = k.size();

	int longest_prefix = longest_common_prefix(k, l->key, depth);
	int kv_creat = (longest_prefix == (min(key_len, l->key.size()) - depth));
	if (!kv_creat) { // Common case, old code
		// Recompute longest prefix from depth. Probably can be optimized

		// New value, we must split the leaf into a node4
		art_node4* new_node = (art_node4*)alloc_node(ART_NODE4);
		// Create a new leaf
		art_leaf* l2 = make_leaf(k, value);
		// Determine longest prefix
		new_node->n.partial_len = longest_prefix;
		memcpy(new_node->n.partial, key + depth, min(ART_MAX_PREFIX_LEN, longest_prefix));
		// Add the leaves to the new node4
		*ref = (art_node*)new_node;
		add_child4(new_node, ref, l->key[depth + longest_prefix], SET_LEAF(l));
		add_child4(new_node, ref, l2->key[depth + longest_prefix], SET_LEAF(l2));

		// Set next and prev. Check if new leaf goes before or after existing leaf
		if (l2->key[depth + longest_prefix] < l->key[depth + longest_prefix]) {
			// New key comes BEFORE existing key
			insert_before(l2, l);
		} else {
			// New key comes AFTER existing key
			insert_after(l2, l);
		}
		return l2;
	}

	// The new key  is a superset of the key in the leaf.
	// So the current leaf becomes the value of the kv node
	if (key_len > l->key.size()) {
		art_node4_kv* new_node = (art_node4_kv*)alloc_kv_node(ART_NODE4_KV);
		// Create a new leaf with the new key
		art_leaf* l2 = make_leaf(k, value);

		// Determine longest prefix
		new_node->n.n.partial_len = longest_prefix;
		memcpy(new_node->n.n.partial, key + depth, min(ART_MAX_PREFIX_LEN, longest_prefix));

		// Add the leaf to the new node4
		*ref = (art_node*)new_node;
		add_child4((art_node4*)&new_node->n, ref, l2->key[depth + longest_prefix], SET_LEAF(l2));

		ART_FAT_NODE_LEAF(&new_node->n.n) = l;
		art_leaf* fat_leaf = ART_FAT_NODE_LEAF(&new_node->n.n);

		// The new key goes in the leaf and its predecessor is the key in the new fat node
		insert_after(l2, fat_leaf);

		return l2;
	} else {
		// The key in the leaf is a supertset of the new key
		// So the leaf stays a leaf and the new key goes in the kv_node
		art_node4_kv* new_node = (art_node4_kv*)alloc_kv_node(ART_NODE4_KV);
		art_leaf* fat_leaf = make_leaf(k, value);
		ART_FAT_NODE_LEAF(&new_node->n.n) = fat_leaf;
		new_node->n.n.partial_len = longest_prefix;

		memcpy(new_node->n.n.partial, key + depth, min(ART_MAX_PREFIX_LEN, longest_prefix));
		*ref = (art_node*)new_node;
		add_child4((art_node4*)&new_node->n, ref, l->key[depth + longest_prefix], SET_LEAF(l));

		// The fat node is the prev of the current leaf
		insert_before(fat_leaf, l);
		return fat_leaf;
	}
}

int art_tree::longest_common_prefix(const KeyRef& k1, const KeyRef& k2, int depth) {
	int max_cmp = min(k1.size(), k2.size()) - depth;
	int idx;
	for (idx = 0; idx < max_cmp; idx++) {
		if (((unsigned char*)k1.begin())[depth + idx] != ((unsigned char*)k2.begin())[depth + idx])
			return idx;
	}
	return idx;
}

void art_tree::insert_before(art_leaf* l_new, art_leaf* l_existing) {
	l_new->next = l_existing;
	if (l_existing) {
		l_new->prev = l_existing->prev;
		if (l_existing->prev) {
			l_existing->prev->next = l_new;
		}
		l_existing->prev = l_new;
	} else {
		l_new->prev = nullptr;
	}
}

void art_tree::insert_after(art_leaf* l_new, art_leaf* l_existing) {
	l_new->prev = l_existing;
	if (l_existing) {
		l_new->next = l_existing->next;
		if (l_existing->next) {
			l_existing->next->prev = l_new;
		}
		l_existing->next = l_new;
	} else {
		l_new->next = nullptr;
	}
}

void art_tree::add_child(art_node* n, art_node** ref, unsigned char c, void* child) {
	switch (n->type) {
	case ART_NODE4_KV:
	case ART_NODE4:
		return add_child4((art_node4*)n, ref, c, child);
	case ART_NODE16_KV:
	case ART_NODE16:
		return add_child16((art_node16*)n, ref, c, child);
	case ART_NODE48_KV:
	case ART_NODE48:
		return add_child48((art_node48*)n, ref, c, child);
	case ART_NODE256_KV:
	case ART_NODE256:
		return add_child256((art_node256*)n, ref, c, child);
	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

void art_tree::add_child256(art_node256* n, art_node** ref, unsigned char c, void* child) {
	(void)ref;
	n->n.num_children++;
	n->children[c] = (art_node*)child;
}

void art_tree::add_child48(art_node48* n, art_node** ref, unsigned char c, void* child) {
	if (n->n.num_children < 48) {
		int pos = 0;
		while (n->children[pos])
			pos++;
		n->children[pos] = (art_node*)child;
		n->keys[c] = pos + 1;
		n->n.num_children++;
	} else {
		art_node256* new_node;
		// Copy the fat leaf pointer if needed
		if (n->n.type == ART_NODE48_KV) {
			new_node = (art_node256*)alloc_node(ART_NODE256_KV);
			ART_FAT_NODE_LEAF(&new_node->n) = ART_FAT_NODE_LEAF(&n->n);
		} else {
			new_node = (art_node256*)alloc_node(ART_NODE256);
		}
		for (int i = 0; i < 256; i++) {
			if (n->keys[i]) {
				new_node->children[i] = n->children[n->keys[i] - 1];
			}
		}
		copy_header((art_node*)new_node, (art_node*)n);
		*ref = (art_node*)new_node;
		add_child256(new_node, ref, c, child);
	}
}

void art_tree::add_child16(art_node16* n, art_node** ref, unsigned char c, void* child) {
	if (n->n.num_children < 16) {
		__m128i cmp;

		// Compare the key to all 16 stored keys
		cmp = _mm_cmplt_epu8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i*)n->keys));

		// Use a mask to ignore children that don't exist
		unsigned mask = (1 << n->n.num_children) - 1;
		unsigned bitfield = _mm_movemask_epi8(cmp) & mask;

		// Check if less than any
		unsigned idx;
		if (bitfield) {
			idx = ctz(bitfield);
			memmove(n->keys + idx + 1, n->keys + idx, n->n.num_children - idx);
			memmove(n->children + idx + 1, n->children + idx, (n->n.num_children - idx) * sizeof(void*));
		} else
			idx = n->n.num_children;

		// Set the child
		n->keys[idx] = c;
		n->children[idx] = (art_node*)child;
		n->n.num_children++;

	} else {
		art_node48* new_node;
		// Check whether this is a fat node
		if (n->n.type == ART_NODE16_KV) {
			new_node = (art_node48*)alloc_node(ART_NODE48_KV);
			ART_FAT_NODE_LEAF(&new_node->n) = ART_FAT_NODE_LEAF(&n->n);
		} else {
			new_node = (art_node48*)alloc_node(ART_NODE48);
		}
		// Copy the child pointers and populate the key map
		memcpy(new_node->children, n->children, sizeof(void*) * n->n.num_children);
		for (int i = 0; i < n->n.num_children; i++) {
			new_node->keys[n->keys[i]] = i + 1;
		}
		copy_header((art_node*)new_node, (art_node*)n);
		*ref = (art_node*)new_node;
		add_child48(new_node, ref, c, child);
	}
}

void art_tree::add_child4(art_node4* n, art_node** ref, unsigned char c, void* child) {
	if (n->n.num_children < 4) {
		int idx;
		for (idx = 0; idx < n->n.num_children; idx++) {
			if (c < n->keys[idx])
				break;
		}

		// Shift to make room
		memmove(n->keys + idx + 1, n->keys + idx, n->n.num_children - idx);
		memmove(n->children + idx + 1, n->children + idx, (n->n.num_children - idx) * sizeof(void*));

		// Insert element
		n->keys[idx] = c;
		n->children[idx] = (art_node*)child;
		n->n.num_children++;

	} else {
		art_node16* new_node;
		// Check whether this is a fat node
		if (n->n.type == ART_NODE4_KV) {
			new_node = (art_node16*)alloc_node(ART_NODE16_KV);
			ART_FAT_NODE_LEAF(&new_node->n) = ART_FAT_NODE_LEAF(&n->n);
		} else {
			new_node = (art_node16*)alloc_node(ART_NODE16);
		}

		// Copy the child pointers and the key map
		memcpy(new_node->children, n->children, sizeof(void*) * n->n.num_children);
		memcpy(new_node->keys, n->keys, sizeof(unsigned char) * n->n.num_children);
		copy_header((art_node*)new_node, (art_node*)n);
		*ref = (art_node*)new_node;
		add_child16(new_node, ref, c, child);
	}
}

// Every node is actually a kv node, but the type is <= NODE256
art_node* art_tree::alloc_node(ART_NODE_TYPE type) {
	const int offset = type > ART_NODE256 ? 0 : ART_NODE256;
	art_node* n = (art_node*)new ((Arena&)*this->arena) uint8_t[node_sizes[offset + type]]();
	n->type = type;
	return n;
}

art_node* art_tree::alloc_kv_node(ART_NODE_TYPE type) {
	art_node* n = (art_node*)new ((Arena&)*this->arena) uint8_t[node_sizes[type]]();
	n->type = type;
	return n;
}

art_leaf* art_tree::make_leaf(const KeyRef& k, void* value) {
	const int key_len = k.size();
	// Allocate contiguous buffer to hold the leaf and the key pointed by the KeyRef
	art_leaf* v = (art_leaf*)new ((Arena&)*this->arena) uint8_t[sizeof(art_leaf) + key_len];
	// copy the key to the proper offset in the buffer
	memcpy(v + 1, k.begin(), key_len);
	KeyRef nkr = KeyRef((const uint8_t*)(v + 1), key_len);
	// create the art_leaf, allocating it on the buffer
	// The KeyRef& passed as argument is the new one, allocated in the buffer
	art_leaf* l = new (v) art_leaf(nkr, value);
	l->prev = nullptr;
	l->next = nullptr;
	l->type = ART_LEAF;
	return l;
}

int art_tree::prefix_mismatch(art_node* n, KeyRef& k, int depth, art_leaf** minout) {
	const int key_len = k.size();
	const unsigned char* key = (const unsigned char*)k.begin();
	int max_cmp = min(min(ART_MAX_PREFIX_LEN, n->partial_len), key_len - depth);
	int idx;
	for (idx = 0; idx < max_cmp; idx++) {
		if (n->partial[idx] != key[depth + idx])
			return idx;
	}

	// If the prefix is short we can avoid finding a leaf
	if (n->partial_len > ART_MAX_PREFIX_LEN) {
		// Prefix is longer than what we've checked, find a leaf
		art_leaf* l = minimum(n);
		*minout = l;
		max_cmp = min(l->key.size(), key_len) - depth;
		for (; idx < max_cmp; idx++) {
			if (l->key.begin()[idx + depth] != k.begin()[depth + idx])
				return idx;
		}
	}
	return idx;
}

// Find the child corresponding to c, if present, and the largest child smaller than c
void art_tree::find_prev(art_node* n, unsigned char c, art_node** out) {
	int i, mask, bitfield;
	*out = nullptr;
	union {
		art_node4* p1;
		art_node16* p2;
		art_node48* p3;
		art_node256* p4;
	} p;
	switch (n->type) {
	case ART_NODE4_KV:
	case ART_NODE4:
		p.p1 = (art_node4*)n;
		for (i = 0; i < n->num_children; i++) {
			if (p.p1->keys[i] < c) {
				*out = p.p1->children[i];
			} else {
				break;
			}
		}
		if (i == 0 && n->type == ART_NODE4_KV) {
			*out = n;
		}

		break;

		{
			__m128i cmp;
		case ART_NODE16_KV:
		case ART_NODE16:

			p.p2 = (art_node16*)n;

			// Compare the key to all 16 stored keys for less than
			cmp = _mm_cmpgt_epu8(_mm_set1_epi8(c), _mm_loadu_si128((__m128i*)p.p2->keys));

			// Use a mask to ignore children that don't exist
			mask = (1 << n->num_children) - 1;
			bitfield = _mm_movemask_epi8(cmp) & mask;
			if (bitfield) {
				// We have at least one bit set to one, so the largest smaller exists
				int one = clz(bitfield);
				one = (sizeof(bitfield) * 8) - one;
				*out = p.p2->children[one - 1];
			} else {
				if (n->type == ART_NODE16_KV) {
					*out = n;
				}
			}

			break;
		}
	case ART_NODE48_KV:
	case ART_NODE48: {
		p.p3 = (art_node48*)n;

		unsigned char cc = c - 1;
		while (cc < 255) {
			i = p.p3->keys[cc];
			if (i) {
				*out = p.p3->children[i - 1];
				break;
			}
			--cc;
		}
		if (cc == 255 && n->type == ART_NODE48_KV) {
			*out = n;
		}
		break;
	}
	case ART_NODE256_KV:
	case ART_NODE256: {
		p.p4 = (art_node256*)n;
		unsigned char cc = c - 1;
		if (c == 0 && n->type == ART_NODE256_KV) {
			*out = n;
		}
		while (cc < 255) {
			if (p.p4->children[cc]) {
				*out = p.p4->children[cc];
				break;
			}
			--cc;
		}
		break;
	}

	default:
		UNSTOPPABLE_ASSERT(false);
	}
}

#endif // ART_IMPL_H