Merge branch 'master' into grv-proxy

CONTRIBUTING.md

@@ -32,7 +32,7 @@ We draw inspiration from the Apache Software Foundation's informal motto: ["comm
 
 The project technical lead is Evan Tschannen (ejt@apple.com).
 
-Members of the Apple FoundationDB team are part of the initial core committers helping review individual contributions; you'll see them commenting on your pull requests. Future committers to the open source project, and the process for adding individuals in this role will be formalized in the future.
+Members of the Apple FoundationDB team are part of the core committers helping review individual contributions; you'll see them commenting on your pull requests.  As the FDB open source community has grown, some members of the community have consistently produced high quality code reviews and other significant contributions to FoundationDB.  The project technical lead maintains a list of external committers that actively contribute in this way, and gives them permission to review and merge pull requests.
 
 ## Contributing
 ### Opening a Pull Request

bindings/java/src/main/com/apple/foundationdb/RangeQuery.java

@@ -44,7 +44,7 @@ import com.apple.foundationdb.async.AsyncUtil;
  *   operation, the remove is not durable until {@code commit()} on the {@code Transaction}
  *   that yielded this query returns <code>true</code>.
  */
-class RangeQuery implements AsyncIterable<KeyValue>, Iterable<KeyValue> {
+class RangeQuery implements AsyncIterable<KeyValue> {
 	private final FDBTransaction tr;
 	private final KeySelector begin;
 	private final KeySelector end;

bindings/java/src/main/com/apple/foundationdb/testing/WorkloadContext.java

@@ -20,8 +20,6 @@
 
 package com.apple.foundationdb.testing;
 
-import java.util.Map;
-
 public class WorkloadContext {
 	long impl;
 

bindings/java/src/main/com/apple/foundationdb/tuple/Tuple.java

@@ -129,7 +129,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
 		}
 		return new Tuple(this, o,
 				(o instanceof Versionstamp && !((Versionstamp)o).isComplete()) ||
-						(o instanceof List<?> && TupleUtil.hasIncompleteVersionstamp(((List)o).stream())) ||
+						(o instanceof List<?> && TupleUtil.hasIncompleteVersionstamp(((List<?>)o).stream())) ||
 						(o instanceof Tuple && ((Tuple) o).hasIncompleteVersionstamp()));
 	}
 

bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java

@@ -788,7 +788,7 @@ class TupleUtil {
 				return hasIncompleteVersionstamp(((Tuple) item).stream());
 			}
 			else if(item instanceof Collection<?>) {
-				return hasIncompleteVersionstamp(((Collection) item).stream());
+				return hasIncompleteVersionstamp(((Collection<?>) item).stream());
 			}
 			else {
 				return false;

flow/Arena.cpp

@@ -20,6 +20,8 @@
 
 #include "Arena.h"
 
+#include "flow/UnitTest.h"
+
 // See https://dox.ipxe.org/memcheck_8h_source.html and https://dox.ipxe.org/valgrind_8h_source.html for an explanation
 // of valgrind client requests
 #ifdef VALGRIND_ARENA
@@ -388,3 +390,158 @@ void ArenaBlock::destroyLeaf() {
 		}
 	}
 }
+
+namespace {
+template <template <class> class VectorRefLike>
+void testRangeBasedForLoop() {
+	VectorRefLike<StringRef> xs;
+	Arena a;
+	int size = deterministicRandom()->randomInt(0, 100);
+	for (int i = 0; i < size; ++i) {
+		xs.push_back_deep(a, StringRef(std::to_string(i)));
+	}
+	ASSERT(xs.size() == size);
+	int i = 0;
+	for (const auto& x : xs) {
+		ASSERT(x == StringRef(std::to_string(i++)));
+	}
+	ASSERT(i == size);
+}
+
+template <template <class> class VectorRefLike>
+void testIteratorIncrement() {
+	VectorRefLike<StringRef> xs;
+	Arena a;
+	int size = deterministicRandom()->randomInt(0, 100);
+	for (int i = 0; i < size; ++i) {
+		xs.push_back_deep(a, StringRef(std::to_string(i)));
+	}
+	ASSERT(xs.size() == size);
+	{
+		int i = 0;
+		for (auto iter = xs.begin(); iter != xs.end();) {
+			ASSERT(*iter++ == StringRef(std::to_string(i++)));
+		}
+		ASSERT(i == size);
+	}
+	{
+		int i = 0;
+		for (auto iter = xs.begin(); iter != xs.end() && i < xs.size() - 1;) {
+			ASSERT(*++iter == StringRef(std::to_string(++i)));
+		}
+	}
+	{
+		int i = 0;
+		for (auto iter = xs.begin(); iter < xs.end();) {
+			ASSERT(*iter == StringRef(std::to_string(i)));
+			iter += 2;
+			i += 2;
+		}
+	}
+	{
+		int i = xs.size() - 1;
+		for (auto iter = xs.end() - 1; iter >= xs.begin();) {
+			ASSERT(*iter == StringRef(std::to_string(i)));
+			iter -= 2;
+			i -= 2;
+		}
+	}
+	{
+		int i = 0;
+		for (auto iter = xs.begin(); iter < xs.end();) {
+			ASSERT(*iter == StringRef(std::to_string(i)));
+			iter = iter + 2;
+			i += 2;
+		}
+	}
+	{
+		int i = xs.size() - 1;
+		for (auto iter = xs.end() - 1; iter >= xs.begin();) {
+			ASSERT(*iter == StringRef(std::to_string(i)));
+			iter = iter - 2;
+			i -= 2;
+		}
+	}
+}
+
+template <template <class> class VectorRefLike>
+void testReverseIterator() {
+	VectorRefLike<StringRef> xs;
+	Arena a;
+	int size = deterministicRandom()->randomInt(0, 100);
+	for (int i = 0; i < size; ++i) {
+		xs.push_back_deep(a, StringRef(std::to_string(i)));
+	}
+	ASSERT(xs.size() == size);
+
+	int i = xs.size() - 1;
+	for (auto iter = xs.rbegin(); iter != xs.rend();) {
+		ASSERT(*iter++ == StringRef(std::to_string(i--)));
+	}
+	ASSERT(i == -1);
+}
+
+template <template <class> class VectorRefLike>
+void testAppend() {
+	VectorRefLike<StringRef> xs;
+	Arena a;
+	int size = deterministicRandom()->randomInt(0, 100);
+	for (int i = 0; i < size; ++i) {
+		xs.push_back_deep(a, StringRef(std::to_string(i)));
+	}
+	VectorRefLike<StringRef> ys;
+	ys.append(a, xs.begin(), xs.size());
+	ASSERT(xs.size() == ys.size());
+	ASSERT(std::equal(xs.begin(), xs.end(), ys.begin()));
+}
+
+template <template <class> class VectorRefLike>
+void testCopy() {
+	Standalone<VectorRefLike<StringRef>> xs;
+	int size = deterministicRandom()->randomInt(0, 100);
+	for (int i = 0; i < size; ++i) {
+		xs.push_back_deep(xs.arena(), StringRef(std::to_string(i)));
+	}
+	Arena a;
+	VectorRefLike<StringRef> ys(a, xs);
+	xs = Standalone<VectorRefLike<StringRef>>();
+	int i = 0;
+	for (const auto& y : ys) {
+		ASSERT(y == StringRef(std::to_string(i++)));
+	}
+	ASSERT(i == size);
+}
+
+template <template <class> class VectorRefLike>
+void testVectorLike() {
+	testRangeBasedForLoop<VectorRefLike>();
+	testIteratorIncrement<VectorRefLike>();
+	testReverseIterator<VectorRefLike>();
+	testAppend<VectorRefLike>();
+	testCopy<VectorRefLike>();
+}
+} // namespace
+
+// Fix number of template parameters
+template <class T>
+using VectorRefProxy = VectorRef<T>;
+TEST_CASE("/flow/Arena/VectorRef") {
+	testVectorLike<VectorRefProxy>();
+	return Void();
+}
+
+// Fix number of template parameters
+template <class T>
+using SmallVectorRefProxy = SmallVectorRef<T>;
+TEST_CASE("/flow/Arena/SmallVectorRef") {
+	testVectorLike<SmallVectorRefProxy>();
+	return Void();
+}
+
+// Fix number of template parameters
+template <class T>
+using SmallVectorRef10Proxy = SmallVectorRef<T, 10>;
+TEST_CASE("/flow/Arena/SmallVectorRef10") {
+	testVectorLike<SmallVectorRef10Proxy>();
+	return Void();
+}

flow/Arena.h

@@ -829,6 +829,15 @@ public:
 		}
 	}
 
+	template <class It>
+	VectorRef(Arena& arena, It first, It last) {
+		if constexpr (flow_ref<T>::value) {
+			append_deep(arena, first, std::distance(first, last));
+		} else {
+			append(arena, first, std::distance(first, last));
+		}
+	}
+
 	VectorRef(T* data, int size) : data(data), m_size(size), m_capacity(size) {}
 	VectorRef(T* data, int size, int capacity) : data(data), m_size(size), m_capacity(capacity) {}
 	// VectorRef( const VectorRef<T>& toCopy ) : data( toCopy.data ), m_size( toCopy.m_size ), m_capacity(
@@ -917,7 +926,8 @@ public:
 		VPS::add(*ptr);
 		m_size++;
 	}
-	void append(Arena& p, const T* begin, int count) {
+	template <class It>
+	void append(Arena& p, It begin, int count) {
 		if (m_size + count > m_capacity) reallocate(p, m_size + count);
 		VPS::invalidate();
 		if (count > 0) {
@@ -1002,6 +1012,8 @@ protected:
 // won't need allocations in these cases.
 template <class T, int InlineMembers = 1>
 class SmallVectorRef {
+	static_assert(InlineMembers >= 0);
+
 public:
 	// types
 	template <bool isConst>
@@ -1012,28 +1024,52 @@ public:
 		int idx = 0;
 
 	public:
-		using Category = std::random_access_iterator_tag;
+		using iterator_category = std::random_access_iterator_tag;
 		using value_type = std::conditional_t<isConst, const T, T>;
 		using difference_type = int;
 		using pointer = value_type*;
 		using reference = value_type&;
 		friend class SmallVectorRef<T, InlineMembers>;
-		template <bool I>
-		friend bool operator<(const iterator_impl<I>&, const iterator_impl<I>&);
-		template <bool I>
-		friend bool operator>(const iterator_impl<I>&, const iterator_impl<I>&);
-		template <bool I>
-		friend bool operator<=(const iterator_impl<I>&, const iterator_impl<I>&);
-		template <bool I>
-		friend bool operator>=(const iterator_impl<I>&, const iterator_impl<I>&);
-		template <bool I>
-		friend self_t operator+(const iterator_impl<I>&, difference_type);
-		template <bool I>
-		friend self_t operator+(difference_type, const self_t&);
-		template <bool I>
-		friend self_t operator-(const iterator_impl<I>&, difference_type);
-		template <bool I>
-		friend difference_type operator-(iterator_impl<I>, self_t);
+		friend bool operator<(const self_t& lhs, const self_t& rhs) {
+			ASSERT(lhs.vec == rhs.vec);
+			return lhs.idx < rhs.idx;
+		}
+		friend bool operator>(const self_t& lhs, const self_t& rhs) {
+			ASSERT(lhs.vec == rhs.vec);
+			return lhs.idx > rhs.idx;
+		}
+		friend bool operator<=(const self_t& lhs, const self_t& rhs) {
+			ASSERT(lhs.vec == rhs.vec);
+			return lhs.idx <= rhs.idx;
+		}
+		friend bool operator>=(const self_t& lhs, const self_t& rhs) {
+			ASSERT(lhs.vec == rhs.vec);
+			return lhs.idx >= rhs.idx;
+		}
+		friend self_t operator+(const self_t& lhs, difference_type diff) {
+			auto res = lhs;
+			res.idx += diff;
+			return res;
+		}
+		friend self_t operator+(difference_type diff, const self_t& lhs) {
+			auto res = lhs;
+			res.idx += diff;
+			return res;
+		}
+		friend self_t operator-(const self_t& lhs, difference_type diff) {
+			auto res = lhs;
+			res.idx -= diff;
+			return res;
+		}
+		friend self_t operator-(difference_type diff, const self_t& lhs) {
+			auto res = lhs;
+			res.idx -= diff;
+			return res;
+		}
+		friend difference_type operator-(const self_t& lhs, const self_t& rhs) {
+			ASSERT(lhs.vec == rhs.vec);
+			return lhs.idx - rhs.idx;
+		}
 
 		self_t& operator++() {
 			++idx;
@@ -1068,7 +1104,7 @@ public:
 			if (i < InlineMembers) {
 				return vec->arr[i];
 			} else {
-				return vec->data[i];
+				return vec->data[i - InlineMembers];
 			}
 		}
 		reference get() const { return get(idx); }
@@ -1094,41 +1130,25 @@ public: // Construction
 
 	template <class T2 = T, int IM = InlineMembers>
 	SmallVectorRef(Arena& arena, const SmallVectorRef<T, IM>& toCopy,
-				   typename std::enable_if<!flow_ref<T2>::value, int>::type = 0)
-		: m_size(toCopy.m_size) {
-		if (toCopy.size() > InlineMembers) {
-			data.resize(arena, toCopy.size() - InlineMembers);
-		}
-		std::copy(toCopy.cbegin(), toCopy.cend(), begin());
+	               typename std::enable_if<!flow_ref<T2>::value, int>::type = 0)
+	  : m_size(0) {
+		append(arena, toCopy.begin(), toCopy.size());
 	}
 
 	template <class T2 = T, int IM = InlineMembers>
 	SmallVectorRef(Arena& arena, const SmallVectorRef<T2, IM>& toCopy,
 	               typename std::enable_if<flow_ref<T2>::value, int>::type = 0)
-	  : m_size(toCopy.m_size) {
-		for (int i = 0; i < toCopy.size(); ++i) {
-			if (i < arr.size()) {
-				new (&arr[i]) T(arena, toCopy[i]);
-			} else {
-				data.push_back_deep(arena, toCopy[i]);
-			}
-		}
+	  : m_size(0) {
+		append_deep(arena, toCopy.begin(), toCopy.size());
 	}
 
 	template <class It>
-	SmallVectorRef(Arena& arena, It first, It last)
-	  : m_size(0) {
-		while (first != last && m_size < InlineMembers) {
-			new (&arr[m_size++]) T(*(first++));
+	SmallVectorRef(Arena& arena, It first, It last) : m_size(0) {
+		if constexpr (flow_ref<T>::value) {
+			append_deep(arena, first, std::distance(first, last));
+		} else {
+			append(arena, first, std::distance(first, last));
 		}
-		while (first != last) {
-			data.push_back(arena, *(first++));
-		}
-	}
-
-	SmallVectorRef(SmallVectorRef<T, InlineMembers>&& o)
-		: m_size(o.m_size), arr(std::move(o.arr)), data(std::move(o.data)) {
-		o.m_size = 0;
 	}
 
 public: // information
@@ -1171,37 +1191,25 @@ public: // Modification
 	void pop_back() {--m_size; }
 
 	template <class It>
-	void append(Arena& arena, It first, It last) {
-		if (first == last) {
-			return;
-		}
-		auto d = std::distance(first, last);
-		if (m_size + d > InlineMembers) {
-			data.reserve(arena, m_size + d - InlineMembers);
-		}
-		while (first != last && m_size < InlineMembers) {
+	void append(Arena& arena, It first, int count) {
+		ASSERT(count >= 0);
+		while (count > 0 && m_size < InlineMembers) {
 			new (&(arr[m_size++])) T(*(first++));
+			--count;
 		}
-		while (first != last) {
-			data.push_back(arena, *(first++));
-		}
+		data.append(arena, first, count);
+		m_size += count;
 	}
 
 	template <class It>
-	void append_deep(Arena& arena, It first, It last) {
-		if (first == last) {
-			return;
-		}
-		auto d = std::distance(first, last);
-		if (m_size + d > InlineMembers) {
-			data.reserve(arena, m_size + d - InlineMembers);
-		}
-		while (first != last && m_size < InlineMembers) {
-			new (&(arr[m_size++])) T(arena, *(first++));
-		}
-		while (first != last) {
-			data.push_back_deep(arena, *(first++));
+	void append_deep(Arena& arena, It first, int count) {
+		ASSERT(count >= 0);
+		while (count > 0 && m_size < InlineMembers) {
+			new (&(arr[m_size++])) T(*(first++));
+			--count;
 		}
+		data.append_deep(arena, first, count);
+		m_size += count;
 	}
 
 public: // iterator access
@@ -1255,66 +1263,6 @@ private:
 	VectorRef<T> data;
 };
 
-template <class T, int InlineMembers, bool isConst>
-bool operator<(const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& lhs,
-               const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& rhs) {
-	return lhs.idx < rhs.idx;
-}
-
-template <class T, int InlineMembers, bool isConst>
-bool operator>(const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& lhs,
-               const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& rhs) {
-	return lhs.idx > rhs.idx;
-}
-
-template <class T, int InlineMembers, bool isConst>
-bool operator<=(const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& lhs,
-                const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& rhs) {
-	return lhs.idx <= rhs.idx;
-}
-
-template <class T, int InlineMembers, bool isConst>
-bool operator>=(const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& lhs,
-                const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& rhs) {
-	return lhs.idx >= rhs.idx;
-}
-
-template <class T, int InlineMembers, bool isConst>
-typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst> operator+(
-    const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& iter,
-    typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>::difference_type diff) {
-	auto res = iter;
-	res.idx += diff;
-	return res;
-}
-
-template <class T, int InlineMembers, bool isConst>
-typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst> operator+(
-    typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>::difference_type diff,
-    const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& iter) {
-	auto res = iter;
-	res.idx += diff;
-	return res;
-}
-
-template <class T, int InlineMembers, bool isConst>
-typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst> operator-(
-    const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& iter,
-    typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>::difference_type diff) {
-	auto res = iter;
-	res.idx -= diff;
-	return res;
-}
-
-template <class T, int InlineMembers, bool isConst>
-typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst> operator-(
-    typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>::difference_type diff,
-    const typename SmallVectorRef<T, InlineMembers>::template iterator_impl<isConst>& iter) {
-	auto res = iter;
-	res.idx -= diff;
-	return res;
-}
-
 template <class T>
 struct Traceable<VectorRef<T>> {
 	constexpr static bool value = Traceable<T>::value;

flow/Tracing.cpp

@@ -31,7 +31,9 @@ struct LogfileTracer : ITracer {
 	TracerType type() const { return TracerType::LOG_FILE; }
 	void trace(Span const& span) override {
 		TraceEvent te(SevInfo, "TracingSpan", span.context);
-		te.detail("Location", span.location.name).detail("Begin", span.begin).detail("End", span.end);
+		te.detail("Location", span.location.name)
+			.detail("Begin", format("%.6f", span.begin))
+			.detail("End", format("%.6f", span.end));
 		if (span.parents.size() == 1) {
 			te.detail("Parent", *span.parents.begin());
 		} else {
@@ -79,6 +81,7 @@ Span& Span::operator=(Span&& o) {
 
 Span::~Span() {
 	if (begin > 0.0) {
+		end = g_network->now();
 		g_tracer->trace(*this);
 	}
 }