wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into mengxu/tls-switch-status-PR

This commit is contained in:
Meng Xu 2019-03-14 10:35:12 -07:00
parent e30e2af1f3 178ebfec30
commit 5a10bf5dfc
74 changed files with 3385 additions and 1451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/java/CMakeLists.txt
View File

@ -54,6 +54,7 @@ set(JAVA_BINDING_SRCS
src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java
src/main/com/apple/foundationdb/tuple/IterableComparator.java
src/main/com/apple/foundationdb/tuple/package-info.java
src/main/com/apple/foundationdb/tuple/StringUtil.java
src/main/com/apple/foundationdb/tuple/Tuple.java
src/main/com/apple/foundationdb/tuple/TupleUtil.java
src/main/com/apple/foundationdb/tuple/Versionstamp.java)
@ -88,6 +89,7 @@ set(JAVA_TESTS_SRCS
src/test/com/apple/foundationdb/test/StackUtils.java
src/test/com/apple/foundationdb/test/TesterArgs.java
src/test/com/apple/foundationdb/test/TestResult.java
src/test/com/apple/foundationdb/test/TuplePerformanceTest.java
src/test/com/apple/foundationdb/test/TupleTest.java
src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java
src/test/com/apple/foundationdb/test/WatchTest.java

7
bindings/java/src/main/com/apple/foundationdb/subspace/Subspace.java
View File

@ -46,8 +46,8 @@ import com.apple.foundationdb.tuple.Versionstamp;
* </p>
*/
public class Subspace {
static final Tuple EMPTY_TUPLE = Tuple.from();
static final byte[] EMPTY_BYTES = new byte[0];
private static final Tuple EMPTY_TUPLE = Tuple.from();
private static final byte[] EMPTY_BYTES = new byte[0];
private final byte[] rawPrefix;
@ -248,8 +248,7 @@ public class Subspace {
* @return the {@link Range} of keyspace corresponding to {@code tuple}
*/
public Range range(Tuple tuple) {
Range p = tuple.range();
return new Range(join(rawPrefix, p.begin), join(rawPrefix, p.end));
return tuple.range(rawPrefix);
}
/**

150
bindings/java/src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java
View File

@ -20,7 +20,6 @@
package com.apple.foundationdb.tuple;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Arrays;
@ -154,7 +153,10 @@ public class ByteArrayUtil {
* @return a newly created array where {@code pattern} replaced with {@code replacement}
*/
public static byte[] replace(byte[] src, byte[] pattern, byte[] replacement) {
return join(replacement, split(src, pattern));
if(src == null) {
return null;
}
return replace(src, 0, src.length, pattern, replacement);
}
/**
@ -171,7 +173,75 @@ public class ByteArrayUtil {
*/
public static byte[] replace(byte[] src, int offset, int length,
byte[] pattern, byte[] replacement) {
return join(replacement, split(src, offset, length, pattern));
if(offset < 0 || offset > src.length) {
throw new IllegalArgumentException("Invalid offset for array pattern replacement");
}
if(length < 0 || offset + length > src.length) {
throw new IllegalArgumentException("Invalid length for array pattern replacement");
}
if(pattern == null || pattern.length == 0) {
return Arrays.copyOfRange(src, offset, offset + length);
}
ByteBuffer dest;
if(replacement == null || replacement.length != pattern.length) {
// Array might change size. This is the "tricky" case.
int newLength = replace(src, offset, length, pattern, replacement, null);
if(newLength != length) {
dest = ByteBuffer.allocate(newLength);
}
else {
// If the array size didn't change, as the pattern and replacement lengths
// differ, it must be the case that there weren't any occurrences of pattern in src
// between offset and offset + length, so we can just return a copy.
return Arrays.copyOfRange(src, offset, offset + length);
}
}
else {
// No matter what, the array will stay the same size as replacement.length = pattern.length
dest = ByteBuffer.allocate(length);
}
replace(src, offset, length, pattern, replacement, dest);
return dest.array();
}
// Replace any occurrences of pattern in src between offset and offset + length with replacement.
// The new array is serialized into dest and the new length is returned.
static int replace(byte[] src, int offset, int length, byte[] pattern, byte[] replacement, ByteBuffer dest) {
if(pattern == null || pattern.length == 0) {
if(dest != null) {
dest.put(src, offset, length);
}
return length;
}
byte patternFirst = pattern[0];
int lastPosition = offset;
int currentPosition = offset;
int newLength = 0;
int replacementLength = replacement == null ? 0 : replacement.length;
while(currentPosition < offset + length) {
if(src[currentPosition] == patternFirst && regionEquals(src, currentPosition, pattern)) {
if(dest != null) {
dest.put(src, lastPosition, currentPosition - lastPosition);
if(replacement != null) {
dest.put(replacement);
}
}
newLength += currentPosition - lastPosition + replacementLength;
currentPosition += pattern.length;
lastPosition = currentPosition;
}
else {
currentPosition++;
}
}
newLength += currentPosition - lastPosition;
if(dest != null) {
dest.put(src, lastPosition, currentPosition - lastPosition);
}
return newLength;
}
/**
@ -203,7 +273,7 @@ public class ByteArrayUtil {
* @return a list of byte arrays from {@code src} now not containing {@code delimiter}
*/
public static List<byte[]> split(byte[] src, int offset, int length, byte[] delimiter) {
List<byte[]> parts = new LinkedList<byte[]>();
List<byte[]> parts = new LinkedList<>();
int idx = offset;
int lastSplitEnd = offset;
while(idx <= (offset+length) - delimiter.length) {
@ -225,14 +295,6 @@ public class ByteArrayUtil {
return parts;
}
static int bisectLeft(BigInteger[] arr, BigInteger i) {
int n = Arrays.binarySearch(arr, i);
if(n >= 0)
return n;
int ip = (n + 1) * -1;
return ip;
}
/**
* Compare byte arrays for equality and ordering purposes. Elements in the array
* are interpreted and compared as unsigned bytes. Neither parameter
@ -277,61 +339,6 @@ public class ByteArrayUtil {
return true;
}
/**
* Scan through an array of bytes to find the first occurrence of a specific value.
*
* @param src array to scan. Must not be {@code null}.
* @param what the value for which to search.
* @param start the index at which to start the search. If this is at or after
* the end of {@code src}, the result will always be {@code -1}.
* @param end the index one past the last entry at which to search
*
* @return return the location of the first instance of {@code value}, or
* {@code -1} if not found.
*/
static int findNext(byte[] src, byte what, int start, int end) {
for(int i = start; i < end; i++) {
if(src[i] == what)
return i;
}
return -1;
}
/**
* Gets the index of the first element after the next occurrence of the byte sequence [nm]
* @param v the bytes to scan through
* @param n first character to find
* @param m second character to find
* @param start the index at which to start the scan
*
* @return the index after the next occurrence of [nm]
*/
static int findTerminator(byte[] v, byte n, byte m, int start) {
return findTerminator(v, n, m, start, v.length);
}
/**
* Gets the index of the first element after the next occurrence of the byte sequence [nm]
* @param v the bytes to scan through
* @param n first character to find
* @param m second character to find
* @param start the index at which to start the scan
* @param end the index at which to stop the search (exclusive)
*
* @return the index after the next occurrence of [nm]
*/
static int findTerminator(byte[] v, byte n, byte m, int start, int end) {
int pos = start;
while(true) {
pos = findNext(v, n, pos, end);
if(pos < 0)
return end;
if(pos + 1 == end || v[pos+1] != m)
return pos;
pos += 2;
}
}
/**
* Computes the first key that would sort outside the range prefixed by {@code key}.
* {@code key} must be non-null, and contain at least some character this is not
@ -418,5 +425,14 @@ public class ByteArrayUtil {
return s.toString();
}
static int nullCount(byte[] val) {
int nulls = 0;
for(int i = 0; i < val.length; i++) {
if(val[i] == 0x00)
nulls += 1;
}
return nulls;
}
private ByteArrayUtil() {}
}

118
bindings/java/src/main/com/apple/foundationdb/tuple/StringUtil.java Normal file
View File

@ -0,0 +1,118 @@
/*
* StringUtil.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.foundationdb.tuple;
final class StringUtil {
private static final char SURROGATE_COUNT = Character.MAX_LOW_SURROGATE - Character.MIN_HIGH_SURROGATE + 1;
private static final char ABOVE_SURROGATES = Character.MAX_VALUE - Character.MAX_LOW_SURROGATE;
static char adjustForSurrogates(char c, String s, int pos) {
if(c > Character.MAX_LOW_SURROGATE) {
return (char)(c - SURROGATE_COUNT);
}
else {
// Validate the UTF-16 string as this can do weird things on invalid strings
if((Character.isHighSurrogate(c) && (pos + 1 >= s.length() || !Character.isLowSurrogate(s.charAt(pos + 1)))) ||
(Character.isLowSurrogate(c) && (pos == 0 || !Character.isHighSurrogate(s.charAt(pos - 1))))) {
throw new IllegalArgumentException("malformed UTF-16 string does not follow high surrogate with low surrogate");
}
return (char)(c + ABOVE_SURROGATES);
}
}
// Compare two strings based on their UTF-8 code point values. Note that Java stores strings
// using UTF-16. However, {@link Tuple}s are encoded using UTF-8. Using unsigned byte comparison,
// UTF-8 strings will sort based on their Unicode codepoints. However, UTF-16 strings <em>almost</em>,
// but not quite, sort that way. This can be addressed by fixing up surrogates. There are 0x800 surrogate
// values and about 0x2000 code points above the maximum surrogate value. For anything that is a surrogate,
// shift it up by 0x2000, and anything that is above the maximum surrogate value, shift it down by 0x800.
// This makes all surrogates sort after all non-surrogates.
//
// See: https://ssl.icu-project.org/docs/papers/utf16_code_point_order.html
static int compareUtf8(String s1, String s2) {
// Ignore common prefix at the beginning which will compare equal regardless of encoding
int pos = 0;
while(pos < s1.length() && pos < s2.length() && s1.charAt(pos) == s2.charAt(pos)) {
pos++;
}
if(pos >= s1.length() || pos >= s2.length()) {
// One string is the prefix of another, so return based on length.
return Integer.compare(s1.length(), s2.length());
}
// Compare first different character
char c1 = s1.charAt(pos);
char c2 = s2.charAt(pos);
// Apply "fix up" for surrogates
if(c1 >= Character.MIN_HIGH_SURROGATE) {
c1 = adjustForSurrogates(c1, s1, pos);
}
if(c2 >= Character.MIN_HIGH_SURROGATE) {
c2 = adjustForSurrogates(c2, s2, pos);
}
return Character.compare(c1, c2);
}
static int packedSize(String s) {
final int strLength = s.length();
int size = 0;
int pos = 0;
while(pos < strLength) {
char c = s.charAt(pos);
if(c == '\0') {
// Null is encoded as \x00\xff
size += 2;
}
else if(c <= 0x7f) {
// ASCII code point. Only 1 byte.
size += 1;
}
else if(c <= 0x07ff) {
// 2 byte code point
size += 2;
}
else if(Character.isHighSurrogate(c)) {
if(pos + 1 < s.length() && Character.isLowSurrogate(s.charAt(pos + 1))) {
// High surrogate followed by low surrogate means the code point
// is between U+10000 and U+10FFFF, so it requires 4 bytes.
size += 4;
pos += 1;
}
else {
throw new IllegalArgumentException("malformed UTF-16 has high surrogate not followed by low surrogate");
}
}
else if(Character.isLowSurrogate(c)) {
throw new IllegalArgumentException("malformed UTF-16 has low surrogate without prior high surrogate");
}
else {
// 3 byte code point
size += 3;
}
pos += 1;
}
return size;
}
private StringUtil() {}
}

407
bindings/java/src/main/com/apple/foundationdb/tuple/Tuple.java
View File

@ -21,11 +21,11 @@
package com.apple.foundationdb.tuple;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
@ -68,18 +68,40 @@ import com.apple.foundationdb.Range;
* This class is not thread safe.
*/
public class Tuple implements Comparable<Tuple>, Iterable<Object> {
private static IterableComparator comparator = new IterableComparator();
private static final IterableComparator comparator = new IterableComparator();
private static final byte[] EMPTY_BYTES = new byte[0];
private List<Object> elements;
List<Object> elements;
private byte[] packed = null;
private int memoizedHash = 0;
private int memoizedPackedSize = -1;
private final boolean incompleteVersionstamp;
private Tuple(List<? extends Object> elements, Object newItem) {
this(elements);
private Tuple(Tuple original, Object newItem, boolean itemHasIncompleteVersionstamp) {
this.elements = new ArrayList<>(original.elements.size() + 1);
this.elements.addAll(original.elements);
this.elements.add(newItem);
incompleteVersionstamp = original.incompleteVersionstamp || itemHasIncompleteVersionstamp;
}
private Tuple(List<? extends Object> elements) {
this.elements = new ArrayList<>(elements);
private Tuple(List<Object> elements) {
this.elements = elements;
incompleteVersionstamp = TupleUtil.hasIncompleteVersionstamp(elements.stream());
}
/**
* Construct a new empty {@code Tuple}. After creation, items can be added
* with calls to the variations of {@code add()}.
*
* @see #from(Object...)
* @see #fromBytes(byte[])
* @see #fromItems(Iterable)
*/
public Tuple() {
elements = Collections.emptyList();
packed = EMPTY_BYTES;
memoizedPackedSize = 0;
incompleteVersionstamp = false;
}
/**
@ -105,7 +127,10 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
!(o instanceof Versionstamp)) {
throw new IllegalArgumentException("Parameter type (" + o.getClass().getName() + ") not recognized");
}
return new Tuple(this.elements, o);
return new Tuple(this, o,
(o instanceof Versionstamp && !((Versionstamp)o).isComplete()) ||
(o instanceof List<?> && TupleUtil.hasIncompleteVersionstamp(((List)o).stream())) ||
(o instanceof Tuple && ((Tuple) o).hasIncompleteVersionstamp()));
}
/**
@ -116,7 +141,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(String s) {
return new Tuple(this.elements, s);
return new Tuple(this, s, false);
}
/**
@ -127,7 +152,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(long l) {
return new Tuple(this.elements, l);
return new Tuple(this, l, false);
}
/**
@ -138,7 +163,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(byte[] b) {
return new Tuple(this.elements, b);
return new Tuple(this, b, false);
}
/**
@ -149,7 +174,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(boolean b) {
return new Tuple(this.elements, b);
return new Tuple(this, b, false);
}
/**
@ -160,7 +185,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(UUID uuid) {
return new Tuple(this.elements, uuid);
return new Tuple(this, uuid, false);
}
/**
@ -176,7 +201,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
if(bi == null) {
throw new NullPointerException("Number types in Tuple cannot be null");
}
return new Tuple(this.elements, bi);
return new Tuple(this, bi, false);
}
/**
@ -187,7 +212,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(float f) {
return new Tuple(this.elements, f);
return new Tuple(this, f, false);
}
/**
@ -198,7 +223,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(double d) {
return new Tuple(this.elements, d);
return new Tuple(this, d, false);
}
/**
@ -210,11 +235,11 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(Versionstamp v) {
return new Tuple(this.elements, v);
return new Tuple(this, v, !v.isComplete());
}
/**
* Creates a copy of this {@code Tuple} with an {@link List} appended as the last element.
* Creates a copy of this {@code Tuple} with a {@link List} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}).
* This adds the list as a single element nested within the outer {@code Tuple}.
*
@ -222,8 +247,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a newly created {@code Tuple}
*/
public Tuple add(List<? extends Object> l) {
return new Tuple(this.elements, l);
public Tuple add(List<?> l) {
return new Tuple(this, l, TupleUtil.hasIncompleteVersionstamp(l.stream()));
}
/**
@ -236,7 +261,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(Tuple t) {
return new Tuple(this.elements, t);
return new Tuple(this, t, t.hasIncompleteVersionstamp());
}
/**
@ -249,7 +274,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(byte[] b, int offset, int length) {
return new Tuple(this.elements, Arrays.copyOfRange(b, offset, offset + length));
return new Tuple(this, Arrays.copyOfRange(b, offset, offset + length), false);
}
/**
@ -260,8 +285,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a newly created {@code Tuple}
*/
public Tuple addAll(List<? extends Object> o) {
List<Object> merged = new ArrayList<Object>(o.size() + this.elements.size());
public Tuple addAll(List<?> o) {
List<Object> merged = new ArrayList<>(o.size() + this.elements.size());
merged.addAll(this.elements);
merged.addAll(o);
return new Tuple(merged);
@ -275,32 +300,88 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple addAll(Tuple other) {
List<Object> merged = new ArrayList<Object>(this.size() + other.size());
List<Object> merged = new ArrayList<>(this.size() + other.size());
merged.addAll(this.elements);
merged.addAll(other.peekItems());
return new Tuple(merged);
merged.addAll(other.elements);
Tuple t = new Tuple(merged);
if(!t.hasIncompleteVersionstamp() && packed != null && other.packed != null) {
t.packed = ByteArrayUtil.join(packed, other.packed);
}
if(memoizedPackedSize >= 0 && other.memoizedPackedSize >= 0) {
t.memoizedPackedSize = memoizedPackedSize + other.memoizedPackedSize;
}
return t;
}
/**
* Get an encoded representation of this {@code Tuple}. Each element is encoded to
* {@code byte}s and concatenated.
* {@code byte}s and concatenated. Note that once a {@code Tuple} has been packed, its
* serialized representation is stored internally so that future calls to this function
* are faster than the initial call.
*
* @return a serialized representation of this {@code Tuple}.
* @return a packed representation of this {@code Tuple}
*/
public byte[] pack() {
return pack(null);
return packInternal(null, true);
}
/**
* Get an encoded representation of this {@code Tuple}. Each element is encoded to
* {@code byte}s and concatenated, and then the prefix supplied is prepended to
* the array.
* the array. Note that once a {@code Tuple} has been packed, its serialized representation
* is stored internally so that future calls to this function are faster than the
* initial call.
*
* @param prefix additional byte-array prefix to prepend to serialized bytes.
* @return a serialized representation of this {@code Tuple} prepended by the {@code prefix}.
* @param prefix additional byte-array prefix to prepend to the packed bytes
* @return a packed representation of this {@code Tuple} prepended by the {@code prefix}
*/
public byte[] pack(byte[] prefix) {
return TupleUtil.pack(elements, prefix);
return packInternal(prefix, true);
}
byte[] packInternal(byte[] prefix, boolean copy) {
if(hasIncompleteVersionstamp()) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
if(packed == null) {
packed = TupleUtil.pack(elements, getPackedSize());
}
boolean hasPrefix = prefix != null && prefix.length > 0;
if(hasPrefix) {
return ByteArrayUtil.join(prefix, packed);
}
else if(copy) {
return Arrays.copyOf(packed, packed.length);
}
else {
return packed;
}
}
/**
* Pack an encoded representation of this {@code Tuple} onto the end of the given {@link ByteBuffer}.
* It is up to the caller to ensure that there is enough space allocated within the buffer
* to avoid {@link java.nio.BufferOverflowException}s. The client may call {@link #getPackedSize()}
* to determine how large this {@code Tuple} will be once packed in order to allocate sufficient memory.
* Note that unlike {@link #pack()}, the serialized representation of this {@code Tuple} is not stored, so
* calling this function multiple times with the same {@code Tuple} requires serializing the {@code Tuple}
* multiple times.
* <br>
* <br>
* This method will throw an error if there are any incomplete {@link Versionstamp}s in this {@code Tuple}.
*
* @param dest the destination {@link ByteBuffer} for the encoded {@code Tuple}
*/
public void packInto(ByteBuffer dest) {
if(hasIncompleteVersionstamp()) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
if(packed == null) {
TupleUtil.pack(dest, elements);
}
else {
dest.put(packed);
}
}
/**
@ -309,7 +390,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* This works the same as the {@link #packWithVersionstamp(byte[]) one-paramter version of this method},
* but it does not add any prefix to the array.
*
* @return a serialized representation of this {@code Tuple} for use with versionstamp ops.
* @return a packed representation of this {@code Tuple} for use with versionstamp ops.
* @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple}
*/
public byte[] packWithVersionstamp() {
@ -322,19 +403,58 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* There must be exactly one incomplete {@link Versionstamp} instance within this
* {@code Tuple} or this will throw an {@link IllegalArgumentException}.
* Each element is encoded to {@code byte}s and concatenated, the prefix
* is then prepended to the array, and then the index of the serialized incomplete
* is then prepended to the array, and then the index of the packed incomplete
* {@link Versionstamp} is appended as a little-endian integer. This can then be passed
* as the key to
* {@link com.apple.foundationdb.Transaction#mutate(com.apple.foundationdb.MutationType, byte[], byte[]) Transaction.mutate()}
* with the {@code SET_VERSIONSTAMPED_KEY} {@link com.apple.foundationdb.MutationType}, and the transaction's
* version will then be filled in at commit time.
* <br>
* <br>
* Note that once a {@code Tuple} has been packed, its serialized representation is stored internally so that
* future calls to this function are faster than the initial call.
*
* @param prefix additional byte-array prefix to prepend to serialized bytes.
* @return a serialized representation of this {@code Tuple} for use with versionstamp ops.
* @param prefix additional byte-array prefix to prepend to packed bytes.
* @return a packed representation of this {@code Tuple} for use with versionstamp ops.
* @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple}
*/
public byte[] packWithVersionstamp(byte[] prefix) {
return TupleUtil.packWithVersionstamp(elements, prefix);
return packWithVersionstampInternal(prefix, true);
}
byte[] packWithVersionstampInternal(byte[] prefix, boolean copy) {
if(!hasIncompleteVersionstamp()) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
}
if(packed == null) {
packed = TupleUtil.packWithVersionstamp(elements, getPackedSize());
}
boolean hasPrefix = prefix != null && prefix.length > 0;
if(hasPrefix) {
byte[] withPrefix = ByteArrayUtil.join(prefix, packed);
TupleUtil.adjustVersionPosition(withPrefix, prefix.length);
return withPrefix;
}
else if(copy) {
return Arrays.copyOf(packed, packed.length);
}
else {
return packed;
}
}
byte[] packMaybeVersionstamp() {
if(packed == null) {
if(hasIncompleteVersionstamp()) {
return packWithVersionstampInternal(null, false);
}
else {
return packInternal(null, false);
}
}
else {
return packed;
}
}
/**
@ -343,7 +463,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return the elements that make up this {@code Tuple}.
*/
public List<Object> getItems() {
return new ArrayList<Object>(elements);
return new ArrayList<>(elements);
}
/**
@ -355,16 +475,6 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return elements.stream();
}
/**
* Returns the internal elements that make up this tuple. For internal use only, as
* modifications to the result will mean that this Tuple is modified.
*
* @return the elements of this Tuple, without copying
*/
private List<Object> peekItems() {
return this.elements;
}
/**
* Gets an {@code Iterator} over the {@code Objects} in this {@code Tuple}. This {@code Iterator} is
* unmodifiable and will throw an exception if {@link Iterator#remove() remove()} is called.
@ -376,25 +486,16 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return Collections.unmodifiableList(this.elements).iterator();
}
/**
* Construct a new empty {@code Tuple}. After creation, items can be added
* with calls the the variations of {@code add()}.
*
* @see #from(Object...)
* @see #fromBytes(byte[])
* @see #fromItems(Iterable)
*/
public Tuple() {
this.elements = new LinkedList<Object>();
}
/**
* Construct a new {@code Tuple} with elements decoded from a supplied {@code byte} array.
* The passed byte array must not be {@code null}.
* The passed byte array must not be {@code null}. This will throw an exception if the passed byte
* array does not represent a valid {@code Tuple}. For example, this will throw an error if it
* encounters an unknown type code or if there is a packed element that appears to be truncated.
*
* @param bytes encoded {@code Tuple} source
*
* @return a new {@code Tuple} constructed by deserializing the provided {@code byte} array
* @throws IllegalArgumentException if {@code bytes} does not represent a valid {@code Tuple}
*/
public static Tuple fromBytes(byte[] bytes) {
return fromBytes(bytes, 0, bytes.length);
@ -402,17 +503,29 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Construct a new {@code Tuple} with elements decoded from a supplied {@code byte} array.
* The passed byte array must not be {@code null}.
* The passed byte array must not be {@code null}. This will throw an exception if the specified slice of
* the passed byte array does not represent a valid {@code Tuple}. For example, this will throw an error
* if it encounters an unknown type code or if there is a packed element that appears to be truncated.
*
* @param bytes encoded {@code Tuple} source
* @param offset starting offset of byte array of encoded data
* @param length length of encoded data within the source
*
* @return a new {@code Tuple} constructed by deserializing the specified slice of the provided {@code byte} array
* @throws IllegalArgumentException if {@code offset} or {@code length} are negative or would exceed the size of
* the array or if {@code bytes} does not represent a valid {@code Tuple}
*/
public static Tuple fromBytes(byte[] bytes, int offset, int length) {
Tuple t = new Tuple();
t.elements = TupleUtil.unpack(bytes, offset, length);
if(offset < 0 || offset > bytes.length) {
throw new IllegalArgumentException("Invalid offset for Tuple deserialization");
}
if(length < 0 || offset + length > bytes.length) {
throw new IllegalArgumentException("Invalid length for Tuple deserialization");
}
byte[] packed = Arrays.copyOfRange(bytes, offset, offset + length);
Tuple t = new Tuple(TupleUtil.unpack(packed));
t.packed = packed;
t.memoizedPackedSize = length;
return t;
}
@ -623,13 +736,14 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
Object o = this.elements.get(index);
if(o == null) {
return null;
} else if(o instanceof Tuple) {
}
else if(o instanceof Tuple) {
return ((Tuple)o).getItems();
} else if(o instanceof List<?>) {
List<Object> ret = new LinkedList<Object>();
ret.addAll((List<? extends Object>)o);
return ret;
} else {
}
else if(o instanceof List<?>) {
return new ArrayList<>((List<?>) o);
}
else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to list");
}
}
@ -650,11 +764,14 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
Object o = this.elements.get(index);
if(o == null) {
return null;
} else if(o instanceof Tuple) {
}
else if(o instanceof Tuple) {
return (Tuple)o;
} else if(o instanceof List<?>) {
return Tuple.fromItems((List<? extends Object>)o);
} else {
}
else if(o instanceof List<?>) {
return Tuple.fromList((List<?>)o);
}
else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to tuple");
}
}
@ -678,15 +795,10 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @throws IllegalStateException if this {@code Tuple} is empty
*/
public Tuple popFront() {
if(elements.size() == 0)
if(elements.isEmpty())
throw new IllegalStateException("Tuple contains no elements");
List<Object> items = new ArrayList<Object>(elements.size() - 1);
for(int i = 1; i < this.elements.size(); i++) {
items.add(this.elements.get(i));
}
return new Tuple(items);
return new Tuple(elements.subList(1, elements.size()));
}
/**
@ -697,15 +809,10 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @throws IllegalStateException if this {@code Tuple} is empty
*/
public Tuple popBack() {
if(elements.size() == 0)
if(elements.isEmpty())
throw new IllegalStateException("Tuple contains no elements");
List<Object> items = new ArrayList<Object>(elements.size() - 1);
for(int i = 0; i < this.elements.size() - 1; i++) {
items.add(this.elements.get(i));
}
return new Tuple(items);
return new Tuple(elements.subList(0, elements.size() - 1));
}
/**
@ -718,15 +825,43 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* Tuple t = Tuple.from("a", "b");
* Range r = t.range();</pre>
* {@code r} includes all tuples ("a", "b", ...)
* <br>
* This function will throw an error if this {@code Tuple} contains an incomplete
* {@link Versionstamp}.
*
* @return the range of keys containing all {@code Tuple}s that have this {@code Tuple}
* as a prefix
* @return the range of keys containing all possible keys that have this {@code Tuple}
* as a strict prefix
*/
public Range range() {
byte[] p = pack();
//System.out.println("Packed tuple is: " + ByteArrayUtil.printable(p));
return range(null);
}
/**
* Returns a range representing all keys that encode {@code Tuple}s strictly starting
* with the given prefix followed by this {@code Tuple}.
* <br>
* <br>
* For example:
* <pre>
* Tuple t = Tuple.from("a", "b");
* Range r = t.range(Tuple.from("c").pack());</pre>
* {@code r} contains all tuples ("c", "a", "b", ...)
* <br>
* This function will throw an error if this {@code Tuple} contains an incomplete
* {@link Versionstamp}.
*
* @param prefix a byte prefix to precede all elements in the range
*
* @return the range of keys containing all possible keys that have {@code prefix}
* followed by this {@code Tuple} as a strict prefix
*/
public Range range(byte[] prefix) {
if(hasIncompleteVersionstamp()) {
throw new IllegalStateException("Tuple with incomplete versionstamp used for range");
}
byte[] p = packInternal(prefix, false);
return new Range(ByteArrayUtil.join(p, new byte[] {0x0}),
ByteArrayUtil.join(p, new byte[] {(byte)0xff}));
ByteArrayUtil.join(p, new byte[] {(byte)0xff}));
}
/**
@ -739,7 +874,41 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* {@code Tuple}
*/
public boolean hasIncompleteVersionstamp() {
return TupleUtil.hasIncompleteVersionstamp(stream());
return incompleteVersionstamp;
}
/**
* Get the number of bytes in the packed representation of this {@code Tuple}. This is done by summing
* the serialized sizes of all of the elements of this {@code Tuple} and does not pack everything
* into a single {@code Tuple}. The return value of this function is stored within this {@code Tuple}
* after this function has been called so that subsequent calls on the same object are fast. This method
* does not validate that there is not more than one incomplete {@link Versionstamp} in this {@code Tuple}.
*
* @return the number of bytes in the packed representation of this {@code Tuple}
*/
public int getPackedSize() {
if(memoizedPackedSize < 0) {
memoizedPackedSize = getPackedSize(false);
}
return memoizedPackedSize;
}
int getPackedSize(boolean nested) {
if(memoizedPackedSize >= 0) {
if(!nested) {
return memoizedPackedSize;
}
int nullCount = 0;
for(Object elem : elements) {
if(elem == null) {
nullCount++;
}
}
return memoizedPackedSize + nullCount;
}
else {
return TupleUtil.getPackedSize(elements, nested);
}
}
/**
@ -756,7 +925,14 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*/
@Override
public int compareTo(Tuple t) {
return comparator.compare(elements, t.elements);
// If either tuple has an incomplete versionstamp, then there is a possibility that the byte order
// is not the semantic comparison order.
if(packed != null && t.packed != null && !hasIncompleteVersionstamp() && !t.hasIncompleteVersionstamp()) {
return ByteArrayUtil.compareUnsigned(packed, t.packed);
}
else {
return comparator.compare(elements, t.elements);
}
}
/**
@ -772,14 +948,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
@Override
public int hashCode() {
if(memoizedHash == 0) {
byte[] packed;
if(hasIncompleteVersionstamp()) {
packed = packWithVersionstamp(null);
}
else {
packed = pack();
}
memoizedHash = Arrays.hashCode(packed);
memoizedHash = Arrays.hashCode(packMaybeVersionstamp());
}
return memoizedHash;
}
@ -857,12 +1026,15 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple fromItems(Iterable<? extends Object> items) {
Tuple t = new Tuple();
for(Object o : items) {
t = t.addObject(o);
public static Tuple fromItems(Iterable<?> items) {
if(items instanceof List<?>) {
return Tuple.fromList((List<?>)items);
}
return t;
List<Object> elements = new ArrayList<>();
for(Object o : items) {
elements.add(o);
}
return new Tuple(elements);
}
/**
@ -875,8 +1047,9 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple fromList(List<? extends Object> items) {
return new Tuple(items);
public static Tuple fromList(List<?> items) {
List<Object> elements = new ArrayList<>(items);
return new Tuple(elements);
}
/**
@ -890,10 +1063,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple fromStream(Stream<? extends Object> items) {
Tuple t = new Tuple();
t.elements = items.collect(Collectors.toList());
return t;
public static Tuple fromStream(Stream<?> items) {
return new Tuple(items.collect(Collectors.toList()));
}
/**
@ -907,7 +1078,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple from(Object... items) {
return fromList(Arrays.asList(items));
return new Tuple(Arrays.asList(items));
}
static void main(String[] args) {
@ -1011,7 +1182,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
}
private static Tuple createTuple(int items) {
List<Object> elements = new ArrayList<Object>(items);
List<Object> elements = new ArrayList<>(items);
for(int i = 0; i < items; i++) {
elements.add(new byte[]{99});
}

927
bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java
View File

File diff suppressed because it is too large Load Diff

4
bindings/java/src/main/com/apple/foundationdb/tuple/Versionstamp.java
View File

@ -94,8 +94,8 @@ public class Versionstamp implements Comparable<Versionstamp> {
private static final byte[] UNSET_TRANSACTION_VERSION = {(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff,
(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff};
private boolean complete;
private byte[] versionBytes;
private final boolean complete;
private final byte[] versionBytes;
/**
* From a byte array, unpack the user version starting at the given position.

6
bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java
View File

@ -412,7 +412,11 @@ public class AsyncStackTester {
return inst.popParams(listSize).thenAcceptAsync(rawElements -> {
List<Tuple> tuples = new ArrayList<>(listSize);
for(Object o : rawElements) {
tuples.add(Tuple.fromBytes((byte[])o));
// Unpacking a tuple keeps around the serialized representation and uses
// it for comparison if it's available. To test semantic comparison, recreate
// the tuple from the item list.
Tuple t = Tuple.fromBytes((byte[])o);
tuples.add(Tuple.fromList(t.getItems()));
}
Collections.sort(tuples);
for(Tuple t : tuples) {

8
bindings/java/src/test/com/apple/foundationdb/test/StackTester.java
View File

@ -368,9 +368,13 @@ public class StackTester {
else if (op == StackOperation.TUPLE_SORT) {
int listSize = StackUtils.getInt(inst.popParam().join());
List<Object> rawElements = inst.popParams(listSize).join();
List<Tuple> tuples = new ArrayList<Tuple>(listSize);
List<Tuple> tuples = new ArrayList<>(listSize);
for(Object o : rawElements) {
tuples.add(Tuple.fromBytes((byte[])o));
// Unpacking a tuple keeps around the serialized representation and uses
// it for comparison if it's available. To test semantic comparison, recreate
// the tuple from the item list.
Tuple t = Tuple.fromBytes((byte[])o);
tuples.add(Tuple.fromList(t.getItems()));
}
Collections.sort(tuples);
for(Tuple t : tuples) {

177
bindings/java/src/test/com/apple/foundationdb/test/TuplePerformanceTest.java
View File

@ -13,17 +13,26 @@ import com.apple.foundationdb.tuple.Versionstamp;
public class TuplePerformanceTest {
private enum GeneratedTypes {
ALL,
LONG,
FLOATING_POINT,
STRING_LIKE
}
private final Random r;
private final int ignoreIterations;
private final int iterations;
private final GeneratedTypes generatedTypes;
public TuplePerformanceTest(Random r, int ignoreIterations, int iterations) {
public TuplePerformanceTest(Random r, int ignoreIterations, int iterations, GeneratedTypes generatedTypes) {
this.r = r;
this.ignoreIterations = ignoreIterations;
this.iterations = iterations;
this.generatedTypes = generatedTypes;
}
public Tuple createTuple(int length) {
public Tuple createMultiTypeTuple(int length) {
List<Object> values = new ArrayList<>(length);
for(int i = 0; i < length; i++) {
double choice = r.nextDouble();
@ -38,7 +47,7 @@ public class TuplePerformanceTest {
else if(choice < 0.3) {
char[] chars = new char[r.nextInt(20)];
for (int j = 0; j < chars.length; j++) {
chars[j] = (char)('a' + r.nextInt(26));
chars[j] = (char) ('a' + r.nextInt(26));
}
values.add(new String(chars));
}
@ -69,7 +78,91 @@ public class TuplePerformanceTest {
values.add(nested);
}
}
return Tuple.from(values);
return Tuple.fromList(values);
}
public Tuple createLongsTuple(int length) {
List<Object> values = new ArrayList<>(length);
for(int i = 0; i < length; i++) {
int byteLength = r.nextInt(Long.BYTES + 1);
long val = 0L;
for(int x = 0; x < byteLength; x++) {
int nextBytes = r.nextInt(256);
val = (val << 8) + nextBytes;
}
values.add(val);
}
return Tuple.fromList(values);
}
public Tuple createFloatingPointTuple(int length) {
List<Object> values = new ArrayList<>(length);
for(int i = 0; i < length; i++) {
double choice = r.nextDouble();
if(choice < 0.40) {
values.add(r.nextFloat());
}
else if(choice < 0.80) {
values.add(r.nextDouble());
}
// These last two are more likely to produce NaN values
else if(choice < 0.90) {
values.add(Float.intBitsToFloat(r.nextInt()));
}
else {
values.add(Double.longBitsToDouble(r.nextLong()));
}
}
return Tuple.fromList(values);
}
public Tuple createStringLikeTuple(int length) {
List<Object> values = new ArrayList<>(length);
for(int i = 0; i < length; i++) {
double choice = r.nextDouble();
if(choice < 0.4) {
byte[] arr = new byte[r.nextInt(20)];
r.nextBytes(arr);
values.add(arr);
}
else if(choice < 0.8) {
// Random ASCII codepoints
int[] codepoints = new int[r.nextInt(20)];
for(int x = 0; x < codepoints.length; x++) {
codepoints[x] = r.nextInt(0x7F);
}
values.add(new String(codepoints, 0, codepoints.length));
}
else if(choice < 0.9) {
// All zeroes
byte[] zeroes = new byte[r.nextInt(20)];
values.add(zeroes);
}
else {
// Random Unicode codepoints
int[] codepoints = new int[r.nextInt(20)];
for(int x = 0; x < codepoints.length; x++) {
codepoints[x] = r.nextInt(0x10FFFF);
}
values.add(new String(codepoints, 0, codepoints.length));
}
}
return Tuple.fromList(values);
}
public Tuple createTuple(int length) {
switch (generatedTypes) {
case ALL:
return createMultiTypeTuple(length);
case LONG:
return createLongsTuple(length);
case FLOATING_POINT:
return createFloatingPointTuple(length);
case STRING_LIKE:
return createStringLikeTuple(length);
default:
throw new IllegalStateException("unknown generated types " + generatedTypes);
}
}
public void run() {
@ -86,6 +179,8 @@ public class TuplePerformanceTest {
long packNanos = 0L;
long unpackNanos = 0L;
long equalsNanos = 0L;
long equalsArrayNanos = 0L;
long sizeNanos = 0L;
long hashNanos = 0L;
long secondHashNanos = 0L;
long subspacePackNanos = 0L;
@ -93,6 +188,9 @@ public class TuplePerformanceTest {
long totalLength = 0L;
long totalBytes = 0L;
for(int i = 0; i < iterations; i++) {
if(i % 100_000 == 0) {
System.out.println(" iteration " + i);
}
int length = r.nextInt(20);
Tuple t = createTuple(length);
@ -100,20 +198,39 @@ public class TuplePerformanceTest {
byte[] serialized = t.pack();
long endNanos = System.nanoTime();
packNanos += endNanos - startNanos;
totalLength += length;
totalBytes += serialized.length;
totalLength += t.size();
totalBytes += t.getPackedSize();
startNanos = System.nanoTime();
Tuple t2 = Tuple.fromBytes(serialized);
endNanos = System.nanoTime();
unpackNanos += endNanos - startNanos;
// Copy items over as if both are packed, their byte arrays are compared
Tuple tCopy = Tuple.fromList(t.getItems());
Tuple t2Copy = Tuple.fromList(t2.getItems());
startNanos = System.nanoTime();
if (!tCopy.equals(t2Copy)) {
throw new RuntimeException("deserialized did not match serialized: " + t + " -- " + t2);
}
endNanos = System.nanoTime();
equalsNanos += endNanos - startNanos;
startNanos = System.nanoTime();
if(!t.equals(t2)) {
throw new RuntimeException("deserialized did not match serialized: " + t + " -- " + t2);
}
endNanos = System.nanoTime();
equalsNanos += endNanos - startNanos;
equalsArrayNanos += endNanos - startNanos;
tCopy = Tuple.fromList(t.getItems());
startNanos = System.nanoTime();
int size = tCopy.getPackedSize();
endNanos = System.nanoTime();
if (size != t.pack().length) {
throw new RuntimeException("packed size did not match actual packed length: " + t + " -- " + " " + tCopy.getPackedSize() + " instead of " + t.getPackedSize());
}
sizeNanos += endNanos - startNanos;
startNanos = System.nanoTime();
byte[] subspacePacked = subspace.pack(t);
@ -126,7 +243,7 @@ public class TuplePerformanceTest {
startNanos = System.nanoTime();
Tuple t3 = subspace.unpack(subspacePacked);
endNanos = System.nanoTime();
if(!t.equals(t3)) {
if (!Tuple.fromList(t.getItems()).equals(Tuple.fromList(t3.getItems())) || !t.equals(t3)) {
throw new RuntimeException("does not unpack equally from subspace");
}
if(!Arrays.equals(t.pack(), t3.pack())) {
@ -149,29 +266,33 @@ public class TuplePerformanceTest {
}
System.out.println("Test ended.");
System.out.printf(" Total elements: %d%n", totalLength);
System.out.printf(" Total bytes: %d kB%n", totalBytes / 1000);
System.out.printf(" Bytes per tuple: %f B%n", totalBytes * 1.0 / iterations);
System.out.printf(" Pack time: %f s%n", packNanos * 1e-9);
System.out.printf(" Pack time per tuple: %f \u03BCs%n", packNanos * 1e-3 / iterations);
System.out.printf(" Pack time per kB: %f \u03BCs%n", packNanos * 1.0 / totalBytes);
System.out.printf(" Serialization rate: %f objects / \u03BCs%n", totalLength * 1000.0 / packNanos);
System.out.printf(" Unpack time: %f s%n", unpackNanos * 1e-9);
System.out.printf(" Unpack time per tuple: %f \u03BCs%n", unpackNanos * 1e-3 / iterations);
System.out.printf(" Equals time: %f s%n", equalsNanos * 1e-9);
System.out.printf(" Equals time per tuple: %f \u03BCs%n", equalsNanos * 1e-3 / iterations);
System.out.printf(" Subspace pack time: %f s%n", subspacePackNanos * 1e-9);
System.out.printf(" Subspace pack time per tuple: %f \u03BCs%n", subspacePackNanos * 1e-3 / iterations);
System.out.printf(" Subspace unpack time: %f s%n", subspaceUnpackNanos * 1e-9);
System.out.printf(" Subspace unpack time per tuple: %f \u03BCs%n", subspaceUnpackNanos * 1e-3 / iterations);
System.out.printf(" Hash time: %f s%n", hashNanos * 1e-9);
System.out.printf(" Hash time per tuple: %f \u03BCs%n", hashNanos * 1e-3 / iterations);
System.out.printf(" Second hash time: %f s%n", secondHashNanos * 1e-9);
System.out.printf(" Second hash time per tuple: %f \u03BCs%n", secondHashNanos * 1e-3 / iterations);
System.out.printf(" Total elements: %d%n", totalLength);
System.out.printf(" Total bytes: %d kB%n", totalBytes / 1000);
System.out.printf(" Bytes per tuple: %f B%n", totalBytes * 1.0 / iterations);
System.out.printf(" Pack time: %f s%n", packNanos * 1e-9);
System.out.printf(" Pack time per tuple: %f \u03BCs%n", packNanos * 1e-3 / iterations);
System.out.printf(" Pack time per kB: %f \u03BCs%n", packNanos * 1.0 / totalBytes);
System.out.printf(" Serialization rate: %f objects / \u03BCs%n", totalLength * 1000.0 / packNanos);
System.out.printf(" Unpack time: %f s%n", unpackNanos * 1e-9);
System.out.printf(" Unpack time per tuple: %f \u03BCs%n", unpackNanos * 1e-3 / iterations);
System.out.printf(" Equals time: %f s%n", equalsNanos * 1e-9);
System.out.printf(" Equals time per tuple: %f \u03BCs%n", equalsNanos * 1e-3 / iterations);
System.out.printf(" Equals time (using packed): %f s%n", equalsArrayNanos * 1e-9);
System.out.printf(" Equals time (using packed) per tuple: %f \u03BCs%n", equalsArrayNanos * 1e-3 / iterations);
System.out.printf(" Size time: %f s%n", sizeNanos * 1e-9);
System.out.printf(" Size time per tuple: %f \u03BCs%n", sizeNanos * 1e-3 / iterations);
System.out.printf(" Subspace pack time: %f s%n", subspacePackNanos * 1e-9);
System.out.printf(" Subspace pack time per tuple: %f \u03BCs%n", subspacePackNanos * 1e-3 / iterations);
System.out.printf(" Subspace unpack time: %f s%n", subspaceUnpackNanos * 1e-9);
System.out.printf(" Subspace unpack time per tuple: %f \u03BCs%n", subspaceUnpackNanos * 1e-3 / iterations);
System.out.printf(" Hash time: %f s%n", hashNanos * 1e-9);
System.out.printf(" Hash time per tuple: %f \u03BCs%n", hashNanos * 1e-3 / iterations);
System.out.printf(" Second hash time: %f s%n", secondHashNanos * 1e-9);
System.out.printf(" Second hash time per tuple: %f \u03BCs%n", secondHashNanos * 1e-3 / iterations);
}
public static void main(String[] args) {
TuplePerformanceTest tester = new TuplePerformanceTest(new Random(), 100_000, 10_000);
TuplePerformanceTest tester = new TuplePerformanceTest(new Random(), 100_000, 10_000_000, GeneratedTypes.ALL);
tester.run();
}
}

916
bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java
View File

@ -20,16 +20,42 @@
package com.apple.foundationdb.test;
import java.math.BigInteger;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Stream;
import com.apple.foundationdb.Database;
import com.apple.foundationdb.FDB;
import com.apple.foundationdb.TransactionContext;
import com.apple.foundationdb.subspace.Subspace;
import com.apple.foundationdb.tuple.ByteArrayUtil;
import com.apple.foundationdb.tuple.Tuple;
import com.apple.foundationdb.tuple.Versionstamp;
public class TupleTest {
private static final byte FF = (byte)0xff;
public static void main(String[] args) throws InterruptedException {
final int reps = 1000;
try {
FDB fdb = FDB.selectAPIVersion(610);
addMethods();
comparisons();
emptyTuple();
incompleteVersionstamps();
intoBuffer();
offsetsAndLengths();
malformedBytes();
replaceTests();
serializedForms();
try(Database db = fdb.open()) {
runTests(reps, db);
}
@ -38,6 +64,896 @@ public class TupleTest {
}
}
private static class TupleSerialization {
private final Tuple tuple;
private final byte[] serialization;
TupleSerialization(Tuple tuple, byte[] serialization) {
this.tuple = tuple;
this.serialization = serialization;
}
static void addAll(List<TupleSerialization> list, Object... args) {
for(int i = 0; i < args.length; i += 2) {
TupleSerialization serialization = new TupleSerialization((Tuple)args[i], (byte[])args[i + 1]);
list.add(serialization);
}
}
}
private static void serializedForms() {
List<TupleSerialization> serializations = new ArrayList<>();
TupleSerialization.addAll(serializations,
Tuple.from(), new byte[0],
Tuple.from(0L), new byte[]{0x14},
Tuple.from(BigInteger.ZERO), new byte[]{0x14},
Tuple.from(1L), new byte[]{0x15, 0x01},
Tuple.from(BigInteger.ONE), new byte[]{0x15, 0x01},
Tuple.from(-1L), new byte[]{0x13, FF - 1},
Tuple.from(BigInteger.ONE.negate()), new byte[]{0x13, FF - 1},
Tuple.from(255L), new byte[]{0x15, FF},
Tuple.from(BigInteger.valueOf(255)), new byte[]{0x15, FF},
Tuple.from(-255L), new byte[]{0x13, 0x00},
Tuple.from(BigInteger.valueOf(-255)), new byte[]{0x13, 0x00},
Tuple.from(256L), new byte[]{0x16, 0x01, 0x00},
Tuple.from(BigInteger.valueOf(256)), new byte[]{0x16, 0x01, 0x00},
Tuple.from(-256L), new byte[]{0x12, FF - 1, FF},
Tuple.from(BigInteger.valueOf(-256)), new byte[]{0x12, FF - 1, FF},
Tuple.from(65536), new byte[]{0x17, 0x01, 0x00, 0x00},
Tuple.from(-65536), new byte[]{0x11, FF - 1, FF, FF},
Tuple.from(Long.MAX_VALUE), new byte[]{0x1C, 0x7f, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(BigInteger.valueOf(Long.MAX_VALUE)), new byte[]{0x1C, 0x7f, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE)), new byte[]{0x1C, (byte)0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE)), new byte[]{0x1C, FF, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(BigInteger.ONE.shiftLeft(64)), new byte[]{0x1D, 0x09, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(-((1L << 32) - 1)), new byte[]{0x10, 0x00, 0x00, 0x00, 0x00},
Tuple.from(BigInteger.ONE.shiftLeft(32).subtract(BigInteger.ONE).negate()), new byte[]{0x10, 0x00, 0x00, 0x00, 0x00},
Tuple.from(Long.MIN_VALUE + 2), new byte[]{0x0C, (byte)0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
Tuple.from(Long.MIN_VALUE + 1), new byte[]{0x0C, (byte)0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(BigInteger.valueOf(Long.MIN_VALUE).add(BigInteger.ONE)), new byte[]{0x0C, (byte)0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(Long.MIN_VALUE), new byte[]{0x0C, 0x7f, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(BigInteger.valueOf(Long.MIN_VALUE)), new byte[]{0x0C, 0x7f, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(BigInteger.valueOf(Long.MIN_VALUE).subtract(BigInteger.ONE)), new byte[]{0x0C, 0x7f, FF, FF, FF, FF, FF, FF, FF - 1},
Tuple.from(BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE).negate()), new byte[]{0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(3.14f), new byte[]{0x20, (byte)0xc0, 0x48, (byte)0xf5, (byte)0xc3},
Tuple.from(-3.14f), new byte[]{0x20, (byte)0x3f, (byte)0xb7, (byte)0x0a, (byte)0x3c},
Tuple.from(3.14), new byte[]{0x21, (byte)0xc0, (byte)0x09, (byte)0x1e, (byte)0xb8, (byte)0x51, (byte)0xeb, (byte)0x85, (byte)0x1f},
Tuple.from(-3.14), new byte[]{0x21, (byte)0x3f, (byte)0xf6, (byte)0xe1, (byte)0x47, (byte)0xae, (byte)0x14, (byte)0x7a, (byte)0xe0},
Tuple.from(0.0f), new byte[]{0x20, (byte)0x80, 0x00, 0x00, 0x00},
Tuple.from(-0.0f), new byte[]{0x20, 0x7f, FF, FF, FF},
Tuple.from(0.0), new byte[]{0x21, (byte)0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(-0.0), new byte[]{0x21, 0x7f, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(Float.POSITIVE_INFINITY), new byte[]{0x20, FF, (byte)0x80, 0x00, 0x00},
Tuple.from(Float.NEGATIVE_INFINITY), new byte[]{0x20, 0x00, 0x7f, FF, FF},
Tuple.from(Double.POSITIVE_INFINITY), new byte[]{0x21, FF, (byte)0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from(Double.NEGATIVE_INFINITY), new byte[]{0x21, 0x00, 0x0f, FF, FF, FF, FF, FF, FF},
Tuple.from(Float.intBitsToFloat(Integer.MAX_VALUE)), new byte[]{0x20, FF, FF, FF, FF},
Tuple.from(Double.longBitsToDouble(Long.MAX_VALUE)), new byte[]{0x21, FF, FF, FF, FF, FF, FF, FF, FF},
Tuple.from(Float.intBitsToFloat(~0)), new byte[]{0x20, 0x00, 0x00, 0x00, 0x00},
Tuple.from(Double.longBitsToDouble(~0L)), new byte[]{0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
Tuple.from((Object)new byte[0]), new byte[]{0x01, 0x00},
Tuple.from((Object)new byte[]{0x01, 0x02, 0x03}), new byte[]{0x01, 0x01, 0x02, 0x03, 0x00},
Tuple.from((Object)new byte[]{0x00, 0x00, 0x00, 0x04}), new byte[]{0x01, 0x00, FF, 0x00, FF, 0x00, FF, 0x04, 0x00},
Tuple.from(""), new byte[]{0x02, 0x00},
Tuple.from("hello"), new byte[]{0x02, 'h', 'e', 'l', 'l', 'o', 0x00},
Tuple.from("\u4e2d\u6587"), new byte[]{0x02, (byte)0xe4, (byte)0xb8, (byte)0xad, (byte)0xe6, (byte)0x96, (byte)0x87, 0x00},
Tuple.from("\u03bc\u03ac\u03b8\u03b7\u03bc\u03b1"), new byte[]{0x02, (byte)0xce, (byte)0xbc, (byte)0xce, (byte)0xac, (byte)0xce, (byte)0xb8, (byte)0xce, (byte)0xb7, (byte)0xce, (byte)0xbc, (byte)0xce, (byte)0xb1, 0x00},
Tuple.from(new String(new int[]{0x1f525}, 0, 1)), new byte[]{0x02, (byte)0xf0, (byte)0x9f, (byte)0x94, (byte)0xa5, 0x00},
Tuple.from("\ud83d\udd25"), new byte[]{0x02, (byte)0xf0, (byte)0x9f, (byte)0x94, (byte)0xa5, 0x00},
Tuple.from("\ud83e\udd6f"), new byte[]{0x02, (byte)0xf0, (byte)0x9f, (byte)0xa5, (byte)0xaf, 0x00},
Tuple.from("\ud83d"), new byte[]{0x02, 0x3f, 0x00},
Tuple.from("\udd25\ud83e\udd6f"), new byte[]{0x02, 0x3f, (byte)0xf0, (byte)0x9f, (byte)0xa5, (byte)0xaf, 0x00}, // malformed string - low surrogate without high surrogate
Tuple.from("a\udd25\ud83e\udd6f"), new byte[]{0x02, 'a', 0x3f, (byte)0xf0, (byte)0x9f, (byte)0xa5, (byte)0xaf, 0x00}, // malformed string - low surrogate without high surrogate
Tuple.from(Tuple.from((Object)null)), new byte[]{0x05, 0x00, FF, 0x00},
Tuple.from(Tuple.from(null, "hello")), new byte[]{0x05, 0x00, FF, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00, 0x00},
Tuple.from(Arrays.asList(null, "hello")), new byte[]{0x05, 0x00, FF, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00, 0x00},
Tuple.from(Tuple.from(null, "hell\0")), new byte[]{0x05, 0x00, FF, 0x02, 'h', 'e', 'l', 'l', 0x00, FF, 0x00, 0x00},
Tuple.from(Arrays.asList(null, "hell\0")), new byte[]{0x05, 0x00, FF, 0x02, 'h', 'e', 'l', 'l', 0x00, FF, 0x00, 0x00},
Tuple.from(Tuple.from((Object)null), "hello"), new byte[]{0x05, 0x00, FF, 0x00, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00},
Tuple.from(Tuple.from((Object)null), "hello", new byte[]{0x01, 0x00}, new byte[0]), new byte[]{0x05, 0x00, FF, 0x00, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00, 0x01, 0x01, 0x00, FF, 0x00, 0x01, 0x00},
Tuple.from(new UUID(0xba5eba11, 0x5ca1ab1e)), new byte[]{0x30, FF, FF, FF, FF, (byte)0xba, 0x5e, (byte)0xba, 0x11, 0x00, 0x00, 0x00, 0x00, 0x5c, (byte)0xa1, (byte)0xab, 0x1e},
Tuple.from(false), new byte[]{0x26},
Tuple.from(true), new byte[]{0x27},
Tuple.from((short)0x3019), new byte[]{0x16, 0x30, 0x19},
Tuple.from((byte)0x03), new byte[]{0x15, 0x03},
Tuple.from(Versionstamp.complete(new byte[]{(byte)0xaa, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03})), new byte[]{0x33, (byte)0xaa, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03, 0x00, 0x00},
Tuple.from(Versionstamp.complete(new byte[]{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}, 657)), new byte[]{0x33, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x02, (byte)0x91}
);
Tuple bigTuple = new Tuple();
List<byte[]> serializedForms = new ArrayList<>();
for(TupleSerialization serialization : serializations) {
bigTuple = bigTuple.addAll(serialization.tuple);
serializedForms.add(serialization.serialization);
}
serializations.add(new TupleSerialization(bigTuple, ByteArrayUtil.join(null, serializedForms)));
for(TupleSerialization serialization : serializations) {
System.out.println("Packing " + serialization.tuple + " (expecting: " + ByteArrayUtil.printable(serialization.serialization) + ")");
if(serialization.tuple.getPackedSize() != serialization.serialization.length) {
throw new RuntimeException("Tuple " + serialization.tuple + " packed size " + serialization.tuple.getPackedSize() + " does not match expected packed size " + serialization.serialization.length);
}
if(!Arrays.equals(serialization.tuple.pack(), serialization.serialization)) {
throw new RuntimeException("Tuple " + serialization.tuple + " has serialization " + ByteArrayUtil.printable(serialization.tuple.pack()) +
" which does not match expected serialization " + ByteArrayUtil.printable(serialization.serialization));
}
if(!Objects.equals(serialization.tuple, Tuple.fromItems(Tuple.fromBytes(serialization.serialization).getItems()))) {
throw new RuntimeException("Tuple " + serialization.tuple + " does not match deserialization " + Tuple.fromBytes(serialization.serialization) +
" which comes from serialization " + ByteArrayUtil.printable(serialization.serialization));
}
}
System.out.println("All tuples had matching serializations");
}
private static void comparisons() {
List<Tuple> tuples = Arrays.asList(
Tuple.from(0L),
Tuple.from(BigInteger.ZERO),
Tuple.from(1L),
Tuple.from(BigInteger.ONE),
Tuple.from(-1L),
Tuple.from(BigInteger.ONE.negate()),
Tuple.from(Long.MAX_VALUE),
Tuple.from(Long.MIN_VALUE),
Tuple.from(BigInteger.valueOf(Long.MIN_VALUE).subtract(BigInteger.ONE)),
Tuple.from(BigInteger.valueOf(Long.MIN_VALUE).shiftLeft(1)),
Tuple.from(-0.0f),
Tuple.from(0.0f),
Tuple.from(-0.0),
Tuple.from(0.0),
Tuple.from(Float.NEGATIVE_INFINITY),
Tuple.from(Double.NEGATIVE_INFINITY),
Tuple.from(Float.NaN),
Tuple.from(Double.NaN),
Tuple.from(Float.intBitsToFloat(Float.floatToIntBits(Float.NaN) + 1)),
Tuple.from(Double.longBitsToDouble(Double.doubleToLongBits(Double.NaN) + 1)),
Tuple.from(Float.intBitsToFloat(Float.floatToIntBits(Float.NaN) + 2)),
Tuple.from(Double.longBitsToDouble(Double.doubleToLongBits(Double.NaN) + 2)),
Tuple.from(Float.intBitsToFloat(Float.floatToIntBits(Float.NaN) ^ Integer.MIN_VALUE)),
Tuple.from(Double.longBitsToDouble(Double.doubleToLongBits(Double.NaN) ^ Long.MIN_VALUE)),
Tuple.from(Float.intBitsToFloat(Float.floatToIntBits(Float.NaN) ^ Integer.MIN_VALUE + 1)),
Tuple.from(Double.longBitsToDouble(Double.doubleToLongBits(Double.NaN) ^ Long.MIN_VALUE + 1)),
Tuple.from(Float.POSITIVE_INFINITY),
Tuple.from(Double.POSITIVE_INFINITY),
Tuple.from((Object)new byte[0]),
Tuple.from((Object)new byte[]{0x00}),
Tuple.from((Object)new byte[]{0x00, FF}),
Tuple.from((Object)new byte[]{0x7f}),
Tuple.from((Object)new byte[]{(byte)0x80}),
Tuple.from(null, new byte[0]),
Tuple.from(null, new byte[]{0x00}),
Tuple.from(null, new byte[]{0x00, FF}),
Tuple.from(null, new byte[]{0x7f}),
Tuple.from(null, new byte[]{(byte)0x80}),
Tuple.from(Tuple.from(null, new byte[0])),
Tuple.from(Tuple.from(null, new byte[]{0x00})),
Tuple.from(Tuple.from(null, new byte[]{0x00, FF})),
Tuple.from(Tuple.from(null, new byte[]{0x7f})),
Tuple.from(Tuple.from(null, new byte[]{(byte)0x80})),
Tuple.from("a"),
Tuple.from("\u03bc\u03ac\u03b8\u03b7\u03bc\u03b1"),
Tuple.from("\u03bc\u03b1\u0301\u03b8\u03b7\u03bc\u03b1"),
Tuple.from("\u4e2d\u6587"),
Tuple.from("\u4e2d\u570B"),
Tuple.from("\ud83d\udd25"),
Tuple.from("\ud83e\udd6f"),
Tuple.from("a\ud83d\udd25"),
Tuple.from("\ufb49"),
Tuple.from("\ud83d\udd25\ufb49"),
Tuple.from("\ud8ed\ud8ed"), // malformed string -- two high surrogates
Tuple.from("\ud8ed\ud8eda"), // malformed string -- two high surrogates
Tuple.from("\udd25\udd25"), // malformed string -- two low surrogates
Tuple.from("a\udd25\ud8ed"), // malformed string -- two low surrogates
Tuple.from("\udd25\ud83e\udd6f"), // malformed string -- low surrogate followed by high then low surrogate
Tuple.from("\udd6f\ud83e\udd6f"), // malformed string -- low surrogate followed by high then low surrogate
Tuple.from(new UUID(-1, 0)),
Tuple.from(new UUID(-1, -1)),
Tuple.from(new UUID(1, -1)),
Tuple.from(new UUID(1, 1)),
Tuple.from(false),
Tuple.from(true),
Tuple.from(Arrays.asList(0, 1, 2)),
Tuple.from(Arrays.asList(0, 1), "hello"),
Tuple.from(Arrays.asList(0, 1), "help"),
Tuple.from(Versionstamp.complete(new byte[]{0x0a, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03})),
Tuple.from(Versionstamp.complete(new byte[]{(byte)0xaa, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03})),
Tuple.from(Versionstamp.complete(new byte[]{(byte)0xaa, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03}, 1)),
Tuple.from(Versionstamp.complete(new byte[]{(byte)0xaa, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03}, 0xa101)),
Tuple.from(Versionstamp.complete(new byte[]{(byte)0xaa, (byte)0xbb, (byte)0xcc, (byte)0xdd, (byte)0xee, FF, 0x00, 0x01, 0x02, 0x03}, 65535))
);
for(Tuple t1 : tuples) {
for(Tuple t2 : tuples) {
System.out.println("Comparing " + t1 + " and " + t2);
// Copy the items over to new tuples to avoid having them use the memoized packed representations
Tuple t1copy = Tuple.fromList(t1.getItems());
Tuple t2copy = Tuple.fromList(t2.getItems());
int semanticComparison = t1copy.compareTo(t2copy);
int byteComparison = ByteArrayUtil.compareUnsigned(t1.pack(), t2.pack());
if(Integer.signum(semanticComparison) != Integer.signum(byteComparison)) {
throw new RuntimeException("Tuple t1 and t2 comparison mismatched: semantic = " + semanticComparison + " while byte order = " + byteComparison);
}
int implicitByteComparison = t1.compareTo(t2);
if(Integer.signum(semanticComparison) != Integer.signum(implicitByteComparison)) {
throw new RuntimeException("Tuple t1 and t2 comparison mismatched: semantic = " + semanticComparison + " while implicit byte order = " + implicitByteComparison);
}
}
}
}
private static void emptyTuple() {
Tuple t = new Tuple();
if(!t.isEmpty()) {
throw new RuntimeException("empty tuple is not empty");
}
if(t.getPackedSize() != 0) {
throw new RuntimeException("empty tuple packed size is not 0");
}
if(t.pack().length != 0) {
throw new RuntimeException("empty tuple is not packed to the empty byte string");
}
}
private static void addMethods() {
List<Tuple> baseTuples = Arrays.asList(
new Tuple(),
Tuple.from(),
Tuple.from((Object)null),
Tuple.from("prefix"),
Tuple.from("prefix", null),
Tuple.from(new UUID(100, 1000)),
Tuple.from(Versionstamp.incomplete(1)),
Tuple.from(Tuple.from(Versionstamp.incomplete(2))),
Tuple.from(Collections.singletonList(Versionstamp.incomplete(3)))
);
List<Object> toAdd = Arrays.asList(
null,
1066L,
BigInteger.valueOf(1066),
-3.14f,
2.71828,
new byte[]{0x01, 0x02, 0x03},
new byte[]{0x01, 0x00, 0x02, 0x00, 0x03},
"hello there",
"hell\0 there",
"\ud83d\udd25",
"\ufb14",
false,
true,
Float.NaN,
Float.intBitsToFloat(Integer.MAX_VALUE),
Double.NaN,
Double.longBitsToDouble(Long.MAX_VALUE),
Versionstamp.complete(new byte[]{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09}, 100),
Versionstamp.incomplete(4),
new UUID(-1, 1),
Tuple.from((Object)null),
Tuple.from("suffix", "tuple"),
Tuple.from("s\0ffix", "tuple"),
Arrays.asList("suffix", "tuple"),
Arrays.asList("suffix", null, "tuple"),
Tuple.from("suffix", null, "tuple"),
Tuple.from("suffix", Versionstamp.incomplete(4), "tuple"),
Arrays.asList("suffix", Arrays.asList("inner", Versionstamp.incomplete(5), "tuple"), "tuple")
);
for(Tuple baseTuple : baseTuples) {
for(Object newItem : toAdd) {
int baseSize = baseTuple.size();
Tuple freshTuple = Tuple.fromStream(Stream.concat(baseTuple.stream(), Stream.of(newItem)));
if(freshTuple.size() != baseSize + 1) {
throw new RuntimeException("freshTuple size was not one larger than base size");
}
Tuple withObjectAdded = baseTuple.addObject(newItem);
if(withObjectAdded.size() != baseSize + 1) {
throw new RuntimeException("withObjectAdded size was not one larger than the base size");
}
// Use the appropriate "add" overload.
Tuple withValueAdded;
if(newItem == null) {
withValueAdded = baseTuple.addObject(null);
}
else if(newItem instanceof byte[]) {
withValueAdded = baseTuple.add((byte[])newItem);
}
else if(newItem instanceof String) {
withValueAdded = baseTuple.add((String)newItem);
}
else if(newItem instanceof Long) {
withValueAdded = baseTuple.add((Long)newItem);
}
else if(newItem instanceof BigInteger) {
withValueAdded = baseTuple.add((BigInteger)newItem);
}
else if(newItem instanceof Float) {
withValueAdded = baseTuple.add((Float)newItem);
}
else if(newItem instanceof Double) {
withValueAdded = baseTuple.add((Double)newItem);
}
else if(newItem instanceof Boolean) {
withValueAdded = baseTuple.add((Boolean)newItem);
}
else if(newItem instanceof UUID) {
withValueAdded = baseTuple.add((UUID)newItem);
}
else if(newItem instanceof Versionstamp) {
withValueAdded = baseTuple.add((Versionstamp)newItem);
}
else if(newItem instanceof List<?>) {
withValueAdded = baseTuple.add((List<?>)newItem);
}
else if(newItem instanceof Tuple) {
withValueAdded = baseTuple.add((Tuple)newItem);
}
else {
throw new RuntimeException("unknown type for tuple serialization " + newItem.getClass());
}
// Use Tuple.addAll, which has optimizations if both tuples have been packed already
// Getting their hash codes memoizes the packed representation.
Tuple newItemTuple = Tuple.from(newItem);
baseTuple.hashCode();
newItemTuple.hashCode();
Tuple withTupleAddedAll = baseTuple.addAll(newItemTuple);
Tuple withListAddedAll = baseTuple.addAll(Collections.singletonList(newItem));
List<Tuple> allTuples = Arrays.asList(freshTuple, withObjectAdded, withValueAdded, withTupleAddedAll, withListAddedAll);
int basePlusNewSize = baseTuple.getPackedSize() + Tuple.from(newItem).getPackedSize();
int freshTuplePackedSize = freshTuple.getPackedSize();
int withObjectAddedPackedSize = withObjectAdded.getPackedSize();
int withValueAddedPackedSize = withValueAdded.getPackedSize();
int withTupleAddedAllPackedSize = withTupleAddedAll.getPackedSize();
int withListAddAllPackedSize = withListAddedAll.getPackedSize();
if(basePlusNewSize != freshTuplePackedSize || basePlusNewSize != withObjectAddedPackedSize ||
basePlusNewSize != withValueAddedPackedSize || basePlusNewSize != withTupleAddedAllPackedSize ||
basePlusNewSize != withListAddAllPackedSize) {
throw new RuntimeException("packed sizes not equivalent");
}
byte[] concatPacked;
byte[] prefixPacked;
byte[] freshPacked;
byte[] objectAddedPacked;
byte[] valueAddedPacked;
byte[] tupleAddedAllPacked;
byte[] listAddedAllPacked;
if(!baseTuple.hasIncompleteVersionstamp() && !Tuple.from(newItem).hasIncompleteVersionstamp()) {
concatPacked = ByteArrayUtil.join(baseTuple.pack(), Tuple.from(newItem).pack());
prefixPacked = Tuple.from(newItem).pack(baseTuple.pack());
freshPacked = freshTuple.pack();
objectAddedPacked = withObjectAdded.pack();
valueAddedPacked = withValueAdded.pack();
tupleAddedAllPacked = withTupleAddedAll.pack();
listAddedAllPacked = withListAddedAll.pack();
for(Tuple t : allTuples) {
try {
t.packWithVersionstamp();
throw new RuntimeException("able to pack tuple without incomplete versionstamp using packWithVersionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
else if(!baseTuple.hasIncompleteVersionstamp() && Tuple.from(newItem).hasIncompleteVersionstamp()) {
concatPacked = newItemTuple.packWithVersionstamp(baseTuple.pack());
try {
prefixPacked = Tuple.from(newItem).packWithVersionstamp(baseTuple.pack());
}
catch(NullPointerException e) {
prefixPacked = Tuple.from(newItem).packWithVersionstamp(baseTuple.pack());
}
freshPacked = freshTuple.packWithVersionstamp();
objectAddedPacked = withObjectAdded.packWithVersionstamp();
valueAddedPacked = withValueAdded.packWithVersionstamp();
tupleAddedAllPacked = withTupleAddedAll.packWithVersionstamp();
listAddedAllPacked = withListAddedAll.packWithVersionstamp();
for(Tuple t : allTuples) {
try {
t.pack();
throw new RuntimeException("able to pack tuple with incomplete versionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
else if(baseTuple.hasIncompleteVersionstamp() && !Tuple.from(newItem).hasIncompleteVersionstamp()) {
concatPacked = baseTuple.addAll(Tuple.from(newItem)).packWithVersionstamp();
prefixPacked = baseTuple.addObject(newItem).packWithVersionstamp();
freshPacked = freshTuple.packWithVersionstamp();
objectAddedPacked = withObjectAdded.packWithVersionstamp();
valueAddedPacked = withValueAdded.packWithVersionstamp();
tupleAddedAllPacked = withTupleAddedAll.packWithVersionstamp();
listAddedAllPacked = withListAddedAll.packWithVersionstamp();
for(Tuple t : allTuples) {
try {
t.pack();
throw new RuntimeException("able to pack tuple with incomplete versionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
else {
for(Tuple t : allTuples) {
try {
t.pack();
throw new RuntimeException("able to pack tuple with two versionstamps using pack");
}
catch(IllegalArgumentException e) {
// eat
}
try {
t.packWithVersionstamp();
throw new RuntimeException("able to pack tuple with two versionstamps using packWithVersionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
try {
t.hashCode();
throw new RuntimeException("able to get hash code of tuple with two versionstamps");
}
catch(IllegalArgumentException e) {
// eat
}
}
concatPacked = null;
prefixPacked = null;
freshPacked = null;
objectAddedPacked = null;
valueAddedPacked = null;
tupleAddedAllPacked = null;
listAddedAllPacked = null;
}
if(!Arrays.equals(concatPacked, freshPacked) ||
!Arrays.equals(freshPacked, prefixPacked) ||
!Arrays.equals(freshPacked, objectAddedPacked) ||
!Arrays.equals(freshPacked, valueAddedPacked) ||
!Arrays.equals(freshPacked, tupleAddedAllPacked) ||
!Arrays.equals(freshPacked, listAddedAllPacked)) {
throw new RuntimeException("packed values are not concatenation of original packings");
}
if(freshPacked != null && freshPacked.length != basePlusNewSize) {
throw new RuntimeException("packed length did not match expectation");
}
if(freshPacked != null) {
if(freshTuple.hashCode() != Arrays.hashCode(freshPacked)) {
throw new IllegalArgumentException("hash code does not match fresh packed");
}
for(Tuple t : allTuples) {
if(t.hashCode() != freshTuple.hashCode()) {
throw new IllegalArgumentException("hash code mismatch");
}
if(Tuple.fromItems(t.getItems()).hashCode() != freshTuple.hashCode()) {
throw new IllegalArgumentException("hash code mismatch after re-compute");
}
}
}
}
}
}
private static void incompleteVersionstamps() {
if(FDB.instance().getAPIVersion() < 520) {
throw new IllegalStateException("cannot run test with API version " + FDB.instance().getAPIVersion());
}
// This is a tricky case where there are two tuples with identical representations but different semantics.
byte[] arr = new byte[0x0100fe];
Arrays.fill(arr, (byte)0x7f); // The actual value doesn't matter, but it can't be zero.
Tuple t1 = Tuple.from(arr, Versionstamp.complete(new byte[]{FF, FF, FF, FF, FF, FF, FF, FF, FF, FF}), new byte[]{0x01, 0x01});
Tuple t2 = Tuple.from(arr, Versionstamp.incomplete());
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal");
}
byte[] bytes1 = t1.pack();
byte[] bytes2 = t2.packWithVersionstamp();
if(!Arrays.equals(bytes1, bytes2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " did not have matching representations");
}
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal with memoized packed representations");
}
// Make sure position information adjustment works.
Tuple t3 = Tuple.from(Versionstamp.incomplete(1));
if(t3.getPackedSize() != 1 + Versionstamp.LENGTH + Integer.BYTES) {
throw new RuntimeException("incomplete versionstamp has incorrect packed size " + t3.getPackedSize());
}
byte[] bytes3 = t3.packWithVersionstamp();
if(ByteBuffer.wrap(bytes3, bytes3.length - Integer.BYTES, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).getInt() != 1) {
throw new RuntimeException("incomplete versionstamp has incorrect position");
}
if(!Tuple.fromBytes(bytes3, 0, bytes3.length - Integer.BYTES).equals(Tuple.from(Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes did not match");
}
Subspace subspace = new Subspace(Tuple.from("prefix"));
byte[] bytes4 = subspace.packWithVersionstamp(t3);
if(ByteBuffer.wrap(bytes4, bytes4.length - Integer.BYTES, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).getInt() != 1 + subspace.getKey().length) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
if(!Tuple.fromBytes(bytes4, 0, bytes4.length - Integer.BYTES).equals(Tuple.from("prefix", Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes with subspace did not match");
}
try {
// At this point, the representation is cached, so an easy bug would be to have it return the already serialized value
t3.pack();
throw new RuntimeException("was able to pack versionstamp with incomplete versionstamp");
} catch(IllegalArgumentException e) {
// eat
}
// Tuples with two incomplete versionstamps somewhere.
List<Tuple> twoIncompleteList = Arrays.asList(
Tuple.from(Versionstamp.incomplete(1), Versionstamp.incomplete(2)),
Tuple.from(Tuple.from(Versionstamp.incomplete(3)), Tuple.from(Versionstamp.incomplete(4))),
new Tuple().add(Versionstamp.incomplete()).add(Versionstamp.incomplete()),
new Tuple().add(Versionstamp.incomplete()).add(3L).add(Versionstamp.incomplete()),
Tuple.from(Tuple.from(Versionstamp.incomplete()), "dummy_string").add(Tuple.from(Versionstamp.incomplete())),
Tuple.from(Arrays.asList(Versionstamp.incomplete(), "dummy_string")).add(Tuple.from(Versionstamp.incomplete())),
Tuple.from(Tuple.from(Versionstamp.incomplete()), "dummy_string").add(Collections.singletonList(Versionstamp.incomplete()))
);
for(Tuple t : twoIncompleteList) {
if(!t.hasIncompleteVersionstamp()) {
throw new RuntimeException("tuple doesn't think it has incomplete versionstamp");
}
if(t.getPackedSize() < 2 * (1 + Versionstamp.LENGTH + Integer.BYTES)) {
throw new RuntimeException("tuple packed size " + t.getPackedSize() + " is smaller than expected");
}
try {
t.pack();
throw new RuntimeException("no error thrown when packing any incomplete versionstamps");
}
catch(IllegalArgumentException e) {
// eat
}
try {
t.packWithVersionstamp();
throw new RuntimeException("no error thrown when packing with versionstamp with two incompletes");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
// Assumes API version < 520
private static void incompleteVersionstamps300() {
if(FDB.instance().getAPIVersion() >= 520) {
throw new IllegalStateException("cannot run test with API version " + FDB.instance().getAPIVersion());
}
Tuple t1 = Tuple.from(Versionstamp.complete(new byte[]{FF, FF, FF, FF, FF, FF, FF, FF, FF, FF}), new byte[]{});
Tuple t2 = Tuple.from(Versionstamp.incomplete());
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal");
}
byte[] bytes1 = t1.pack();
byte[] bytes2 = t2.packWithVersionstamp();
if(!Arrays.equals(bytes1, bytes2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " did not have matching representations");
}
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal with memoized packed representations");
}
// Make sure position information adjustment works.
Tuple t3 = Tuple.from(Versionstamp.incomplete(1));
if(t3.getPackedSize() != 1 + Versionstamp.LENGTH + Short.BYTES) {
throw new RuntimeException("incomplete versionstamp has incorrect packed size " + t3.getPackedSize());
}
byte[] bytes3 = t3.packWithVersionstamp();
if(ByteBuffer.wrap(bytes3, bytes3.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() != 1) {
throw new RuntimeException("incomplete versionstamp has incorrect position");
}
if(!Tuple.fromBytes(bytes3, 0, bytes3.length - Short.BYTES).equals(Tuple.from(Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes did not match");
}
Subspace subspace = new Subspace(Tuple.from("prefix"));
byte[] bytes4 = subspace.packWithVersionstamp(t3);
if(ByteBuffer.wrap(bytes4, bytes4.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() != 1 + subspace.getKey().length) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
if(!Tuple.fromBytes(bytes4, 0, bytes4.length - Short.BYTES).equals(Tuple.from("prefix", Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes with subspace did not match");
}
// Make sure an offset > 0xFFFF throws an error.
Tuple t4 = Tuple.from(Versionstamp.incomplete(2));
byte[] bytes5 = t4.packWithVersionstamp(); // Get bytes memoized.
if(ByteBuffer.wrap(bytes5, bytes5.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() != 1) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
byte[] bytes6 = t4.packWithVersionstamp(new byte[0xfffe]); // Offset is 0xffff
if(!Arrays.equals(Arrays.copyOfRange(bytes5, 0, 1 + Versionstamp.LENGTH), Arrays.copyOfRange(bytes6, 0xfffe, 0xffff + Versionstamp.LENGTH))) {
throw new RuntimeException("area before versionstamp offset did not match");
}
if((ByteBuffer.wrap(bytes6, bytes6.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() & 0xffff) != 0xffff) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
try {
t4.packWithVersionstamp(new byte[0xffff]); // Offset is 0x10000
throw new RuntimeException("able to pack versionstamp with offset that is too large");
}
catch(IllegalArgumentException e) {
// eat
}
// Same as before, but packed representation is not memoized.
try {
Tuple.from(Versionstamp.incomplete(3)).packWithVersionstamp(new byte[0xffff]); // Offset is 0x10000
throw new RuntimeException("able to pack versionstamp with offset that is too large");
}
catch(IllegalArgumentException e) {
// eat
}
}
private static void malformedBytes() {
List<byte[]> malformedSequences = Arrays.asList(
new byte[]{0x01, (byte)0xde, (byte)0xad, (byte)0xc0, (byte)0xde}, // no termination character for byte array
new byte[]{0x01, (byte)0xde, (byte)0xad, 0x00, FF, (byte)0xc0, (byte)0xde}, // no termination character but null in middle
new byte[]{0x02, 'h', 'e', 'l', 'l', 'o'}, // no termination character for string
new byte[]{0x02, 'h', 'e', 'l', 0x00, FF, 'l', 'o'}, // no termination character but null in the middle
// Invalid UTF-8 decodes malformed as U+FFFD rather than throwing an error
// new byte[]{0x02, 'u', 't', 'f', 0x08, (byte)0x80, 0x00}, // invalid utf-8 code point start character
// new byte[]{0x02, 'u', 't', 'f', 0x08, (byte)0xc0, 0x01, 0x00}, // invalid utf-8 code point second character
new byte[]{0x05, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00}, // no termination character for nested tuple
new byte[]{0x05, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00, 0x00, FF, 0x02, 't', 'h', 'e', 'r', 'e', 0x00}, // no termination character for nested tuple but null in the middle
new byte[]{0x16, 0x01}, // integer truncation
new byte[]{0x12, 0x01}, // integer truncation
new byte[]{0x1d, 0x09, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, // integer truncation
new byte[]{0x0b, 0x09 ^ FF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, // integer truncation
new byte[]{0x20, 0x01, 0x02, 0x03}, // float truncation
new byte[]{0x21, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}, // double truncation
new byte[]{0x30, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}, // UUID truncation
new byte[]{0x33, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b}, // versionstamp truncation
new byte[]{FF} // unknown start code
);
for(byte[] sequence : malformedSequences) {
try {
Tuple t = Tuple.fromBytes(sequence);
throw new RuntimeException("Able to unpack " + ByteArrayUtil.printable(sequence) + " into " + t);
}
catch(IllegalArgumentException e) {
System.out.println("Error for " + ByteArrayUtil.printable(sequence) + ": " + e.getMessage());
}
}
// Perfectly good byte sequences, but using the offset and length to remove terminal bytes
List<byte[]> wellFormedSequences = Arrays.asList(
Tuple.from((Object)new byte[]{0x01, 0x02}).pack(),
Tuple.from("hello").pack(),
Tuple.from("hell\0").pack(),
Tuple.from(1066L).pack(),
Tuple.from(-1066L).pack(),
Tuple.from(BigInteger.ONE.shiftLeft(Long.SIZE + 1)).pack(),
Tuple.from(BigInteger.ONE.shiftLeft(Long.SIZE + 1).negate()).pack(),
Tuple.from(-3.14f).pack(),
Tuple.from(2.71828).pack(),
Tuple.from(new UUID(1066L, 1415L)).pack(),
Tuple.from(Versionstamp.fromBytes(new byte[]{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c})).pack()
);
for(byte[] sequence : wellFormedSequences) {
try {
Tuple t = Tuple.fromBytes(sequence, 0, sequence.length - 1);
throw new RuntimeException("Able to unpack " + ByteArrayUtil.printable(sequence) + " into " + t + " without last character");
}
catch(IllegalArgumentException e) {
System.out.println("Error for " + ByteArrayUtil.printable(sequence) + ": " + e.getMessage());
}
}
}
private static void offsetsAndLengths() {
List<Tuple> tuples = Arrays.asList(
new Tuple(),
Tuple.from((Object)null),
Tuple.from(null, new byte[]{0x10, 0x66}),
Tuple.from("dummy_string"),
Tuple.from(1066L)
);
Tuple allTuples = tuples.stream().reduce(new Tuple(), Tuple::addAll);
byte[] allTupleBytes = allTuples.pack();
// Unpack each tuple individually using their lengths
int offset = 0;
for(Tuple t : tuples) {
int length = t.getPackedSize();
Tuple unpacked = Tuple.fromBytes(allTupleBytes, offset, length);
if(!unpacked.equals(t)) {
throw new RuntimeException("unpacked tuple " + unpacked + " does not match serialized tuple " + t);
}
offset += length;
}
// Unpack successive pairs of tuples.
offset = 0;
for(int i = 0; i < tuples.size() - 1; i++) {
Tuple combinedTuple = tuples.get(i).addAll(tuples.get(i + 1));
Tuple unpacked = Tuple.fromBytes(allTupleBytes, offset, combinedTuple.getPackedSize());
if(!unpacked.equals(combinedTuple)) {
throw new RuntimeException("unpacked tuple " + unpacked + " does not match combined tuple " + combinedTuple);
}
offset += tuples.get(i).getPackedSize();
}
// Allow an offset to equal the length of the array, but essentially only a zero-length is allowed there.
Tuple emptyAtEndTuple = Tuple.fromBytes(allTupleBytes, allTupleBytes.length, 0);
if(!emptyAtEndTuple.isEmpty()) {
throw new RuntimeException("tuple with no bytes is not empty");
}
try {
Tuple.fromBytes(allTupleBytes, -1, 4);
throw new RuntimeException("able to give negative offset to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, allTupleBytes.length + 1, 4);
throw new RuntimeException("able to give offset larger than array to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, 0, -1);
throw new RuntimeException("able to give negative length to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, 0, allTupleBytes.length + 1);
throw new RuntimeException("able to give length larger than array to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, allTupleBytes.length / 2, allTupleBytes.length / 2 + 2);
throw new RuntimeException("able to exceed array length in fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
}
private static void intoBuffer() {
Tuple t = Tuple.from("hello", 3.14f, "world");
ByteBuffer buffer = ByteBuffer.allocate("hello".length() + 2 + Float.BYTES + 1 + "world".length() + 2);
t.packInto(buffer);
if(!Arrays.equals(t.pack(), buffer.array())) {
throw new RuntimeException("buffer and tuple do not match");
}
buffer = ByteBuffer.allocate(t.getPackedSize() + 2);
buffer.order(ByteOrder.LITTLE_ENDIAN);
t.packInto(buffer);
if(!Arrays.equals(ByteArrayUtil.join(t.pack(), new byte[]{0x00, 0x00}), buffer.array())) {
throw new RuntimeException("buffer and tuple do not match");
}
if(!buffer.order().equals(ByteOrder.LITTLE_ENDIAN)) {
throw new RuntimeException("byte order changed");
}
buffer = ByteBuffer.allocate(t.getPackedSize() + 2);
buffer.put((byte)0x01).put((byte)0x02);
t.packInto(buffer);
if(!Arrays.equals(t.pack(new byte[]{0x01, 0x02}), buffer.array())) {
throw new RuntimeException("buffer and tuple do not match");
}
buffer = ByteBuffer.allocate(t.getPackedSize() - 1);
try {
t.packInto(buffer);
throw new RuntimeException("able to pack into buffer that was too small");
}
catch(BufferOverflowException e) {
// eat
}
Tuple tCopy = Tuple.fromItems(t.getItems()); // remove memoized stuff
buffer = ByteBuffer.allocate(t.getPackedSize() - 1);
try {
tCopy.packInto(buffer);
throw new RuntimeException("able to pack into buffer that was too small");
}
catch(BufferOverflowException e) {
// eat
}
Tuple tWithIncomplete = Tuple.from(Versionstamp.incomplete(3));
buffer = ByteBuffer.allocate(tWithIncomplete.getPackedSize());
try {
tWithIncomplete.packInto(buffer);
throw new RuntimeException("able to pack incomplete versionstamp into buffer");
}
catch(IllegalArgumentException e) {
// eat
}
if(buffer.arrayOffset() != 0) {
throw new RuntimeException("offset changed after unsuccessful pack with incomplete versionstamp");
}
}
// These should be in ArrayUtilTest, but those can't be run at the moment, so here they go.
private static void replaceTests() {
List<byte[]> arrays = Arrays.asList(
new byte[]{0x01, 0x02, 0x01, 0x02}, new byte[]{0x01, 0x02}, new byte[]{0x03, 0x04}, new byte[]{0x03, 0x04, 0x03, 0x04},
new byte[]{0x01, 0x02, 0x01, 0x02}, new byte[]{0x01, 0x02}, new byte[]{0x03}, new byte[]{0x03, 0x03},
new byte[]{0x01, 0x02, 0x01, 0x02}, new byte[]{0x01, 0x02}, new byte[]{0x03, 0x04, 0x05}, new byte[]{0x03, 0x04, 0x05, 0x03, 0x04, 0x05},
new byte[]{0x00, 0x01, 0x02, 0x00, 0x01, 0x02, 0x00}, new byte[]{0x01, 0x02}, new byte[]{0x03, 0x04, 0x05}, new byte[]{0x00, 0x03, 0x04, 0x05, 0x00, 0x03, 0x04, 0x05, 0x00},
new byte[]{0x01, 0x01, 0x01, 0x01}, new byte[]{0x01, 0x02}, new byte[]{0x03, 0x04}, new byte[]{0x01, 0x01, 0x01, 0x01},
new byte[]{0x01, 0x01, 0x01, 0x01}, new byte[]{0x01, 0x02}, new byte[]{0x03}, new byte[]{0x01, 0x01, 0x01, 0x01},
new byte[]{0x01, 0x01, 0x01, 0x01}, new byte[]{0x01, 0x02}, new byte[]{0x03, 0x04, 0x05}, new byte[]{0x01, 0x01, 0x01, 0x01},
new byte[]{0x01, 0x01, 0x01, 0x01, 0x01}, new byte[]{0x01, 0x01}, new byte[]{0x03, 0x04, 0x05}, new byte[]{0x03, 0x04, 0x05, 0x03, 0x04, 0x05, 0x01},
new byte[]{0x01, 0x01, 0x01, 0x01, 0x01}, new byte[]{0x01, 0x01}, new byte[]{0x03, 0x04}, new byte[]{0x03, 0x04, 0x03, 0x04, 0x01},
new byte[]{0x01, 0x01, 0x01, 0x01, 0x01}, new byte[]{0x01, 0x01}, new byte[]{0x03}, new byte[]{0x03, 0x03, 0x01},
new byte[]{0x01, 0x02, 0x01, 0x02}, new byte[]{0x01, 0x02}, null, new byte[0],
new byte[]{0x01, 0x02, 0x01, 0x02}, new byte[]{0x01, 0x02}, new byte[0], new byte[0],
new byte[]{0x01, 0x02, 0x01, 0x02}, null, new byte[]{0x04}, new byte[]{0x01, 0x02, 0x01, 0x02},
new byte[]{0x01, 0x02, 0x01, 0x02}, new byte[0], new byte[]{0x04}, new byte[]{0x01, 0x02, 0x01, 0x02},
null, new byte[]{0x01, 0x02}, new byte[]{0x04}, null
);
for(int i = 0; i < arrays.size(); i += 4) {
byte[] src = arrays.get(i);
byte[] pattern = arrays.get(i + 1);
byte[] replacement = arrays.get(i + 2);
byte[] expectedResults = arrays.get(i + 3);
byte[] results = ByteArrayUtil.replace(src, pattern, replacement);
if(!Arrays.equals(results, expectedResults)) {
throw new RuntimeException("results " + ByteArrayUtil.printable(results) + " did not match expected results " +
ByteArrayUtil.printable(expectedResults) + " when replacing " + ByteArrayUtil.printable(pattern) +
" with " + ByteArrayUtil.printable(replacement) + " in " + ByteArrayUtil.printable(src));
}
if(src != null && src == results) {
throw new RuntimeException("src and results array are pointer-equal when replacing " + ByteArrayUtil.printable(pattern) +
" with " + ByteArrayUtil.printable(replacement) + " in " + ByteArrayUtil.printable(src));
}
}
try {
ByteArrayUtil.replace(null, 0, 1, new byte[]{0x00}, new byte[]{0x00, FF});
throw new RuntimeException("able to replace null bytes");
}
catch(NullPointerException e) {
// eat
}
try {
ByteArrayUtil.replace(new byte[]{0x00, 0x01}, -1, 2, new byte[]{0x00}, new byte[]{0x00, FF});
throw new RuntimeException("able to use negative offset");
}
catch(IllegalArgumentException e) {
// eat
}
try {
ByteArrayUtil.replace(new byte[]{0x00, 0x01}, 3, 2, new byte[]{0x00}, new byte[]{0x00, FF});
throw new RuntimeException("able to use offset after end of array");
}
catch(IllegalArgumentException e) {
// eat
}
try {
ByteArrayUtil.replace(new byte[]{0x00, 0x01}, 1, -1, new byte[]{0x00}, new byte[]{0x00, FF});
throw new RuntimeException("able to use negative length");
}
catch(IllegalArgumentException e) {
// eat
}
try {
ByteArrayUtil.replace(new byte[]{0x00, 0x01}, 1, 2, new byte[]{0x00}, new byte[]{0x00, FF});
throw new RuntimeException("able to give length that exceeds end of the array");
}
catch(IllegalArgumentException e) {
// eat
}
}
private static void runTests(final int reps, TransactionContext db) {
System.out.println("Running tests...");
long start = System.currentTimeMillis();

15
documentation/sphinx/source/administration.rst
View File

@ -141,6 +141,21 @@ Any client connected to FoundationDB can access information about its cluster fi
* To get the path to the cluster file, read the key ``\xFF\xFF/cluster_file_path``.
* To get the contents of the cluster file, read the key ``\xFF\xFF/connection_string``.
.. _ipv6-support:
IPv6 Support
============
FoundationDB (since v6.1) can accept network connections from clients connecting over IPv6. IPv6 address/port pair is represented as ``[IP]:PORT``, e.g. "[::1]:4800", "[abcd::dead:beef]:4500".
1) The cluster file can contain mix of IPv6 and IPv6 addresses. For example::
description:ID@127.0.0.1:4500,[::1]:4500,...
2) Starting ``fdbserver`` with IPv6::
$ /path/to/fdbserver -C fdb.cluster -p \[::1\]:4500
.. _adding-machines-to-a-cluster:
Adding machines to a cluster

1
documentation/sphinx/source/mr-status.rst
View File

@ -340,6 +340,7 @@ cluster.messages log_servers_error Time
cluster.messages transaction_start_timeout Unable to start transaction after __ seconds.
cluster.messages unreachable_master_worker Unable to locate the master worker.
cluster.messages unreachable_dataDistributor_worker Unable to locate the data distributor worker.
cluster.messages unreachable_ratekeeper_worker Unable to locate the ratekeeper worker.
cluster.messages unreachable_processes The cluster has some unreachable processes.
cluster.messages unreadable_configuration Unable to read database configuration.
cluster.messages layer_status_incomplete Some or all of the layers subdocument could not be read.

17
documentation/sphinx/source/release-notes.rst
View File

@ -7,7 +7,9 @@ Release Notes
Features
--------
* Improved replication mechanism, a new hierarchical replication technique that further significantly reduces the frequency of data loss events even when multiple machines (e.g., fault-tolerant zones in the current code) permanently fail at the same time. `(PR #964) <https://github.com/apple/foundationdb/pull/964>`.
* Improved replication mechanism, a new hierarchical replication technique that further significantly reduces the frequency of data loss events even when multiple machines (e.g., fault-tolerant zones in the current code) permanently fail at the same time. `(PR #964) <https://github.com/apple/foundationdb/pull/964>`_.
* Added background actor to remove redundant teams from team collection so that the healthy team number is guaranteed not exceeding the desired number. `(PR #1139) <https://github.com/apple/foundationdb/pull/1139>`_
* Show the number of connected coordinators per client in JSON status `(PR #1222) <https://github.com/apple/foundationdb/pull/1222>`_
@ -17,6 +19,11 @@ Features
* Batch priority transactions are now limited separately by ratekeeper and will be throttled at lower levels of cluster saturation. This makes it possible to run a more intense background load at saturation without significantly affecting normal priority transactions. It is still recommended not to run excessive loads at batch priority. `(PR #1198) <https://github.com/apple/foundationdb/pull/1198>`_
* Restore now requires the destnation cluster to be specified explicitly to avoid confusion. `(PR #1240) <https://github.com/apple/foundationdb/pull/1240>`_
* Restore target version can now be specified by timestamp if the original cluster is available. `(PR #1240) <https://github.com/apple/foundationdb/pull/1240>`_
* Separate data distribution out from master as a new role. `(PR #1062) <https://github.com/apple/foundationdb/pull/1062>`_
* Separate rate keeper out from data distribution as a new role. `(PR ##1176) <https://github.com/apple/foundationdb/pull/1176>`_
* Added a new atomic op `CompareAndClear`. `(PR #1105) <https://github.com/apple/foundationdb/pull/1105>`_
* Added support for IPv6. `(PR #1176) https://github.com/apple/foundationdb/pull/1178`_
* FDB can now simultaneously listen to TLS and unencrypted ports to facilitate smoother migration to TLS. `(PR #1157) https://github.com/apple/foundationdb/pull/1157`_
Performance
-----------
@ -27,6 +34,7 @@ Fixes
-----
* Python: Creating a ``SingleFloat`` for the tuple layer didn't work with integers. `(PR #1216) <https://github.com/apple/foundationdb/pull/1216>`_
* Added `DISABLE_POSIX_KERNEL_AIO` knob to fallback to libeio instead of kernel async I/O (KAIO) for systems that do not support KAIO or O_DIRECT flag. `(PR #1283) https://github.com/apple/foundationdb/pull/1283`_
Status
------
@ -41,6 +49,10 @@ Bindings
* Java: Deprecated ``FDB.createCluster`` and ``Cluster``. The preferred way to get a ``Database`` is by using ``FDB.open``, which should work in both new and old API versions. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Java: Removed ``Cluster(long cPtr, Executor executor)`` constructor. This is API breaking for any code that has subclassed the ``Cluster`` class and is not protected by API versioning. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Java: Several methods relevant to read-only transactions have been moved into the ``ReadTransaction`` interface.
* Java: Tuples now cache previous hash codes and equality checking no longer requires packing the underlying Tuples. `(PR #1166) <https://github.com/apple/foundationdb/pull/1166>`_
* Java: Tuple performance has been improved to use fewer allocations when packing and unpacking. `(Issue #1206) <https://github.com/apple/foundationdb/issues/1206>`_
* Java: Unpacking a Tuple with a byte array or string that is missing the end-of-string character now throws an error. `(Issue #671) <https://github.com/apple/foundationdb/issues/671>`_
* Java: Unpacking a Tuple constrained to a subset of the underlying array now throws an error when it encounters a truncated integer. `(Issue #672) <https://github.com/apple/foundationdb/issues/672>`_
* Ruby: Removed ``FDB.init``, ``FDB.create_cluster``, and ``FDB.Cluster``. ``FDB.open`` no longer accepts a ``database_name`` parameter. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Golang: Deprecated ``fdb.StartNetwork``, ``fdb.Open``, ``fdb.MustOpen``, and ``fdb.CreateCluster`` and added ``fdb.OpenDatabase`` and ``fdb.MustOpenDatabase``. The preferred way to start the network and get a ``Database`` is by using ``FDB.OpenDatabase`` or ``FDB.OpenDefault``. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_
* Flow: Removed ``API::createCluster`` and ``Cluster`` and added ``API::createDatabase``. The new way to get a ``Database`` is by using ``API::createDatabase``. `(PR #942) <https://github.com/apple/foundationdb/pull/942>`_ `(PR #1215) <https://github.com/apple/foundationdb/pull/1215>`_
@ -50,11 +62,14 @@ Bindings
* Flow: Changed ``Transaction::setVersion`` to ``Transaction::setReadVersion``. `(PR #1215) <https://github.com/apple/foundationdb/pull/1215>`_
* Flow: On update to this version of the Flow bindings, client code will fail to build due to the changes in the API, irrespective of the API version used. Client code must be updated to use the new bindings API. These changes affect the bindings only and won't impact compatibility with different versions of the cluster. `(PR #1215) <https://github.com/apple/foundationdb/pull/1215>`_
* Golang: Added ``fdb.Printable`` to print a human-readable string for a given byte array. Add ``Key.String()``, which converts the ``Key`` to a ``string`` using the ``Printable`` function. `(PR #1010) <https://github.com/apple/foundationdb/pull/1010>`_
* Golang: Tuples now support ``Versionstamp`` operations. `(PR #1187) <https://github.com/apple/foundationdb/pull/1187>`_
* Python: Python signal handling didn't work when waiting on a future. In particular, pressing Ctrl-C would not successfully interrupt the program. `(PR #1138) <https://github.com/apple/foundationdb/pull/1138>`_
Other Changes
-------------
* Migrated to Boost 1.67. `(PR #1242) https://github.com/apple/foundationdb/pull/1242`_
Earlier release notes
---------------------
* :doc:`6.0 (API Version 600) </old-release-notes/release-notes-600>`

48
documentation/sphinx/source/tls.rst
View File

@ -29,10 +29,52 @@ This will configure the new cluster to communicate with TLS.
.. note:: Depending on your operating system, version and configuration, there may be a firewall in place that prevents external access to certain ports. If necessary, please consult the appropriate documentation for your OS and ensure that all machines in your cluster can reach the ports configured in your :ref:`configuration file <foundationdb-conf>`.
.. _converting-existing-cluster:
.. _converting-existing-cluster-after-6.1:
Converting an existing cluster to use TLS
=========================================
Converting an existing cluster to use TLS (since v6.1)
======================================================
Since version 6.1, FoundationDB clusters can be converted to TLS without downtime. FoundationDB server can listen to TLS and unencrypted traffic simultaneously on two separate ports. As a result, FDB clusters can live migrate to TLS:
1) Restart each FoundationDB server individually, but with an additional listen address for TLS traffic::
/path/to/fdbserver -C fdb.cluster -p 127.0.0.1:4500 -p 127.0.0.1:4600:tls
Since, the server still listens to unencrypted traffic and the cluster file still contains the old address, rest of the processes will be able to talk to this new process.
2) Once all processes are listening to both TLS and unencrypted traffic, switch one or more coordinator to use TLS. Therefore, if the old coordinator list was ``127.0.0.1:4500,127.0.0.1:4501,127.0.0.1:4502``, the new one would be something like ``127.0.0.1:4600:tls,127.0.0.1:4501,127.0.0.1:4502``. Switching few coordinators to TLS at a time allows a smoother migration and a window to find out clients who do not yet have TLS configured. The number of coordinators each client can connect to can be seen via ``fdbstatus`` (look for ``connected_coordinators`` field in ``clients``)::
"clients" : {
"count" : 2,
"supported_versions" : [
{
"client_version" : "6.1.0",
"connected_clients" : [
{
"address" : "127.0.0.1:42916",
"connected_coordinators": 3,
"log_group" : "default"
},
{
"address" : "127.0.0.1:42918",
"connected_coordinators": 2,
"log_group" : "default"
}
]
}, ...
]
}
3) If there exist a client (e.g., the client 127.0.0.1:42918 in the above example) that cannot connect to all coordinators after a coordinator is switched to TLS, it mean the client does not set up its TLS correctly. System operator should notify the client to correct the client's TLS configuration. Otherwise, when all coordinators are switched to TLS ports, the client will loose connection.
4) Repeat (2) and (3) until all the addresses in coordinator list are TLS.
5) Restart each FoundationDB server, but only with one public address that listens to TLS traffic only.
.. _converting-existing-cluster-before-6.1:
Converting an existing cluster to use TLS (< v6.1)
==================================================
Enabling TLS on an existing (non-TLS) cluster cannot be accomplished without downtime because all processes must have TLS enabled to communicate. At startup, each server process enables TLS if the addresses in its cluster file are TLS-enabled. As a result, server processes must be stopped and restarted to convert them to use TLS. To convert the cluster to TLS in the most conservative way:

28
fdbclient/DatabaseConfiguration.cpp
View File

@ -38,7 +38,7 @@ void DatabaseConfiguration::resetInternal() {
autoDesiredTLogCount = CLIENT_KNOBS->DEFAULT_AUTO_LOGS;
usableRegions = 1;
regions.clear();
tLogPolicy = storagePolicy = remoteTLogPolicy = IRepPolicyRef();
tLogPolicy = storagePolicy = remoteTLogPolicy = Reference<IReplicationPolicy>();
remoteDesiredTLogCount = -1;
remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0;
}
@ -48,7 +48,7 @@ void parse( int* i, ValueRef const& v ) {
*i = atoi(v.toString().c_str());
}
void parseReplicationPolicy(IRepPolicyRef* policy, ValueRef const& v) {
void parseReplicationPolicy(Reference<IReplicationPolicy>* policy, ValueRef const& v) {
BinaryReader reader(v, IncludeVersion());
serializeReplicationPolicy(reader, *policy);
}
@ -91,35 +91,35 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
info.satelliteTLogReplicationFactor = 1;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = IRepPolicyRef(new PolicyOne());
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
} else if(satelliteReplication == "one_satellite_double") {
info.satelliteTLogReplicationFactor = 2;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "one_satellite_triple") {
info.satelliteTLogReplicationFactor = 3;
info.satelliteTLogUsableDcs = 1;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "two_satellite_safe") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 0;
info.satelliteTLogPolicy = IRepPolicyRef(new PolicyAcross(2, "dcid", IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(satelliteReplication == "two_satellite_fast") {
info.satelliteTLogReplicationFactor = 4;
info.satelliteTLogUsableDcs = 2;
info.satelliteTLogWriteAntiQuorum = 2;
info.satelliteTLogPolicy = IRepPolicyRef(new PolicyAcross(2, "dcid", IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))));
info.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
info.satelliteTLogReplicationFactorFallback = 2;
info.satelliteTLogUsableDcsFallback = 1;
info.satelliteTLogWriteAntiQuorumFallback = 0;
info.satelliteTLogPolicyFallback = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
info.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else {
throw invalid_option();
}
@ -141,20 +141,20 @@ void parse( std::vector<RegionInfo>* regions, ValueRef const& v ) {
void DatabaseConfiguration::setDefaultReplicationPolicy() {
if(!storagePolicy) {
storagePolicy = IRepPolicyRef(new PolicyAcross(storageTeamSize, "zoneid", IRepPolicyRef(new PolicyOne())));
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageTeamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(!tLogPolicy) {
tLogPolicy = IRepPolicyRef(new PolicyAcross(tLogReplicationFactor, "zoneid", IRepPolicyRef(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(tLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(remoteTLogReplicationFactor > 0 && !remoteTLogPolicy) {
remoteTLogPolicy = IRepPolicyRef(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", IRepPolicyRef(new PolicyOne())));
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(remoteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
for(auto& r : regions) {
if(r.satelliteTLogReplicationFactor > 0 && !r.satelliteTLogPolicy) {
r.satelliteTLogPolicy = IRepPolicyRef(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", IRepPolicyRef(new PolicyOne())));
r.satelliteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactor, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
if(r.satelliteTLogReplicationFactorFallback > 0 && !r.satelliteTLogPolicyFallback) {
r.satelliteTLogPolicyFallback = IRepPolicyRef(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", IRepPolicyRef(new PolicyOne())));
r.satelliteTLogPolicyFallback = Reference<IReplicationPolicy>(new PolicyAcross(r.satelliteTLogReplicationFactorFallback, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
}
}
}

12
fdbclient/DatabaseConfiguration.h
View File

@ -49,13 +49,13 @@ struct RegionInfo {
Key dcId;
int32_t priority;
IRepPolicyRef satelliteTLogPolicy;
Reference<IReplicationPolicy> satelliteTLogPolicy;
int32_t satelliteDesiredTLogCount;
int32_t satelliteTLogReplicationFactor;
int32_t satelliteTLogWriteAntiQuorum;
int32_t satelliteTLogUsableDcs;
IRepPolicyRef satelliteTLogPolicyFallback;
Reference<IReplicationPolicy> satelliteTLogPolicyFallback;
int32_t satelliteTLogReplicationFactorFallback;
int32_t satelliteTLogWriteAntiQuorumFallback;
int32_t satelliteTLogUsableDcsFallback;
@ -157,7 +157,7 @@ struct DatabaseConfiguration {
int32_t autoResolverCount;
// TLogs
IRepPolicyRef tLogPolicy;
Reference<IReplicationPolicy> tLogPolicy;
int32_t desiredTLogCount;
int32_t autoDesiredTLogCount;
int32_t tLogWriteAntiQuorum;
@ -167,7 +167,7 @@ struct DatabaseConfiguration {
TLogSpillType tLogSpillType;
// Storage Servers
IRepPolicyRef storagePolicy;
Reference<IReplicationPolicy> storagePolicy;
int32_t storageTeamSize;
KeyValueStoreType storageServerStoreType;
@ -175,7 +175,7 @@ struct DatabaseConfiguration {
int32_t desiredLogRouterCount;
int32_t remoteDesiredTLogCount;
int32_t remoteTLogReplicationFactor;
IRepPolicyRef remoteTLogPolicy;
Reference<IReplicationPolicy> remoteTLogPolicy;
//Data centers
int32_t usableRegions;
@ -195,7 +195,7 @@ struct DatabaseConfiguration {
if(desired == -1) return autoDesiredTLogCount; return desired;
}
int32_t getRemoteTLogReplicationFactor() const { if(remoteTLogReplicationFactor == 0) return tLogReplicationFactor; return remoteTLogReplicationFactor; }
IRepPolicyRef getRemoteTLogPolicy() const { if(remoteTLogReplicationFactor == 0) return tLogPolicy; return remoteTLogPolicy; }
Reference<IReplicationPolicy> getRemoteTLogPolicy() const { if(remoteTLogReplicationFactor == 0) return tLogPolicy; return remoteTLogPolicy; }
bool operator == ( DatabaseConfiguration const& rhs ) const {
const_cast<DatabaseConfiguration*>(this)->makeConfigurationImmutable();

8
fdbclient/FDBTypes.h
View File

@ -737,6 +737,7 @@ struct HealthMetrics {
int64_t worstStorageDurabilityLag;
int64_t worstTLogQueue;
double tpsLimit;
bool batchLimited;
std::map<UID, StorageStats> storageStats;
std::map<UID, int64_t> tLogQueue;
@ -745,6 +746,7 @@ struct HealthMetrics {
, worstStorageDurabilityLag(0)
, worstTLogQueue(0)
, tpsLimit(0.0)
, batchLimited(false)
{}
void update(const HealthMetrics& hm, bool detailedInput, bool detailedOutput)
@ -753,6 +755,7 @@ struct HealthMetrics {
worstStorageDurabilityLag = hm.worstStorageDurabilityLag;
worstTLogQueue = hm.worstTLogQueue;
tpsLimit = hm.tpsLimit;
batchLimited = hm.batchLimited;
if (!detailedOutput) {
storageStats.clear();
@ -769,13 +772,14 @@ struct HealthMetrics {
worstStorageDurabilityLag == r.worstStorageDurabilityLag &&
worstTLogQueue == r.worstTLogQueue &&
storageStats == r.storageStats &&
tLogQueue == r.tLogQueue
tLogQueue == r.tLogQueue &&
batchLimited == r.batchLimited
);
}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, worstStorageQueue, worstStorageDurabilityLag, worstTLogQueue, tpsLimit, storageStats, tLogQueue);
serializer(ar, worstStorageQueue, worstStorageDurabilityLag, worstTLogQueue, tpsLimit, batchLimited, storageStats, tLogQueue);
}
};

44
fdbclient/ManagementAPI.actor.cpp
View File

@ -99,42 +99,42 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
}
std::string redundancy, log_replicas;
IRepPolicyRef storagePolicy;
IRepPolicyRef tLogPolicy;
Reference<IReplicationPolicy> storagePolicy;
Reference<IReplicationPolicy> tLogPolicy;
bool redundancySpecified = true;
if (mode == "single") {
redundancy="1";
log_replicas="1";
storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyOne());
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
} else if(mode == "double" || mode == "fast_recovery_double") {
redundancy="2";
log_replicas="2";
storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(mode == "triple" || mode == "fast_recovery_triple") {
redundancy="3";
log_replicas="3";
storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(mode == "three_datacenter" || mode == "multi_dc") {
redundancy="6";
log_replicas="4";
storagePolicy = IRepPolicyRef(new PolicyAcross(3, "dcid",
IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "dcid",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "dcid",
IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
} else if(mode == "three_datacenter_fallback") {
redundancy="4";
log_replicas="4";
storagePolicy = tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "dcid", IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))));
storagePolicy = tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "dcid", Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))));
} else if(mode == "three_data_hall") {
redundancy="3";
log_replicas="4";
storagePolicy = IRepPolicyRef(new PolicyAcross(3, "data_hall", IRepPolicyRef(new PolicyOne())));
tLogPolicy = IRepPolicyRef(new PolicyAcross(2, "data_hall",
IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))
storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
tLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
} else
redundancySpecified = false;
@ -154,29 +154,29 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
}
std::string remote_redundancy, remote_log_replicas;
IRepPolicyRef remoteTLogPolicy;
Reference<IReplicationPolicy> remoteTLogPolicy;
bool remoteRedundancySpecified = true;
if (mode == "remote_default") {
remote_redundancy="0";
remote_log_replicas="0";
remoteTLogPolicy = IRepPolicyRef();
remoteTLogPolicy = Reference<IReplicationPolicy>();
} else if (mode == "remote_single") {
remote_redundancy="1";
remote_log_replicas="1";
remoteTLogPolicy = IRepPolicyRef(new PolicyOne());
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyOne());
} else if(mode == "remote_double") {
remote_redundancy="2";
remote_log_replicas="2";
remoteTLogPolicy = IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())));
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(mode == "remote_triple") {
remote_redundancy="3";
remote_log_replicas="3";
remoteTLogPolicy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
} else if(mode == "remote_three_data_hall") { //FIXME: not tested in simulation
remote_redundancy="3";
remote_log_replicas="4";
remoteTLogPolicy = IRepPolicyRef(new PolicyAcross(2, "data_hall",
IRepPolicyRef(new PolicyAcross(2, "zoneid", IRepPolicyRef(new PolicyOne())))
remoteTLogPolicy = Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall",
Reference<IReplicationPolicy>(new PolicyAcross(2, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())))
));
} else
remoteRedundancySpecified = false;
@ -212,7 +212,7 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
auto p = configKeysPrefix.toString();
if(!outConf.count(p + "storage_replication_policy") && outConf.count(p + "storage_replicas")) {
int storageCount = stoi(outConf[p + "storage_replicas"]);
IRepPolicyRef storagePolicy = IRepPolicyRef(new PolicyAcross(storageCount, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> storagePolicy = Reference<IReplicationPolicy>(new PolicyAcross(storageCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
BinaryWriter policyWriter(IncludeVersion());
serializeReplicationPolicy(policyWriter, storagePolicy);
outConf[p+"storage_replication_policy"] = policyWriter.toStringRef().toString();
@ -220,7 +220,7 @@ ConfigurationResult::Type buildConfiguration( std::vector<StringRef> const& mode
if(!outConf.count(p + "log_replication_policy") && outConf.count(p + "log_replicas")) {
int logCount = stoi(outConf[p + "log_replicas"]);
IRepPolicyRef logPolicy = IRepPolicyRef(new PolicyAcross(logCount, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> logPolicy = Reference<IReplicationPolicy>(new PolicyAcross(logCount, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
BinaryWriter policyWriter(IncludeVersion());
serializeReplicationPolicy(policyWriter, logPolicy);
outConf[p+"log_replication_policy"] = policyWriter.toStringRef().toString();

1
fdbclient/NativeAPI.actor.cpp
View File

@ -35,7 +35,6 @@
#include "fdbclient/MutationList.h"
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/MonitorLeader.h"
#include "fdbclient/Knobs.h"
#if defined(CMAKE_BUILD) || !defined(WIN32)
#include "versions.h"
#endif

3
fdbclient/Schemas.cpp
View File

@ -52,6 +52,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
"test"
]
},
"degraded":true,
"roles":[
{
"query_queue_max":0,
@ -281,6 +282,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
],
"datacenter_version_difference":0,
"degraded_processes":0,
"database_available":true,
"database_locked":false,
"generation":2,
@ -325,6 +327,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema(
"$enum":[
"unreachable_master_worker",
"unreachable_dataDistributor_worker",
"unreachable_ratekeeper_worker",
"unreadable_configuration",
"full_replication_timeout",
"client_issues",

4
fdbrpc/FlowTransport.actor.cpp
View File

@ -215,6 +215,10 @@ struct ConnectPacket {
uint16_t flags;
uint8_t canonicalRemoteIp6[16];
ConnectPacket() {
memset(this, 0, sizeof(*this));
}
IPAddress canonicalRemoteIp() const {
if (isIPv6()) {
IPAddress::IPAddressStore store;

49
fdbrpc/Locality.cpp
View File

@ -164,26 +164,35 @@ ProcessClass::Fitness ProcessClass::machineClassFitness( ClusterRole role ) cons
}
case ProcessClass::DataDistributor:
switch( _class ) {
case ProcessClass::DataDistributorClass:
return ProcessClass::BestFit;
case ProcessClass::StatelessClass:
return ProcessClass::GoodFit;
case ProcessClass::MasterClass:
return ProcessClass::OkayFit;
case ProcessClass::ResolutionClass:
return ProcessClass::OkayFit;
case ProcessClass::TransactionClass:
return ProcessClass::OkayFit;
case ProcessClass::ProxyClass:
return ProcessClass::OkayFit;
case ProcessClass::UnsetClass:
return ProcessClass::UnsetFit;
case ProcessClass::CoordinatorClass:
return ProcessClass::NeverAssign;
case ProcessClass::TesterClass:
return ProcessClass::NeverAssign;
default:
return ProcessClass::WorstFit;
case ProcessClass::DataDistributorClass:
return ProcessClass::BestFit;
case ProcessClass::StatelessClass:
return ProcessClass::GoodFit;
case ProcessClass::MasterClass:
return ProcessClass::OkayFit;
case ProcessClass::UnsetClass:
return ProcessClass::UnsetFit;
case ProcessClass::CoordinatorClass:
case ProcessClass::TesterClass:
return ProcessClass::NeverAssign;
default:
return ProcessClass::WorstFit;
}
case ProcessClass::RateKeeper:
switch( _class ) {
case ProcessClass::RateKeeperClass:
return ProcessClass::BestFit;
case ProcessClass::StatelessClass:
return ProcessClass::GoodFit;
case ProcessClass::MasterClass:
return ProcessClass::OkayFit;
case ProcessClass::UnsetClass:
return ProcessClass::UnsetFit;
case ProcessClass::CoordinatorClass:
case ProcessClass::TesterClass:
return ProcessClass::NeverAssign;
default:
return ProcessClass::WorstFit;
}
default:
return ProcessClass::NeverAssign;

7
fdbrpc/Locality.h
View File

@ -26,9 +26,9 @@
struct ProcessClass {
// This enum is stored in restartInfo.ini for upgrade tests, so be very careful about changing the existing items!
enum ClassType { UnsetClass, StorageClass, TransactionClass, ResolutionClass, TesterClass, ProxyClass, MasterClass, StatelessClass, LogClass, ClusterControllerClass, LogRouterClass, DataDistributorClass, CoordinatorClass, InvalidClass = -1 };
enum ClassType { UnsetClass, StorageClass, TransactionClass, ResolutionClass, TesterClass, ProxyClass, MasterClass, StatelessClass, LogClass, ClusterControllerClass, LogRouterClass, DataDistributorClass, CoordinatorClass, RateKeeperClass, InvalidClass = -1 };
enum Fitness { BestFit, GoodFit, UnsetFit, OkayFit, WorstFit, ExcludeFit, NeverAssign }; //cannot be larger than 7 because of leader election mask
enum ClusterRole { Storage, TLog, Proxy, Master, Resolver, LogRouter, ClusterController, DataDistributor, NoRole };
enum ClusterRole { Storage, TLog, Proxy, Master, Resolver, LogRouter, ClusterController, DataDistributor, RateKeeper, NoRole };
enum ClassSource { CommandLineSource, AutoSource, DBSource, InvalidSource = -1 };
int16_t _class;
int16_t _source;
@ -50,6 +50,7 @@ public:
else if (s=="cluster_controller") _class = ClusterControllerClass;
else if (s=="data_distributor") _class = DataDistributorClass;
else if (s=="coordinator") _class = CoordinatorClass;
else if (s=="ratekeeper") _class = RateKeeperClass;
else _class = InvalidClass;
}
@ -67,6 +68,7 @@ public:
else if (classStr=="cluster_controller") _class = ClusterControllerClass;
else if (classStr=="data_distributor") _class = DataDistributorClass;
else if (classStr=="coordinator") _class = CoordinatorClass;
else if (classStr=="ratekeeper") _class = RateKeeperClass;
else _class = InvalidClass;
if (sourceStr=="command_line") _source = CommandLineSource;
@ -99,6 +101,7 @@ public:
case ClusterControllerClass: return "cluster_controller";
case DataDistributorClass: return "data_distributor";
case CoordinatorClass: return "coordinator";
case RateKeeperClass: return "ratekeeper";
default: return "invalid";
}
}

7
fdbrpc/Net2FileSystem.cpp
View File

@ -58,7 +58,12 @@ Future< Reference<class IAsyncFile> > Net2FileSystem::open( std::string filename
Future<Reference<IAsyncFile>> f;
#ifdef __linux__
if ( (flags & IAsyncFile::OPEN_UNBUFFERED) && !(flags & IAsyncFile::OPEN_NO_AIO) )
// In the vast majority of cases, we wish to use Kernel AIO. However, some systems
// dont properly support don’t properly support kernel async I/O without O_DIRECT
// or AIO at all. In such cases, DISABLE_POSIX_KERNEL_AIO knob can be enabled to fallback to
// EIO instead of Kernel AIO.
if ((flags & IAsyncFile::OPEN_UNBUFFERED) && !(flags & IAsyncFile::OPEN_NO_AIO) &&
!FLOW_KNOBS->DISABLE_POSIX_KERNEL_AIO)
f = AsyncFileKAIO::open(filename, flags, mode, NULL);
else
#endif

32
fdbrpc/Replication.h
View File

@ -36,23 +36,23 @@ public:
virtual void delref() { ReferenceCounted<LocalitySet>::delref(); }
bool selectReplicas(
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results)
{
LocalitySetRef fromServers = LocalitySetRef::addRef(this);
Reference<LocalitySet> fromServers = Reference<LocalitySet>::addRef(this);
return policy->selectReplicas(fromServers, alsoServers, results);
}
bool selectReplicas(
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityEntry> & results)
{ return selectReplicas(policy, std::vector<LocalityEntry>(), results); }
bool validate(
IRepPolicyRef const& policy) const
Reference<IReplicationPolicy> const& policy) const
{
LocalitySetRef const solutionSet = LocalitySetRef::addRef((LocalitySet*) this);
Reference<LocalitySet> const solutionSet = Reference<LocalitySet>::addRef((LocalitySet*) this);
return policy->validate(solutionSet);
}
@ -159,7 +159,7 @@ public:
}
static void staticDisplayEntries(
LocalitySetRef const& fromServers,
Reference<LocalitySet> const& fromServers,
std::vector<LocalityEntry> const& entryArray,
const char* name = "zone")
{
@ -174,8 +174,8 @@ public:
// the specified value for the given key
// The returned LocalitySet contains the LocalityRecords that have the same value as
// the indexValue under the same indexKey (e.g., zoneid)
LocalitySetRef restrict(AttribKey indexKey, AttribValue indexValue ) {
LocalitySetRef localitySet;
Reference<LocalitySet> restrict(AttribKey indexKey, AttribValue indexValue ) {
Reference<LocalitySet> localitySet;
LocalityCacheRecord searchRecord(AttribRecord(indexKey, indexValue), localitySet);
auto itKeyValue = std::lower_bound(_cacheArray.begin(), _cacheArray.end(), searchRecord, LocalityCacheRecord::compareKeyValue);
@ -185,7 +185,7 @@ public:
localitySet = itKeyValue->_resultset;
}
else {
localitySet = LocalitySetRef(new LocalitySet(*_localitygroup));
localitySet = Reference<LocalitySet>(new LocalitySet(*_localitygroup));
_cachemisses ++;
// If the key is not within the current key set, skip it because no items within
// the current entry array has the key
@ -213,8 +213,8 @@ public:
}
// This function is used to create an subset containing the specified entries
LocalitySetRef restrict(std::vector<LocalityEntry> const& entryArray) {
LocalitySetRef localitySet(new LocalitySet(*_localitygroup));
Reference<LocalitySet> restrict(std::vector<LocalityEntry> const& entryArray) {
Reference<LocalitySet> localitySet(new LocalitySet(*_localitygroup));
for (auto& entry : entryArray) {
localitySet->add(getRecordViaEntry(entry), *this);
}
@ -453,8 +453,8 @@ protected:
// This class stores the cache record for each entry within the locality set
struct LocalityCacheRecord {
AttribRecord _attribute;
LocalitySetRef _resultset;
LocalityCacheRecord(AttribRecord const& attribute, LocalitySetRef resultset):_attribute(attribute),_resultset(resultset){}
Reference<LocalitySet> _resultset;
LocalityCacheRecord(AttribRecord const& attribute, Reference<LocalitySet> resultset):_attribute(attribute),_resultset(resultset){}
LocalityCacheRecord(LocalityCacheRecord const& source):_attribute(source._attribute),_resultset(source._resultset){}
virtual ~LocalityCacheRecord(){}
LocalityCacheRecord& operator=(LocalityCacheRecord const& source) {
@ -584,7 +584,7 @@ struct LocalityMap : public LocalityGroup {
virtual ~LocalityMap() {}
bool selectReplicas(
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry>& entryResults,
std::vector<V*> & results)
@ -601,7 +601,7 @@ struct LocalityMap : public LocalityGroup {
}
bool selectReplicas(
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityEntry> const& alsoServers,
std::vector<V*> & results)
{
@ -610,7 +610,7 @@ struct LocalityMap : public LocalityGroup {
}
bool selectReplicas(
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<V*> & results)
{ return selectReplicas(policy, std::vector<LocalityEntry>(), results); }

36
fdbrpc/ReplicationPolicy.cpp
View File

@ -24,14 +24,14 @@
bool IReplicationPolicy::selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> & results )
{
return selectReplicas(fromServers, std::vector<LocalityEntry>(), results);
}
bool IReplicationPolicy::validate(
LocalitySetRef const& solutionSet ) const
Reference<LocalitySet> const& solutionSet ) const
{
return validate(solutionSet->getEntries(), solutionSet);
}
@ -40,7 +40,7 @@ bool IReplicationPolicy::validateFull(
bool solved,
std::vector<LocalityEntry> const& solutionSet,
std::vector<LocalityEntry> const& alsoServers,
LocalitySetRef const& fromServers )
Reference<LocalitySet> const& fromServers )
{
bool valid = true;
std::vector<LocalityEntry> totalSolution(solutionSet);
@ -105,7 +105,7 @@ bool IReplicationPolicy::validateFull(
}
bool PolicyOne::selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results )
{
@ -131,12 +131,12 @@ bool PolicyOne::selectReplicas(
bool PolicyOne::validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const
Reference<LocalitySet> const& fromServers ) const
{
return ((solutionSet.size() > 0) && (fromServers->size() > 0));
}
PolicyAcross::PolicyAcross(int count, std::string const& attribKey, IRepPolicyRef const policy):
PolicyAcross::PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy):
_count(count),_attribKey(attribKey),_policy(policy)
{
return;
@ -150,7 +150,7 @@ PolicyAcross::~PolicyAcross()
// Debug purpose only
// Trace all record entries to help debug
// fromServers is the servers locality to be printed out.
void IReplicationPolicy::traceLocalityRecords(LocalitySetRef const& fromServers) {
void IReplicationPolicy::traceLocalityRecords(Reference<LocalitySet> const& fromServers) {
std::vector<Reference<LocalityRecord>> const& recordArray = fromServers->getRecordArray();
TraceEvent("LocalityRecordArray").detail("Size", recordArray.size());
for (auto& record : recordArray) {
@ -158,7 +158,7 @@ void IReplicationPolicy::traceLocalityRecords(LocalitySetRef const& fromServers)
}
}
void IReplicationPolicy::traceOneLocalityRecord(Reference<LocalityRecord> record, LocalitySetRef const& fromServers) {
void IReplicationPolicy::traceOneLocalityRecord(Reference<LocalityRecord> record, Reference<LocalitySet> const& fromServers) {
int localityEntryIndex = record->_entryIndex._id;
Reference<KeyValueMap> const& dataMap = record->_dataMap;
std::vector<AttribRecord> const& keyValueArray = dataMap->_keyvaluearray;
@ -185,7 +185,7 @@ void IReplicationPolicy::traceOneLocalityRecord(Reference<LocalityRecord> record
// return true if the team satisfies the policy; false otherwise
bool PolicyAcross::validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const
Reference<LocalitySet> const& fromServers ) const
{
bool valid = true;
int count = 0;
@ -262,7 +262,7 @@ bool PolicyAcross::validate(
// that should be excluded from being selected as replicas.
// FIXME: Simplify this function, such as removing unnecessary printf
bool PolicyAcross::selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results )
{
@ -437,7 +437,7 @@ bool PolicyAcross::selectReplicas(
bool PolicyAnd::validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const
Reference<LocalitySet> const& fromServers ) const
{
bool valid = true;
for (auto& policy : _policies) {
@ -450,7 +450,7 @@ bool PolicyAnd::validate(
}
bool PolicyAnd::selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results )
{
@ -486,26 +486,26 @@ bool PolicyAnd::selectReplicas(
return passed;
}
void testPolicySerialization(IRepPolicyRef& policy) {
void testPolicySerialization(Reference<IReplicationPolicy>& policy) {
std::string policyInfo = policy->info();
BinaryWriter writer(IncludeVersion());
serializeReplicationPolicy(writer, policy);
BinaryReader reader(writer.getData(), writer.getLength(), IncludeVersion());
IRepPolicyRef copy;
Reference<IReplicationPolicy> copy;
serializeReplicationPolicy(reader, copy);
ASSERT(policy->info() == copy->info());
}
void testReplicationPolicy(int nTests) {
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(1, "data_hall", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(1, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())));
testPolicySerialization(policy);
policy = IRepPolicyRef(new PolicyAnd({
IRepPolicyRef(new PolicyAcross(2, "data_center", IRepPolicyRef(new PolicyAcross(3, "rack", IRepPolicyRef(new PolicyOne()))))),
IRepPolicyRef(new PolicyAcross(2, "data_center", IRepPolicyRef(new PolicyAcross(2, "data_hall", IRepPolicyRef(new PolicyOne())))))
policy = Reference<IReplicationPolicy>(new PolicyAnd({
Reference<IReplicationPolicy>(new PolicyAcross(2, "data_center", Reference<IReplicationPolicy>(new PolicyAcross(3, "rack", Reference<IReplicationPolicy>(new PolicyOne()))))),
Reference<IReplicationPolicy>(new PolicyAcross(2, "data_center", Reference<IReplicationPolicy>(new PolicyAcross(2, "data_hall", Reference<IReplicationPolicy>(new PolicyOne())))))
}));
testPolicySerialization(policy);

64
fdbrpc/ReplicationPolicy.h
View File

@ -26,7 +26,7 @@
#include "fdbrpc/ReplicationTypes.h"
template <class Ar>
void serializeReplicationPolicy(Ar& ar, IRepPolicyRef& policy);
void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy);
extern void testReplicationPolicy(int nTests);
@ -40,36 +40,36 @@ struct IReplicationPolicy : public ReferenceCounted<IReplicationPolicy> {
virtual int maxResults() const = 0;
virtual int depth() const = 0;
virtual bool selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results ) = 0;
virtual void traceLocalityRecords(LocalitySetRef const& fromServers);
virtual void traceOneLocalityRecord(Reference<LocalityRecord> record, LocalitySetRef const& fromServers);
virtual void traceLocalityRecords(Reference<LocalitySet> const& fromServers);
virtual void traceOneLocalityRecord(Reference<LocalityRecord> record, Reference<LocalitySet> const& fromServers);
virtual bool validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const = 0;
Reference<LocalitySet> const& fromServers ) const = 0;
bool operator == ( const IReplicationPolicy& r ) const { return info() == r.info(); }
bool operator != ( const IReplicationPolicy& r ) const { return info() != r.info(); }
template <class Ar>
void serialize(Ar& ar) {
IRepPolicyRef refThis(this);
Reference<IReplicationPolicy> refThis(this);
serializeReplicationPolicy(ar, refThis);
refThis->delref_no_destroy();
}
// Utility functions
bool selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> & results );
bool validate(
LocalitySetRef const& solutionSet ) const;
Reference<LocalitySet> const& solutionSet ) const;
bool validateFull(
bool solved,
std::vector<LocalityEntry> const& solutionSet,
std::vector<LocalityEntry> const& alsoServers,
LocalitySetRef const& fromServers );
Reference<LocalitySet> const& fromServers );
// Returns a set of the attributes that this policy uses in selection and validation.
std::set<std::string> attributeKeys() const
@ -78,7 +78,7 @@ struct IReplicationPolicy : public ReferenceCounted<IReplicationPolicy> {
};
template <class Archive>
inline void load( Archive& ar, IRepPolicyRef& value ) {
inline void load( Archive& ar, Reference<IReplicationPolicy>& value ) {
bool present = (value.getPtr());
ar >> present;
if (present) {
@ -90,11 +90,11 @@ inline void load( Archive& ar, IRepPolicyRef& value ) {
}
template <class Archive>
inline void save( Archive& ar, const IRepPolicyRef& value ) {
inline void save( Archive& ar, const Reference<IReplicationPolicy>& value ) {
bool present = (value.getPtr());
ar << present;
if (present) {
serializeReplicationPolicy(ar, (IRepPolicyRef&) value);
serializeReplicationPolicy(ar, (Reference<IReplicationPolicy>&) value);
}
}
@ -107,9 +107,9 @@ struct PolicyOne : IReplicationPolicy, public ReferenceCounted<PolicyOne> {
virtual int depth() const { return 1; }
virtual bool validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const;
Reference<LocalitySet> const& fromServers ) const;
virtual bool selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results );
template <class Ar>
@ -119,7 +119,7 @@ struct PolicyOne : IReplicationPolicy, public ReferenceCounted<PolicyOne> {
};
struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross> {
PolicyAcross(int count, std::string const& attribKey, IRepPolicyRef const policy);
PolicyAcross(int count, std::string const& attribKey, Reference<IReplicationPolicy> const policy);
virtual ~PolicyAcross();
virtual std::string name() const { return "Across"; }
virtual std::string info() const
@ -128,9 +128,9 @@ struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross>
virtual int depth() const { return 1 + _policy->depth(); }
virtual bool validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const;
Reference<LocalitySet> const& fromServers ) const;
virtual bool selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results );
@ -149,18 +149,18 @@ struct PolicyAcross : IReplicationPolicy, public ReferenceCounted<PolicyAcross>
protected:
int _count;
std::string _attribKey;
IRepPolicyRef _policy;
Reference<IReplicationPolicy> _policy;
// Cache temporary members
std::vector<AttribValue> _usedValues;
std::vector<LocalityEntry> _newResults;
LocalitySetRef _selected;
Reference<LocalitySet> _selected;
VectorRef<std::pair<int,int>> _addedResults;
Arena _arena;
};
struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
PolicyAnd(std::vector<IRepPolicyRef> policies): _policies(policies), _sortedPolicies(policies)
PolicyAnd(std::vector<Reference<IReplicationPolicy>> policies): _policies(policies), _sortedPolicies(policies)
{
// Sort the policy array
std::sort(_sortedPolicies.begin(), _sortedPolicies.end(), PolicyAnd::comparePolicy);
@ -194,14 +194,14 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
}
virtual bool validate(
std::vector<LocalityEntry> const& solutionSet,
LocalitySetRef const& fromServers ) const;
Reference<LocalitySet> const& fromServers ) const;
virtual bool selectReplicas(
LocalitySetRef & fromServers,
Reference<LocalitySet> & fromServers,
std::vector<LocalityEntry> const& alsoServers,
std::vector<LocalityEntry> & results );
static bool comparePolicy(const IRepPolicyRef& rhs, const IRepPolicyRef& lhs)
static bool comparePolicy(const Reference<IReplicationPolicy>& rhs, const Reference<IReplicationPolicy>& lhs)
{ return (lhs->maxResults() < rhs->maxResults()) || (!(rhs->maxResults() < lhs->maxResults()) && (lhs->depth() < rhs->depth())); }
template <class Ar>
@ -219,18 +219,18 @@ struct PolicyAnd : IReplicationPolicy, public ReferenceCounted<PolicyAnd> {
}
virtual void attributeKeys(std::set<std::string> *set) const override
{ for (const IRepPolicyRef& r : _policies) { r->attributeKeys(set); } }
{ for (const Reference<IReplicationPolicy>& r : _policies) { r->attributeKeys(set); } }
protected:
std::vector<IRepPolicyRef> _policies;
std::vector<IRepPolicyRef> _sortedPolicies;
std::vector<Reference<IReplicationPolicy>> _policies;
std::vector<Reference<IReplicationPolicy>> _sortedPolicies;
};
extern int testReplication();
template <class Ar>
void serializeReplicationPolicy(Ar& ar, IRepPolicyRef& policy) {
void serializeReplicationPolicy(Ar& ar, Reference<IReplicationPolicy>& policy) {
if(Ar::isDeserializing) {
StringRef name;
serializer(ar, name);
@ -238,20 +238,20 @@ void serializeReplicationPolicy(Ar& ar, IRepPolicyRef& policy) {
if(name == LiteralStringRef("One")) {
PolicyOne* pointer = new PolicyOne();
pointer->serialize(ar);
policy = IRepPolicyRef(pointer);
policy = Reference<IReplicationPolicy>(pointer);
}
else if(name == LiteralStringRef("Across")) {
PolicyAcross* pointer = new PolicyAcross(0, "", IRepPolicyRef());
PolicyAcross* pointer = new PolicyAcross(0, "", Reference<IReplicationPolicy>());
pointer->serialize(ar);
policy = IRepPolicyRef(pointer);
policy = Reference<IReplicationPolicy>(pointer);
}
else if(name == LiteralStringRef("And")) {
PolicyAnd* pointer = new PolicyAnd({});
pointer->serialize(ar);
policy = IRepPolicyRef(pointer);
policy = Reference<IReplicationPolicy>(pointer);
}
else if(name == LiteralStringRef("None")) {
policy = IRepPolicyRef();
policy = Reference<IReplicationPolicy>();
}
else {
TraceEvent(SevError, "SerializingInvalidPolicyType")

3
fdbrpc/ReplicationTypes.h
View File

@ -34,9 +34,6 @@ struct LocalityRecord;
struct StringToIntMap;
struct IReplicationPolicy;
typedef Reference<LocalitySet> LocalitySetRef;
typedef Reference<IReplicationPolicy> IRepPolicyRef;
extern int g_replicationdebug;
struct AttribKey {

142
fdbrpc/ReplicationUtils.cpp
View File

@ -27,8 +27,8 @@
double ratePolicy(
LocalitySetRef & localitySet,
IRepPolicyRef const& policy,
Reference<LocalitySet> & localitySet,
Reference<IReplicationPolicy> const& policy,
unsigned int nTestTotal)
{
double rating = -1.0;
@ -85,14 +85,14 @@ double ratePolicy(
bool findBestPolicySet(
std::vector<LocalityEntry>& bestResults,
LocalitySetRef & localitySet,
IRepPolicyRef const& policy,
Reference<LocalitySet> & localitySet,
Reference<IReplicationPolicy> const& policy,
unsigned int nMinItems,
unsigned int nSelectTests,
unsigned int nPolicyTests)
{
bool bSucceeded = true;
LocalitySetRef bestLocalitySet, testLocalitySet;
Reference<LocalitySet> bestLocalitySet, testLocalitySet;
std::vector<LocalityEntry> results;
double testRate, bestRate = -1.0;
@ -162,15 +162,15 @@ bool findBestPolicySet(
bool findBestUniquePolicySet(
std::vector<LocalityEntry>& bestResults,
LocalitySetRef & localitySet,
IRepPolicyRef const& policy,
Reference<LocalitySet> & localitySet,
Reference<IReplicationPolicy> const& policy,
StringRef localityUniquenessKey,
unsigned int nMinItems,
unsigned int nSelectTests,
unsigned int nPolicyTests)
{
bool bSucceeded = true;
LocalitySetRef bestLocalitySet, testLocalitySet;
Reference<LocalitySet> bestLocalitySet, testLocalitySet;
std::vector<LocalityEntry> results;
double testRate, bestRate = -1.0;
@ -262,7 +262,7 @@ bool findBestUniquePolicySet(
bool validateAllCombinations(
std::vector<LocalityData> & offendingCombo,
LocalityGroup const& localitySet,
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityData> const& newItems,
unsigned int nCombinationSize,
bool bCheckIfValid)
@ -286,25 +286,39 @@ bool validateAllCombinations(
}
else
{
bool bIsValidGroup;
LocalityGroup localityGroup;
bool bIsValidGroup;
Reference<LocalitySet> localSet = Reference<LocalitySet>( new LocalityGroup() );
LocalityGroup* localGroup = (LocalityGroup*) localSet.getPtr();
localGroup->deep_copy(localitySet);
std::vector<LocalityEntry> localityGroupEntries = localGroup->getEntries();
int originalSize = localityGroupEntries.size();
for (int i = 0; i < newItems.size(); ++i) {
localGroup->add(newItems[i]);
}
std::string bitmask(nCombinationSize, 1); // K leading 1's
bitmask.resize(newItems.size(), 0); // N-K trailing 0's
std::vector<LocalityEntry> resultEntries;
do
{
localityGroup.deep_copy(localitySet);
localityGroupEntries.resize(originalSize);
// [0..N-1] integers
for (int i = 0; i < newItems.size(); ++i) {
for (int i = 0; i < bitmask.size(); ++i) {
if (bitmask[i]) {
localityGroup.add(newItems[i]);
localityGroupEntries.push_back(localGroup->getEntry(originalSize + i));
}
}
// Check if the group combination passes validation
bIsValidGroup = localityGroup.validate(policy);
resultEntries.clear();
// Run the policy, assert if unable to satisfy
bool result = localSet->selectReplicas(policy, localityGroupEntries, resultEntries);
ASSERT(result);
bIsValidGroup = resultEntries.size() == 0;
if (((bCheckIfValid) &&
(!bIsValidGroup) ) ||
@ -319,7 +333,7 @@ bool validateAllCombinations(
}
if (g_replicationdebug > 2) {
printf("Invalid group\n");
localityGroup.DisplayEntries();
localGroup->DisplayEntries();
}
if (g_replicationdebug > 3) {
printf("Full set\n");
@ -337,7 +351,7 @@ bool validateAllCombinations(
bool validateAllCombinations(
LocalityGroup const& localitySet,
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityData> const& newItems,
unsigned int nCombinationSize,
bool bCheckIfValid)
@ -358,10 +372,10 @@ repTestType convertToTestType(int iValue) {
return sValue;
}
LocalitySetRef createTestLocalityMap(std::vector<repTestType>& indexes, int dcTotal,
Reference<LocalitySet> createTestLocalityMap(std::vector<repTestType>& indexes, int dcTotal,
int szTotal, int rackTotal, int slotTotal, int independentItems, int independentTotal)
{
LocalitySetRef buildServer(new LocalityMap<repTestType>());
Reference<LocalitySet> buildServer(new LocalityMap<repTestType>());
LocalityMap<repTestType>* serverMap = (LocalityMap<repTestType>*) buildServer.getPtr();
int serverValue, dcLoop, szLoop, rackLoop, slotLoop;
std::string dcText, szText, rackText, slotText, independentName, independentText;
@ -442,8 +456,8 @@ LocalitySetRef createTestLocalityMap(std::vector<repTestType>& indexes, int dcTo
}
bool testPolicy(
LocalitySetRef servers,
IRepPolicyRef const& policy,
Reference<LocalitySet> servers,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityEntry> const& including,
bool validate)
{
@ -506,109 +520,109 @@ bool testPolicy(
}
bool testPolicy(
LocalitySetRef servers,
IRepPolicyRef const& policy,
Reference<LocalitySet> servers,
Reference<IReplicationPolicy> const& policy,
bool validate)
{
return testPolicy(servers, policy, emptyEntryArray, validate);
}
std::vector<IRepPolicyRef> const& getStaticPolicies()
std::vector<Reference<IReplicationPolicy>> const& getStaticPolicies()
{
static std::vector<IRepPolicyRef> staticPolicies;
static std::vector<Reference<IReplicationPolicy>> staticPolicies;
if (staticPolicies.empty())
{
staticPolicies = {
IRepPolicyRef( new PolicyOne() ),
Reference<IReplicationPolicy>( new PolicyOne() ),
// 1 'dc^2 x 1'
IRepPolicyRef( new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyOne() ) ) ),
Reference<IReplicationPolicy>( new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyOne() ) ) ),
// 2 'dc^3 x 1'
IRepPolicyRef( new PolicyAcross(3, "dc", IRepPolicyRef( new PolicyOne() ) ) ),
Reference<IReplicationPolicy>( new PolicyAcross(3, "dc", Reference<IReplicationPolicy>( new PolicyOne() ) ) ),
// 3 'sz^3 x 1'
IRepPolicyRef( new PolicyAcross(3, "sz", IRepPolicyRef( new PolicyOne() ) ) ),
Reference<IReplicationPolicy>( new PolicyAcross(3, "sz", Reference<IReplicationPolicy>( new PolicyOne() ) ) ),
// 4 'dc^1 x az^3 x 1'
IRepPolicyRef( new PolicyAcross(1, "dc", IRepPolicyRef( new PolicyAcross(3, "az", IRepPolicyRef( new PolicyOne() ))) ) ),
Reference<IReplicationPolicy>( new PolicyAcross(1, "dc", Reference<IReplicationPolicy>( new PolicyAcross(3, "az", Reference<IReplicationPolicy>( new PolicyOne() ))) ) ),
// 5 '(sz^3 x rack^2 x 1) + (dc^2 x az^3 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(3, "sz", IRepPolicyRef(new PolicyAcross(2, "rack", IRepPolicyRef(new PolicyOne() ))))), IRepPolicyRef(new PolicyAcross(2, "dc", IRepPolicyRef(new PolicyAcross(3, "az", IRepPolicyRef(new PolicyOne()) ))) )} ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyAcross(2, "rack", Reference<IReplicationPolicy>(new PolicyOne() ))))), Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>(new PolicyAcross(3, "az", Reference<IReplicationPolicy>(new PolicyOne()) ))) )} ) ),
// 6 '(sz^1 x 1)'
IRepPolicyRef( new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne())) ),
Reference<IReplicationPolicy>( new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne())) ),
// 7 '(sz^1 x 1) + (sz^1 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
// 8 '(sz^2 x 1) + (sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
// 9 '(dc^1 x sz^2 x 1)'
IRepPolicyRef( new PolicyAcross(1, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))),
Reference<IReplicationPolicy>( new PolicyAcross(1, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
//10 '(dc^2 x sz^2 x 1)'
IRepPolicyRef( new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))),
Reference<IReplicationPolicy>( new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))),
//11 '(dc^1 x sz^2 x 1) + (dc^2 x sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(1, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))), IRepPolicyRef(new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))), Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) } ) ),
//12 '(dc^2 x sz^2 x 1) + (dc^1 x sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))), IRepPolicyRef(new PolicyAcross(1, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))), Reference<IReplicationPolicy>(new PolicyAcross(1, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) } ) ),
//13 '(sz^2 x 1) + (dc^1 x sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(1, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(1, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) } ) ),
//14 '(sz^2 x 1) + (dc^2 x sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) } ) ),
//15 '(sz^3 x 1) + (dc^2 x sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(3, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))))) } ) ),
//16 '(sz^1 x 1) + (sz^2 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
//17 '(sz^2 x 1) + (sz^3 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(3, "sz", IRepPolicyRef(new PolicyOne()))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
//18 '(sz^1 x 1) + (sz^2 x 1) + (sz^3 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(3, "sz", IRepPolicyRef(new PolicyOne()))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
//19 '(sz^1 x 1) + (machine^1 x 1)'
IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(1, "zoneid", IRepPolicyRef(new PolicyOne()))) } ) ),
Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(1, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
// '(dc^1 x 1) + (sz^1 x 1) + (machine^1 x 1)'
// IRepPolicyRef( new PolicyAnd( { IRepPolicyRef(new PolicyAcross(1, "dc", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(1, "zoneid", IRepPolicyRef(new PolicyOne()))) } ) ),
// Reference<IReplicationPolicy>( new PolicyAnd( { Reference<IReplicationPolicy>(new PolicyAcross(1, "dc", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(1, "zoneid", Reference<IReplicationPolicy>(new PolicyOne()))) } ) ),
// '(dc^1 x sz^3 x 1)'
IRepPolicyRef( new PolicyAcross(1, "dc", IRepPolicyRef( new PolicyAcross(3, "sz", IRepPolicyRef(new PolicyOne())))) ),
Reference<IReplicationPolicy>( new PolicyAcross(1, "dc", Reference<IReplicationPolicy>( new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne())))) ),
// '(dc^2 x sz^3 x 1)'
IRepPolicyRef( new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(3, "sz", IRepPolicyRef(new PolicyOne())))) ),
Reference<IReplicationPolicy>( new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(3, "sz", Reference<IReplicationPolicy>(new PolicyOne())))) ),
// '(dc^2 x az^3 x 1)'
IRepPolicyRef( new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(3, "az", IRepPolicyRef(new PolicyOne())))) ),
Reference<IReplicationPolicy>( new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(3, "az", Reference<IReplicationPolicy>(new PolicyOne())))) ),
// '(sz^1 x 1) + (dc^2 x az^3 x 1)'
IRepPolicyRef( new PolicyAnd({IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "dc", IRepPolicyRef( new PolicyAcross(3, "az", IRepPolicyRef(new PolicyOne())))))}) ),
Reference<IReplicationPolicy>( new PolicyAnd({Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "dc", Reference<IReplicationPolicy>( new PolicyAcross(3, "az", Reference<IReplicationPolicy>(new PolicyOne())))))}) ),
// 'dc^1 x (az^2 x 1) + (sz^2 x 1)'
// IRepPolicyRef( new PolicyAcross(1, "dc", IRepPolicyRef(new PolicyAnd({IRepPolicyRef(new PolicyAcross(2, "az", IRepPolicyRef(new PolicyOne()))), IRepPolicyRef(new PolicyAcross(2, "sz", IRepPolicyRef(new PolicyOne())))}))) ),
// Reference<IReplicationPolicy>( new PolicyAcross(1, "dc", Reference<IReplicationPolicy>(new PolicyAnd({Reference<IReplicationPolicy>(new PolicyAcross(2, "az", Reference<IReplicationPolicy>(new PolicyOne()))), Reference<IReplicationPolicy>(new PolicyAcross(2, "sz", Reference<IReplicationPolicy>(new PolicyOne())))}))) ),
// Require backtracking
IRepPolicyRef( new PolicyAcross(8, "zoneid", IRepPolicyRef(new PolicyAcross(1, "az", IRepPolicyRef(new PolicyOne()))) ) ),
IRepPolicyRef( new PolicyAcross(8, "zoneid", IRepPolicyRef(new PolicyAcross(1, "sz", IRepPolicyRef(new PolicyOne()))) ) )
Reference<IReplicationPolicy>( new PolicyAcross(8, "zoneid", Reference<IReplicationPolicy>(new PolicyAcross(1, "az", Reference<IReplicationPolicy>(new PolicyOne()))) ) ),
Reference<IReplicationPolicy>( new PolicyAcross(8, "zoneid", Reference<IReplicationPolicy>(new PolicyAcross(1, "sz", Reference<IReplicationPolicy>(new PolicyOne()))) ) )
};
}
return staticPolicies;
}
IRepPolicyRef const randomAcrossPolicy(LocalitySet const& serverSet)
Reference<IReplicationPolicy> const randomAcrossPolicy(LocalitySet const& serverSet)
{
int usedKeyTotal, keysUsed, keyIndex, valueTotal, maxValueTotal, maxKeyTotal, skips, lastKeyIndex;
std::vector<std::string> keyArray(serverSet.getGroupKeyMap()->_lookuparray);
@ -616,7 +630,7 @@ IRepPolicyRef const randomAcrossPolicy(LocalitySet const& serverSet)
AttribKey indexKey;
Optional<AttribValue> keyValue;
std::string keyText;
IRepPolicyRef policy(new PolicyOne());
Reference<IReplicationPolicy> policy(new PolicyOne());
// Determine the number of keys to used within the policy
usedKeyTotal = g_random->randomInt(1, keyArray.size()+1);
@ -669,7 +683,7 @@ IRepPolicyRef const randomAcrossPolicy(LocalitySet const& serverSet)
}
valueTotal = g_random->randomInt(1, valueSet.size()+2);
if ((valueTotal > maxValueTotal) && (g_random->random01() > .25)) valueTotal = maxValueTotal;
policy = IRepPolicyRef( new PolicyAcross(valueTotal, keyText, policy) );
policy = Reference<IReplicationPolicy>( new PolicyAcross(valueTotal, keyText, policy) );
if (g_replicationdebug > 1) {
printf(" item%3d: (%3d =>%3d) %-10s =>%4d\n", keysUsed+1, keyIndex, indexKey._id, keyText.c_str(), valueTotal);
}
@ -725,8 +739,8 @@ int testReplication()
int policyMin = policyMinEnv ? atoi(policyMinEnv) : 2;
int policyIndex, testCounter, alsoSize, debugBackup, maxAlsoSize;
std::vector<repTestType> serverIndexes;
LocalitySetRef testServers;
std::vector<IRepPolicyRef> policies;
Reference<LocalitySet> testServers;
std::vector<Reference<IReplicationPolicy>> policies;
std::vector<LocalityEntry> alsoServers, bestSet;
int totalErrors = 0;
@ -819,12 +833,12 @@ void filterLocalityDataForPolicy(const std::set<std::string>& keys, LocalityData
}
}
void filterLocalityDataForPolicy(IRepPolicyRef policy, LocalityData* ld) {
void filterLocalityDataForPolicy(Reference<IReplicationPolicy> policy, LocalityData* ld) {
if (!policy) return;
filterLocalityDataForPolicy(policy->attributeKeys(), ld);
}
void filterLocalityDataForPolicy(IRepPolicyRef policy, std::vector<LocalityData>* vld) {
void filterLocalityDataForPolicy(Reference<IReplicationPolicy> policy, std::vector<LocalityData>* vld) {
if (!policy) return;
std::set<std::string> keys = policy->attributeKeys();
for (LocalityData& ld : *vld) {

20
fdbrpc/ReplicationUtils.h
View File

@ -34,22 +34,22 @@ extern repTestType convertToTestType(int iValue);
extern int testReplication();
extern double ratePolicy(
LocalitySetRef & localitySet,
IRepPolicyRef const& policy,
Reference<LocalitySet> & localitySet,
Reference<IReplicationPolicy> const& policy,
unsigned int nSelectTests);
extern bool findBestPolicySet(
std::vector<LocalityEntry>& bestResults,
LocalitySetRef & localitySet,
IRepPolicyRef const& policy,
Reference<LocalitySet> & localitySet,
Reference<IReplicationPolicy> const& policy,
unsigned int nMinItems,
unsigned int nSelectTests,
unsigned int nPolicyTests);
extern bool findBestUniquePolicySet(
std::vector<LocalityEntry>& bestResults,
LocalitySetRef & localitySet,
IRepPolicyRef const& policy,
Reference<LocalitySet> & localitySet,
Reference<IReplicationPolicy> const& policy,
StringRef localityUniquenessKey,
unsigned int nMinItems,
unsigned int nSelectTests,
@ -60,20 +60,20 @@ extern bool findBestUniquePolicySet(
extern bool validateAllCombinations(
std::vector<LocalityData> & offendingCombo,
LocalityGroup const& localitySet,
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityData> const& newItems,
unsigned int nCombinationSize,
bool bCheckIfValid = true);
extern bool validateAllCombinations(
LocalityGroup const& localitySet,
IRepPolicyRef const& policy,
Reference<IReplicationPolicy> const& policy,
std::vector<LocalityData> const& newItems,
unsigned int nCombinationSize,
bool bCheckIfValid = true);
/// Remove all pieces of locality information from the LocalityData that will not be used when validating the policy.
void filterLocalityDataForPolicy(IRepPolicyRef policy, LocalityData* ld);
void filterLocalityDataForPolicy(IRepPolicyRef policy, std::vector<LocalityData>* vld);
void filterLocalityDataForPolicy(Reference<IReplicationPolicy> policy, LocalityData* ld);
void filterLocalityDataForPolicy(Reference<IReplicationPolicy> policy, std::vector<LocalityData>* vld);
#endif

11
fdbrpc/simulator.h
View File

@ -99,6 +99,7 @@ public:
case ProcessClass::LogRouterClass: return false;
case ProcessClass::ClusterControllerClass: return false;
case ProcessClass::DataDistributorClass: return false;
case ProcessClass::RateKeeperClass: return false;
default: return false;
}
}
@ -279,11 +280,11 @@ public:
std::set<NetworkAddress> protectedAddresses;
std::map<NetworkAddress, ProcessInfo*> currentlyRebootingProcesses;
class ClusterConnectionString* extraDB;
IRepPolicyRef storagePolicy;
IRepPolicyRef tLogPolicy;
Reference<IReplicationPolicy> storagePolicy;
Reference<IReplicationPolicy> tLogPolicy;
int32_t tLogWriteAntiQuorum;
Optional<Standalone<StringRef>> primaryDcId;
IRepPolicyRef remoteTLogPolicy;
Reference<IReplicationPolicy> remoteTLogPolicy;
int32_t usableRegions;
std::string disablePrimary;
std::string disableRemote;
@ -291,8 +292,8 @@ public:
bool allowLogSetKills;
Optional<Standalone<StringRef>> remoteDcId;
bool hasSatelliteReplication;
IRepPolicyRef satelliteTLogPolicy;
IRepPolicyRef satelliteTLogPolicyFallback;
Reference<IReplicationPolicy> satelliteTLogPolicy;
Reference<IReplicationPolicy> satelliteTLogPolicyFallback;
int32_t satelliteTLogWriteAntiQuorum;
int32_t satelliteTLogWriteAntiQuorumFallback;
std::vector<Optional<Standalone<StringRef>>> primarySatelliteDcIds;

2
fdbserver/CMakeLists.txt
View File

@ -56,7 +56,7 @@ set(FDBSERVER_SRCS
QuietDatabase.actor.cpp
QuietDatabase.h
Ratekeeper.actor.cpp
Ratekeeper.h
RatekeeperInterface.h
RecoveryState.h
Restore.actor.cpp
RestoreInterface.h

497
fdbserver/ClusterController.actor.cpp
View File

File diff suppressed because it is too large Load Diff

12
fdbserver/ClusterRecruitmentInterface.h
View File

@ -168,15 +168,17 @@ struct RegisterWorkerRequest {
ClusterControllerPriorityInfo priorityInfo;
Generation generation;
Optional<DataDistributorInterface> distributorInterf;
Optional<RatekeeperInterface> ratekeeperInterf;
ReplyPromise<RegisterWorkerReply> reply;
bool degraded;
RegisterWorkerRequest() : priorityInfo(ProcessClass::UnsetFit, false, ClusterControllerPriorityInfo::FitnessUnknown) {}
RegisterWorkerRequest(WorkerInterface wi, ProcessClass initialClass, ProcessClass processClass, ClusterControllerPriorityInfo priorityInfo, Generation generation, Optional<DataDistributorInterface> ddInterf) :
wi(wi), initialClass(initialClass), processClass(processClass), priorityInfo(priorityInfo), generation(generation), distributorInterf(ddInterf) {}
RegisterWorkerRequest() : priorityInfo(ProcessClass::UnsetFit, false, ClusterControllerPriorityInfo::FitnessUnknown), degraded(false) {}
RegisterWorkerRequest(WorkerInterface wi, ProcessClass initialClass, ProcessClass processClass, ClusterControllerPriorityInfo priorityInfo, Generation generation, Optional<DataDistributorInterface> ddInterf, Optional<RatekeeperInterface> rkInterf, bool degraded) :
wi(wi), initialClass(initialClass), processClass(processClass), priorityInfo(priorityInfo), generation(generation), distributorInterf(ddInterf), ratekeeperInterf(rkInterf), degraded(degraded) {}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, wi, initialClass, processClass, priorityInfo, generation, distributorInterf, reply);
serializer(ar, wi, initialClass, processClass, priorityInfo, generation, distributorInterf, ratekeeperInterf, reply, degraded);
}
};
@ -184,7 +186,7 @@ struct GetWorkersRequest {
enum { TESTER_CLASS_ONLY = 0x1, NON_EXCLUDED_PROCESSES_ONLY = 0x2 };
int flags;
ReplyPromise<vector<std::pair<WorkerInterface, ProcessClass>>> reply;
ReplyPromise<vector<WorkerDetails>> reply;
GetWorkersRequest() : flags(0) {}
explicit GetWorkersRequest(int fl) : flags(fl) {}

2
fdbserver/DBCoreState.h
View File

@ -41,7 +41,7 @@ struct CoreTLogSet {
int32_t tLogWriteAntiQuorum; // The write anti quorum previously used to write to tLogs, which might be different from the anti quorum suggested by the current configuration going forward!
int32_t tLogReplicationFactor; // The replication factor previously used to write to tLogs, which might be different from the current configuration
std::vector< LocalityData > tLogLocalities; // Stores the localities of the log servers
IRepPolicyRef tLogPolicy;
Reference<IReplicationPolicy> tLogPolicy;
bool isLocal;
int8_t locality;
Version startVersion;

146
fdbserver/DataDistribution.actor.cpp
View File

@ -29,7 +29,6 @@
#include "fdbserver/WaitFailure.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/IKeyValueStore.h"
#include "fdbserver/Ratekeeper.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbrpc/Replication.h"
#include "flow/UnitTest.h"
@ -570,7 +569,6 @@ struct DDTeamCollection : ReferenceCounted<DDTeamCollection> {
PromiseStream<UID> removedServers;
std::set<UID> recruitingIds; // The IDs of the SS which are being recruited
std::set<NetworkAddress> recruitingLocalities;
Optional<PromiseStream< std::pair<UID, Optional<StorageServerInterface>> >> serverChanges;
Future<Void> initialFailureReactionDelay;
Future<Void> initializationDoneActor;
Promise<Void> serverTrackerErrorOut;
@ -629,13 +627,12 @@ struct DDTeamCollection : ReferenceCounted<DDTeamCollection> {
Reference<ShardsAffectedByTeamFailure> const& shardsAffectedByTeamFailure,
DatabaseConfiguration configuration, std::vector<Optional<Key>> includedDCs,
Optional<std::vector<Optional<Key>>> otherTrackedDCs,
Optional<PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>> const& serverChanges,
Future<Void> readyToStart, Reference<AsyncVar<bool>> zeroHealthyTeams, bool primary,
Reference<AsyncVar<bool>> processingUnhealthy)
: cx(cx), distributorId(distributorId), lock(lock), output(output),
shardsAffectedByTeamFailure(shardsAffectedByTeamFailure), doBuildTeams(true), teamBuilder(Void()),
badTeamRemover(Void()), redundantTeamRemover(Void()), configuration(configuration),
serverChanges(serverChanges), readyToStart(readyToStart),
readyToStart(readyToStart),
checkTeamDelay(delay(SERVER_KNOBS->CHECK_TEAM_DELAY, TaskDataDistribution)),
initialFailureReactionDelay(
delayed(readyToStart, SERVER_KNOBS->INITIAL_FAILURE_REACTION_DELAY, TaskDataDistribution)),
@ -2839,10 +2836,6 @@ ACTOR Future<Void> storageServerTracker(
state Future<KeyValueStoreType> storeTracker = keyValueStoreTypeTracker( self, server );
state bool hasWrongStoreTypeOrDC = false;
if(self->serverChanges.present()) {
self->serverChanges.get().send( std::make_pair(server->id, server->lastKnownInterface) );
}
try {
loop {
status.isUndesired = false;
@ -2933,9 +2926,6 @@ ACTOR Future<Void> storageServerTracker(
when( wait( failureTracker ) ) {
// The server is failed AND all data has been removed from it, so permanently remove it.
TraceEvent("StatusMapChange", self->distributorId).detail("ServerID", server->id).detail("Status", "Removing");
if(self->serverChanges.present()) {
self->serverChanges.get().send( std::make_pair(server->id, Optional<StorageServerInterface>()) );
}
if(server->updated.canBeSet()) {
server->updated.send(Void());
@ -3040,9 +3030,6 @@ ACTOR Future<Void> storageServerTracker(
}
interfaceChanged = server->onInterfaceChanged;
if(self->serverChanges.present()) {
self->serverChanges.get().send( std::make_pair(server->id, server->lastKnownInterface) );
}
// We rely on the old failureTracker being actorCancelled since the old actor now has a pointer to an invalid location
status = ServerStatus( status.isFailed, status.isUndesired, server->lastKnownInterface.locality );
@ -3460,13 +3447,39 @@ ACTOR Future<Void> pollMoveKeysLock( Database cx, MoveKeysLock lock ) {
}
}
ACTOR Future<Void> dataDistribution(
Reference<AsyncVar<struct ServerDBInfo>> db,
UID myId,
PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges,
double* lastLimited)
struct DataDistributorData : NonCopyable, ReferenceCounted<DataDistributorData> {
Reference<AsyncVar<struct ServerDBInfo>> dbInfo;
UID ddId;
PromiseStream<Future<Void>> addActor;
DataDistributorData(Reference<AsyncVar<ServerDBInfo>> const& db, UID id) : dbInfo(db), ddId(id) {}
};
ACTOR Future<Void> monitorBatchLimitedTime(Reference<AsyncVar<ServerDBInfo>> db, double* lastLimited) {
loop {
wait( delay(SERVER_KNOBS->METRIC_UPDATE_RATE) );
state Reference<ProxyInfo> proxies(new ProxyInfo(db->get().client.proxies, db->get().myLocality));
choose {
when (wait(db->onChange())) {}
when (GetHealthMetricsReply reply = wait(proxies->size() ?
loadBalance(proxies, &MasterProxyInterface::getHealthMetrics, GetHealthMetricsRequest(false))
: Never())) {
if (reply.healthMetrics.batchLimited) {
*lastLimited = now();
}
}
}
}
}
ACTOR Future<Void> dataDistribution(Reference<DataDistributorData> self)
{
state Database cx = openDBOnServer(db, TaskDataDistributionLaunch, true, true);
state double lastLimited = 0;
self->addActor.send( monitorBatchLimitedTime(self->dbInfo, &lastLimited) );
state Database cx = openDBOnServer(self->dbInfo, TaskDataDistributionLaunch, true, true);
cx->locationCacheSize = SERVER_KNOBS->DD_LOCATION_CACHE_SIZE;
//cx->setOption( FDBDatabaseOptions::LOCATION_CACHE_SIZE, StringRef((uint8_t*) &SERVER_KNOBS->DD_LOCATION_CACHE_SIZE, 8) );
@ -3481,10 +3494,10 @@ ACTOR Future<Void> dataDistribution(
loop {
try {
loop {
TraceEvent("DDInitTakingMoveKeysLock", myId);
MoveKeysLock lock_ = wait( takeMoveKeysLock( cx, myId ) );
TraceEvent("DDInitTakingMoveKeysLock", self->ddId);
MoveKeysLock lock_ = wait( takeMoveKeysLock( cx, self->ddId ) );
lock = lock_;
TraceEvent("DDInitTookMoveKeysLock", myId);
TraceEvent("DDInitTookMoveKeysLock", self->ddId);
DatabaseConfiguration configuration_ = wait( getDatabaseConfiguration(cx) );
configuration = configuration_;
@ -3498,7 +3511,7 @@ ACTOR Future<Void> dataDistribution(
remoteDcIds.push_back( regions[1].dcId );
}
TraceEvent("DDInitGotConfiguration", myId).detail("Conf", configuration.toString());
TraceEvent("DDInitGotConfiguration", self->ddId).detail("Conf", configuration.toString());
state Transaction tr(cx);
loop {
@ -3528,24 +3541,24 @@ ACTOR Future<Void> dataDistribution(
}
}
TraceEvent("DDInitUpdatedReplicaKeys", myId);
Reference<InitialDataDistribution> initData_ = wait( getInitialDataDistribution(cx, myId, lock, configuration.usableRegions > 1 ? remoteDcIds : std::vector<Optional<Key>>() ) );
TraceEvent("DDInitUpdatedReplicaKeys", self->ddId);
Reference<InitialDataDistribution> initData_ = wait( getInitialDataDistribution(cx, self->ddId, lock, configuration.usableRegions > 1 ? remoteDcIds : std::vector<Optional<Key>>() ) );
initData = initData_;
if(initData->shards.size() > 1) {
TraceEvent("DDInitGotInitialDD", myId)
TraceEvent("DDInitGotInitialDD", self->ddId)
.detail("B", printable(initData->shards.end()[-2].key))
.detail("E", printable(initData->shards.end()[-1].key))
.detail("Src", describe(initData->shards.end()[-2].primarySrc))
.detail("Dest", describe(initData->shards.end()[-2].primaryDest))
.trackLatest("InitialDD");
} else {
TraceEvent("DDInitGotInitialDD", myId).detail("B","").detail("E", "").detail("Src", "[no items]").detail("Dest", "[no items]").trackLatest("InitialDD");
TraceEvent("DDInitGotInitialDD", self->ddId).detail("B","").detail("E", "").detail("Src", "[no items]").detail("Dest", "[no items]").trackLatest("InitialDD");
}
if (initData->mode) break; // mode may be set true by system operator using fdbcli
TraceEvent("DataDistributionDisabled", myId);
TraceEvent("DataDistributionDisabled", self->ddId);
TraceEvent("MovingData", myId)
TraceEvent("MovingData", self->ddId)
.detail( "InFlight", 0 )
.detail( "InQueue", 0 )
.detail( "AverageShardSize", -1 )
@ -3554,8 +3567,8 @@ ACTOR Future<Void> dataDistribution(
.detail( "HighestPriority", 0 )
.trackLatest( "MovingData" );
TraceEvent("TotalDataInFlight", myId).detail("Primary", true).detail("TotalBytes", 0).detail("UnhealthyServers", 0).detail("HighestPriority", 0).trackLatest("TotalDataInFlight");
TraceEvent("TotalDataInFlight", myId).detail("Primary", false).detail("TotalBytes", 0).detail("UnhealthyServers", 0).detail("HighestPriority", configuration.usableRegions > 1 ? 0 : -1).trackLatest("TotalDataInFlightRemote");
TraceEvent("TotalDataInFlight", self->ddId).detail("Primary", true).detail("TotalBytes", 0).detail("UnhealthyServers", 0).detail("HighestPriority", 0).trackLatest("TotalDataInFlight");
TraceEvent("TotalDataInFlight", self->ddId).detail("Primary", false).detail("TotalBytes", 0).detail("UnhealthyServers", 0).detail("HighestPriority", configuration.usableRegions > 1 ? 0 : -1).trackLatest("TotalDataInFlightRemote");
wait( waitForDataDistributionEnabled(cx) );
TraceEvent("DataDistributionEnabled");
@ -3573,12 +3586,12 @@ ACTOR Future<Void> dataDistribution(
state Reference<ShardsAffectedByTeamFailure> shardsAffectedByTeamFailure( new ShardsAffectedByTeamFailure );
state int shard = 0;
for(; shard<initData->shards.size() - 1; shard++) {
for (; shard < initData->shards.size() - 1; shard++) {
KeyRangeRef keys = KeyRangeRef(initData->shards[shard].key, initData->shards[shard+1].key);
shardsAffectedByTeamFailure->defineShard(keys);
std::vector<ShardsAffectedByTeamFailure::Team> teams;
teams.push_back(ShardsAffectedByTeamFailure::Team(initData->shards[shard].primarySrc, true));
if(configuration.usableRegions > 1) {
if (configuration.usableRegions > 1) {
teams.push_back(ShardsAffectedByTeamFailure::Team(initData->shards[shard].remoteSrc, false));
}
if(g_network->isSimulated()) {
@ -3587,11 +3600,11 @@ ACTOR Future<Void> dataDistribution(
}
shardsAffectedByTeamFailure->moveShard(keys, teams);
if(initData->shards[shard].hasDest) {
if (initData->shards[shard].hasDest) {
// This shard is already in flight. Ideally we should use dest in sABTF and generate a dataDistributionRelocator directly in
// DataDistributionQueue to track it, but it's easier to just (with low priority) schedule it for movement.
bool unhealthy = initData->shards[shard].primarySrc.size() != configuration.storageTeamSize;
if(!unhealthy && configuration.usableRegions > 1) {
if (!unhealthy && configuration.usableRegions > 1) {
unhealthy = initData->shards[shard].remoteSrc.size() != configuration.storageTeamSize;
}
output.send( RelocateShard( keys, unhealthy ? PRIORITY_TEAM_UNHEALTHY : PRIORITY_RECOVER_MOVE ) );
@ -3620,20 +3633,20 @@ ACTOR Future<Void> dataDistribution(
}
actors.push_back( pollMoveKeysLock(cx, lock) );
actors.push_back( reportErrorsExcept( dataDistributionTracker( initData, cx, output, shardsAffectedByTeamFailure, getShardMetrics, getAverageShardBytes.getFuture(), readyToStart, anyZeroHealthyTeams, myId ), "DDTracker", myId, &normalDDQueueErrors() ) );
actors.push_back( reportErrorsExcept( dataDistributionQueue( cx, output, input.getFuture(), getShardMetrics, processingUnhealthy, tcis, shardsAffectedByTeamFailure, lock, getAverageShardBytes, myId, storageTeamSize, lastLimited ), "DDQueue", myId, &normalDDQueueErrors() ) );
actors.push_back( reportErrorsExcept( dataDistributionTracker( initData, cx, output, shardsAffectedByTeamFailure, getShardMetrics, getAverageShardBytes.getFuture(), readyToStart, anyZeroHealthyTeams, self->ddId ), "DDTracker", self->ddId, &normalDDQueueErrors() ) );
actors.push_back( reportErrorsExcept( dataDistributionQueue( cx, output, input.getFuture(), getShardMetrics, processingUnhealthy, tcis, shardsAffectedByTeamFailure, lock, getAverageShardBytes, self->ddId, storageTeamSize, &lastLimited ), "DDQueue", self->ddId, &normalDDQueueErrors() ) );
vector<DDTeamCollection*> teamCollectionsPtrs;
Reference<DDTeamCollection> primaryTeamCollection( new DDTeamCollection(cx, myId, lock, output, shardsAffectedByTeamFailure, configuration, primaryDcId, configuration.usableRegions > 1 ? remoteDcIds : std::vector<Optional<Key>>(), serverChanges, readyToStart.getFuture(), zeroHealthyTeams[0], true, processingUnhealthy) );
Reference<DDTeamCollection> primaryTeamCollection( new DDTeamCollection(cx, self->ddId, lock, output, shardsAffectedByTeamFailure, configuration, primaryDcId, configuration.usableRegions > 1 ? remoteDcIds : std::vector<Optional<Key>>(), readyToStart.getFuture(), zeroHealthyTeams[0], true, processingUnhealthy) );
teamCollectionsPtrs.push_back(primaryTeamCollection.getPtr());
if (configuration.usableRegions > 1) {
Reference<DDTeamCollection> remoteTeamCollection( new DDTeamCollection(cx, myId, lock, output, shardsAffectedByTeamFailure, configuration, remoteDcIds, Optional<std::vector<Optional<Key>>>(), serverChanges, readyToStart.getFuture() && remoteRecovered(db), zeroHealthyTeams[1], false, processingUnhealthy) );
Reference<DDTeamCollection> remoteTeamCollection( new DDTeamCollection(cx, self->ddId, lock, output, shardsAffectedByTeamFailure, configuration, remoteDcIds, Optional<std::vector<Optional<Key>>>(), readyToStart.getFuture() && remoteRecovered(self->dbInfo), zeroHealthyTeams[1], false, processingUnhealthy) );
teamCollectionsPtrs.push_back(remoteTeamCollection.getPtr());
remoteTeamCollection->teamCollections = teamCollectionsPtrs;
actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( remoteTeamCollection, initData, tcis[1], db ), "DDTeamCollectionSecondary", myId, &normalDDQueueErrors() ) );
actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( remoteTeamCollection, initData, tcis[1], self->dbInfo ), "DDTeamCollectionSecondary", self->ddId, &normalDDQueueErrors() ) );
}
primaryTeamCollection->teamCollections = teamCollectionsPtrs;
actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( primaryTeamCollection, initData, tcis[0], db ), "DDTeamCollectionPrimary", myId, &normalDDQueueErrors() ) );
actors.push_back( reportErrorsExcept( dataDistributionTeamCollection( primaryTeamCollection, initData, tcis[0], self->dbInfo ), "DDTeamCollectionPrimary", self->ddId, &normalDDQueueErrors() ) );
actors.push_back(yieldPromiseStream(output.getFuture(), input));
wait( waitForAll( actors ) );
@ -3651,15 +3664,6 @@ ACTOR Future<Void> dataDistribution(
}
}
struct DataDistributorData : NonCopyable, ReferenceCounted<DataDistributorData> {
Reference<AsyncVar<struct ServerDBInfo>> dbInfo;
UID ddId;
PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > ddStorageServerChanges;
PromiseStream<Future<Void>> addActor;
DataDistributorData(Reference<AsyncVar<ServerDBInfo>> const& db, UID id) : dbInfo(db), ddId(id) {}
};
static std::set<int> const& normalDataDistributorErrors() {
static std::set<int> s;
if (s.empty()) {
@ -3672,31 +3676,14 @@ static std::set<int> const& normalDataDistributorErrors() {
return s;
}
static std::set<int> const& normalRateKeeperErrors() {
static std::set<int> s;
if (s.empty()) {
s.insert( error_code_worker_removed );
s.insert( error_code_broken_promise );
s.insert( error_code_actor_cancelled );
s.insert( error_code_please_reboot );
}
return s;
}
ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncVar<struct ServerDBInfo>> db ) {
state UID lastClusterControllerID(0,0);
state Reference<DataDistributorData> self( new DataDistributorData(db, di.id()) );
state Future<Void> collection = actorCollection( self->addActor.getFuture() );
TraceEvent("DataDistributor_Starting", di.id());
self->addActor.send( waitFailureServer(di.waitFailure.getFuture()) );
try {
TraceEvent("DataDistributor_Running", di.id());
state PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > ddStorageServerChanges;
state double lastLimited = 0;
state Future<Void> distributor = reportErrorsExcept( dataDistribution( self->dbInfo, di.id(), ddStorageServerChanges, &lastLimited ), "DataDistribution", di.id(), &normalDataDistributorErrors() );
self->addActor.send( reportErrorsExcept( rateKeeper( self->dbInfo, ddStorageServerChanges, di.getRateInfo.getFuture(), &lastLimited ), "Ratekeeper", di.id(), &normalRateKeeperErrors() ) );
self->addActor.send( waitFailureServer(di.waitFailure.getFuture()) );
state Future<Void> distributor = reportErrorsExcept( dataDistribution(self), "DataDistribution", di.id(), &normalDataDistributorErrors() );
wait( distributor || collection );
}
@ -3711,7 +3698,7 @@ ACTOR Future<Void> dataDistributor(DataDistributorInterface di, Reference<AsyncV
return Void();
}
DDTeamCollection* testTeamCollection(int teamSize, IRepPolicyRef policy, int processCount) {
DDTeamCollection* testTeamCollection(int teamSize, Reference<IReplicationPolicy> policy, int processCount) {
Database database = DatabaseContext::create(
Reference<AsyncVar<ClientDBInfo>>(new AsyncVar<ClientDBInfo>()),
Never(),
@ -3732,7 +3719,6 @@ DDTeamCollection* testTeamCollection(int teamSize, IRepPolicyRef policy, int pro
conf,
{},
{},
PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>(),
Future<Void>(Void()),
Reference<AsyncVar<bool>>( new AsyncVar<bool>(true) ),
true,
@ -3754,7 +3740,7 @@ DDTeamCollection* testTeamCollection(int teamSize, IRepPolicyRef policy, int pro
return collection;
}
DDTeamCollection* testMachineTeamCollection(int teamSize, IRepPolicyRef policy, int processCount) {
DDTeamCollection* testMachineTeamCollection(int teamSize, Reference<IReplicationPolicy> policy, int processCount) {
Database database = DatabaseContext::create(Reference<AsyncVar<ClientDBInfo>>(new AsyncVar<ClientDBInfo>()),
Never(), LocalityData(), false);
@ -3765,7 +3751,7 @@ DDTeamCollection* testMachineTeamCollection(int teamSize, IRepPolicyRef policy,
DDTeamCollection* collection =
new DDTeamCollection(database, UID(0, 0), MoveKeysLock(), PromiseStream<RelocateShard>(),
Reference<ShardsAffectedByTeamFailure>(new ShardsAffectedByTeamFailure()), conf, {}, {},
PromiseStream<std::pair<UID, Optional<StorageServerInterface>>>(), Future<Void>(Void()),
Future<Void>(Void()),
Reference<AsyncVar<bool>>(new AsyncVar<bool>(true)), true,
Reference<AsyncVar<bool>>(new AsyncVar<bool>(false)));
@ -3806,7 +3792,7 @@ TEST_CASE("DataDistribution/AddTeamsBestOf/UseMachineID") {
int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(teamSize, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
state DDTeamCollection* collection = testMachineTeamCollection(teamSize, policy, processSize);
int result = collection->addTeamsBestOf(30, desiredTeams, maxTeams);
@ -3826,7 +3812,7 @@ TEST_CASE("DataDistribution/AddTeamsBestOf/NotUseMachineID") {
int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(teamSize, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(teamSize, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
state DDTeamCollection* collection = testMachineTeamCollection(teamSize, policy, processSize);
if (collection == NULL) {
@ -3844,7 +3830,7 @@ TEST_CASE("DataDistribution/AddTeamsBestOf/NotUseMachineID") {
}
TEST_CASE("DataDistribution/AddAllTeams/isExhaustive") {
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
state int processSize = 10;
state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
@ -3863,7 +3849,7 @@ TEST_CASE("DataDistribution/AddAllTeams/isExhaustive") {
}
TEST_CASE("/DataDistribution/AddAllTeams/withLimit") {
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
state int processSize = 10;
state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
@ -3881,7 +3867,7 @@ TEST_CASE("/DataDistribution/AddAllTeams/withLimit") {
TEST_CASE("/DataDistribution/AddTeamsBestOf/SkippingBusyServers") {
wait(Future<Void>(Void()));
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
state int processSize = 10;
state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;
@ -3911,7 +3897,7 @@ TEST_CASE("/DataDistribution/AddTeamsBestOf/SkippingBusyServers") {
TEST_CASE("/DataDistribution/AddTeamsBestOf/NotEnoughServers") {
wait(Future<Void>(Void()));
IRepPolicyRef policy = IRepPolicyRef(new PolicyAcross(3, "zoneid", IRepPolicyRef(new PolicyOne())));
Reference<IReplicationPolicy> policy = Reference<IReplicationPolicy>(new PolicyAcross(3, "zoneid", Reference<IReplicationPolicy>(new PolicyOne())));
state int processSize = 5;
state int desiredTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * processSize;
state int maxTeams = SERVER_KNOBS->MAX_TEAMS_PER_SERVER * processSize;

1
fdbserver/DataDistribution.actor.h
View File

@ -253,5 +253,6 @@ int64_t getMaxShardSize( double dbSizeEstimate );
class DDTeamCollection;
ACTOR Future<Void> teamRemover(DDTeamCollection* self);
ACTOR Future<Void> teamRemoverPeriodic(DDTeamCollection* self);
ACTOR Future<vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(Transaction* tr);
#endif

37
fdbserver/DataDistributorInterface.h
View File

@ -21,21 +21,19 @@
#ifndef FDBSERVER_DATADISTRIBUTORINTERFACE_H
#define FDBSERVER_DATADISTRIBUTORINTERFACE_H
#include "fdbclient/FDBTypes.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Locality.h"
struct DataDistributorInterface {
RequestStream<ReplyPromise<Void>> waitFailure;
RequestStream<struct GetRateInfoRequest> getRateInfo;
struct LocalityData locality;
DataDistributorInterface() {}
explicit DataDistributorInterface(const struct LocalityData& l) : locality(l) {}
void initEndpoints() {}
UID id() const { return getRateInfo.getEndpoint().token; }
NetworkAddress address() const { return getRateInfo.getEndpoint().getPrimaryAddress(); }
UID id() const { return waitFailure.getEndpoint().token; }
NetworkAddress address() const { return waitFailure.getEndpoint().getPrimaryAddress(); }
bool operator== (const DataDistributorInterface& r) const {
return id() == r.id();
}
@ -45,36 +43,7 @@ struct DataDistributorInterface {
template <class Archive>
void serialize(Archive& ar) {
serializer(ar, waitFailure, getRateInfo, locality);
}
};
struct GetRateInfoRequest {
UID requesterID;
int64_t totalReleasedTransactions;
int64_t batchReleasedTransactions;
bool detailed;
ReplyPromise<struct GetRateInfoReply> reply;
GetRateInfoRequest() {}
GetRateInfoRequest(UID const& requesterID, int64_t totalReleasedTransactions, int64_t batchReleasedTransactions, bool detailed)
: requesterID(requesterID), totalReleasedTransactions(totalReleasedTransactions), batchReleasedTransactions(batchReleasedTransactions), detailed(detailed) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, requesterID, totalReleasedTransactions, batchReleasedTransactions, detailed, reply);
}
};
struct GetRateInfoReply {
double transactionRate;
double batchTransactionRate;
double leaseDuration;
HealthMetrics healthMetrics;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, transactionRate, batchTransactionRate, leaseDuration, healthMetrics);
serializer(ar, waitFailure, locality);
}
};

7
fdbserver/Knobs.cpp
View File

@ -73,7 +73,10 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( DISK_QUEUE_ADAPTER_MAX_SWITCH_TIME, 5.0 );
init( TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES, 2e9 ); if ( randomize && BUGGIFY ) TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES = 2e6;
init( DISK_QUEUE_FILE_EXTENSION_BYTES, 10<<20 ); // BUGGIFYd per file within the DiskQueue
init( DISK_QUEUE_FILE_SHRINK_BYTES, 100<<20 ); // BUGGIFYd per file within the DiskQueue
init( DISK_QUEUE_FILE_SHRINK_BYTES, 100<<20 ); // BUGGIFYd per file within the DiskQueue
init( TLOG_DEGRADED_DELAY_COUNT, 5 );
init( TLOG_DEGRADED_DURATION, 5.0 );
init( TLOG_DEGRADED_RESET_INTERVAL, 48*60*60 ); if ( randomize && BUGGIFY ) TLOG_DEGRADED_RESET_INTERVAL = 10;
// Data distribution queue
init( HEALTH_POLL_TIME, 1.0 );
@ -307,11 +310,13 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
init( WAIT_FOR_GOOD_REMOTE_RECRUITMENT_DELAY, 5.0 );
init( ATTEMPT_RECRUITMENT_DELAY, 0.035 );
init( WAIT_FOR_DISTRIBUTOR_JOIN_DELAY, 1.0 );
init( WAIT_FOR_RATEKEEPER_JOIN_DELAY, 1.0 );
init( WORKER_FAILURE_TIME, 1.0 ); if( randomize && BUGGIFY ) WORKER_FAILURE_TIME = 10.0;
init( CHECK_OUTSTANDING_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) CHECK_OUTSTANDING_INTERVAL = 0.001;
init( VERSION_LAG_METRIC_INTERVAL, 0.5 ); if( randomize && BUGGIFY ) VERSION_LAG_METRIC_INTERVAL = 10.0;
init( MAX_VERSION_DIFFERENCE, 20 * VERSIONS_PER_SECOND );
init( FORCE_RECOVERY_CHECK_DELAY, 5.0 );
init( RATEKEEPER_FAILURE_TIME, 1.0 );
init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0;
init( EXPECTED_MASTER_FITNESS, ProcessClass::UnsetFit );

5
fdbserver/Knobs.h
View File

@ -77,6 +77,9 @@ public:
int64_t TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES;
int64_t DISK_QUEUE_FILE_EXTENSION_BYTES; // When we grow the disk queue, by how many bytes should it grow?
int64_t DISK_QUEUE_FILE_SHRINK_BYTES; // When we shrink the disk queue, by how many bytes should it shrink?
int TLOG_DEGRADED_DELAY_COUNT;
double TLOG_DEGRADED_DURATION;
double TLOG_DEGRADED_RESET_INTERVAL;
// Data distribution queue
double HEALTH_POLL_TIME;
@ -248,12 +251,14 @@ public:
double WAIT_FOR_GOOD_REMOTE_RECRUITMENT_DELAY;
double ATTEMPT_RECRUITMENT_DELAY;
double WAIT_FOR_DISTRIBUTOR_JOIN_DELAY;
double WAIT_FOR_RATEKEEPER_JOIN_DELAY;
double WORKER_FAILURE_TIME;
double CHECK_OUTSTANDING_INTERVAL;
double INCOMPATIBLE_PEERS_LOGGING_INTERVAL;
double VERSION_LAG_METRIC_INTERVAL;
int64_t MAX_VERSION_DIFFERENCE;
double FORCE_RECOVERY_CHECK_DELAY;
double RATEKEEPER_FAILURE_TIME;
// Knobs used to select the best policy (via monte carlo)
int POLICY_RATING_TESTS; // number of tests per policy (in order to compare)

10
fdbserver/LogSystem.h
View File

@ -40,8 +40,8 @@ public:
int32_t tLogReplicationFactor;
std::vector< LocalityData > tLogLocalities; // Stores the localities of the log servers
TLogVersion tLogVersion;
IRepPolicyRef tLogPolicy;
LocalitySetRef logServerSet;
Reference<IReplicationPolicy> tLogPolicy;
Reference<LocalitySet> logServerSet;
std::vector<int> logIndexArray;
std::vector<LocalityEntry> logEntryArray;
bool isLocal;
@ -84,7 +84,7 @@ public:
used_servers.insert(std::make_pair(0,i));
}
LocalitySetRef serverSet = Reference<LocalitySet>(new LocalityMap<std::pair<int,int>>());
Reference<LocalitySet> serverSet = Reference<LocalitySet>(new LocalityMap<std::pair<int,int>>());
LocalityMap<std::pair<int,int>>* serverMap = (LocalityMap<std::pair<int,int>>*) serverSet.getPtr();
std::vector<std::pair<int,int>> resultPairs;
for(int loc = 0; loc < satelliteTagLocations.size(); loc++) {
@ -189,7 +189,7 @@ public:
void updateLocalitySet( vector<LocalityData> const& localities ) {
LocalityMap<int>* logServerMap;
logServerSet = LocalitySetRef(new LocalityMap<int>());
logServerSet = Reference<LocalitySet>(new LocalityMap<int>());
logServerMap = (LocalityMap<int>*) logServerSet.getPtr();
logEntryArray.clear();
@ -412,7 +412,7 @@ struct ILogSystem {
int tLogReplicationFactor;
MergedPeekCursor( vector< Reference<ILogSystem::IPeekCursor> > const& serverCursors, Version begin );
MergedPeekCursor( std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers, int bestServer, int readQuorum, Tag tag, Version begin, Version end, bool parallelGetMore, std::vector<LocalityData> const& tLogLocalities, IRepPolicyRef const tLogPolicy, int tLogReplicationFactor );
MergedPeekCursor( std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers, int bestServer, int readQuorum, Tag tag, Version begin, Version end, bool parallelGetMore, std::vector<LocalityData> const& tLogLocalities, Reference<IReplicationPolicy> const tLogPolicy, int tLogReplicationFactor );
MergedPeekCursor( vector< Reference<IPeekCursor> > const& serverCursors, LogMessageVersion const& messageVersion, int bestServer, int readQuorum, Optional<LogMessageVersion> nextVersion, Reference<LogSet> logSet, int tLogReplicationFactor );
virtual Reference<IPeekCursor> cloneNoMore();

2
fdbserver/LogSystemConfig.h
View File

@ -61,7 +61,7 @@ struct TLogSet {
int32_t tLogWriteAntiQuorum, tLogReplicationFactor;
std::vector< LocalityData > tLogLocalities; // Stores the localities of the log servers
TLogVersion tLogVersion;
IRepPolicyRef tLogPolicy;
Reference<IReplicationPolicy> tLogPolicy;
bool isLocal;
int8_t locality;
Version startVersion;

2
fdbserver/LogSystemPeekCursor.actor.cpp
View File

@ -273,7 +273,7 @@ ILogSystem::MergedPeekCursor::MergedPeekCursor( vector< Reference<ILogSystem::IP
}
ILogSystem::MergedPeekCursor::MergedPeekCursor( std::vector<Reference<AsyncVar<OptionalInterface<TLogInterface>>>> const& logServers, int bestServer, int readQuorum, Tag tag, Version begin, Version end,
bool parallelGetMore, std::vector< LocalityData > const& tLogLocalities, IRepPolicyRef const tLogPolicy, int tLogReplicationFactor )
bool parallelGetMore, std::vector< LocalityData > const& tLogLocalities, Reference<IReplicationPolicy> const tLogPolicy, int tLogReplicationFactor )
: bestServer(bestServer), readQuorum(readQuorum), tag(tag), currentCursor(0), hasNextMessage(false), messageVersion(begin), randomID(g_random->randomUniqueID()), tLogReplicationFactor(tLogReplicationFactor) {
if(tLogPolicy) {
logSet = Reference<LogSet>( new LogSet() );

29
fdbserver/MasterProxyServer.actor.cpp
View File

@ -76,17 +76,6 @@ struct ProxyStats {
}
};
ACTOR template <class T>
Future<Void> forwardValue(Promise<T> out, Future<T> in)
{
// Like forwardPromise, but throws on error
T t = wait(in);
out.send(t);
return Void();
}
int getBytes(Promise<Version> const& r) { return 0; }
ACTOR Future<Void> getRate(UID myID, Reference<AsyncVar<ServerDBInfo>> db, int64_t* inTransactionCount, int64_t* inBatchTransactionCount, double* outTransactionRate,
double* outBatchTransactionRate, GetHealthMetricsReply* healthMetricsReply, GetHealthMetricsReply* detailedHealthMetricsReply) {
state Future<Void> nextRequestTimer = Never();
@ -94,21 +83,17 @@ ACTOR Future<Void> getRate(UID myID, Reference<AsyncVar<ServerDBInfo>> db, int64
state Future<GetRateInfoReply> reply = Never();
state double lastDetailedReply = 0.0; // request detailed metrics immediately
state bool expectingDetailedReply = false;
state int64_t lastTC = 0;
if (db->get().distributor.present()) {
nextRequestTimer = Void();
}
if (db->get().ratekeeper.present()) nextRequestTimer = Void();
loop choose {
when ( wait( db->onChange() ) ) {
if ( db->get().distributor.present() ) {
TraceEvent("Proxy_DataDistributorChanged", myID)
.detail("DDID", db->get().distributor.get().id());
nextRequestTimer = Void(); // trigger GetRate request
if ( db->get().ratekeeper.present() ) {
TraceEvent("Proxy_RatekeeperChanged", myID)
.detail("RKID", db->get().ratekeeper.get().id());
nextRequestTimer = Void(); // trigger GetRate request
} else {
TraceEvent("Proxy_DataDistributorDied", myID);
TraceEvent("Proxy_RatekeeperDied", myID);
nextRequestTimer = Never();
reply = Never();
}
@ -116,7 +101,7 @@ ACTOR Future<Void> getRate(UID myID, Reference<AsyncVar<ServerDBInfo>> db, int64
when ( wait( nextRequestTimer ) ) {
nextRequestTimer = Never();
bool detailed = now() - lastDetailedReply > SERVER_KNOBS->DETAILED_METRIC_UPDATE_RATE;
reply = brokenPromiseToNever(db->get().distributor.get().getRateInfo.getReply(GetRateInfoRequest(myID, *inTransactionCount, *inBatchTransactionCount, detailed)));
reply = brokenPromiseToNever(db->get().ratekeeper.get().getRateInfo.getReply(GetRateInfoRequest(myID, *inTransactionCount, *inBatchTransactionCount, detailed)));
expectingDetailedReply = detailed;
}
when ( GetRateInfoReply rep = wait(reply) ) {

29
fdbserver/OldTLogServer_6_0.actor.cpp
View File

@ -33,6 +33,7 @@
#include "fdbrpc/FailureMonitor.h"
#include "fdbserver/IDiskQueue.h"
#include "fdbrpc/sim_validation.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/LogSystem.h"
#include "fdbserver/WaitFailure.h"
@ -273,10 +274,12 @@ struct TLogData : NonCopyable {
FlowLock concurrentLogRouterReads;
FlowLock persistentDataCommitLock;
TLogData(UID dbgid, IKeyValueStore* persistentData, IDiskQueue * persistentQueue, Reference<AsyncVar<ServerDBInfo>> const& dbInfo)
Reference<AsyncVar<bool>> degraded;
TLogData(UID dbgid, IKeyValueStore* persistentData, IDiskQueue * persistentQueue, Reference<AsyncVar<ServerDBInfo>> dbInfo, Reference<AsyncVar<bool>> degraded)
: dbgid(dbgid), instanceID(g_random->randomUniqueID().first()),
persistentData(persistentData), rawPersistentQueue(persistentQueue), persistentQueue(new TLogQueue(persistentQueue, dbgid)),
dbInfo(dbInfo), queueCommitBegin(0), queueCommitEnd(0),
dbInfo(dbInfo), degraded(degraded), queueCommitBegin(0), queueCommitEnd(0),
diskQueueCommitBytes(0), largeDiskQueueCommitBytes(false), bytesInput(0), bytesDurable(0), overheadBytesInput(0), overheadBytesDurable(0),
concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS)
{
@ -1087,6 +1090,19 @@ ACTOR Future<Void> tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere
return Void();
}
ACTOR Future<Void> watchDegraded(TLogData* self) {
//This delay is divided into multiple delays to avoid marking the tlog as degraded because of a single SlowTask
state int loopCount = 0;
while(loopCount < SERVER_KNOBS->TLOG_DEGRADED_DELAY_COUNT) {
wait(delay(SERVER_KNOBS->TLOG_DEGRADED_DURATION/SERVER_KNOBS->TLOG_DEGRADED_DELAY_COUNT, TaskLowPriority));
loopCount++;
}
TraceEvent(SevWarnAlways, "TLogDegraded", self->dbgid);
TEST(true); //6.0 TLog degraded
self->degraded->set(true);
return Void();
}
ACTOR Future<Void> doQueueCommit( TLogData* self, Reference<LogData> logData ) {
state Version ver = logData->version.get();
state Version commitNumber = self->queueCommitBegin+1;
@ -1098,7 +1114,12 @@ ACTOR Future<Void> doQueueCommit( TLogData* self, Reference<LogData> logData ) {
self->diskQueueCommitBytes = 0;
self->largeDiskQueueCommitBytes.set(false);
state Future<Void> degraded = watchDegraded(self);
wait(c);
if(g_network->isSimulated() && !g_simulator.speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
wait(delay(6.0));
}
degraded.cancel();
wait(self->queueCommitEnd.whenAtLeast(commitNumber-1));
//Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
@ -2052,8 +2073,8 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
}
// New tLog (if !recoverFrom.size()) or restore from network
ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQueue, Reference<AsyncVar<ServerDBInfo>> db, LocalityData locality, PromiseStream<InitializeTLogRequest> tlogRequests, UID tlogId, bool restoreFromDisk, Promise<Void> oldLog, Promise<Void> recovered ) {
state TLogData self( tlogId, persistentData, persistentQueue, db );
ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQueue, Reference<AsyncVar<ServerDBInfo>> db, LocalityData locality, PromiseStream<InitializeTLogRequest> tlogRequests, UID tlogId, bool restoreFromDisk, Promise<Void> oldLog, Promise<Void> recovered, Reference<AsyncVar<bool>> degraded) {
state TLogData self( tlogId, persistentData, persistentQueue, db, degraded );
state Future<Void> error = actorCollection( self.sharedActors.getFuture() );
TraceEvent("SharedTlog", tlogId);

30
fdbserver/QuietDatabase.actor.cpp
View File

@ -30,10 +30,10 @@
#include "fdbclient/ManagementAPI.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR Future<vector<std::pair<WorkerInterface, ProcessClass>>> getWorkers( Reference<AsyncVar<ServerDBInfo>> dbInfo, int flags = 0 ) {
ACTOR Future<vector<WorkerDetails>> getWorkers( Reference<AsyncVar<ServerDBInfo>> dbInfo, int flags = 0 ) {
loop {
choose {
when( vector<std::pair<WorkerInterface, ProcessClass>> w = wait( brokenPromiseToNever( dbInfo->get().clusterInterface.getWorkers.getReply( GetWorkersRequest( flags ) ) ) ) ) {
when( vector<WorkerDetails> w = wait( brokenPromiseToNever( dbInfo->get().clusterInterface.getWorkers.getReply( GetWorkersRequest( flags ) ) ) ) ) {
return w;
}
when( wait( dbInfo->onChange() ) ) {}
@ -46,12 +46,12 @@ ACTOR Future<WorkerInterface> getMasterWorker( Database cx, Reference<AsyncVar<S
TraceEvent("GetMasterWorker").detail("Stage", "GettingWorkers");
loop {
state vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( dbInfo ) );
state vector<WorkerDetails> workers = wait( getWorkers( dbInfo ) );
for( int i = 0; i < workers.size(); i++ ) {
if( workers[i].first.address() == dbInfo->get().master.address() ) {
TraceEvent("GetMasterWorker").detail("Stage", "GotWorkers").detail("MasterId", dbInfo->get().master.id()).detail("WorkerId", workers[i].first.id());
return workers[i].first;
if( workers[i].interf.address() == dbInfo->get().master.address() ) {
TraceEvent("GetMasterWorker").detail("Stage", "GotWorkers").detail("MasterId", dbInfo->get().master.id()).detail("WorkerId", workers[i].interf.id());
return workers[i].interf;
}
}
@ -69,15 +69,15 @@ ACTOR Future<WorkerInterface> getDataDistributorWorker( Database cx, Reference<A
TraceEvent("GetDataDistributorWorker").detail("Stage", "GettingWorkers");
loop {
state vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( dbInfo ) );
state vector<WorkerDetails> workers = wait( getWorkers( dbInfo ) );
if (!dbInfo->get().distributor.present()) continue;
for( int i = 0; i < workers.size(); i++ ) {
if( workers[i].first.address() == dbInfo->get().distributor.get().address() ) {
if( workers[i].interf.address() == dbInfo->get().distributor.get().address() ) {
TraceEvent("GetDataDistributorWorker").detail("Stage", "GotWorkers")
.detail("DataDistributorId", dbInfo->get().distributor.get().id())
.detail("WorkerId", workers[i].first.id());
return workers[i].first;
.detail("WorkerId", workers[i].interf.id());
return workers[i].interf;
}
}
@ -128,10 +128,10 @@ int64_t getQueueSize( const TraceEventFields& md ) {
ACTOR Future<int64_t> getMaxTLogQueueSize( Database cx, Reference<AsyncVar<ServerDBInfo>> dbInfo ) {
TraceEvent("MaxTLogQueueSize").detail("Stage", "ContactingLogs");
state std::vector<std::pair<WorkerInterface, ProcessClass>> workers = wait(getWorkers(dbInfo));
state std::vector<WorkerDetails> workers = wait(getWorkers(dbInfo));
std::map<NetworkAddress, WorkerInterface> workersMap;
for(auto worker : workers) {
workersMap[worker.first.address()] = worker.first;
workersMap[worker.interf.address()] = worker.interf;
}
state std::vector<Future<TraceEventFields>> messages;
@ -189,14 +189,14 @@ ACTOR Future<int64_t> getMaxStorageServerQueueSize( Database cx, Reference<Async
TraceEvent("MaxStorageServerQueueSize").detail("Stage", "ContactingStorageServers");
Future<std::vector<StorageServerInterface>> serversFuture = getStorageServers(cx);
state Future<std::vector<std::pair<WorkerInterface, ProcessClass>>> workersFuture = getWorkers(dbInfo);
state Future<std::vector<WorkerDetails>> workersFuture = getWorkers(dbInfo);
state std::vector<StorageServerInterface> servers = wait(serversFuture);
state std::vector<std::pair<WorkerInterface, ProcessClass>> workers = wait(workersFuture);
state std::vector<WorkerDetails> workers = wait(workersFuture);
std::map<NetworkAddress, WorkerInterface> workersMap;
for(auto worker : workers) {
workersMap[worker.first.address()] = worker.first;
workersMap[worker.interf.address()] = worker.interf;
}
state std::vector<Future<TraceEventFields>> messages;

2
fdbserver/QuietDatabase.h
View File

@ -35,7 +35,7 @@ Future<int64_t> getDataDistributionQueueSize( Database const &cx, Reference<Asyn
Future<bool> getTeamCollectionValid(Database const& cx, WorkerInterface const&);
Future<bool> getTeamCollectionValid(Database const& cx, Reference<AsyncVar<struct ServerDBInfo>> const&);
Future<vector<StorageServerInterface>> getStorageServers( Database const& cx, bool const &use_system_priority = false);
Future<vector<std::pair<WorkerInterface, ProcessClass>>> getWorkers( Reference<AsyncVar<ServerDBInfo>> const& dbInfo, int const& flags = 0 );
Future<vector<WorkerDetails>> getWorkers( Reference<AsyncVar<ServerDBInfo>> const& dbInfo, int const& flags = 0 );
Future<WorkerInterface> getMasterWorker( Database const& cx, Reference<AsyncVar<ServerDBInfo>> const& dbInfo );
Future<Void> repairDeadDatacenter(Database const& cx, Reference<AsyncVar<ServerDBInfo>> const& dbInfo, std::string const& context);

182
fdbserver/Ratekeeper.actor.cpp
View File

@ -3,7 +3,7 @@
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
* Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,13 +19,14 @@
*/
#include "flow/IndexedSet.h"
#include "fdbserver/Ratekeeper.h"
#include "fdbrpc/FailureMonitor.h"
#include "fdbserver/Knobs.h"
#include "fdbrpc/Smoother.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbrpc/simulator.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbserver/Knobs.h"
#include "fdbserver/DataDistribution.actor.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/WaitFailure.h"
#include "flow/actorcompiler.h" // This must be the last #include.
enum limitReason_t {
@ -146,7 +147,7 @@ struct TransactionCounts {
TransactionCounts() : total(0), batch(0), time(0) {}
};
struct Ratekeeper {
struct RatekeeperData {
Map<UID, StorageQueueInfo> storageQueueInfo;
Map<UID, TLogQueueInfo> tlogQueueInfo;
@ -154,16 +155,16 @@ struct Ratekeeper {
Smoother smoothReleasedTransactions, smoothBatchReleasedTransactions, smoothTotalDurableBytes;
HealthMetrics healthMetrics;
DatabaseConfiguration configuration;
PromiseStream<Future<Void>> addActor;
Int64MetricHandle actualTpsMetric;
double lastWarning;
double* lastLimited;
RatekeeperLimits normalLimits;
RatekeeperLimits batchLimits;
Ratekeeper() : smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
RatekeeperData() : smoothReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothBatchReleasedTransactions(SERVER_KNOBS->SMOOTHING_AMOUNT), smoothTotalDurableBytes(SERVER_KNOBS->SLOW_SMOOTHING_AMOUNT),
actualTpsMetric(LiteralStringRef("Ratekeeper.ActualTPS")),
lastWarning(0),
normalLimits("", SERVER_KNOBS->TARGET_BYTES_PER_STORAGE_SERVER, SERVER_KNOBS->SPRING_BYTES_STORAGE_SERVER, SERVER_KNOBS->TARGET_BYTES_PER_TLOG, SERVER_KNOBS->SPRING_BYTES_TLOG, SERVER_KNOBS->MAX_TL_SS_VERSION_DIFFERENCE),
@ -172,7 +173,7 @@ struct Ratekeeper {
};
//SOMEDAY: template trackStorageServerQueueInfo and trackTLogQueueInfo into one function
ACTOR Future<Void> trackStorageServerQueueInfo( Ratekeeper* self, StorageServerInterface ssi ) {
ACTOR Future<Void> trackStorageServerQueueInfo( RatekeeperData* self, StorageServerInterface ssi ) {
self->storageQueueInfo.insert( mapPair(ssi.id(), StorageQueueInfo(ssi.id(), ssi.locality) ) );
state Map<UID, StorageQueueInfo>::iterator myQueueInfo = self->storageQueueInfo.find(ssi.id());
TraceEvent("RkTracking", ssi.id());
@ -217,7 +218,7 @@ ACTOR Future<Void> trackStorageServerQueueInfo( Ratekeeper* self, StorageServerI
}
}
ACTOR Future<Void> trackTLogQueueInfo( Ratekeeper* self, TLogInterface tli ) {
ACTOR Future<Void> trackTLogQueueInfo( RatekeeperData* self, TLogInterface tli ) {
self->tlogQueueInfo.insert( mapPair(tli.id(), TLogQueueInfo(tli.id()) ) );
state Map<UID, TLogQueueInfo>::iterator myQueueInfo = self->tlogQueueInfo.find(tli.id());
TraceEvent("RkTracking", tli.id());
@ -270,7 +271,7 @@ ACTOR Future<Void> splitError( Future<Void> in, Promise<Void> errOut ) {
}
ACTOR Future<Void> trackEachStorageServer(
Ratekeeper* self,
RatekeeperData* self,
FutureStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges )
{
state Map<UID, Future<Void>> actors;
@ -289,7 +290,47 @@ ACTOR Future<Void> trackEachStorageServer(
}
}
void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
ACTOR Future<Void> monitorServerListChange(
Reference<AsyncVar<ServerDBInfo>> dbInfo,
PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges) {
state Database db = openDBOnServer(dbInfo, TaskRateKeeper, true, true);
state std::map<UID, StorageServerInterface> oldServers;
state Transaction tr(db);
loop {
try {
vector<std::pair<StorageServerInterface, ProcessClass>> results = wait(getServerListAndProcessClasses(&tr));
std::map<UID, StorageServerInterface> newServers;
for (int i = 0; i < results.size(); i++) {
const StorageServerInterface& ssi = results[i].first;
const UID serverId = ssi.id();
newServers[serverId] = ssi;
if (oldServers.count(serverId)) {
if (ssi.getValue.getEndpoint() != oldServers[serverId].getValue.getEndpoint()) {
serverChanges.send( std::make_pair(serverId, Optional<StorageServerInterface>(ssi)) );
}
oldServers.erase(serverId);
} else {
serverChanges.send( std::make_pair(serverId, Optional<StorageServerInterface>(ssi)) );
}
}
for (const auto& it : oldServers) {
serverChanges.send( std::make_pair(it.first, Optional<StorageServerInterface>()) );
}
oldServers.swap(newServers);
tr = Transaction(db);
wait(delay(SERVER_KNOBS->SERVER_LIST_DELAY));
} catch(Error& e) {
wait( tr.onError(e) );
}
}
}
void updateRate(RatekeeperData* self, RatekeeperLimits* limits) {
//double controlFactor = ; // dt / eFoldingTime
double actualTps = self->smoothReleasedTransactions.smoothRate();
@ -297,7 +338,7 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
// SOMEDAY: Remove the max( 1.0, ... ) since the below calculations _should_ be able to recover back up from this value
actualTps = std::max( std::max( 1.0, actualTps ), self->smoothTotalDurableBytes.smoothRate() / CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT );
limits.tpsLimit = std::numeric_limits<double>::infinity();
limits->tpsLimit = std::numeric_limits<double>::infinity();
UID reasonID = UID();
limitReason_t limitReason = limitReason_t::unlimited;
@ -323,9 +364,9 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
worstFreeSpaceStorageServer = std::min(worstFreeSpaceStorageServer, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace);
int64_t springBytes = std::max<int64_t>(1, std::min<int64_t>(limits.storageSpringBytes, (ss.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
int64_t targetBytes = std::max<int64_t>(1, std::min(limits.storageTargetBytes, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace));
if (targetBytes != limits.storageTargetBytes) {
int64_t springBytes = std::max<int64_t>(1, std::min<int64_t>(limits->storageSpringBytes, (ss.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
int64_t targetBytes = std::max<int64_t>(1, std::min(limits->storageTargetBytes, (int64_t)ss.smoothFreeSpace.smoothTotal() - minFreeSpace));
if (targetBytes != limits->storageTargetBytes) {
if (minFreeSpace == SERVER_KNOBS->MIN_FREE_SPACE) {
ssLimitReason = limitReason_t::storage_server_min_free_space;
} else {
@ -389,9 +430,9 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
storageTpsLimitReverseIndex.insert(std::make_pair(limitTps, &ss));
if(limitTps < limits.tpsLimit && (ssLimitReason == limitReason_t::storage_server_min_free_space || ssLimitReason == limitReason_t::storage_server_min_free_space_ratio)) {
if (limitTps < limits->tpsLimit && (ssLimitReason == limitReason_t::storage_server_min_free_space || ssLimitReason == limitReason_t::storage_server_min_free_space_ratio)) {
reasonID = ss.id;
limits.tpsLimit = limitTps;
limits->tpsLimit = limitTps;
limitReason = ssLimitReason;
}
@ -402,19 +443,19 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
self->healthMetrics.worstStorageDurabilityLag = worstStorageDurabilityLagStorageServer;
std::set<Optional<Standalone<StringRef>>> ignoredMachines;
for(auto ss = storageTpsLimitReverseIndex.begin(); ss != storageTpsLimitReverseIndex.end() && ss->first < limits.tpsLimit; ++ss) {
if(ignoredMachines.size() < std::min(self->configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
for (auto ss = storageTpsLimitReverseIndex.begin(); ss != storageTpsLimitReverseIndex.end() && ss->first < limits->tpsLimit; ++ss) {
if (ignoredMachines.size() < std::min(self->configuration.storageTeamSize - 1, SERVER_KNOBS->MAX_MACHINES_FALLING_BEHIND)) {
ignoredMachines.insert(ss->second->locality.zoneId());
continue;
}
if(ignoredMachines.count(ss->second->locality.zoneId()) > 0) {
if (ignoredMachines.count(ss->second->locality.zoneId()) > 0) {
continue;
}
limitingStorageQueueStorageServer = ss->second->lastReply.bytesInput - ss->second->smoothDurableBytes.smoothTotal();
limits.tpsLimit = ss->first;
limitReason = ssReasons[storageTpsLimitReverseIndex.begin()->second->id];
limits->tpsLimit = ss->first;
reasonID = storageTpsLimitReverseIndex.begin()->second->id; // Although we aren't controlling based on the worst SS, we still report it as the limiting process
limitReason = ssReasons[reasonID];
break;
}
@ -426,27 +467,27 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
{
Version minSSVer = std::numeric_limits<Version>::max();
Version minLimitingSSVer = std::numeric_limits<Version>::max();
for(auto i = self->storageQueueInfo.begin(); i != self->storageQueueInfo.end(); ++i) {
auto& ss = i->value;
for (const auto& it : self->storageQueueInfo) {
auto& ss = it.value;
if (!ss.valid) continue;
minSSVer = std::min(minSSVer, ss.lastReply.version);
// Machines that ratekeeper isn't controlling can fall arbitrarily far behind
if(ignoredMachines.count(i->value.locality.zoneId()) == 0) {
if (ignoredMachines.count(it.value.locality.zoneId()) == 0) {
minLimitingSSVer = std::min(minLimitingSSVer, ss.lastReply.version);
}
}
Version maxTLVer = std::numeric_limits<Version>::min();
for(auto i = self->tlogQueueInfo.begin(); i != self->tlogQueueInfo.end(); ++i) {
auto& tl = i->value;
for(const auto& it : self->tlogQueueInfo) {
auto& tl = it.value;
if (!tl.valid) continue;
maxTLVer = std::max(maxTLVer, tl.lastReply.v);
}
// writeToReadLatencyLimit: 0 = infinte speed; 1 = TL durable speed ; 2 = half TL durable speed
writeToReadLatencyLimit = ((maxTLVer - minLimitingSSVer) - limits.maxVersionDifference/2) / (limits.maxVersionDifference/4);
writeToReadLatencyLimit = ((maxTLVer - minLimitingSSVer) - limits->maxVersionDifference/2) / (limits->maxVersionDifference/4);
worstVersionLag = std::max((Version)0, maxTLVer - minSSVer);
limitingVersionLag = std::max((Version)0, maxTLVer - minLimitingSSVer);
}
@ -454,8 +495,8 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
int64_t worstFreeSpaceTLog = std::numeric_limits<int64_t>::max();
int64_t worstStorageQueueTLog = 0;
int tlcount = 0;
for(auto i = self->tlogQueueInfo.begin(); i != self->tlogQueueInfo.end(); ++i) {
auto& tl = i->value;
for (auto& it : self->tlogQueueInfo) {
auto& tl = it.value;
if (!tl.valid) continue;
++tlcount;
@ -465,9 +506,9 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace);
int64_t springBytes = std::max<int64_t>(1, std::min<int64_t>(limits.logSpringBytes, (tl.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
int64_t targetBytes = std::max<int64_t>(1, std::min(limits.logTargetBytes, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace));
if (targetBytes != limits.logTargetBytes) {
int64_t springBytes = std::max<int64_t>(1, std::min<int64_t>(limits->logSpringBytes, (tl.smoothFreeSpace.smoothTotal() - minFreeSpace) * 0.2));
int64_t targetBytes = std::max<int64_t>(1, std::min(limits->logTargetBytes, (int64_t)tl.smoothFreeSpace.smoothTotal() - minFreeSpace));
if (targetBytes != limits->logTargetBytes) {
if (minFreeSpace == SERVER_KNOBS->MIN_FREE_SPACE) {
tlogLimitReason = limitReason_t::log_server_min_free_space;
} else {
@ -487,7 +528,7 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
}
reasonID = tl.id;
limitReason = limitReason_t::log_server_min_free_space;
limits.tpsLimit = 0.0;
limits->tpsLimit = 0.0;
}
double targetRateRatio = std::min( ( b + springBytes ) / (double)springBytes, 2.0 );
@ -505,8 +546,8 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
if (targetRateRatio < .75) //< FIXME: KNOB for 2.0
x = std::max(x, 0.95);
double lim = actualTps * x;
if (lim < limits.tpsLimit){
limits.tpsLimit = lim;
if (lim < limits->tpsLimit){
limits->tpsLimit = lim;
reasonID = tl.id;
limitReason = tlogLimitReason;
}
@ -515,8 +556,8 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
// Don't let any tlogs use up its target bytes faster than its MVCC window!
double x = ((targetBytes - springBytes) / ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS)/SERVER_KNOBS->VERSIONS_PER_SECOND) + 2.0)) / inputRate;
double lim = actualTps * x;
if (lim < limits.tpsLimit){
limits.tpsLimit = lim;
if (lim < limits->tpsLimit){
limits->tpsLimit = lim;
reasonID = tl.id;
limitReason = limitReason_t::log_server_mvcc_write_bandwidth;
}
@ -525,10 +566,10 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
self->healthMetrics.worstTLogQueue = worstStorageQueueTLog;
limits.tpsLimit = std::max(limits.tpsLimit, 0.0);
limits->tpsLimit = std::max(limits->tpsLimit, 0.0);
if(g_network->isSimulated() && g_simulator.speedUpSimulation) {
limits.tpsLimit = std::max(limits.tpsLimit, 100.0);
limits->tpsLimit = std::max(limits->tpsLimit, 100.0);
}
int64_t totalDiskUsageBytes = 0;
@ -539,13 +580,13 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
if (s.value.valid)
totalDiskUsageBytes += s.value.lastReply.storageBytes.used;
limits.tpsLimitMetric = std::min(limits.tpsLimit, 1e6);
limits.reasonMetric = limitReason;
limits->tpsLimitMetric = std::min(limits->tpsLimit, 1e6);
limits->reasonMetric = limitReason;
if (g_random->random01() < 0.1) {
std::string name = "RkUpdate" + limits.context;
std::string name = "RkUpdate" + limits->context;
TraceEvent(name.c_str())
.detail("TPSLimit", limits.tpsLimit)
.detail("TPSLimit", limits->tpsLimit)
.detail("Reason", limitReason)
.detail("ReasonServerID", reasonID)
.detail("ReleasedTPS", self->smoothReleasedTransactions.smoothRate())
@ -566,7 +607,7 @@ void updateRate( Ratekeeper* self, RatekeeperLimits &limits ) {
}
}
ACTOR Future<Void> configurationMonitor( Ratekeeper* self, Reference<AsyncVar<ServerDBInfo>> dbInfo ) {
ACTOR Future<Void> configurationMonitor(Reference<AsyncVar<ServerDBInfo>> dbInfo, DatabaseConfiguration* conf) {
state Database cx = openDBOnServer(dbInfo, TaskDefaultEndpoint, true, true);
loop {
state ReadYourWritesTransaction tr(cx);
@ -578,7 +619,7 @@ ACTOR Future<Void> configurationMonitor( Ratekeeper* self, Reference<AsyncVar<Se
Standalone<RangeResultRef> results = wait( tr.getRange( configKeys, CLIENT_KNOBS->TOO_MANY ) );
ASSERT( !results.more && results.size() < CLIENT_KNOBS->TOO_MANY );
self->configuration.fromKeyValues( (VectorRef<KeyValueRef>) results );
conf->fromKeyValues( (VectorRef<KeyValueRef>) results );
state Future<Void> watchFuture = tr.watch(moveKeysLockOwnerKey);
wait( tr.commit() );
@ -591,21 +632,21 @@ ACTOR Future<Void> configurationMonitor( Ratekeeper* self, Reference<AsyncVar<Se
}
}
ACTOR Future<Void> rateKeeper(
Reference<AsyncVar<ServerDBInfo>> dbInfo,
PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges,
FutureStream< struct GetRateInfoRequest > getRateInfo,
double* lastLimited)
{
state Ratekeeper self;
state Future<Void> track = trackEachStorageServer( &self, serverChanges.getFuture() );
ACTOR Future<Void> rateKeeper(RatekeeperInterface rkInterf, Reference<AsyncVar<ServerDBInfo>> dbInfo) {
state RatekeeperData self;
state Future<Void> timeout = Void();
state std::vector<Future<Void>> actors;
state std::vector<Future<Void>> tlogTrackers;
state std::vector<TLogInterface> tlogInterfs;
state Promise<Void> err;
state Future<Void> configMonitor = configurationMonitor(&self, dbInfo);
self.lastLimited = lastLimited;
state Future<Void> collection = actorCollection( self.addActor.getFuture() );
TraceEvent("Ratekeeper_Starting", rkInterf.id());
self.addActor.send( waitFailureServer(rkInterf.waitFailure.getFuture()) );
self.addActor.send( configurationMonitor(dbInfo, &self.configuration) );
PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > serverChanges;
self.addActor.send( monitorServerListChange(dbInfo, serverChanges) );
self.addActor.send( trackEachStorageServer(&self, serverChanges.getFuture()) );
TraceEvent("RkTLogQueueSizeParameters").detail("Target", SERVER_KNOBS->TARGET_BYTES_PER_TLOG).detail("Spring", SERVER_KNOBS->SPRING_BYTES_TLOG)
.detail("Rate", (SERVER_KNOBS->TARGET_BYTES_PER_TLOG - SERVER_KNOBS->SPRING_BYTES_TLOG) / ((((double)SERVER_KNOBS->MAX_READ_TRANSACTION_LIFE_VERSIONS) / SERVER_KNOBS->VERSIONS_PER_SECOND) + 2.0));
@ -617,18 +658,14 @@ ACTOR Future<Void> rateKeeper(
for( int i = 0; i < tlogInterfs.size(); i++ )
tlogTrackers.push_back( splitError( trackTLogQueueInfo(&self, tlogInterfs[i]), err ) );
loop{
choose {
when (wait( track )) { break; }
try {
state bool lastLimited = false;
loop choose {
when (wait( timeout )) {
updateRate(&self, self.normalLimits);
updateRate(&self, self.batchLimits);
if(self.smoothReleasedTransactions.smoothRate() > SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit) {
*self.lastLimited = now();
}
updateRate(&self, &self.normalLimits);
updateRate(&self, &self.batchLimits);
lastLimited = self.smoothReleasedTransactions.smoothRate() > SERVER_KNOBS->LAST_LIMITED_RATIO * self.batchLimits.tpsLimit;
double tooOld = now() - 1.0;
for(auto p=self.proxy_transactionCounts.begin(); p!=self.proxy_transactionCounts.end(); ) {
if (p->second.time < tooOld)
@ -638,7 +675,7 @@ ACTOR Future<Void> rateKeeper(
}
timeout = delayJittered(SERVER_KNOBS->METRIC_UPDATE_RATE);
}
when (GetRateInfoRequest req = waitNext(getRateInfo)) {
when (GetRateInfoRequest req = waitNext(rkInterf.getRateInfo.getFuture())) {
GetRateInfoReply reply;
auto& p = self.proxy_transactionCounts[ req.requesterID ];
@ -660,6 +697,7 @@ ACTOR Future<Void> rateKeeper(
reply.healthMetrics.update(self.healthMetrics, true, req.detailed);
reply.healthMetrics.tpsLimit = self.normalLimits.tpsLimit;
reply.healthMetrics.batchLimited = lastLimited;
req.reply.send( reply );
}
@ -672,8 +710,14 @@ ACTOR Future<Void> rateKeeper(
tlogTrackers.push_back( splitError( trackTLogQueueInfo(&self, tlogInterfs[i]), err ) );
}
}
when(wait(configMonitor)) {}
when ( wait(collection) ) {
ASSERT(false);
throw internal_error();
}
}
}
catch (Error& err) {
TraceEvent("Ratekeeper_Died", rkInterf.id()).error(err, true);
}
return Void();
}

35
fdbserver/Ratekeeper.h
View File

@ -1,35 +0,0 @@
/*
* Ratekeeper.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_RATEKEEPER_H
#define FDBSERVER_RATEKEEPER_H
#pragma once
#include "fdbserver/MasterInterface.h"
#include "fdbserver/TLogInterface.h"
#include "fdbclient/DatabaseConfiguration.h"
Future<Void> rateKeeper(
Reference<AsyncVar<struct ServerDBInfo>> const& dbInfo,
PromiseStream< std::pair<UID, Optional<StorageServerInterface>> > const& serverChanges, // actually an input, but we don't want broken_promise
FutureStream< struct GetRateInfoRequest > const& getRateInfo,
double* const& lastLimited);
#endif

81
fdbserver/RatekeeperInterface.h Normal file
View File

@ -0,0 +1,81 @@
/*
* RatekeeperInterface.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBSERVER_RATEKEEPERINTERFACE_H
#define FDBSERVER_RATEKEEPERINTERFACE_H
#include "fdbclient/FDBTypes.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbrpc/Locality.h"
struct RatekeeperInterface {
RequestStream<ReplyPromise<Void>> waitFailure;
RequestStream<struct GetRateInfoRequest> getRateInfo;
struct LocalityData locality;
RatekeeperInterface() {}
explicit RatekeeperInterface(const struct LocalityData& l) : locality(l) {}
void initEndpoints() {}
UID id() const { return getRateInfo.getEndpoint().token; }
NetworkAddress address() const { return getRateInfo.getEndpoint().getPrimaryAddress(); }
bool operator== (const RatekeeperInterface& r) const {
return id() == r.id();
}
bool operator!= (const RatekeeperInterface& r) const {
return !(*this == r);
}
template <class Archive>
void serialize(Archive& ar) {
serializer(ar, waitFailure, getRateInfo, locality);
}
};
struct GetRateInfoRequest {
UID requesterID;
int64_t totalReleasedTransactions;
int64_t batchReleasedTransactions;
bool detailed;
ReplyPromise<struct GetRateInfoReply> reply;
GetRateInfoRequest() {}
GetRateInfoRequest(UID const& requesterID, int64_t totalReleasedTransactions, int64_t batchReleasedTransactions, bool detailed)
: requesterID(requesterID), totalReleasedTransactions(totalReleasedTransactions), batchReleasedTransactions(batchReleasedTransactions), detailed(detailed) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, requesterID, totalReleasedTransactions, batchReleasedTransactions, detailed, reply);
}
};
struct GetRateInfoReply {
double transactionRate;
double batchTransactionRate;
double leaseDuration;
HealthMetrics healthMetrics;
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, transactionRate, batchTransactionRate, leaseDuration, healthMetrics);
}
};
#endif //FDBSERVER_RATEKEEPERINTERFACE_H

4
fdbserver/ServerDBInfo.h
View File

@ -26,6 +26,7 @@
#include "fdbserver/DataDistributorInterface.h"
#include "fdbserver/MasterInterface.h"
#include "fdbserver/LogSystemConfig.h"
#include "fdbserver/RatekeeperInterface.h"
#include "fdbserver/RecoveryState.h"
#include "fdbserver/LatencyBandConfig.h"
@ -39,6 +40,7 @@ struct ServerDBInfo {
ClientDBInfo client; // After a successful recovery, eventually proxies that communicate with it
Optional<DataDistributorInterface> distributor; // The best guess of current data distributor.
MasterInterface master; // The best guess as to the most recent master, which might still be recovering
Optional<RatekeeperInterface> ratekeeper;
vector<ResolverInterface> resolvers;
DBRecoveryCount recoveryCount; // A recovery count from DBCoreState. A successful master recovery increments it twice; unsuccessful recoveries may increment it once. Depending on where the current master is in its recovery process, this might not have been written by the current master.
RecoveryState recoveryState;
@ -55,7 +57,7 @@ struct ServerDBInfo {
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, id, clusterInterface, client, distributor, master, resolvers, recoveryCount, recoveryState, masterLifetime, logSystemConfig, priorCommittedLogServers, latencyBandConfig);
serializer(ar, id, clusterInterface, client, distributor, master, ratekeeper, resolvers, recoveryCount, recoveryState, masterLifetime, logSystemConfig, priorCommittedLogServers, latencyBandConfig);
}
};

141
fdbserver/Status.actor.cpp
View File

@ -100,12 +100,12 @@ ACTOR static Future< Optional<TraceEventFields> > latestEventOnWorker(WorkerInte
}
}
ACTOR static Future< Optional< std::pair<WorkerEvents, std::set<std::string>> > > latestEventOnWorkers(std::vector<std::pair<WorkerInterface, ProcessClass>> workers, std::string eventName) {
ACTOR static Future< Optional< std::pair<WorkerEvents, std::set<std::string>> > > latestEventOnWorkers(std::vector<WorkerDetails> workers, std::string eventName) {
try {
state vector<Future<ErrorOr<TraceEventFields>>> eventTraces;
for (int c = 0; c < workers.size(); c++) {
EventLogRequest req = eventName.size() > 0 ? EventLogRequest(Standalone<StringRef>(eventName)) : EventLogRequest();
eventTraces.push_back(errorOr(timeoutError(workers[c].first.eventLogRequest.getReply(req), 2.0)));
eventTraces.push_back(errorOr(timeoutError(workers[c].interf.eventLogRequest.getReply(req), 2.0)));
}
wait(waitForAll(eventTraces));
@ -116,11 +116,11 @@ ACTOR static Future< Optional< std::pair<WorkerEvents, std::set<std::string>> >
for (int i = 0; i < eventTraces.size(); i++) {
const ErrorOr<TraceEventFields>& v = eventTraces[i].get();
if (v.isError()){
failed.insert(workers[i].first.address().toString());
results[workers[i].first.address()] = TraceEventFields();
failed.insert(workers[i].interf.address().toString());
results[workers[i].interf.address()] = TraceEventFields();
}
else {
results[workers[i].first.address()] = v.get();
results[workers[i].interf.address()] = v.get();
}
}
@ -135,26 +135,26 @@ ACTOR static Future< Optional< std::pair<WorkerEvents, std::set<std::string>> >
throw;
}
}
static Future< Optional< std::pair<WorkerEvents, std::set<std::string>> > > latestErrorOnWorkers(std::vector<std::pair<WorkerInterface, ProcessClass>> workers) {
static Future< Optional< std::pair<WorkerEvents, std::set<std::string>> > > latestErrorOnWorkers(std::vector<WorkerDetails> workers) {
return latestEventOnWorkers( workers, "" );
}
static Optional<std::pair<WorkerInterface, ProcessClass>> getWorker(std::vector<std::pair<WorkerInterface, ProcessClass>> const& workers, NetworkAddress const& address) {
static Optional<WorkerDetails> getWorker(std::vector<WorkerDetails> const& workers, NetworkAddress const& address) {
try {
for (int c = 0; c < workers.size(); c++)
if (address == workers[c].first.address())
if (address == workers[c].interf.address())
return workers[c];
return Optional<std::pair<WorkerInterface, ProcessClass>>();
return Optional<WorkerDetails>();
}
catch (Error &e){
return Optional<std::pair<WorkerInterface, ProcessClass>>();
return Optional<WorkerDetails>();
}
}
static Optional<std::pair<WorkerInterface, ProcessClass>> getWorker(std::map<NetworkAddress, std::pair<WorkerInterface, ProcessClass>> const& workersMap, NetworkAddress const& address) {
static Optional<WorkerDetails> getWorker(std::map<NetworkAddress, WorkerDetails> const& workersMap, NetworkAddress const& address) {
auto itr = workersMap.find(address);
if(itr == workersMap.end()) {
return Optional<std::pair<WorkerInterface, ProcessClass>>();
return Optional<WorkerDetails>();
}
return itr->second;
@ -261,7 +261,7 @@ static JsonBuilderObject getError(const TraceEventFields& errorFields) {
return statusObj;
}
static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics, vector<std::pair<WorkerInterface, ProcessClass>> workers, Optional<DatabaseConfiguration> configuration, std::set<std::string> *incomplete_reasons) {
static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics, vector<WorkerDetails> workers, Optional<DatabaseConfiguration> configuration, std::set<std::string> *incomplete_reasons) {
JsonBuilderObject machineMap;
double metric;
int failed = 0;
@ -274,9 +274,9 @@ static JsonBuilderObject machineStatusFetcher(WorkerEvents mMetrics, vector<std:
std::map<std::string, JsonBuilderObject> machineJsonMap;
for (auto const& worker : workers){
locality[worker.first.address()] = worker.first.locality;
if (worker.first.locality.dcId().present())
dcIds[worker.first.address()] = worker.first.locality.dcId().get().printable();
locality[worker.interf.address()] = worker.interf.locality;
if (worker.interf.locality.dcId().present())
dcIds[worker.interf.address()] = worker.interf.locality.dcId().get().printable();
}
for(auto it = mMetrics.begin(); it != mMetrics.end(); it++) {
@ -540,7 +540,7 @@ struct RolesInfo {
ACTOR static Future<JsonBuilderObject> processStatusFetcher(
Reference<AsyncVar<struct ServerDBInfo>> db,
std::vector<std::pair<WorkerInterface, ProcessClass>> workers,
std::vector<WorkerDetails> workers,
WorkerEvents pMetrics,
WorkerEvents mMetrics,
WorkerEvents errors,
@ -581,13 +581,13 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
}
state std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo> machineMemoryUsage;
state std::vector<std::pair<WorkerInterface, ProcessClass>>::iterator workerItr;
state std::vector<WorkerDetails>::iterator workerItr;
for(workerItr = workers.begin(); workerItr != workers.end(); ++workerItr) {
wait(yield());
state std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo>::iterator memInfo = machineMemoryUsage.insert(std::make_pair(workerItr->first.locality.machineId(), MachineMemoryInfo())).first;
state std::map<Optional<Standalone<StringRef>>, MachineMemoryInfo>::iterator memInfo = machineMemoryUsage.insert(std::make_pair(workerItr->interf.locality.machineId(), MachineMemoryInfo())).first;
try {
ASSERT(pMetrics.count(workerItr->first.address()));
const TraceEventFields& processMetrics = pMetrics[workerItr->first.address()];
ASSERT(pMetrics.count(workerItr->interf.address()));
const TraceEventFields& processMetrics = pMetrics[workerItr->interf.address()];
if(memInfo->second.valid()) {
if(processMetrics.size() > 0) {
@ -647,10 +647,10 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
wait(yield());
state JsonBuilderObject statusObj;
try {
ASSERT(pMetrics.count(workerItr->first.address()));
ASSERT(pMetrics.count(workerItr->interf.address()));
NetworkAddress address = workerItr->first.address();
const TraceEventFields& event = pMetrics[workerItr->first.address()];
NetworkAddress address = workerItr->interf.address();
const TraceEventFields& event = pMetrics[workerItr->interf.address()];
statusObj["address"] = address.toString();
JsonBuilderObject memoryObj;
@ -661,7 +661,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
std::string MachineID = event.getValue("MachineID");
statusObj["machine_id"] = MachineID;
statusObj["locality"] = getLocalityInfo(workerItr->first.locality);
statusObj["locality"] = getLocalityInfo(workerItr->interf.locality);
statusObj.setKeyRawNumber("uptime_seconds",event.getValue("UptimeSeconds"));
@ -750,7 +750,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
double availableMemory;
availableMemory = mMetrics[address].getDouble("AvailableMemory");
auto machineMemInfo = machineMemoryUsage[workerItr->first.locality.machineId()];
auto machineMemInfo = machineMemoryUsage[workerItr->interf.locality.machineId()];
if (machineMemInfo.valid()) {
ASSERT(machineMemInfo.numProcesses > 0);
int64_t memory = (availableMemory + machineMemInfo.memoryUsage) / machineMemInfo.numProcesses;
@ -794,16 +794,18 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
statusObj["excluded"] = configuration.get().isExcludedServer(address);
}
statusObj["class_type"] = workerItr->second.toString();
statusObj["class_source"] = workerItr->second.sourceString();
statusObj["class_type"] = workerItr->processClass.toString();
statusObj["class_source"] = workerItr->processClass.sourceString();
if(workerItr->degraded) {
statusObj["degraded"] = true;
}
}
catch (Error& e){
// Something strange occurred, process list is incomplete but what was built so far, if anything, will be returned.
incomplete_reasons->insert("Cannot retrieve all process status information.");
}
processMap[printable(workerItr->first.locality.processId())] = statusObj;
processMap[printable(workerItr->interf.locality.processId())] = statusObj;
}
return processMap;
}
@ -850,11 +852,11 @@ static JsonBuilderObject clientStatusFetcher(ClientVersionMap clientVersionMap,
return clientStatus;
}
ACTOR static Future<JsonBuilderObject> recoveryStateStatusFetcher(std::pair<WorkerInterface, ProcessClass> mWorker, int workerCount, std::set<std::string> *incomplete_reasons, int* statusCode) {
ACTOR static Future<JsonBuilderObject> recoveryStateStatusFetcher(WorkerDetails mWorker, int workerCount, std::set<std::string> *incomplete_reasons, int* statusCode) {
state JsonBuilderObject message;
try {
TraceEventFields md = wait( timeoutError(mWorker.first.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0) );
TraceEventFields md = wait( timeoutError(mWorker.interf.eventLogRequest.getReply( EventLogRequest( LiteralStringRef("MasterRecoveryState") ) ), 1.0) );
state int mStatusCode = md.getInt("StatusCode");
if (mStatusCode < 0 || mStatusCode >= RecoveryStatus::END)
throw attribute_not_found();
@ -1103,18 +1105,18 @@ static JsonBuilderObject configurationFetcher(Optional<DatabaseConfiguration> co
return statusObj;
}
ACTOR static Future<JsonBuilderObject> dataStatusFetcher(std::pair<WorkerInterface, ProcessClass> ddWorker, int *minReplicasRemaining) {
ACTOR static Future<JsonBuilderObject> dataStatusFetcher(WorkerDetails ddWorker, int *minReplicasRemaining) {
state JsonBuilderObject statusObjData;
try {
std::vector<Future<TraceEventFields>> futures;
// TODO: Should this be serial?
futures.push_back(timeoutError(ddWorker.first.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStarting"))), 1.0));
futures.push_back(timeoutError(ddWorker.first.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStats"))), 1.0));
futures.push_back(timeoutError(ddWorker.first.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MovingData"))), 1.0));
futures.push_back(timeoutError(ddWorker.first.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlight"))), 1.0));
futures.push_back(timeoutError(ddWorker.first.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlightRemote"))), 1.0));
futures.push_back(timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStarting"))), 1.0));
futures.push_back(timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("DDTrackerStats"))), 1.0));
futures.push_back(timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("MovingData"))), 1.0));
futures.push_back(timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlight"))), 1.0));
futures.push_back(timeoutError(ddWorker.interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("TotalDataInFlightRemote"))), 1.0));
std::vector<TraceEventFields> dataInfo = wait(getAll(futures));
@ -1327,16 +1329,16 @@ ACTOR static Future<vector<std::pair<MasterProxyInterface, EventMap>>> getProxie
return results;
}
static int getExtraTLogEligibleMachines(const vector<std::pair<WorkerInterface, ProcessClass>>& workers, const DatabaseConfiguration& configuration) {
static int getExtraTLogEligibleMachines(const vector<WorkerDetails>& workers, const DatabaseConfiguration& configuration) {
std::set<StringRef> allMachines;
std::map<Key,std::set<StringRef>> dcId_machine;
for(auto const& worker : workers) {
if(worker.second.machineClassFitness(ProcessClass::TLog) < ProcessClass::NeverAssign
&& !configuration.isExcludedServer(worker.first.address()))
if(worker.processClass.machineClassFitness(ProcessClass::TLog) < ProcessClass::NeverAssign
&& !configuration.isExcludedServer(worker.interf.address()))
{
allMachines.insert(worker.first.locality.zoneId().get());
if(worker.first.locality.dcId().present()) {
dcId_machine[worker.first.locality.dcId().get()].insert(worker.first.locality.zoneId().get());
allMachines.insert(worker.interf.locality.zoneId().get());
if(worker.interf.locality.dcId().present()) {
dcId_machine[worker.interf.locality.dcId().get()].insert(worker.interf.locality.zoneId().get());
}
}
}
@ -1390,7 +1392,7 @@ JsonBuilderObject getPerfLimit(TraceEventFields const& ratekeeper, double transP
return perfLimit;
}
ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, vector<std::pair<WorkerInterface, ProcessClass>> workers, std::pair<WorkerInterface, ProcessClass> mWorker, std::pair<WorkerInterface, ProcessClass> ddWorker,
ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(Reference<AsyncVar<struct ServerDBInfo>> db, vector<WorkerDetails> workers, WorkerDetails mWorker, WorkerDetails rkWorker,
JsonBuilderObject *qos, JsonBuilderObject *data_overlay, std::set<std::string> *incomplete_reasons, Future<ErrorOr<vector<std::pair<StorageServerInterface, EventMap>>>> storageServerFuture)
{
state JsonBuilderObject statusObj;
@ -1401,14 +1403,14 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(Reference<AsyncVar<
// Writes and conflicts
try {
vector<Future<TraceEventFields>> proxyStatFutures;
std::map<NetworkAddress, std::pair<WorkerInterface, ProcessClass>> workersMap;
std::map<NetworkAddress, WorkerDetails> workersMap;
for (auto const& w : workers) {
workersMap[w.first.address()] = w;
workersMap[w.interf.address()] = w;
}
for (auto &p : db->get().client.proxies) {
auto worker = getWorker(workersMap, p.address());
if (worker.present())
proxyStatFutures.push_back(timeoutError(worker.get().first.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("ProxyMetrics"))), 1.0));
proxyStatFutures.push_back(timeoutError(worker.get().interf.eventLogRequest.getReply(EventLogRequest(LiteralStringRef("ProxyMetrics"))), 1.0));
else
throw all_alternatives_failed(); // We need data from all proxies for this result to be trustworthy
}
@ -1442,8 +1444,8 @@ ACTOR static Future<JsonBuilderObject> workloadStatusFetcher(Reference<AsyncVar<
// Transactions
try {
state TraceEventFields ratekeeper = wait( timeoutError(ddWorker.first.eventLogRequest.getReply( EventLogRequest(LiteralStringRef("RkUpdate") ) ), 1.0) );
TraceEventFields batchRatekeeper = wait( timeoutError(ddWorker.first.eventLogRequest.getReply( EventLogRequest(LiteralStringRef("RkUpdateBatch") ) ), 1.0) );
state TraceEventFields ratekeeper = wait( timeoutError(rkWorker.interf.eventLogRequest.getReply( EventLogRequest(LiteralStringRef("RkUpdate") ) ), 1.0) );
TraceEventFields batchRatekeeper = wait( timeoutError(rkWorker.interf.eventLogRequest.getReply( EventLogRequest(LiteralStringRef("RkUpdateBatch") ) ), 1.0) );
double tpsLimit = ratekeeper.getDouble("TPSLimit");
double batchTpsLimit = batchRatekeeper.getDouble("TPSLimit");
@ -1599,7 +1601,7 @@ static JsonBuilderArray oldTlogFetcher(int* oldLogFaultTolerance, Reference<Asyn
return oldTlogsArray;
}
static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, std::vector<std::pair<WorkerInterface, ProcessClass>>& workers, int extraTlogEligibleMachines, int minReplicasRemaining) {
static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration configuration, ServerCoordinators coordinators, std::vector<WorkerDetails>& workers, int extraTlogEligibleMachines, int minReplicasRemaining) {
JsonBuilderObject statusObj;
// without losing data
@ -1608,7 +1610,7 @@ static JsonBuilderObject faultToleranceStatusFetcher(DatabaseConfiguration confi
std::map<NetworkAddress, StringRef> workerZones;
for(auto& worker : workers) {
workerZones[worker.first.address()] = worker.first.locality.zoneId().orDefault(LiteralStringRef(""));
workerZones[worker.interf.address()] = worker.interf.locality.zoneId().orDefault(LiteralStringRef(""));
}
std::map<StringRef, int> coordinatorZoneCounts;
for(auto& coordinator : coordinators.ccf->getConnectionString().coordinators()) {
@ -1803,7 +1805,7 @@ ACTOR Future<JsonBuilderObject> lockedStatusFetcher(Reference<AsyncVar<struct Se
ACTOR Future<StatusReply> clusterGetStatus(
Reference<AsyncVar<struct ServerDBInfo>> db,
Database cx,
vector<std::pair<WorkerInterface, ProcessClass>> workers,
vector<WorkerDetails> workers,
ProcessIssuesMap workerIssues,
ProcessIssuesMap clientIssues,
ClientVersionMap clientVersionMap,
@ -1817,19 +1819,20 @@ ACTOR Future<StatusReply> clusterGetStatus(
// Check if master worker is present
state JsonBuilderArray messages;
state std::set<std::string> status_incomplete_reasons;
state std::pair<WorkerInterface, ProcessClass> mWorker;
state std::pair<WorkerInterface, ProcessClass> ddWorker; // DataDistributor worker
state WorkerDetails mWorker;
state WorkerDetails ddWorker; // DataDistributor worker
state WorkerDetails rkWorker; // RateKeeper worker
try {
// Get the master Worker interface
Optional<std::pair<WorkerInterface, ProcessClass>> _mWorker = getWorker( workers, db->get().master.address() );
Optional<WorkerDetails> _mWorker = getWorker( workers, db->get().master.address() );
if (_mWorker.present()) {
mWorker = _mWorker.get();
} else {
messages.push_back(JsonString::makeMessage("unreachable_master_worker", "Unable to locate the master worker."));
}
// Get the DataDistributor worker interface
Optional<std::pair<WorkerInterface, ProcessClass>> _ddWorker;
Optional<WorkerDetails> _ddWorker;
if (db->get().distributor.present()) {
_ddWorker = getWorker( workers, db->get().distributor.get().address() );
}
@ -1840,6 +1843,18 @@ ACTOR Future<StatusReply> clusterGetStatus(
ddWorker = _ddWorker.get();
}
// Get the RateKeeper worker interface
Optional<WorkerDetails> _rkWorker;
if (db->get().ratekeeper.present()) {
_rkWorker = getWorker( workers, db->get().ratekeeper.get().address() );
}
if (!db->get().ratekeeper.present() || !_rkWorker.present()) {
messages.push_back(JsonString::makeMessage("unreachable_ratekeeper_worker", "Unable to locate the ratekeeper worker."));
} else {
rkWorker = _rkWorker.get();
}
// Get latest events for various event types from ALL workers
// WorkerEvents is a map of worker's NetworkAddress to its event string
// The pair represents worker responses and a set of worker NetworkAddress strings which did not respond
@ -1933,7 +1948,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
// in status output is important to give context to error messages in status that reference a storage server role ID.
state std::unordered_map<NetworkAddress, WorkerInterface> address_workers;
for (auto const& worker : workers) {
address_workers[worker.first.address()] = worker.first;
address_workers[worker.interf.address()] = worker.interf;
}
state Future<ErrorOr<vector<std::pair<StorageServerInterface, EventMap>>>> storageServerFuture = errorOr(getStorageServersAndMetrics(cx, address_workers));
@ -1943,7 +1958,7 @@ ACTOR Future<StatusReply> clusterGetStatus(
state int minReplicasRemaining = -1;
std::vector<Future<JsonBuilderObject>> futures2;
futures2.push_back(dataStatusFetcher(ddWorker, &minReplicasRemaining));
futures2.push_back(workloadStatusFetcher(db, workers, mWorker, ddWorker, &qos, &data_overlay, &status_incomplete_reasons, storageServerFuture));
futures2.push_back(workloadStatusFetcher(db, workers, mWorker, rkWorker, &qos, &data_overlay, &status_incomplete_reasons, storageServerFuture));
futures2.push_back(layerStatusFetcher(cx, &messages, &status_incomplete_reasons));
futures2.push_back(lockedStatusFetcher(db, &messages, &status_incomplete_reasons));
@ -2036,6 +2051,14 @@ ACTOR Future<StatusReply> clusterGetStatus(
statusObj["incompatible_connections"] = incompatibleConnectionsArray;
statusObj["datacenter_version_difference"] = datacenterVersionDifference;
int totalDegraded = 0;
for(auto& it : workers) {
if(it.degraded) {
totalDegraded++;
}
}
statusObj["degraded_processes"] = totalDegraded;
if (!recoveryStateStatus.empty())
statusObj["recovery_state"] = recoveryStateStatus;

2
fdbserver/Status.h
View File

@ -38,7 +38,7 @@ struct ClientStatusInfo {
ClientStatusInfo(std::string const& traceLogGroup, int const connectedCoordinatorsNum) : traceLogGroup(traceLogGroup), connectedCoordinatorsNum(connectedCoordinatorsNum) {}
};
Future<StatusReply> clusterGetStatus( Reference<AsyncVar<struct ServerDBInfo>> const& db, Database const& cx, vector<std::pair<WorkerInterface, ProcessClass>> const& workers,
Future<StatusReply> clusterGetStatus( Reference<AsyncVar<struct ServerDBInfo>> const& db, Database const& cx, vector<WorkerDetails> const& workers,
ProcessIssuesMap const& workerIssues, ProcessIssuesMap const& clientIssues, ClientVersionMap const& clientVersionMap, std::map<NetworkAddress, struct ClientStatusInfo> const& clientStatusInfoMap,
ServerCoordinators const& coordinators, std::vector<NetworkAddress> const& incompatibleConnections, Version const& datacenterVersionDifference );

29
fdbserver/TLogServer.actor.cpp
View File

@ -33,6 +33,7 @@
#include "fdbrpc/FailureMonitor.h"
#include "fdbserver/IDiskQueue.h"
#include "fdbrpc/sim_validation.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/LogSystem.h"
#include "fdbserver/WaitFailure.h"
@ -299,10 +300,12 @@ struct TLogData : NonCopyable {
FlowLock concurrentLogRouterReads;
FlowLock persistentDataCommitLock;
TLogData(UID dbgid, IKeyValueStore* persistentData, IDiskQueue * persistentQueue, Reference<AsyncVar<ServerDBInfo>> const& dbInfo)
Reference<AsyncVar<bool>> degraded;
TLogData(UID dbgid, IKeyValueStore* persistentData, IDiskQueue * persistentQueue, Reference<AsyncVar<ServerDBInfo>> dbInfo, Reference<AsyncVar<bool>> degraded)
: dbgid(dbgid), instanceID(g_random->randomUniqueID().first()),
persistentData(persistentData), rawPersistentQueue(persistentQueue), persistentQueue(new TLogQueue(persistentQueue, dbgid)),
dbInfo(dbInfo), queueCommitBegin(0), queueCommitEnd(0),
dbInfo(dbInfo), degraded(degraded), queueCommitBegin(0), queueCommitEnd(0),
diskQueueCommitBytes(0), largeDiskQueueCommitBytes(false), bytesInput(0), bytesDurable(0), overheadBytesInput(0), overheadBytesDurable(0),
peekMemoryLimiter(SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES),
concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS)
@ -1363,6 +1366,19 @@ ACTOR Future<Void> tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere
return Void();
}
ACTOR Future<Void> watchDegraded(TLogData* self) {
//This delay is divided into multiple delays to avoid marking the tlog as degraded because of a single SlowTask
state int loopCount = 0;
while(loopCount < SERVER_KNOBS->TLOG_DEGRADED_DELAY_COUNT) {
wait(delay(SERVER_KNOBS->TLOG_DEGRADED_DURATION/SERVER_KNOBS->TLOG_DEGRADED_DELAY_COUNT, TaskLowPriority));
loopCount++;
}
TraceEvent(SevWarnAlways, "TLogDegraded", self->dbgid);
TEST(true); //TLog degraded
self->degraded->set(true);
return Void();
}
ACTOR Future<Void> doQueueCommit( TLogData* self, Reference<LogData> logData ) {
state Version ver = logData->version.get();
state Version commitNumber = self->queueCommitBegin+1;
@ -1374,7 +1390,12 @@ ACTOR Future<Void> doQueueCommit( TLogData* self, Reference<LogData> logData ) {
self->diskQueueCommitBytes = 0;
self->largeDiskQueueCommitBytes.set(false);
state Future<Void> degraded = watchDegraded(self);
wait(c);
if(g_network->isSimulated() && !g_simulator.speedUpSimulation && BUGGIFY_WITH_PROB(0.0001)) {
wait(delay(6.0));
}
degraded.cancel();
wait(self->queueCommitEnd.whenAtLeast(commitNumber-1));
//Calling check_yield instead of yield to avoid a destruction ordering problem in simulation
@ -2328,8 +2349,8 @@ ACTOR Future<Void> tLogStart( TLogData* self, InitializeTLogRequest req, Localit
}
// New tLog (if !recoverFrom.size()) or restore from network
ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQueue, Reference<AsyncVar<ServerDBInfo>> db, LocalityData locality, PromiseStream<InitializeTLogRequest> tlogRequests, UID tlogId, bool restoreFromDisk, Promise<Void> oldLog, Promise<Void> recovered ) {
state TLogData self( tlogId, persistentData, persistentQueue, db );
ACTOR Future<Void> tLog( IKeyValueStore* persistentData, IDiskQueue* persistentQueue, Reference<AsyncVar<ServerDBInfo>> db, LocalityData locality, PromiseStream<InitializeTLogRequest> tlogRequests, UID tlogId, bool restoreFromDisk, Promise<Void> oldLog, Promise<Void> recovered, Reference<AsyncVar<bool>> degraded ) {
state TLogData self( tlogId, persistentData, persistentQueue, db, degraded );
state Future<Void> error = actorCollection( self.sharedActors.getFuture() );
TraceEvent("SharedTlog", tlogId);

25
fdbserver/TagPartitionedLogSystem.actor.cpp
View File

@ -531,12 +531,12 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
}
if(begin >= lastBegin) {
TraceEvent("TLogPeekRemoteBestOnly", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("BestSet", bestSet).detail("BestSetStart", lastBegin).detail("LogRouterIds", tLogs[bestSet]->logRouterString());
return Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, begin, getPeekEnd(), false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) );
return Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, begin, getPeekEnd(), false, std::vector<LocalityData>(), Reference<IReplicationPolicy>(), 0 ) );
} else {
std::vector< Reference<ILogSystem::IPeekCursor> > cursors;
std::vector< LogMessageVersion > epochEnds;
TraceEvent("TLogPeekRemoteAddingBest", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("BestSet", bestSet).detail("BestSetStart", lastBegin).detail("LogRouterIds", tLogs[bestSet]->logRouterString());
cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, lastBegin, getPeekEnd(), false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ) );
cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( tLogs[bestSet]->logRouters, -1, (int)tLogs[bestSet]->logRouters.size(), tag, lastBegin, getPeekEnd(), false, std::vector<LocalityData>(), Reference<IReplicationPolicy>(), 0 ) ) );
int i = 0;
while(begin < lastBegin) {
if(i == oldLogData.size()) {
@ -565,7 +565,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
TraceEvent("TLogPeekRemoteAddingOldBest", dbgid).detail("Tag", tag.toString()).detail("Begin", begin).detail("BestOldSet", bestOldSet).detail("LogRouterIds", oldLogData[i].tLogs[bestOldSet]->logRouterString())
.detail("LastBegin", lastBegin).detail("ThisBegin", thisBegin).detail("BestStartVer", oldLogData[i].tLogs[bestOldSet]->startVersion);
cursors.push_back( Reference<ILogSystem::MergedPeekCursor>( new ILogSystem::MergedPeekCursor( oldLogData[i].tLogs[bestOldSet]->logRouters, -1, (int)oldLogData[i].tLogs[bestOldSet]->logRouters.size(), tag,
thisBegin, lastBegin, false, std::vector<LocalityData>(), IRepPolicyRef(), 0 ) ) );
thisBegin, lastBegin, false, std::vector<LocalityData>(), Reference<IReplicationPolicy>(), 0 ) ) );
epochEnds.push_back(LogMessageVersion(lastBegin));
lastBegin = thisBegin;
}
@ -959,24 +959,17 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
wait( quorum( alive, std::min(logSet->tLogReplicationFactor, numPresent - logSet->tLogWriteAntiQuorum) ) );
state Reference<LocalityGroup> locked(new LocalityGroup());
state std::vector<bool> responded(alive.size());
for (int i = 0; i < alive.size(); i++) {
responded[i] = false;
}
state std::vector<LocalityEntry> aliveEntries;
state std::vector<bool> responded(alive.size(), false);
loop {
for (int i = 0; i < alive.size(); i++) {
if (!responded[i] && alive[i].isReady() && !alive[i].isError()) {
locked->add(logSet->tLogLocalities[i]);
aliveEntries.push_back(logSet->logEntryArray[i]);
responded[i] = true;
}
}
bool quorum_obtained = locked->validate(logSet->tLogPolicy);
// We intentionally skip considering antiquorums, as the CPU cost of doing so is prohibitive.
if (logSet->tLogReplicationFactor == 1 && locked->size() > 0) {
ASSERT(quorum_obtained);
}
if (quorum_obtained) {
if (logSet->satisfiesPolicy(aliveEntries)) {
return Void();
}
@ -1566,7 +1559,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
}
ACTOR static Future<Void> recruitOldLogRouters( TagPartitionedLogSystem* self, vector<WorkerInterface> workers, LogEpoch recoveryCount, int8_t locality, Version startVersion,
std::vector<LocalityData> tLogLocalities, IRepPolicyRef tLogPolicy, bool forRemote ) {
std::vector<LocalityData> tLogLocalities, Reference<IReplicationPolicy> tLogPolicy, bool forRemote ) {
state vector<vector<Future<TLogInterface>>> logRouterInitializationReplies;
state vector<Future<TLogInterface>> allReplies;
int nextRouter = 0;

40
fdbserver/WorkerInterface.actor.h
View File

@ -28,6 +28,7 @@
#include "fdbserver/DataDistributorInterface.h"
#include "fdbserver/MasterInterface.h"
#include "fdbserver/TLogInterface.h"
#include "fdbserver/RatekeeperInterface.h"
#include "fdbserver/ResolverInterface.h"
#include "fdbclient/StorageServerInterface.h"
#include "fdbserver/TesterInterface.actor.h"
@ -46,6 +47,7 @@ struct WorkerInterface {
RequestStream< struct RecruitMasterRequest > master;
RequestStream< struct InitializeMasterProxyRequest > masterProxy;
RequestStream< struct InitializeDataDistributorRequest > dataDistributor;
RequestStream< struct InitializeRatekeeperRequest > ratekeeper;
RequestStream< struct InitializeResolverRequest > resolver;
RequestStream< struct InitializeStorageRequest > storage;
RequestStream< struct InitializeLogRouterRequest > logRouter;
@ -68,7 +70,21 @@ struct WorkerInterface {
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, clientInterface, locality, tLog, master, masterProxy, dataDistributor, resolver, storage, logRouter, debugPing, coordinationPing, waitFailure, setMetricsRate, eventLogRequest, traceBatchDumpRequest, testerInterface, diskStoreRequest);
serializer(ar, clientInterface, locality, tLog, master, masterProxy, dataDistributor, ratekeeper, resolver, storage, logRouter, debugPing, coordinationPing, waitFailure, setMetricsRate, eventLogRequest, traceBatchDumpRequest, testerInterface, diskStoreRequest);
}
};
struct WorkerDetails {
WorkerInterface interf;
ProcessClass processClass;
bool degraded;
WorkerDetails() : degraded(false) {}
WorkerDetails(const WorkerInterface& interf, ProcessClass processClass, bool degraded) : interf(interf), processClass(processClass), degraded(degraded) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, interf, processClass, degraded);
}
};
@ -104,7 +120,7 @@ struct InitializeLogRouterRequest {
Tag routerTag;
Version startVersion;
std::vector<LocalityData> tLogLocalities;
IRepPolicyRef tLogPolicy;
Reference<IReplicationPolicy> tLogPolicy;
int8_t locality;
ReplyPromise<struct TLogInterface> reply;
@ -145,12 +161,26 @@ struct InitializeDataDistributorRequest {
UID reqId;
ReplyPromise<DataDistributorInterface> reply;
InitializeDataDistributorRequest() {}
explicit InitializeDataDistributorRequest(UID uid) : reqId(uid) {}
template <class Ar>
void serialize( Ar& ar ) {
serializer(ar, reqId, reply);
}
};
struct InitializeRatekeeperRequest {
UID reqId;
ReplyPromise<RatekeeperInterface> reply;
InitializeRatekeeperRequest() {}
explicit InitializeRatekeeperRequest(UID uid) : reqId(uid) {}
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, reqId, reply);
}
};
struct InitializeResolverRequest {
uint64_t recoveryCount;
int proxyCount;
@ -300,6 +330,7 @@ struct Role {
static const Role TESTER;
static const Role LOG_ROUTER;
static const Role DATA_DISTRIBUTOR;
static const Role RATE_KEEPER;
std::string roleName;
std::string abbreviation;
@ -352,7 +383,7 @@ ACTOR Future<Void> masterProxyServer(MasterProxyInterface proxy, InitializeMaste
ACTOR Future<Void> tLog(IKeyValueStore* persistentData, IDiskQueue* persistentQueue,
Reference<AsyncVar<ServerDBInfo>> db, LocalityData locality,
PromiseStream<InitializeTLogRequest> tlogRequests, UID tlogId, bool restoreFromDisk,
Promise<Void> oldLog, Promise<Void> recovered); // changes tli->id() to be the recovered ID
Promise<Void> oldLog, Promise<Void> recovered, Reference<AsyncVar<bool>> degraded); // changes tli->id() to be the recovered ID
ACTOR Future<Void> monitorServerDBInfo(Reference<AsyncVar<Optional<ClusterControllerFullInterface>>> ccInterface,
Reference<ClusterConnectionFile> ccf, LocalityData locality,
Reference<AsyncVar<ServerDBInfo>> dbInfo);
@ -361,6 +392,7 @@ ACTOR Future<Void> resolver(ResolverInterface proxy, InitializeResolverRequest i
ACTOR Future<Void> logRouter(TLogInterface interf, InitializeLogRouterRequest req,
Reference<AsyncVar<ServerDBInfo>> db);
ACTOR Future<Void> dataDistributor(DataDistributorInterface ddi, Reference<AsyncVar<ServerDBInfo>> db);
ACTOR Future<Void> rateKeeper(RatekeeperInterface rki, Reference<AsyncVar<ServerDBInfo>> db);
void registerThreadForProfiling();
void updateCpuProfiler(ProfilerRequest req);
@ -373,7 +405,7 @@ namespace oldTLog_6_0 {
ACTOR Future<Void> tLog(IKeyValueStore* persistentData, IDiskQueue* persistentQueue,
Reference<AsyncVar<ServerDBInfo>> db, LocalityData locality,
PromiseStream<InitializeTLogRequest> tlogRequests, UID tlogId, bool restoreFromDisk,
Promise<Void> oldLog, Promise<Void> recovered);
Promise<Void> oldLog, Promise<Void> recovered, Reference<AsyncVar<bool>> degraded);
}
typedef decltype(&tLog) TLogFn;

2
fdbserver/fdbserver.vcxproj
View File

@ -191,7 +191,7 @@
<EnableCompile>false</EnableCompile>
</ActorCompiler>
<ClInclude Include="QuietDatabase.h" />
<ClInclude Include="Ratekeeper.h" />
<ClInclude Include="RatekeeperInterface.h" />
<ClInclude Include="RecoveryState.h" />
<ClInclude Include="ResolverInterface.h" />
<ClInclude Include="RestoreInterface.h" />

3
fdbserver/fdbserver.vcxproj.filters
View File

@ -310,6 +310,7 @@
<ItemGroup>
<ClInclude Include="ConflictSet.h" />
<ClInclude Include="DataDistribution.actor.h" />
<ClInclude Include="DataDistributorInterface.h" />
<ClInclude Include="MoveKeys.actor.h" />
<ClInclude Include="pubsub.h" />
<ClInclude Include="Knobs.h" />
@ -343,7 +344,7 @@
</ClInclude>
<ClInclude Include="LeaderElection.h" />
<ClInclude Include="StorageMetrics.h" />
<ClInclude Include="Ratekeeper.h" />
<ClInclude Include="RatekeeperInterface.h" />
<ClInclude Include="Status.h" />
<ClInclude Include="IDiskQueue.h" />
<ClInclude Include="CoroFlow.h" />

1
fdbserver/masterserver.actor.cpp
View File

@ -31,7 +31,6 @@
#include <iterator>
#include "fdbserver/WaitFailure.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/Ratekeeper.h"
#include "fdbserver/ClusterRecruitmentInterface.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/CoordinatedState.h"

6
fdbserver/tester.actor.cpp
View File

@ -1095,11 +1095,11 @@ ACTOR Future<Void> runTests( Reference<AsyncVar<Optional<struct ClusterControlle
int minTestersExpected, StringRef startingConfiguration, LocalityData locality ) {
state int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
state vector<std::pair<WorkerInterface, ProcessClass>> workers;
state vector<WorkerDetails> workers;
loop {
choose {
when( vector<std::pair<WorkerInterface, ProcessClass>> w = wait( cc->get().present() ? brokenPromiseToNever( cc->get().get().getWorkers.getReply( GetWorkersRequest( flags ) ) ) : Never() ) ) {
when( vector<WorkerDetails> w = wait( cc->get().present() ? brokenPromiseToNever( cc->get().get().getWorkers.getReply( GetWorkersRequest( flags ) ) ) : Never() ) ) {
if (w.size() >= minTestersExpected) {
workers = w;
break;
@ -1116,7 +1116,7 @@ ACTOR Future<Void> runTests( Reference<AsyncVar<Optional<struct ClusterControlle
vector<TesterInterface> ts;
for(int i=0; i<workers.size(); i++)
ts.push_back(workers[i].first.testerInterface);
ts.push_back(workers[i].interf.testerInterface);
wait( runTests( cc, ci, ts, tests, startingConfiguration, locality) );
return Void();

40
fdbserver/worker.actor.cpp
View File

@ -349,22 +349,26 @@ ACTOR Future<Void> registrationClient(
WorkerInterface interf,
Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo,
ProcessClass initialClass,
Reference<AsyncVar<Optional<DataDistributorInterface>>> ddInterf) {
Reference<AsyncVar<Optional<DataDistributorInterface>>> ddInterf,
Reference<AsyncVar<Optional<RatekeeperInterface>>> rkInterf,
Reference<AsyncVar<bool>> degraded) {
// Keeps the cluster controller (as it may be re-elected) informed that this worker exists
// The cluster controller uses waitFailureClient to find out if we die, and returns from registrationReply (requiring us to re-register)
// The registration request piggybacks optional distributor interface if it exists.
state Generation requestGeneration = 0;
state ProcessClass processClass = initialClass;
loop {
RegisterWorkerRequest request(interf, initialClass, processClass, asyncPriorityInfo->get(), requestGeneration++, ddInterf->get());
RegisterWorkerRequest request(interf, initialClass, processClass, asyncPriorityInfo->get(), requestGeneration++, ddInterf->get(), rkInterf->get(), degraded->get());
Future<RegisterWorkerReply> registrationReply = ccInterface->get().present() ? brokenPromiseToNever( ccInterface->get().get().registerWorker.getReply(request) ) : Never();
choose {
when ( RegisterWorkerReply reply = wait( registrationReply )) {
processClass = reply.processClass;
asyncPriorityInfo->set( reply.priorityInfo );
}
when ( wait( ccInterface->onChange() )) { }
when ( wait( ccInterface->onChange() )) {}
when ( wait( ddInterf->onChange() ) ) {}
when ( wait( rkInterf->onChange() ) ) {}
when ( wait( degraded->onChange() ) ) {}
}
}
}
@ -610,6 +614,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
Reference<AsyncVar<ClusterControllerPriorityInfo>> asyncPriorityInfo, ProcessClass initialClass, std::string folder, int64_t memoryLimit, std::string metricsConnFile, std::string metricsPrefix, Promise<Void> recoveredDiskFiles) {
state PromiseStream< ErrorInfo > errors;
state Reference<AsyncVar<Optional<DataDistributorInterface>>> ddInterf( new AsyncVar<Optional<DataDistributorInterface>>() );
state Reference<AsyncVar<Optional<RatekeeperInterface>>> rkInterf( new AsyncVar<Optional<RatekeeperInterface>>() );
state Future<Void> handleErrors = workerHandleErrors( errors.getFuture() ); // Needs to be stopped last
state ActorCollection errorForwarders(false);
state Future<Void> loggingTrigger = Void();
@ -619,6 +624,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
state WorkerCache<InitializeStorageReply> storageCache;
state Reference<AsyncVar<ServerDBInfo>> dbInfo( new AsyncVar<ServerDBInfo>(ServerDBInfo()) );
state Future<Void> metricsLogger;
state Reference<AsyncVar<bool>> degraded( new AsyncVar<bool>(false) );
// tLogFnForOptions() can return a function that doesn't correspond with the FDB version that the
// TLogVersion represents. This can be done if the newer TLog doesn't support a requested option.
// As (store type, spill type) can map to the same TLogFn across multiple TLogVersions, we need to
@ -643,6 +649,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
}
}
errorForwarders.add( resetAfter(degraded, SERVER_KNOBS->TLOG_DEGRADED_RESET_INTERVAL, false));
errorForwarders.add( loadedPonger( interf.debugPing.getFuture() ) );
errorForwarders.add( waitFailureServer( interf.waitFailure.getFuture() ) );
errorForwarders.add( monitorServerDBInfo( ccInterface, connFile, locality, dbInfo ) );
@ -735,7 +742,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
auto& logData = sharedLogs[std::make_tuple(s.tLogOptions.version, s.storeType, s.tLogOptions.spillType)];
// FIXME: Shouldn't if logData.first isValid && !isReady, shouldn't we
// be sending a fake InitializeTLogRequest rather than calling tLog() ?
Future<Void> tl = tLogFn( kv, queue, dbInfo, locality, !logData.first.isValid() || logData.first.isReady() ? logData.second : PromiseStream<InitializeTLogRequest>(), s.storeID, true, oldLog, recovery );
Future<Void> tl = tLogFn( kv, queue, dbInfo, locality, !logData.first.isValid() || logData.first.isReady() ? logData.second : PromiseStream<InitializeTLogRequest>(), s.storeID, true, oldLog, recovery, degraded );
recoveries.push_back(recovery.getFuture());
tl = handleIOErrors( tl, kv, s.storeID );
@ -756,7 +763,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
wait(waitForAll(recoveries));
recoveredDiskFiles.send(Void());
errorForwarders.add( registrationClient( ccInterface, interf, asyncPriorityInfo, initialClass, ddInterf ) );
errorForwarders.add( registrationClient( ccInterface, interf, asyncPriorityInfo, initialClass, ddInterf, rkInterf, degraded ) );
TraceEvent("RecoveriesComplete", interf.id());
@ -829,6 +836,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
TEST(true); // Recruited while already a data distributor.
} else {
startRole( Role::DATA_DISTRIBUTOR, recruited.id(), interf.id() );
DUMPTOKEN( recruited.waitFailure );
Future<Void> dataDistributorProcess = dataDistributor( recruited, dbInfo );
errorForwarders.add( forwardError( errors, Role::DATA_DISTRIBUTOR, recruited.id(), setWhenDoneOrError( dataDistributorProcess, ddInterf, Optional<DataDistributorInterface>() ) ) );
@ -837,6 +845,25 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
TraceEvent("DataDistributorReceived", req.reqId).detail("DataDistributorId", recruited.id());
req.reply.send(recruited);
}
when ( InitializeRatekeeperRequest req = waitNext(interf.ratekeeper.getFuture()) ) {
RatekeeperInterface recruited(locality);
recruited.initEndpoints();
if (rkInterf->get().present()) {
recruited = rkInterf->get().get();
TEST(true); // Recruited while already a ratekeeper.
} else {
startRole(Role::RATE_KEEPER, recruited.id(), interf.id());
DUMPTOKEN( recruited.waitFailure );
DUMPTOKEN( recruited.getRateInfo );
Future<Void> ratekeeper = rateKeeper( recruited, dbInfo );
errorForwarders.add( forwardError( errors, Role::RATE_KEEPER, recruited.id(), setWhenDoneOrError( ratekeeper, rkInterf, Optional<RatekeeperInterface>() ) ) );
rkInterf->set(Optional<RatekeeperInterface>(recruited));
}
TraceEvent("Ratekeeper_InitRequest", req.reqId).detail("RatekeeperId", recruited.id());
req.reply.send(recruited);
}
when( InitializeTLogRequest req = waitNext(interf.tLog.getFuture()) ) {
// For now, there's a one-to-one mapping of spill type to TLogVersion.
// With future work, a particular version of the TLog can support multiple
@ -868,7 +895,7 @@ ACTOR Future<Void> workerServer( Reference<ClusterConnectionFile> connFile, Refe
filesClosed.add( data->onClosed() );
filesClosed.add( queue->onClosed() );
logData.first = tLogFn( data, queue, dbInfo, locality, logData.second, logId, false, Promise<Void>(), Promise<Void>() );
logData.first = tLogFn( data, queue, dbInfo, locality, logData.second, logId, false, Promise<Void>(), Promise<Void>(), degraded );
logData.first = handleIOErrors( logData.first, data, logId );
logData.first = handleIOErrors( logData.first, queue, logId );
errorForwarders.add( forwardError( errors, Role::SHARED_TRANSACTION_LOG, logId, logData.first ) );
@ -1244,3 +1271,4 @@ const Role Role::CLUSTER_CONTROLLER("ClusterController", "CC");
const Role Role::TESTER("Tester", "TS");
const Role Role::LOG_ROUTER("LogRouter", "LR");
const Role Role::DATA_DISTRIBUTOR("DataDistributor", "DD");
const Role Role::RATE_KEEPER("RateKeeper", "RK");

87
fdbserver/workloads/ConsistencyCheck.actor.cpp
View File

@ -1089,26 +1089,26 @@ struct ConsistencyCheckWorkload : TestWorkload
//Returns false if any worker that should have a storage server does not have one
ACTOR Future<bool> checkForStorage(Database cx, DatabaseConfiguration configuration, ConsistencyCheckWorkload *self)
{
state vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
state vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
state vector<StorageServerInterface> storageServers = wait( getStorageServers( cx ) );
std::set<Optional<Key>> missingStorage;
for( int i = 0; i < workers.size(); i++ ) {
if( !configuration.isExcludedServer(workers[i].first.address()) &&
( workers[i].second == ProcessClass::StorageClass || workers[i].second == ProcessClass::UnsetClass ) ) {
if( !configuration.isExcludedServer(workers[i].interf.address()) &&
( workers[i].processClass == ProcessClass::StorageClass || workers[i].processClass == ProcessClass::UnsetClass ) ) {
bool found = false;
for( int j = 0; j < storageServers.size(); j++ ) {
if( storageServers[j].address() == workers[i].first.address() ) {
if( storageServers[j].address() == workers[i].interf.address() ) {
found = true;
break;
}
}
if( !found ) {
TraceEvent("ConsistencyCheck_NoStorage")
.detail("Address", workers[i].first.address())
.detail("Address", workers[i].interf.address())
.detail("ProcessClassEqualToStorageClass",
(int)(workers[i].second == ProcessClass::StorageClass));
missingStorage.insert(workers[i].first.locality.dcId());
(int)(workers[i].processClass == ProcessClass::StorageClass));
missingStorage.insert(workers[i].interf.locality.dcId());
}
}
}
@ -1125,12 +1125,12 @@ struct ConsistencyCheckWorkload : TestWorkload
}
ACTOR Future<bool> checkForExtraDataStores(Database cx, ConsistencyCheckWorkload *self) {
state vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
state vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
state vector<StorageServerInterface> storageServers = wait( getStorageServers( cx ) );
auto& db = self->dbInfo->get();
state std::vector<TLogInterface> logs = db.logSystemConfig.allPresentLogs();
state std::vector<std::pair<WorkerInterface, ProcessClass>>::iterator itr;
state std::vector<WorkerDetails>::iterator itr;
state bool foundExtraDataStore = false;
state std::map<NetworkAddress, std::set<UID>> statefulProcesses;
@ -1142,19 +1142,19 @@ struct ConsistencyCheckWorkload : TestWorkload
}
for(itr = workers.begin(); itr != workers.end(); ++itr) {
ErrorOr<Standalone<VectorRef<UID>>> stores = wait(itr->first.diskStoreRequest.getReplyUnlessFailedFor(DiskStoreRequest(false), 2, 0));
ErrorOr<Standalone<VectorRef<UID>>> stores = wait(itr->interf.diskStoreRequest.getReplyUnlessFailedFor(DiskStoreRequest(false), 2, 0));
if(stores.isError()) {
TraceEvent("ConsistencyCheck_GetDataStoreFailure").error(stores.getError()).detail("Address", itr->first.address());
TraceEvent("ConsistencyCheck_GetDataStoreFailure").error(stores.getError()).detail("Address", itr->interf.address());
self->testFailure("Failed to get data stores");
return false;
}
for(auto id : stores.get()) {
if(!statefulProcesses[itr->first.address()].count(id)) {
TraceEvent("ConsistencyCheck_ExtraDataStore").detail("Address", itr->first.address()).detail("DataStoreID", id);
if(!statefulProcesses[itr->interf.address()].count(id)) {
TraceEvent("ConsistencyCheck_ExtraDataStore").detail("Address", itr->interf.address()).detail("DataStoreID", id);
if(g_network->isSimulated()) {
TraceEvent("ConsistencyCheck_RebootProcess").detail("Address", itr->first.address()).detail("DataStoreID", id);
g_simulator.rebootProcess(g_simulator.getProcessByAddress(itr->first.address()), ISimulator::RebootProcess);
TraceEvent("ConsistencyCheck_RebootProcess").detail("Address", itr->interf.address()).detail("DataStoreID", id);
g_simulator.rebootProcess(g_simulator.getProcessByAddress(itr->interf.address()), ISimulator::RebootProcess);
}
foundExtraDataStore = true;
@ -1172,17 +1172,17 @@ struct ConsistencyCheckWorkload : TestWorkload
//Returns true if the worker at the given address has the specified machineClass or has an unset class
//The interfaceType paramater is used in a TraceEvent, should be something like (Master, MasterProxy, StorageServer, ...)
bool workerHasClass(vector<std::pair<WorkerInterface, ProcessClass>> workers, NetworkAddress address, ProcessClass::ClassType machineClass, std::string interfaceType)
bool workerHasClass(vector<WorkerDetails> workers, NetworkAddress address, ProcessClass::ClassType machineClass, std::string interfaceType)
{
//Search all workers until the correct one is found
for(int i = 0; i < workers.size(); i++)
{
if(workers[i].first.address() == address)
if(workers[i].interf.address() == address)
{
if(workers[i].second == machineClass || workers[i].second == ProcessClass::UnsetClass)
if(workers[i].processClass == machineClass || workers[i].processClass == ProcessClass::UnsetClass)
return true;
TraceEvent("ConsistencyCheck_InvalidClassType").detail("RequestedClass", workers[i].second.toString())
TraceEvent("ConsistencyCheck_InvalidClassType").detail("RequestedClass", workers[i].processClass.toString())
.detail("ActualClass", ProcessClass(machineClass, ProcessClass::CommandLineSource).toString()).detail("InterfaceType", interfaceType);
return false;
@ -1200,16 +1200,16 @@ struct ConsistencyCheckWorkload : TestWorkload
if(g_simulator.extraDB)
return true;
vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
std::set<NetworkAddress> workerAddresses;
for( auto it : workers ) {
ISimulator::ProcessInfo* info = g_simulator.getProcessByAddress(it.first.address());
ISimulator::ProcessInfo* info = g_simulator.getProcessByAddress(it.interf.address());
if(!info || info->failed) {
TraceEvent("ConsistencyCheck_FailedWorkerInList").detail("Addr", it.first.address());
TraceEvent("ConsistencyCheck_FailedWorkerInList").detail("Addr", it.interf.address());
return false;
}
workerAddresses.insert( NetworkAddress(it.first.address().ip, it.first.address().port, true, false) );
workerAddresses.insert( NetworkAddress(it.interf.address().ip, it.interf.address().port, true, false) );
}
vector<ISimulator::ProcessInfo*> all = g_simulator.getAllProcesses();
@ -1281,34 +1281,33 @@ struct ConsistencyCheckWorkload : TestWorkload
}
}
typedef std::pair<WorkerInterface, ProcessClass> WorkerClassPair;
//Returns true if all machines in the cluster that specified a desired class are operating in that class
ACTOR Future<bool> checkUsingDesiredClasses(Database cx, ConsistencyCheckWorkload *self) {
state Optional<Key> expectedPrimaryDcId;
state Optional<Key> expectedRemoteDcId;
state DatabaseConfiguration config = wait(getDatabaseConfiguration(cx));
state vector<WorkerClassPair> allWorkers = wait(getWorkers(self->dbInfo));
state vector<WorkerClassPair> nonExcludedWorkers = wait(getWorkers(self->dbInfo, GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY));
state vector<WorkerDetails> allWorkers = wait(getWorkers(self->dbInfo));
state vector<WorkerDetails> nonExcludedWorkers = wait(getWorkers(self->dbInfo, GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY));
auto& db = self->dbInfo->get();
std::map<NetworkAddress, WorkerClassPair> allWorkerProcessMap;
std::map<NetworkAddress, WorkerDetails> allWorkerProcessMap;
std::map<Optional<Key>, std::vector<ProcessClass::ClassType>> dcToAllClassTypes;
for (auto worker : allWorkers) {
allWorkerProcessMap[worker.first.address()] = worker;
Optional<Key> dc = worker.first.locality._data[LocalityData::keyDcId];
allWorkerProcessMap[worker.interf.address()] = worker;
Optional<Key> dc = worker.interf.locality._data[LocalityData::keyDcId];
if (!dcToAllClassTypes.count(dc))
dcToAllClassTypes.insert({});
dcToAllClassTypes[dc].push_back(worker.second.classType());
dcToAllClassTypes[dc].push_back(worker.processClass.classType());
}
std::map<NetworkAddress, WorkerClassPair> nonExcludedWorkerProcessMap;
std::map<NetworkAddress, WorkerDetails> nonExcludedWorkerProcessMap;
std::map<Optional<Key>, std::vector<ProcessClass::ClassType>> dcToNonExcludedClassTypes;
for (auto worker : nonExcludedWorkers) {
nonExcludedWorkerProcessMap[worker.first.address()] = worker;
Optional<Key> dc = worker.first.locality._data[LocalityData::keyDcId];
nonExcludedWorkerProcessMap[worker.interf.address()] = worker;
Optional<Key> dc = worker.interf.locality._data[LocalityData::keyDcId];
if (!dcToNonExcludedClassTypes.count(dc))
dcToNonExcludedClassTypes.insert({});
dcToNonExcludedClassTypes[dc].push_back(worker.second.classType());
dcToNonExcludedClassTypes[dc].push_back(worker.processClass.classType());
}
if (!allWorkerProcessMap.count(db.clusterInterface.clientInterface.address())) {
@ -1320,8 +1319,8 @@ struct ConsistencyCheckWorkload : TestWorkload
return false;
}
Optional<Key> ccDcId = allWorkerProcessMap[db.clusterInterface.clientInterface.address()].first.locality._data[LocalityData::keyDcId];
Optional<Key> masterDcId = allWorkerProcessMap[db.master.address()].first.locality._data[LocalityData::keyDcId];
Optional<Key> ccDcId = allWorkerProcessMap[db.clusterInterface.clientInterface.address()].interf.locality._data[LocalityData::keyDcId];
Optional<Key> masterDcId = allWorkerProcessMap[db.master.address()].interf.locality._data[LocalityData::keyDcId];
if (ccDcId != masterDcId) {
TraceEvent("ConsistencyCheck_CCAndMasterNotInSameDC").detail("ClusterControllerDcId", getOptionalString(ccDcId)).detail("MasterDcId", getOptionalString(masterDcId));
@ -1351,8 +1350,8 @@ struct ConsistencyCheckWorkload : TestWorkload
// Check CC
ProcessClass::Fitness bestClusterControllerFitness = getBestAvailableFitness(dcToNonExcludedClassTypes[ccDcId], ProcessClass::ClusterController);
if (!nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) || nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].second.machineClassFitness(ProcessClass::ClusterController) != bestClusterControllerFitness) {
TraceEvent("ConsistencyCheck_ClusterControllerNotBest").detail("BestClusterControllerFitness", bestClusterControllerFitness).detail("ExistingClusterControllerFit", nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) ? nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].second.machineClassFitness(ProcessClass::ClusterController) : -1);
if (!nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) || nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].processClass.machineClassFitness(ProcessClass::ClusterController) != bestClusterControllerFitness) {
TraceEvent("ConsistencyCheck_ClusterControllerNotBest").detail("BestClusterControllerFitness", bestClusterControllerFitness).detail("ExistingClusterControllerFit", nonExcludedWorkerProcessMap.count(db.clusterInterface.clientInterface.address()) ? nonExcludedWorkerProcessMap[db.clusterInterface.clientInterface.address()].processClass.machineClassFitness(ProcessClass::ClusterController) : -1);
return false;
}
@ -1365,16 +1364,16 @@ struct ConsistencyCheckWorkload : TestWorkload
}
}
if ((!nonExcludedWorkerProcessMap.count(db.master.address()) && bestMasterFitness != ProcessClass::ExcludeFit) || nonExcludedWorkerProcessMap[db.master.address()].second.machineClassFitness(ProcessClass::Master) != bestMasterFitness) {
TraceEvent("ConsistencyCheck_MasterNotBest").detail("BestMasterFitness", bestMasterFitness).detail("ExistingMasterFit", nonExcludedWorkerProcessMap.count(db.master.address()) ? nonExcludedWorkerProcessMap[db.master.address()].second.machineClassFitness(ProcessClass::Master) : -1);
if ((!nonExcludedWorkerProcessMap.count(db.master.address()) && bestMasterFitness != ProcessClass::ExcludeFit) || nonExcludedWorkerProcessMap[db.master.address()].processClass.machineClassFitness(ProcessClass::Master) != bestMasterFitness) {
TraceEvent("ConsistencyCheck_MasterNotBest").detail("BestMasterFitness", bestMasterFitness).detail("ExistingMasterFit", nonExcludedWorkerProcessMap.count(db.master.address()) ? nonExcludedWorkerProcessMap[db.master.address()].processClass.machineClassFitness(ProcessClass::Master) : -1);
return false;
}
// Check proxy
ProcessClass::Fitness bestMasterProxyFitness = getBestAvailableFitness(dcToNonExcludedClassTypes[masterDcId], ProcessClass::Proxy);
for (auto masterProxy : db.client.proxies) {
if (!nonExcludedWorkerProcessMap.count(masterProxy.address()) || nonExcludedWorkerProcessMap[masterProxy.address()].second.machineClassFitness(ProcessClass::Proxy) != bestMasterProxyFitness) {
TraceEvent("ConsistencyCheck_ProxyNotBest").detail("BestMasterProxyFitness", bestMasterProxyFitness).detail("ExistingMasterProxyFitness", nonExcludedWorkerProcessMap.count(masterProxy.address()) ? nonExcludedWorkerProcessMap[masterProxy.address()].second.machineClassFitness(ProcessClass::Proxy) : -1);
if (!nonExcludedWorkerProcessMap.count(masterProxy.address()) || nonExcludedWorkerProcessMap[masterProxy.address()].processClass.machineClassFitness(ProcessClass::Proxy) != bestMasterProxyFitness) {
TraceEvent("ConsistencyCheck_ProxyNotBest").detail("BestMasterProxyFitness", bestMasterProxyFitness).detail("ExistingMasterProxyFitness", nonExcludedWorkerProcessMap.count(masterProxy.address()) ? nonExcludedWorkerProcessMap[masterProxy.address()].processClass.machineClassFitness(ProcessClass::Proxy) : -1);
return false;
}
}
@ -1382,8 +1381,8 @@ struct ConsistencyCheckWorkload : TestWorkload
// Check resolver
ProcessClass::Fitness bestResolverFitness = getBestAvailableFitness(dcToNonExcludedClassTypes[masterDcId], ProcessClass::Resolver);
for (auto resolver : db.resolvers) {
if (!nonExcludedWorkerProcessMap.count(resolver.address()) || nonExcludedWorkerProcessMap[resolver.address()].second.machineClassFitness(ProcessClass::Resolver) != bestResolverFitness) {
TraceEvent("ConsistencyCheck_ResolverNotBest").detail("BestResolverFitness", bestResolverFitness).detail("ExistingResolverFitness", nonExcludedWorkerProcessMap.count(resolver.address()) ? nonExcludedWorkerProcessMap[resolver.address()].second.machineClassFitness(ProcessClass::Resolver) : -1);
if (!nonExcludedWorkerProcessMap.count(resolver.address()) || nonExcludedWorkerProcessMap[resolver.address()].processClass.machineClassFitness(ProcessClass::Resolver) != bestResolverFitness) {
TraceEvent("ConsistencyCheck_ResolverNotBest").detail("BestResolverFitness", bestResolverFitness).detail("ExistingResolverFitness", nonExcludedWorkerProcessMap.count(resolver.address()) ? nonExcludedWorkerProcessMap[resolver.address()].processClass.machineClassFitness(ProcessClass::Resolver) : -1);
return false;
}
}

6
fdbserver/workloads/CpuProfiler.actor.cpp
View File

@ -69,11 +69,11 @@ struct CpuProfilerWorkload : TestWorkload
//If we are turning the profiler on, get a list of workers in the system
if(enabled)
{
vector<std::pair<WorkerInterface, ProcessClass>> _workers = wait( getWorkers( self->dbInfo ) );
vector<WorkerDetails> _workers = wait( getWorkers( self->dbInfo ) );
vector<WorkerInterface> workers;
for(int i = 0; i < _workers.size(); i++) {
if (self->roles.empty() || std::find(self->roles.cbegin(), self->roles.cend(), _workers[i].second.toString()) != self->roles.cend()) {
workers.push_back(_workers[i].first);
if (self->roles.empty() || std::find(self->roles.cbegin(), self->roles.cend(), _workers[i].processClass.toString()) != self->roles.cend()) {
workers.push_back(_workers[i].interf);
}
}
self->profilingWorkers = workers;

4
fdbserver/workloads/LogMetrics.actor.cpp
View File

@ -54,12 +54,12 @@ struct LogMetricsWorkload : TestWorkload {
ACTOR Future<Void> setSystemRate( LogMetricsWorkload *self, Database cx, uint32_t rate ) {
// set worker interval and ss interval
state BinaryWriter br(Unversioned());
vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
//vector<Future<Void>> replies;
TraceEvent("RateChangeTrigger");
SetMetricsLogRateRequest req(rate);
for(int i = 0; i < workers.size(); i++) {
workers[i].first.setMetricsRate.send( req );
workers[i].interf.setMetricsRate.send( req );
}
//wait( waitForAll( replies ) );

6
fdbserver/workloads/Performance.actor.cpp
View File

@ -103,11 +103,11 @@ struct PerformanceWorkload : TestWorkload {
//FIXME: does not use testers which are recruited on workers
ACTOR Future<vector<TesterInterface>> getTesters( PerformanceWorkload *self) {
state vector<std::pair<WorkerInterface, ProcessClass>> workers;
state vector<WorkerDetails> workers;
loop {
choose {
when( vector<std::pair<WorkerInterface, ProcessClass>> w = wait( brokenPromiseToNever( self->dbInfo->get().clusterInterface.getWorkers.getReply( GetWorkersRequest( GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY ) ) ) ) ) {
when( vector<WorkerDetails> w = wait( brokenPromiseToNever( self->dbInfo->get().clusterInterface.getWorkers.getReply( GetWorkersRequest( GetWorkersRequest::TESTER_CLASS_ONLY | GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY ) ) ) ) ) {
workers = w;
break;
}
@ -117,7 +117,7 @@ struct PerformanceWorkload : TestWorkload {
vector<TesterInterface> ts;
for(int i=0; i<workers.size(); i++)
ts.push_back(workers[i].first.testerInterface);
ts.push_back(workers[i].interf.testerInterface);
return ts;
}

8
fdbserver/workloads/Ping.actor.cpp
View File

@ -180,10 +180,10 @@ struct PingWorkload : TestWorkload {
}
ACTOR Future<Void> workerPinger( PingWorkload* self ) {
vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
vector<RequestStream<LoadedPingRequest>> peers;
for(int i=0; i<workers.size(); i++)
peers.push_back( workers[i].first.debugPing );
peers.push_back( workers[i].interf.debugPing );
vector<Future<Void>> pingers;
for(int i=0; i<self->actorCount; i++)
pingers.push_back( self->pinger( self, peers ) );
@ -208,9 +208,9 @@ struct PingWorkload : TestWorkload {
state Future<Void> collection = actorCollection( addActor.getFuture() );
if( self->workerBroadcast ) {
vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
for( int i=0; i<workers.size(); i++ )
endpoints.push_back( workers[i].first.debugPing );
endpoints.push_back( workers[i].interf.debugPing );
} else {
vector<PingWorkloadInterface> peers = wait( self->fetchInterfaces( self, cx ) );
for( int i=0; i<peers.size(); i++ )

4
fdbserver/workloads/ReadWrite.actor.cpp
View File

@ -225,11 +225,11 @@ struct ReadWriteWorkload : KVWorkload {
ACTOR static Future<bool> traceDumpWorkers( Reference<AsyncVar<ServerDBInfo>> db ) {
try {
loop {
ErrorOr<vector<std::pair<WorkerInterface, ProcessClass>>> workerList = wait( db->get().clusterInterface.getWorkers.tryGetReply( GetWorkersRequest() ) );
ErrorOr<vector<WorkerDetails>> workerList = wait( db->get().clusterInterface.getWorkers.tryGetReply( GetWorkersRequest() ) );
if( workerList.present() ) {
std::vector<Future<ErrorOr<Void>>> dumpRequests;
for( int i = 0; i < workerList.get().size(); i++)
dumpRequests.push_back( workerList.get()[i].first.traceBatchDumpRequest.tryGetReply( TraceBatchDumpRequest() ) );
dumpRequests.push_back( workerList.get()[i].interf.traceBatchDumpRequest.tryGetReply( TraceBatchDumpRequest() ) );
wait( waitForAll( dumpRequests ) );
return true;
}

10
fdbserver/workloads/TargetedKill.actor.cpp
View File

@ -61,14 +61,14 @@ struct TargetedKillWorkload : TestWorkload {
return Void();
}
state vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
state vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
int killed = 0;
for( int i = 0; i < workers.size(); i++ ) {
if( workers[i].first.master.getEndpoint().getPrimaryAddress() == address ||
( self->killAllMachineProcesses && workers[i].first.master.getEndpoint().getPrimaryAddress().ip == address.ip && workers[i].second != ProcessClass::TesterClass ) ) {
TraceEvent("WorkerKill").detail("TargetedMachine", address).detail("Worker", workers[i].first.id());
workers[i].first.clientInterface.reboot.send( RebootRequest() );
if( workers[i].interf.master.getEndpoint().getPrimaryAddress() == address ||
( self->killAllMachineProcesses && workers[i].interf.master.getEndpoint().getPrimaryAddress().ip == address.ip && workers[i].processClass != ProcessClass::TesterClass ) ) {
TraceEvent("WorkerKill").detail("TargetedMachine", address).detail("Worker", workers[i].interf.id());
workers[i].interf.clientInterface.reboot.send( RebootRequest() );
}
}

6
fdbserver/workloads/WorkerErrors.actor.cpp
View File

@ -42,10 +42,10 @@ struct WorkerErrorsWorkload : TestWorkload {
virtual void getMetrics( vector<PerfMetric>& m ) {}
ACTOR Future< std::vector< TraceEventFields > > latestEventOnWorkers( std::vector<std::pair<WorkerInterface, ProcessClass>> workers ) {
ACTOR Future< std::vector< TraceEventFields > > latestEventOnWorkers( std::vector<WorkerDetails> workers ) {
state vector<Future<TraceEventFields>> eventTraces;
for(int c = 0; c < workers.size(); c++) {
eventTraces.push_back( workers[c].first.eventLogRequest.getReply( EventLogRequest() ) );
eventTraces.push_back( workers[c].interf.eventLogRequest.getReply( EventLogRequest() ) );
}
wait( timeoutError( waitForAll( eventTraces ), 2.0 ) );
@ -59,7 +59,7 @@ struct WorkerErrorsWorkload : TestWorkload {
}
ACTOR Future<Void> _start(Database cx, WorkerErrorsWorkload *self) {
state vector<std::pair<WorkerInterface, ProcessClass>> workers = wait( getWorkers( self->dbInfo ) );
state vector<WorkerDetails> workers = wait( getWorkers( self->dbInfo ) );
std::vector<TraceEventFields> errors = wait( self->latestEventOnWorkers( workers ) );
for(auto e : errors) {
printf("%s\n", e.toString().c_str());

1
flow/Knobs.cpp
View File

@ -79,6 +79,7 @@ FlowKnobs::FlowKnobs(bool randomize, bool isSimulated) {
init( MIN_SUBMIT, 10 );
init( PAGE_WRITE_CHECKSUM_HISTORY, 0 ); if( randomize && BUGGIFY ) PAGE_WRITE_CHECKSUM_HISTORY = 10000000;
init( DISABLE_POSIX_KERNEL_AIO, 0 );
//AsyncFileNonDurable
init( MAX_PRIOR_MODIFICATION_DELAY, 1.0 ); if( randomize && BUGGIFY ) MAX_PRIOR_MODIFICATION_DELAY = 10.0;

1
flow/Knobs.h
View File

@ -98,6 +98,7 @@ public:
int MIN_SUBMIT;
int PAGE_WRITE_CHECKSUM_HISTORY;
int DISABLE_POSIX_KERNEL_AIO;
//AsyncFileNonDurable
double MAX_PRIOR_MODIFICATION_DELAY;

24
flow/genericactors.actor.h
View File

@ -775,6 +775,30 @@ Future<Void> setAfter( Reference<AsyncVar<T>> var, double time, T val ) {
return Void();
}
ACTOR template <class T>
Future<Void> resetAfter( Reference<AsyncVar<T>> var, double time, T val ) {
state bool isEqual = var->get() == val;
state Future<Void> resetDelay = isEqual ? Never() : delay(time);
loop {
choose {
when( wait( resetDelay ) ) {
var->set( val );
isEqual = true;
resetDelay = Never();
}
when( wait( var->onChange() ) ) {}
}
if( isEqual && var->get() != val ) {
isEqual = false;
resetDelay = delay(time);
}
if( !isEqual && var->get() == val ) {
isEqual = true;
resetDelay = Never();
}
}
}
ACTOR template <class T>
Future<Void> setWhenDoneOrError( Future<Void> condition, Reference<AsyncVar<T>> var, T val ) {
try {

1
flow/network.h
View File

@ -67,6 +67,7 @@ enum {
TaskUnknownEndpoint = 4000,
TaskMoveKeys = 3550,
TaskDataDistributionLaunch = 3530,
TaskRateKeeper = 3510,
TaskDataDistribution = 3500,
TaskDiskWrite = 3010,
TaskUpdateStorage = 3000,