wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

memoize incomplete versionstamp information in Tuples ; add more tests

This commit is contained in:
Alec Grieser 2019-02-27 20:25:30 -08:00
parent 663d750e1d
commit 39fd30330f
No known key found for this signature in database
GPG Key ID: CAF63551C60D3462
6 changed files with 862 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/java/src/main/com/apple/foundationdb/subspace/Subspace.java
View File

@ -46,8 +46,8 @@ import com.apple.foundationdb.tuple.Versionstamp;
* </p>
*/
public class Subspace {
static final Tuple EMPTY_TUPLE = Tuple.from();
static final byte[] EMPTY_BYTES = new byte[0];
private static final Tuple EMPTY_TUPLE = Tuple.from();
private static final byte[] EMPTY_BYTES = new byte[0];
private final byte[] rawPrefix;
@ -248,8 +248,7 @@ public class Subspace {
* @return the {@link Range} of keyspace corresponding to {@code tuple}
*/
public Range range(Tuple tuple) {
Range p = tuple.range();
return new Range(join(rawPrefix, p.begin), join(rawPrefix, p.end));
return tuple.range(rawPrefix);
}
/**

2
bindings/java/src/main/com/apple/foundationdb/tuple/IterableComparator.java
View File

@ -34,7 +34,7 @@ import java.util.Iterator;
* tuple1.compareTo(tuple2)
* == new IterableComparator().compare(tuple1, tuple2)
* == new IterableComparator().compare(tuple1.getItems(), tuple2.getItems()),
* == ByteArrayUtil.compareUnsigned(tuple1.packInternal(), tuple2.packInternal())}
* == ByteArrayUtil.compareUnsigned(tuple1.pack(), tuple2.pack())}
* </pre>
*
* <p>

291
bindings/java/src/main/com/apple/foundationdb/tuple/Tuple.java
View File

@ -21,11 +21,11 @@
package com.apple.foundationdb.tuple;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
@ -69,19 +69,39 @@ import com.apple.foundationdb.Range;
*/
public class Tuple implements Comparable<Tuple>, Iterable<Object> {
private static final IterableComparator comparator = new IterableComparator();
private static final byte[] EMPTY_BYTES = new byte[0];
private List<Object> elements;
private int memoizedHash = 0;
List<Object> elements;
private byte[] packed = null;
private int memoizedHash = 0;
private int memoizedPackedSize = -1;
private final boolean incompleteVersionstamp;
private Tuple(List<? extends Object> elements, Object newItem) {
this(elements);
private Tuple(Tuple original, Object newItem, boolean itemHasIncompleteVersionstamp) {
this.elements = new ArrayList<>(original.elements.size() + 1);
this.elements.addAll(original.elements);
this.elements.add(newItem);
incompleteVersionstamp = original.incompleteVersionstamp || itemHasIncompleteVersionstamp;
}
private Tuple(List<? extends Object> elements) {
this.elements = new ArrayList<>(elements);
private Tuple(List<Object> elements) {
this.elements = elements;
incompleteVersionstamp = TupleUtil.hasIncompleteVersionstamp(elements.stream());
}
/**
* Construct a new empty {@code Tuple}. After creation, items can be added
* with calls to the variations of {@code add()}.
*
* @see #from(Object...)
* @see #fromBytes(byte[])
* @see #fromItems(Iterable)
*/
public Tuple() {
elements = Collections.emptyList();
packed = EMPTY_BYTES;
memoizedPackedSize = 0;
incompleteVersionstamp = false;
}
/**
@ -107,7 +127,10 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
!(o instanceof Versionstamp)) {
throw new IllegalArgumentException("Parameter type (" + o.getClass().getName() + ") not recognized");
}
return new Tuple(this.elements, o);
return new Tuple(this, o,
(o instanceof Versionstamp && !((Versionstamp)o).isComplete()) ||
(o instanceof List<?> && TupleUtil.hasIncompleteVersionstamp(((List)o).stream())) ||
(o instanceof Tuple && ((Tuple) o).hasIncompleteVersionstamp()));
}
/**
@ -118,7 +141,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(String s) {
return new Tuple(this.elements, s);
return new Tuple(this, s, false);
}
/**
@ -129,7 +152,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(long l) {
return new Tuple(this.elements, l);
return new Tuple(this, l, false);
}
/**
@ -140,7 +163,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(byte[] b) {
return new Tuple(this.elements, b);
return new Tuple(this, b, false);
}
/**
@ -151,7 +174,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(boolean b) {
return new Tuple(this.elements, b);
return new Tuple(this, b, false);
}
/**
@ -162,7 +185,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(UUID uuid) {
return new Tuple(this.elements, uuid);
return new Tuple(this, uuid, false);
}
/**
@ -178,7 +201,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
if(bi == null) {
throw new NullPointerException("Number types in Tuple cannot be null");
}
return new Tuple(this.elements, bi);
return new Tuple(this, bi, false);
}
/**
@ -189,7 +212,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(float f) {
return new Tuple(this.elements, f);
return new Tuple(this, f, false);
}
/**
@ -200,7 +223,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(double d) {
return new Tuple(this.elements, d);
return new Tuple(this, d, false);
}
/**
@ -212,11 +235,11 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(Versionstamp v) {
return new Tuple(this.elements, v);
return new Tuple(this, v, !v.isComplete());
}
/**
* Creates a copy of this {@code Tuple} with an {@link List} appended as the last element.
* Creates a copy of this {@code Tuple} with a {@link List} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}).
* This adds the list as a single element nested within the outer {@code Tuple}.
*
@ -224,8 +247,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a newly created {@code Tuple}
*/
public Tuple add(List<? extends Object> l) {
return new Tuple(this.elements, l);
public Tuple add(List<?> l) {
return new Tuple(this, l, TupleUtil.hasIncompleteVersionstamp(l.stream()));
}
/**
@ -238,7 +261,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(Tuple t) {
return new Tuple(this.elements, t);
return new Tuple(this, t, t.hasIncompleteVersionstamp());
}
/**
@ -251,7 +274,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a newly created {@code Tuple}
*/
public Tuple add(byte[] b, int offset, int length) {
return new Tuple(this.elements, Arrays.copyOfRange(b, offset, offset + length));
return new Tuple(this, Arrays.copyOfRange(b, offset, offset + length), false);
}
/**
@ -262,7 +285,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a newly created {@code Tuple}
*/
public Tuple addAll(List<? extends Object> o) {
public Tuple addAll(List<?> o) {
List<Object> merged = new ArrayList<>(o.size() + this.elements.size());
merged.addAll(this.elements);
merged.addAll(o);
@ -279,8 +302,15 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
public Tuple addAll(Tuple other) {
List<Object> merged = new ArrayList<>(this.size() + other.size());
merged.addAll(this.elements);
merged.addAll(other.peekItems());
return new Tuple(merged);
merged.addAll(other.elements);
Tuple t = new Tuple(merged);
if(!t.hasIncompleteVersionstamp() && packed != null && other.packed != null) {
t.packed = ByteArrayUtil.join(packed, other.packed);
}
if(memoizedPackedSize >= 0 && other.memoizedPackedSize >= 0) {
t.memoizedPackedSize = memoizedPackedSize + other.memoizedPackedSize;
}
return t;
}
/**
@ -306,29 +336,44 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
}
byte[] packInternal(byte[] prefix, boolean copy) {
boolean hasPrefix = prefix != null && prefix.length > 1;
if(packed == null) {
byte[] result = TupleUtil.pack(elements, prefix, getPackedSize());
if(hasPrefix) {
packed = Arrays.copyOfRange(result, prefix.length, result.length);
memoizedPackedSize = packed.length;
return result;
}
else {
packed = result;
memoizedPackedSize = packed.length;
}
if(hasIncompleteVersionstamp()) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
if(packed == null) {
packed = TupleUtil.pack(elements, getPackedSize());
}
boolean hasPrefix = prefix != null && prefix.length > 0;
if(hasPrefix) {
return ByteArrayUtil.join(prefix, packed);
}
else if(copy) {
return Arrays.copyOf(packed, packed.length);
}
else {
if(copy) {
return Arrays.copyOf(packed, packed.length);
}
else {
return packed;
}
return packed;
}
}
/**
* Pack an encoded representation of this {@code Tuple} onto the end of the given {@link ByteBuffer}.
* It is up to the caller to ensure that there is enough space allocated within the buffer
* to avoid {@link java.nio.BufferOverflowException}s. The client may call {@link #getPackedSize()}
* to determine how large this {@code Tuple} will be once packed in order to allocate sufficient memory.
* <br>
* <br>
* This method will throw an error if there are any incomplete {@link Versionstamp}s in this {@code Tuple}.
*
* @param dest the destination {@link ByteBuffer} for the encoded {@code Tuple}
*/
public void packInto(ByteBuffer dest) {
if(hasIncompleteVersionstamp()) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
if(packed == null) {
TupleUtil.pack(dest, elements);
}
else {
dest.put(packed);
}
}
@ -363,37 +408,27 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple}
*/
public byte[] packWithVersionstamp(byte[] prefix) {
return TupleUtil.packWithVersionstamp(elements, prefix, getPackedSize());
return packWithVersionstampInternal(prefix, true);
}
byte[] packWithVersionstampInternal(byte[] prefix, boolean copy) {
boolean hasPrefix = prefix != null && prefix.length > 0;
if(packed == null) {
byte[] result = TupleUtil.packWithVersionstamp(elements, prefix, getPackedSize());
if(hasPrefix) {
byte[] withoutPrefix = Arrays.copyOfRange(result, prefix.length, result.length);
TupleUtil.adjustVersionPosition(packed, -1 * prefix.length);
packed = withoutPrefix;
memoizedPackedSize = packed.length;
return result;
}
else {
packed = result;
memoizedPackedSize = packed.length;
}
if(!hasIncompleteVersionstamp()) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
}
if(packed == null) {
packed = TupleUtil.packWithVersionstamp(elements, getPackedSize());
}
boolean hasPrefix = prefix != null && prefix.length > 0;
if(hasPrefix) {
byte[] withPrefix = ByteArrayUtil.join(prefix, packed);
TupleUtil.adjustVersionPosition(withPrefix, prefix.length);
return withPrefix;
}
else if(copy) {
return Arrays.copyOf(packed, packed.length);
}
else {
if(copy) {
return Arrays.copyOf(packed, packed.length);
}
else {
return packed;
}
return packed;
}
}
@ -429,16 +464,6 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return elements.stream();
}
/**
* Returns the internal elements that make up this tuple. For internal use only, as
* modifications to the result will mean that this Tuple is modified.
*
* @return the elements of this Tuple, without copying
*/
private List<Object> peekItems() {
return this.elements;
}
/**
* Gets an {@code Iterator} over the {@code Objects} in this {@code Tuple}. This {@code Iterator} is
* unmodifiable and will throw an exception if {@link Iterator#remove() remove()} is called.
@ -450,18 +475,6 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return Collections.unmodifiableList(this.elements).iterator();
}
/**
* Construct a new empty {@code Tuple}. After creation, items can be added
* with calls the the variations of {@code add()}.
*
* @see #from(Object...)
* @see #fromBytes(byte[])
* @see #fromItems(Iterable)
*/
public Tuple() {
this.elements = new LinkedList<>();
}
/**
* Construct a new {@code Tuple} with elements decoded from a supplied {@code byte} array.
* The passed byte array must not be {@code null}.
@ -485,9 +498,15 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a new {@code Tuple} constructed by deserializing the specified slice of the provided {@code byte} array
*/
public static Tuple fromBytes(byte[] bytes, int offset, int length) {
Tuple t = new Tuple();
t.elements = TupleUtil.unpack(bytes, offset, length);
t.packed = Arrays.copyOfRange(bytes, offset, offset + length);
if(offset < 0 || offset > bytes.length) {
throw new IllegalArgumentException("Invalid offset for Tuple deserialization");
}
if(length < 0 || offset + length > bytes.length) {
throw new IllegalArgumentException("Invalid length for Tuple deserialization");
}
byte[] packed = Arrays.copyOfRange(bytes, offset, offset + length);
Tuple t = new Tuple(TupleUtil.unpack(packed));
t.packed = packed;
t.memoizedPackedSize = length;
return t;
}
@ -732,7 +751,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return (Tuple)o;
}
else if(o instanceof List<?>) {
return Tuple.fromItems((List<?>)o);
return Tuple.fromList((List<?>)o);
}
else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to tuple");
@ -761,11 +780,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
if(elements.isEmpty())
throw new IllegalStateException("Tuple contains no elements");
List<Object> items = new ArrayList<>(elements.size() - 1);
for(int i = 1; i < this.elements.size(); i++) {
items.add(this.elements.get(i));
}
return new Tuple(items);
return new Tuple(elements.subList(1, elements.size()));
}
/**
@ -779,11 +794,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
if(elements.isEmpty())
throw new IllegalStateException("Tuple contains no elements");
List<Object> items = new ArrayList<>(elements.size() - 1);
for(int i = 0; i < this.elements.size() - 1; i++) {
items.add(this.elements.get(i));
}
return new Tuple(items);
return new Tuple(elements.subList(0, elements.size() - 1));
}
/**
@ -800,17 +811,39 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* This function will throw an error if this {@code Tuple} contains an incomplete
* {@link Versionstamp}.
*
* @return the range of keys containing all {@code Tuple}s that have this {@code Tuple}
* as a prefix
* @return the range of keys containing all possible keys that have this {@code Tuple}
* as a strict prefix
*/
public Range range() {
return range(null);
}
/**
* Returns a range representing all keys that encode {@code Tuple}s strictly starting
* with the given prefix followed by this {@code Tuple}.
* <br>
* <br>
* For example:
* <pre>
* Tuple t = Tuple.from("a", "b");
* Range r = t.range(Tuple.from("c").pack());</pre>
* {@code r} contains all tuples ("c", "a", "b", ...)
* <br>
* This function will throw an error if this {@code Tuple} contains an incomplete
* {@link Versionstamp}.
*
* @param prefix a byte prefix to precede all elements in the range
*
* @return the range of keys containing all possible keys that have {@code prefix}
* followed by this {@code Tuple} as a strict prefix
*/
public Range range(byte[] prefix) {
if(hasIncompleteVersionstamp()) {
throw new IllegalStateException("Tuple with incomplete versionstamp used for range");
}
byte[] p = packInternal(null, false);
//System.out.println("Packed tuple is: " + ByteArrayUtil.printable(p));
byte[] p = packInternal(prefix, false);
return new Range(ByteArrayUtil.join(p, new byte[] {0x0}),
ByteArrayUtil.join(p, new byte[] {(byte)0xff}));
ByteArrayUtil.join(p, new byte[] {(byte)0xff}));
}
/**
@ -823,7 +856,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* {@code Tuple}
*/
public boolean hasIncompleteVersionstamp() {
return TupleUtil.hasIncompleteVersionstamp(stream());
return incompleteVersionstamp;
}
/**
@ -843,7 +876,21 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
}
int getPackedSize(boolean nested) {
return TupleUtil.getPackedSize(elements, nested);
if(memoizedPackedSize >= 0) {
if(!nested) {
return memoizedPackedSize;
}
int nullCount = 0;
for(Object elem : elements) {
if(elem == null) {
nullCount++;
}
}
return memoizedPackedSize + nullCount;
}
else {
return TupleUtil.getPackedSize(elements, nested);
}
}
/**
@ -860,7 +907,9 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*/
@Override
public int compareTo(Tuple t) {
if(packed != null && t.packed != null) {
// If either tuple has an incomplete versionstamp, then there is a possibility that the byte order
// is not the semantic comparison order.
if(packed != null && t.packed != null && !hasIncompleteVersionstamp() && !t.hasIncompleteVersionstamp()) {
return ByteArrayUtil.compareUnsigned(packed, t.packed);
}
else {
@ -959,12 +1008,15 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple fromItems(Iterable<? extends Object> items) {
Tuple t = new Tuple();
for(Object o : items) {
t = t.addObject(o);
public static Tuple fromItems(Iterable<?> items) {
if(items instanceof List<?>) {
return Tuple.fromList((List<?>)items);
}
return t;
List<Object> elements = new ArrayList<>();
for(Object o : items) {
elements.add(o);
}
return new Tuple(elements);
}
/**
@ -977,8 +1029,9 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple fromList(List<? extends Object> items) {
return new Tuple(items);
public static Tuple fromList(List<?> items) {
List<Object> elements = new ArrayList<>(items);
return new Tuple(elements);
}
/**
@ -992,10 +1045,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple fromStream(Stream<? extends Object> items) {
Tuple t = new Tuple();
t.elements = items.collect(Collectors.toList());
return t;
public static Tuple fromStream(Stream<?> items) {
return new Tuple(items.collect(Collectors.toList()));
}
/**
@ -1009,7 +1060,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a new {@code Tuple} with the given items as its elements
*/
public static Tuple from(Object... items) {
return fromList(Arrays.asList(items));
return new Tuple(Arrays.asList(items));
}
static void main(String[] args) {

117
bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java
View File

@ -21,6 +21,7 @@
package com.apple.foundationdb.tuple;
import java.math.BigInteger;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.Charset;
@ -89,7 +90,7 @@ class TupleUtil {
x += 1;
}
}
throw new IllegalArgumentException("no terminator found for bytes starting at " + from);
throw new IllegalArgumentException("No terminator found for bytes starting at " + from);
}
}
@ -135,6 +136,7 @@ class TupleUtil {
else {
ByteArrayUtil.replace(encoded, 0, encoded.length, NULL_ARR, NULL_ESCAPED_ARR, encodedBytes);
}
totalLength += encoded.length + nullCount;
return this;
}
@ -157,6 +159,10 @@ class TupleUtil {
}
}
private static boolean useOldVersionOffsetFormat() {
return FDB.instance().getAPIVersion() < 520;
}
// These four functions are for adjusting the encoding of floating point numbers so
// that when their byte representation is written out in big-endian order, unsigned
// lexicographic byte comparison orders the values in the same way as the semantic
@ -165,32 +171,32 @@ class TupleUtil {
// in the case that the number is positive. For these purposes, 0.0 is positive and -0.0
// is negative.
static int encodeFloatBits(float f) {
private static int encodeFloatBits(float f) {
int intBits = Float.floatToRawIntBits(f);
return (intBits < 0) ? (~intBits) : (intBits ^ Integer.MIN_VALUE);
}
static long encodeDoubleBits(double d) {
private static long encodeDoubleBits(double d) {
long longBits = Double.doubleToRawLongBits(d);
return (longBits < 0L) ? (~longBits) : (longBits ^ Long.MIN_VALUE);
}
static float decodeFloatBits(int i) {
private static float decodeFloatBits(int i) {
int origBits = (i >= 0) ? (~i) : (i ^ Integer.MIN_VALUE);
return Float.intBitsToFloat(origBits);
}
static double decodeDoubleBits(long l) {
private static double decodeDoubleBits(long l) {
long origBits = (l >= 0) ? (~l) : (l ^ Long.MIN_VALUE);
return Double.longBitsToDouble(origBits);
}
// Get the minimal number of bytes in the representation of a long.
static int minimalByteCount(long i) {
private static int minimalByteCount(long i) {
return (Long.SIZE + 7 - Long.numberOfLeadingZeros(i >= 0 ? i : -i)) / 8;
}
static int minimalByteCount(BigInteger i) {
private static int minimalByteCount(BigInteger i) {
int bitLength = (i.compareTo(BigInteger.ZERO) >= 0) ? i.bitLength() : i.negate().bitLength();
return (bitLength + 7) / 8;
}
@ -221,7 +227,7 @@ class TupleUtil {
}
static void adjustVersionPosition(byte[] packed, int delta) {
if(FDB.instance().getAPIVersion() < 520) {
if(useOldVersionOffsetFormat()) {
adjustVersionPosition300(packed, delta);
}
else {
@ -285,7 +291,7 @@ class TupleUtil {
else if(t instanceof List<?>)
encode(state, (List<?>)t);
else if(t instanceof Tuple)
encode(state, ((Tuple)t).getItems());
encode(state, (Tuple)t);
else
throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName());
}
@ -409,6 +415,10 @@ class TupleUtil {
state.add(nil);
}
static void encode(EncodeState state, Tuple value) {
encode(state, value.elements);
}
static void decode(DecodeState state, byte[] rep, int pos, int last) {
//System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos);
@ -491,8 +501,8 @@ class TupleUtil {
int n = positive ? code - INT_ZERO_CODE : INT_ZERO_CODE - code;
int end = start + n;
if(rep.length < last) {
throw new RuntimeException("Invalid tuple (possible truncation)");
if(last < end) {
throw new IllegalArgumentException("Invalid tuple (possible truncation)");
}
if(positive && (n < Long.BYTES || rep[start] > 0)) {
@ -530,12 +540,16 @@ class TupleUtil {
}
}
else if(code == VERSIONSTAMP_CODE) {
if(start + Versionstamp.LENGTH > last) {
throw new IllegalArgumentException("Invalid tuple (possible truncation)");
}
Versionstamp val = Versionstamp.fromBytes(Arrays.copyOfRange(rep, start, start + Versionstamp.LENGTH));
state.add(val, start + Versionstamp.LENGTH);
}
else if(code == NESTED_CODE) {
DecodeState subResult = new DecodeState();
int endPos = start;
boolean foundEnd = false;
while(endPos < last) {
if(rep[endPos] == nil) {
if(endPos + 1 < last && rep[endPos+1] == (byte)0xff) {
@ -543,6 +557,7 @@ class TupleUtil {
endPos += 2;
} else {
endPos += 1;
foundEnd = true;
break;
}
} else {
@ -550,6 +565,9 @@ class TupleUtil {
endPos = subResult.end;
}
}
if(!foundEnd) {
throw new IllegalArgumentException("No terminator found for nested tuple starting at " + start);
}
state.add(subResult.values, endPos);
}
else {
@ -558,6 +576,10 @@ class TupleUtil {
}
static int compareItems(Object item1, Object item2) {
if(item1 == item2) {
// If we have pointer equality, just return 0 immediately.
return 0;
}
int code1 = TupleUtil.getCodeFor(item1);
int code2 = TupleUtil.getCodeFor(item2);
@ -603,14 +625,14 @@ class TupleUtil {
}
}
if(code1 == FLOAT_CODE) {
// This is done for the same reason that double comparison is done
// that way.
// This is done over vanilla float comparison basically to handle NaNs
// sorting correctly.
int fbits1 = encodeFloatBits((Float)item1);
int fbits2 = encodeFloatBits((Float)item2);
return Integer.compareUnsigned(fbits1, fbits2);
}
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// This is done over vanilla double comparison basically to handle NaNs
// sorting correctly.
long dbits1 = encodeDoubleBits((Double)item1);
long dbits2 = encodeDoubleBits((Double)item2);
@ -637,58 +659,57 @@ class TupleUtil {
throw new IllegalArgumentException("Unknown tuple data type: " + item1.getClass());
}
static List<Object> unpack(byte[] bytes, int start, int length) {
DecodeState decodeState = new DecodeState();
int pos = start;
int end = start + length;
while(pos < end) {
decode(decodeState, bytes, pos, end);
pos = decodeState.end;
static List<Object> unpack(byte[] bytes) {
try {
DecodeState decodeState = new DecodeState();
int pos = 0;
int end = bytes.length;
while (pos < end) {
decode(decodeState, bytes, pos, end);
pos = decodeState.end;
}
return decodeState.values;
}
catch(IndexOutOfBoundsException | BufferOverflowException e) {
throw new IllegalArgumentException("Invalid tuple (possible truncation)", e);
}
return decodeState.values;
}
static void encodeAll(EncodeState state, List<Object> items, byte[] prefix) {
if(prefix != null) {
state.add(prefix);
}
static void encodeAll(EncodeState state, List<Object> items) {
for(Object t : items) {
encode(state, t);
}
//System.out.println("Joining whole tuple...");
}
static byte[] pack(List<Object> items, byte[] prefix, int expectedSize) {
ByteBuffer dest = ByteBuffer.allocate(expectedSize + (prefix != null ? prefix.length : 0));
static void pack(ByteBuffer dest, List<Object> items) {
ByteOrder origOrder = dest.order();
EncodeState state = new EncodeState(dest);
if(prefix != null) {
state.add(prefix);
}
encodeAll(state, items, prefix);
encodeAll(state, items);
dest.order(origOrder);
if(state.versionPos >= 0) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple packInternal");
}
else {
return dest.array();
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
}
static byte[] packWithVersionstamp(List<Object> items, byte[] prefix, int expectedSize) {
ByteBuffer dest = ByteBuffer.allocate(expectedSize + (prefix != null ? prefix.length : 0));
static byte[] pack(List<Object> items, int expectedSize) {
ByteBuffer dest = ByteBuffer.allocate(expectedSize);
pack(dest, items);
return dest.array();
}
static byte[] packWithVersionstamp(List<Object> items, int expectedSize) {
ByteBuffer dest = ByteBuffer.allocate(expectedSize);
EncodeState state = new EncodeState(dest);
if(prefix != null) {
state.add(prefix);
}
encodeAll(state, items, prefix);
encodeAll(state, items);
if(state.versionPos < 0) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple packInternal with versionstamp");
}
else {
if(state.versionPos > 0xffff) {
if(useOldVersionOffsetFormat() && state.versionPos > 0xffff) {
throw new IllegalArgumentException("Tuple has incomplete version at position " + state.versionPos + " which is greater than the maximum " + 0xffff);
}
dest.order(ByteOrder.LITTLE_ENDIAN);
if (FDB.instance().getAPIVersion() < 520) {
if (useOldVersionOffsetFormat()) {
dest.putShort((short)state.versionPos);
} else {
dest.putInt(state.versionPos);
@ -740,7 +761,7 @@ class TupleUtil {
packedSize += 1 + Versionstamp.LENGTH;
Versionstamp versionstamp = (Versionstamp)item;
if(!versionstamp.isComplete()) {
int suffixSize = FDB.instance().getAPIVersion() < 520 ? Short.BYTES : Integer.BYTES;
int suffixSize = useOldVersionOffsetFormat() ? Short.BYTES : Integer.BYTES;
packedSize += suffixSize;
}
}
@ -776,7 +797,7 @@ class TupleUtil {
public static void main(String[] args) {
try {
byte[] bytes = pack(Collections.singletonList(4), null, 2);
byte[] bytes = pack(Collections.singletonList(4), 2);
DecodeState result = new DecodeState();
decode(result, bytes, 0, bytes.length);
int val = ((Number)result.values.get(0)).intValue();
@ -788,7 +809,7 @@ class TupleUtil {
}
try {
byte[] bytes = pack(Collections.singletonList("\u021Aest \u0218tring"), null, 15);
byte[] bytes = pack(Collections.singletonList("\u021Aest \u0218tring"), 15);
DecodeState result = new DecodeState();
decode(result, bytes, 0, bytes.length);
String string = (String)result.values.get(0);

4
bindings/java/src/main/com/apple/foundationdb/tuple/Versionstamp.java
View File

@ -94,8 +94,8 @@ public class Versionstamp implements Comparable<Versionstamp> {
private static final byte[] UNSET_TRANSACTION_VERSION = {(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff,
(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff};
private boolean complete;
private byte[] versionBytes;
private final boolean complete;
private final byte[] versionBytes;
/**
* From a byte array, unpack the user version starting at the given position.

620
bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java
View File

@ -21,13 +21,21 @@
package com.apple.foundationdb.test;
import java.math.BigInteger;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Stream;
import com.apple.foundationdb.Database;
import com.apple.foundationdb.FDB;
import com.apple.foundationdb.TransactionContext;
import com.apple.foundationdb.subspace.Subspace;
import com.apple.foundationdb.tuple.ByteArrayUtil;
import com.apple.foundationdb.tuple.Tuple;
import com.apple.foundationdb.tuple.Versionstamp;
@ -38,15 +46,19 @@ public class TupleTest {
public static void main(String[] args) throws InterruptedException {
final int reps = 1000;
try {
// FDB fdb = FDB.selectAPIVersion(610);
serializedForms();
FDB fdb = FDB.selectAPIVersion(610);
addMethods();
comparisons();
emptyTuple();
incompleteVersionstamps();
intoBuffer();
offsetsAndLengths();
malformedBytes();
replaceTests();
/*
serializedForms();
try(Database db = fdb.open()) {
runTests(reps, db);
}
*/
} catch(Throwable t) {
t.printStackTrace();
}
@ -269,6 +281,606 @@ public class TupleTest {
}
}
private static void emptyTuple() {
Tuple t = new Tuple();
if(!t.isEmpty()) {
throw new RuntimeException("empty tuple is not empty");
}
if(t.getPackedSize() != 0) {
throw new RuntimeException("empty tuple packed size is not 0");
}
if(t.pack().length != 0) {
throw new RuntimeException("empty tuple is not packed to the empty byte string");
}
}
private static void addMethods() {
List<Tuple> baseTuples = Arrays.asList(
new Tuple(),
Tuple.from(),
Tuple.from((Object)null),
Tuple.from("prefix"),
Tuple.from("prefix", null),
Tuple.from(new UUID(100, 1000)),
Tuple.from(Versionstamp.incomplete(1)),
Tuple.from(Tuple.from(Versionstamp.incomplete(2))),
Tuple.from(Collections.singletonList(Versionstamp.incomplete(3)))
);
List<Object> toAdd = Arrays.asList(
null,
1066L,
BigInteger.valueOf(1066),
-3.14f,
2.71828,
new byte[]{0x01, 0x02, 0x03},
new byte[]{0x01, 0x00, 0x02, 0x00, 0x03},
"hello there",
"hell\0 there",
"\ud83d\udd25",
"\ufb14",
false,
true,
Float.NaN,
Float.intBitsToFloat(Integer.MAX_VALUE),
Double.NaN,
Double.longBitsToDouble(Long.MAX_VALUE),
Versionstamp.complete(new byte[]{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09}, 100),
Versionstamp.incomplete(4),
new UUID(-1, 1),
Tuple.from((Object)null),
Tuple.from("suffix", "tuple"),
Tuple.from("s\0ffix", "tuple"),
Arrays.asList("suffix", "tuple"),
Arrays.asList("suffix", null, "tuple"),
Tuple.from("suffix", null, "tuple"),
Tuple.from("suffix", Versionstamp.incomplete(4), "tuple"),
Arrays.asList("suffix", Arrays.asList("inner", Versionstamp.incomplete(5), "tuple"), "tuple")
);
for(Tuple baseTuple : baseTuples) {
for(Object newItem : toAdd) {
int baseSize = baseTuple.size();
Tuple freshTuple = Tuple.fromStream(Stream.concat(baseTuple.stream(), Stream.of(newItem)));
if(freshTuple.size() != baseSize + 1) {
throw new RuntimeException("freshTuple size was not one larger than base size");
}
Tuple withObjectAdded = baseTuple.addObject(newItem);
if(withObjectAdded.size() != baseSize + 1) {
throw new RuntimeException("withObjectAdded size was not one larger than the base size");
}
// Use the appropriate "add" overload.
Tuple withValueAdded;
if(newItem == null) {
withValueAdded = baseTuple.addObject(null);
}
else if(newItem instanceof byte[]) {
withValueAdded = baseTuple.add((byte[])newItem);
}
else if(newItem instanceof String) {
withValueAdded = baseTuple.add((String)newItem);
}
else if(newItem instanceof Long) {
withValueAdded = baseTuple.add((Long)newItem);
}
else if(newItem instanceof BigInteger) {
withValueAdded = baseTuple.add((BigInteger)newItem);
}
else if(newItem instanceof Float) {
withValueAdded = baseTuple.add((Float)newItem);
}
else if(newItem instanceof Double) {
withValueAdded = baseTuple.add((Double)newItem);
}
else if(newItem instanceof Boolean) {
withValueAdded = baseTuple.add((Boolean)newItem);
}
else if(newItem instanceof UUID) {
withValueAdded = baseTuple.add((UUID)newItem);
}
else if(newItem instanceof Versionstamp) {
withValueAdded = baseTuple.add((Versionstamp)newItem);
}
else if(newItem instanceof List<?>) {
withValueAdded = baseTuple.add((List<?>)newItem);
}
else if(newItem instanceof Tuple) {
withValueAdded = baseTuple.add((Tuple)newItem);
}
else {
throw new RuntimeException("unknown type for tuple serialization " + newItem.getClass());
}
// Use Tuple.addAll, which has optimizations if both tuples have been packed already
// Getting their hash codes memoizes the packed representation.
Tuple newItemTuple = Tuple.from(newItem);
baseTuple.hashCode();
newItemTuple.hashCode();
Tuple withTupleAddedAll = baseTuple.addAll(newItemTuple);
Tuple withListAddedAll = baseTuple.addAll(Collections.singletonList(newItem));
List<Tuple> allTuples = Arrays.asList(freshTuple, withObjectAdded, withValueAdded, withTupleAddedAll, withListAddedAll);
int basePlusNewSize = baseTuple.getPackedSize() + Tuple.from(newItem).getPackedSize();
int freshTuplePackedSize = freshTuple.getPackedSize();
int withObjectAddedPackedSize = withObjectAdded.getPackedSize();
int withValueAddedPackedSize = withValueAdded.getPackedSize();
int withTupleAddedAllPackedSize = withTupleAddedAll.getPackedSize();
int withListAddAllPackedSize = withListAddedAll.getPackedSize();
if(basePlusNewSize != freshTuplePackedSize || basePlusNewSize != withObjectAddedPackedSize ||
basePlusNewSize != withValueAddedPackedSize || basePlusNewSize != withTupleAddedAllPackedSize ||
basePlusNewSize != withListAddAllPackedSize) {
throw new RuntimeException("packed sizes not equivalent");
}
byte[] concatPacked;
byte[] prefixPacked;
byte[] freshPacked;
byte[] objectAddedPacked;
byte[] valueAddedPacked;
byte[] tupleAddedAllPacked;
byte[] listAddedAllPacked;
if(!baseTuple.hasIncompleteVersionstamp() && !Tuple.from(newItem).hasIncompleteVersionstamp()) {
concatPacked = ByteArrayUtil.join(baseTuple.pack(), Tuple.from(newItem).pack());
prefixPacked = Tuple.from(newItem).pack(baseTuple.pack());
freshPacked = freshTuple.pack();
objectAddedPacked = withObjectAdded.pack();
valueAddedPacked = withValueAdded.pack();
tupleAddedAllPacked = withTupleAddedAll.pack();
listAddedAllPacked = withListAddedAll.pack();
for(Tuple t : allTuples) {
try {
t.packWithVersionstamp();
throw new RuntimeException("able to pack tuple without incomplete versionstamp using packWithVersionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
else if(!baseTuple.hasIncompleteVersionstamp() && Tuple.from(newItem).hasIncompleteVersionstamp()) {
concatPacked = newItemTuple.packWithVersionstamp(baseTuple.pack());
try {
prefixPacked = Tuple.from(newItem).packWithVersionstamp(baseTuple.pack());
}
catch(NullPointerException e) {
prefixPacked = Tuple.from(newItem).packWithVersionstamp(baseTuple.pack());
}
freshPacked = freshTuple.packWithVersionstamp();
objectAddedPacked = withObjectAdded.packWithVersionstamp();
valueAddedPacked = withValueAdded.packWithVersionstamp();
tupleAddedAllPacked = withTupleAddedAll.packWithVersionstamp();
listAddedAllPacked = withListAddedAll.packWithVersionstamp();
for(Tuple t : allTuples) {
try {
t.pack();
throw new RuntimeException("able to pack tuple with incomplete versionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
else if(baseTuple.hasIncompleteVersionstamp() && !Tuple.from(newItem).hasIncompleteVersionstamp()) {
concatPacked = baseTuple.addAll(Tuple.from(newItem)).packWithVersionstamp();
prefixPacked = baseTuple.addObject(newItem).packWithVersionstamp();
freshPacked = freshTuple.packWithVersionstamp();
objectAddedPacked = withObjectAdded.packWithVersionstamp();
valueAddedPacked = withValueAdded.packWithVersionstamp();
tupleAddedAllPacked = withTupleAddedAll.packWithVersionstamp();
listAddedAllPacked = withListAddedAll.packWithVersionstamp();
for(Tuple t : allTuples) {
try {
t.pack();
throw new RuntimeException("able to pack tuple with incomplete versionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
else {
for(Tuple t : allTuples) {
try {
t.pack();
throw new RuntimeException("able to pack tuple with two versionstamps using pack");
}
catch(IllegalArgumentException e) {
// eat
}
try {
t.packWithVersionstamp();
throw new RuntimeException("able to pack tuple with two versionstamps using packWithVersionstamp");
}
catch(IllegalArgumentException e) {
// eat
}
try {
t.hashCode();
throw new RuntimeException("able to get hash code of tuple with two versionstamps");
}
catch(IllegalArgumentException e) {
// eat
}
}
concatPacked = null;
prefixPacked = null;
freshPacked = null;
objectAddedPacked = null;
valueAddedPacked = null;
tupleAddedAllPacked = null;
listAddedAllPacked = null;
}
if(!Arrays.equals(concatPacked, freshPacked) ||
!Arrays.equals(freshPacked, prefixPacked) ||
!Arrays.equals(freshPacked, objectAddedPacked) ||
!Arrays.equals(freshPacked, valueAddedPacked) ||
!Arrays.equals(freshPacked, tupleAddedAllPacked) ||
!Arrays.equals(freshPacked, listAddedAllPacked)) {
throw new RuntimeException("packed values are not concatenation of original packings");
}
if(freshPacked != null && freshPacked.length != basePlusNewSize) {
throw new RuntimeException("packed length did not match expectation");
}
if(freshPacked != null) {
if(freshTuple.hashCode() != Arrays.hashCode(freshPacked)) {
throw new IllegalArgumentException("hash code does not match fresh packed");
}
for(Tuple t : allTuples) {
if(t.hashCode() != freshTuple.hashCode()) {
throw new IllegalArgumentException("hash code mismatch");
}
if(Tuple.fromItems(t.getItems()).hashCode() != freshTuple.hashCode()) {
throw new IllegalArgumentException("hash code mismatch after re-compute");
}
}
}
}
}
}
private static void incompleteVersionstamps() {
if(FDB.instance().getAPIVersion() < 520) {
throw new IllegalStateException("cannot run test with API version " + FDB.instance().getAPIVersion());
}
// This is a tricky case where there are two tuples with identical representations but different semantics.
byte[] arr = new byte[0x0100fe];
Arrays.fill(arr, (byte)0x7f); // The actual value doesn't matter, but it can't be zero.
Tuple t1 = Tuple.from(arr, Versionstamp.complete(new byte[]{FF, FF, FF, FF, FF, FF, FF, FF, FF, FF}), new byte[]{0x01, 0x01});
Tuple t2 = Tuple.from(arr, Versionstamp.incomplete());
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal");
}
byte[] bytes1 = t1.pack();
byte[] bytes2 = t2.packWithVersionstamp();
if(!Arrays.equals(bytes1, bytes2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " did not have matching representations");
}
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal with memoized packed representations");
}
// Make sure position information adjustment works.
Tuple t3 = Tuple.from(Versionstamp.incomplete(1));
if(t3.getPackedSize() != 1 + Versionstamp.LENGTH + Integer.BYTES) {
throw new RuntimeException("incomplete versionstamp has incorrect packed size " + t3.getPackedSize());
}
byte[] bytes3 = t3.packWithVersionstamp();
if(ByteBuffer.wrap(bytes3, bytes3.length - Integer.BYTES, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).getInt() != 1) {
throw new RuntimeException("incomplete versionstamp has incorrect position");
}
if(!Tuple.fromBytes(bytes3, 0, bytes3.length - Integer.BYTES).equals(Tuple.from(Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes did not match");
}
Subspace subspace = new Subspace(Tuple.from("prefix"));
byte[] bytes4 = subspace.packWithVersionstamp(t3);
if(ByteBuffer.wrap(bytes4, bytes4.length - Integer.BYTES, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).getInt() != 1 + subspace.getKey().length) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
if(!Tuple.fromBytes(bytes4, 0, bytes4.length - Integer.BYTES).equals(Tuple.from("prefix", Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes with subspace did not match");
}
try {
// At this point, the representation is cached, so an easy bug would be to have it return the already serialized value
t3.pack();
throw new RuntimeException("was able to pack versionstamp with incomplete versionstamp");
} catch(IllegalArgumentException e) {
// eat
}
// Tuples with two incomplete versionstamps somewhere.
List<Tuple> twoIncompleteList = Arrays.asList(
Tuple.from(Versionstamp.incomplete(1), Versionstamp.incomplete(2)),
Tuple.from(Tuple.from(Versionstamp.incomplete(3)), Tuple.from(Versionstamp.incomplete(4))),
new Tuple().add(Versionstamp.incomplete()).add(Versionstamp.incomplete()),
new Tuple().add(Versionstamp.incomplete()).add(3L).add(Versionstamp.incomplete()),
Tuple.from(Tuple.from(Versionstamp.incomplete()), "dummy_string").add(Tuple.from(Versionstamp.incomplete())),
Tuple.from(Arrays.asList(Versionstamp.incomplete(), "dummy_string")).add(Tuple.from(Versionstamp.incomplete())),
Tuple.from(Tuple.from(Versionstamp.incomplete()), "dummy_string").add(Collections.singletonList(Versionstamp.incomplete()))
);
for(Tuple t : twoIncompleteList) {
if(!t.hasIncompleteVersionstamp()) {
throw new RuntimeException("tuple doesn't think it has incomplete versionstamp");
}
if(t.getPackedSize() < 2 * (1 + Versionstamp.LENGTH + Integer.BYTES)) {
throw new RuntimeException("tuple packed size " + t.getPackedSize() + " is smaller than expected");
}
try {
t.pack();
throw new RuntimeException("no error thrown when packing any incomplete versionstamps");
}
catch(IllegalArgumentException e) {
// eat
}
try {
t.packWithVersionstamp();
throw new RuntimeException("no error thrown when packing with versionstamp with two incompletes");
}
catch(IllegalArgumentException e) {
// eat
}
}
}
// Assumes API version < 520
private static void incompleteVersionstamps300() {
if(FDB.instance().getAPIVersion() >= 520) {
throw new IllegalStateException("cannot run test with API version " + FDB.instance().getAPIVersion());
}
Tuple t1 = Tuple.from(Versionstamp.complete(new byte[]{FF, FF, FF, FF, FF, FF, FF, FF, FF, FF}), new byte[]{});
Tuple t2 = Tuple.from(Versionstamp.incomplete());
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal");
}
byte[] bytes1 = t1.pack();
byte[] bytes2 = t2.packWithVersionstamp();
if(!Arrays.equals(bytes1, bytes2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " did not have matching representations");
}
if(t1.equals(t2)) {
throw new RuntimeException("tuples " + t1 + " and " + t2 + " compared equal with memoized packed representations");
}
// Make sure position information adjustment works.
Tuple t3 = Tuple.from(Versionstamp.incomplete(1));
if(t3.getPackedSize() != 1 + Versionstamp.LENGTH + Short.BYTES) {
throw new RuntimeException("incomplete versionstamp has incorrect packed size " + t3.getPackedSize());
}
byte[] bytes3 = t3.packWithVersionstamp();
if(ByteBuffer.wrap(bytes3, bytes3.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() != 1) {
throw new RuntimeException("incomplete versionstamp has incorrect position");
}
if(!Tuple.fromBytes(bytes3, 0, bytes3.length - Short.BYTES).equals(Tuple.from(Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes did not match");
}
Subspace subspace = new Subspace(Tuple.from("prefix"));
byte[] bytes4 = subspace.packWithVersionstamp(t3);
if(ByteBuffer.wrap(bytes4, bytes4.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() != 1 + subspace.getKey().length) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
if(!Tuple.fromBytes(bytes4, 0, bytes4.length - Short.BYTES).equals(Tuple.from("prefix", Versionstamp.incomplete(1)))) {
throw new RuntimeException("unpacked bytes with subspace did not match");
}
// Make sure an offset > 0xFFFF throws an error.
Tuple t4 = Tuple.from(Versionstamp.incomplete(2));
byte[] bytes5 = t4.packWithVersionstamp(); // Get bytes memoized.
if(ByteBuffer.wrap(bytes5, bytes5.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() != 1) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
byte[] bytes6 = t4.packWithVersionstamp(new byte[0xfffe]); // Offset is 0xffff
if(!Arrays.equals(Arrays.copyOfRange(bytes5, 0, 1 + Versionstamp.LENGTH), Arrays.copyOfRange(bytes6, 0xfffe, 0xffff + Versionstamp.LENGTH))) {
throw new RuntimeException("area before versionstamp offset did not match");
}
if((ByteBuffer.wrap(bytes6, bytes6.length - Short.BYTES, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).getShort() & 0xffff) != 0xffff) {
throw new RuntimeException("incomplete versionstamp has incorrect position with prefix");
}
try {
t4.packWithVersionstamp(new byte[0xffff]); // Offset is 0x10000
throw new RuntimeException("able to pack versionstamp with offset that is too large");
}
catch(IllegalArgumentException e) {
// eat
}
// Same as before, but packed representation is not memoized.
try {
Tuple.from(Versionstamp.incomplete(3)).packWithVersionstamp(new byte[0xffff]); // Offset is 0x10000
throw new RuntimeException("able to pack versionstamp with offset that is too large");
}
catch(IllegalArgumentException e) {
// eat
}
}
private static void malformedBytes() {
List<byte[]> malformedSequences = Arrays.asList(
new byte[]{0x01, (byte)0xde, (byte)0xad, (byte)0xc0, (byte)0xde}, // no termination character for byte array
new byte[]{0x01, (byte)0xde, (byte)0xad, 0x00, FF, (byte)0xc0, (byte)0xde}, // no termination character but null in middle
new byte[]{0x02, 'h', 'e', 'l', 'l', 'o'}, // no termination character for string
new byte[]{0x02, 'h', 'e', 'l', 0x00, FF, 'l', 'o'}, // no termination character but null in the middle
// Invalid UTF-8 decodes malformed as U+FFFD rather than throwing an error
// new byte[]{0x02, 'u', 't', 'f', 0x08, (byte)0x80, 0x00}, // invalid utf-8 code point start character
// new byte[]{0x02, 'u', 't', 'f', 0x08, (byte)0xc0, 0x01, 0x00}, // invalid utf-8 code point second character
new byte[]{0x05, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00}, // no termination character for nested tuple
new byte[]{0x05, 0x02, 'h', 'e', 'l', 'l', 'o', 0x00, 0x00, FF, 0x02, 't', 'h', 'e', 'r', 'e', 0x00}, // no termination character for nested tuple but null in the middle
new byte[]{0x16, 0x01}, // integer truncation
new byte[]{0x12, 0x01}, // integer truncation
new byte[]{0x1d, 0x09, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, // integer truncation
new byte[]{0x0b, 0x09 ^ FF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, // integer truncation
new byte[]{0x20, 0x01, 0x02, 0x03}, // float truncation
new byte[]{0x21, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}, // double truncation
new byte[]{0x30, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}, // UUID truncation
new byte[]{0x33, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b}, // versionstamp truncation
new byte[]{FF} // unknown start code
);
for(byte[] sequence : malformedSequences) {
try {
Tuple t = Tuple.fromBytes(sequence);
throw new RuntimeException("Able to unpack " + ByteArrayUtil.printable(sequence) + " into " + t);
}
catch(IllegalArgumentException e) {
System.out.println("Error for " + ByteArrayUtil.printable(sequence) + ": " + e.getMessage());
}
}
// Perfectly good byte sequences, but using the offset and length to remove terminal bytes
List<byte[]> wellFormedSequences = Arrays.asList(
Tuple.from((Object)new byte[]{0x01, 0x02}).pack(),
Tuple.from("hello").pack(),
Tuple.from("hell\0").pack(),
Tuple.from(1066L).pack(),
Tuple.from(-1066L).pack(),
Tuple.from(BigInteger.ONE.shiftLeft(Long.SIZE + 1)).pack(),
Tuple.from(BigInteger.ONE.shiftLeft(Long.SIZE + 1).negate()).pack(),
Tuple.from(-3.14f).pack(),
Tuple.from(2.71828).pack(),
Tuple.from(new UUID(1066L, 1415L)).pack(),
Tuple.from(Versionstamp.fromBytes(new byte[]{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c})).pack()
);
for(byte[] sequence : wellFormedSequences) {
try {
Tuple t = Tuple.fromBytes(sequence, 0, sequence.length - 1);
throw new RuntimeException("Able to unpack " + ByteArrayUtil.printable(sequence) + " into " + t + " without last character");
}
catch(IllegalArgumentException e) {
System.out.println("Error for " + ByteArrayUtil.printable(sequence) + ": " + e.getMessage());
}
}
}
private static void offsetsAndLengths() {
List<Tuple> tuples = Arrays.asList(
new Tuple(),
Tuple.from((Object)null),
Tuple.from(null, new byte[]{0x10, 0x66}),
Tuple.from("dummy_string"),
Tuple.from(1066L)
);
Tuple allTuples = tuples.stream().reduce(new Tuple(), Tuple::addAll);
byte[] allTupleBytes = allTuples.pack();
// Unpack each tuple individually using their lengths
int offset = 0;
for(Tuple t : tuples) {
int length = t.getPackedSize();
Tuple unpacked = Tuple.fromBytes(allTupleBytes, offset, length);
if(!unpacked.equals(t)) {
throw new RuntimeException("unpacked tuple " + unpacked + " does not match serialized tuple " + t);
}
offset += length;
}
// Unpack successive pairs of tuples.
offset = 0;
for(int i = 0; i < tuples.size() - 1; i++) {
Tuple combinedTuple = tuples.get(i).addAll(tuples.get(i + 1));
Tuple unpacked = Tuple.fromBytes(allTupleBytes, offset, combinedTuple.getPackedSize());
if(!unpacked.equals(combinedTuple)) {
throw new RuntimeException("unpacked tuple " + unpacked + " does not match combined tuple " + combinedTuple);
}
offset += tuples.get(i).getPackedSize();
}
// Allow an offset to equal the length of the array, but essentially only a zero-length is allowed there.
Tuple emptyAtEndTuple = Tuple.fromBytes(allTupleBytes, allTupleBytes.length, 0);
if(!emptyAtEndTuple.isEmpty()) {
throw new RuntimeException("tuple with no bytes is not empty");
}
try {
Tuple.fromBytes(allTupleBytes, -1, 4);
throw new RuntimeException("able to give negative offset to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, allTupleBytes.length + 1, 4);
throw new RuntimeException("able to give offset larger than array to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, 0, -1);
throw new RuntimeException("able to give negative length to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, 0, allTupleBytes.length + 1);
throw new RuntimeException("able to give length larger than array to fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
try {
Tuple.fromBytes(allTupleBytes, allTupleBytes.length / 2, allTupleBytes.length / 2 + 2);
throw new RuntimeException("able to exceed array length in fromBytes");
}
catch(IllegalArgumentException e) {
// eat
}
}
private static void intoBuffer() {
Tuple t = Tuple.from("hello", 3.14f, "world");
ByteBuffer buffer = ByteBuffer.allocate("hello".length() + 2 + Float.BYTES + 1 + "world".length() + 2);
t.packInto(buffer);
if(!Arrays.equals(t.pack(), buffer.array())) {
throw new RuntimeException("buffer and tuple do not match");
}
buffer = ByteBuffer.allocate(t.getPackedSize() + 2);
buffer.order(ByteOrder.LITTLE_ENDIAN);
t.packInto(buffer);
if(!Arrays.equals(ByteArrayUtil.join(t.pack(), new byte[]{0x00, 0x00}), buffer.array())) {
throw new RuntimeException("buffer and tuple do not match");
}
if(!buffer.order().equals(ByteOrder.LITTLE_ENDIAN)) {
throw new RuntimeException("byte order changed");
}
buffer = ByteBuffer.allocate(t.getPackedSize() + 2);
buffer.put((byte)0x01).put((byte)0x02);
t.packInto(buffer);
if(!Arrays.equals(t.pack(new byte[]{0x01, 0x02}), buffer.array())) {
throw new RuntimeException("buffer and tuple do not match");
}
buffer = ByteBuffer.allocate(t.getPackedSize() - 1);
try {
t.packInto(buffer);
throw new RuntimeException("able to pack into buffer that was too small");
}
catch(BufferOverflowException e) {
// eat
}
Tuple tCopy = Tuple.fromItems(t.getItems()); // remove memoized stuff
buffer = ByteBuffer.allocate(t.getPackedSize() - 1);
try {
tCopy.packInto(buffer);
throw new RuntimeException("able to pack into buffer that was too small");
}
catch(BufferOverflowException e) {
// eat
}
Tuple tWithIncomplete = Tuple.from(Versionstamp.incomplete(3));
buffer = ByteBuffer.allocate(tWithIncomplete.getPackedSize());
try {
tWithIncomplete.packInto(buffer);
throw new RuntimeException("able to pack incomplete versionstamp into buffer");
}
catch(IllegalArgumentException e) {
// eat
}
if(buffer.arrayOffset() != 0) {
throw new RuntimeException("offset changed after unsuccessful pack with incomplete versionstamp");
}
}
// These should be in ArrayUtilTest, but those can't be run at the moment, so here they go.
private static void replaceTests() {
List<byte[]> arrays = Arrays.asList(