wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tuple encoding now has fewer copies ; various java 8 vs java 6 inconsistency fixes

This commit is contained in:
Alec Grieser 2017-10-25 10:53:33 -07:00
parent 1855f876db
commit 957243a88e
4 changed files with 308 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
bindings/java/src-completable/main/com/apple/foundationdb/tuple/Tuple.java
View File

@ -45,7 +45,7 @@ import com.apple.foundationdb.Range;
* <h3>Types</h3>
* A {@code Tuple} can
* contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, {@link UUID}s,
* {@code boolean}s, {@link List}s, other {@code Tuple}s, and {@code null}.
* {@code boolean}s, {@link List}s, {@link Versionstamp}s, other {@code Tuple}s, and {@code null}.
* {@link Float} and {@link Double} instances will be serialized as single- and double-precision
* numbers respectively, and {@link BigInteger}s within the range [{@code -2^2040+1},
* {@code 2^2040-1}] are serialized without loss of precision (those outside the range
@ -167,8 +167,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a copy of this {@code Tuple} with a {@link BigInteger} appended as the last element.
* As {@link Tuple}s cannot contain {@code null} numeric types, a {@link NullPointerException}
* is raised if a {@code null} argument is passed.
* As {@link Tuple}s cannot contain {@code null} numeric types, a {@link NullPointerException}
* is raised if a {@code null} argument is passed.
*
* @param bi the {@link BigInteger} to append
*
@ -217,8 +217,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a copy of this {@code Tuple} with an {@link List} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}).
* This adds the list as a single elemented nested within the outer {@code Tuple}.
* This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}).
* This adds the list as a single element nested within the outer {@code Tuple}.
*
* @param l the {@link List} to append
*
@ -230,8 +230,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a copy of this {@code Tuple} with a {@code Tuple} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(Tuple) Tuple.addAll}).
* This adds the list as a single elemented nested within the outer {@code Tuple}.
* This does not add the elements individually (for that, use {@link Tuple#addAll(Tuple) Tuple.addAll}).
* This adds the list as a single element nested within the outer {@code Tuple}.
*
* @param t the {@code Tuple} to append
*
@ -302,11 +302,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a serialized representation of this {@code Tuple} prepended by the {@code prefix}.
*/
public byte[] pack(byte[] prefix) {
TupleUtil.EncodeResult encoded = TupleUtil.pack(elements, prefix);
if(encoded.versionPos > 0) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
return encoded.data;
return TupleUtil.pack(elements, prefix);
}
/**
@ -323,7 +319,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
}
/**
* Get an encoded representation of this {@code Tuple} for use with.
* Get an encoded representation of this {@code Tuple} for use with
* {@link com.apple.foundationdb.MutationType#SET_VERSIONSTAMPED_KEY MutationType.SET_VERSIONSTAMPED_KEY}.
* There must be exactly one incomplete {@link Versionstamp} instance within this
* {@code Tuple} or this will throw an {@link IllegalArgumentException}.
@ -340,14 +336,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple}
*/
public byte[] packWithVersionstamp(byte[] prefix) {
TupleUtil.EncodeResult encoded = TupleUtil.pack(elements, prefix);
if(encoded.versionPos < 0) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
}
return ByteBuffer.allocate(encoded.data.length + 2).order(ByteOrder.LITTLE_ENDIAN)
.put(encoded.data)
.putShort((short)encoded.versionPos)
.array();
return TupleUtil.packWithVersionstamp(elements, prefix);
}
/**
@ -567,7 +556,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Gets an indexed item as a {@link UUID}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@code UUID}.
* The element at the index may not be {@code null}.
* The element at the index may be {@code null}.
*
* @param index the location of the item to return
*

240
bindings/java/src-completable/main/com/apple/foundationdb/tuple/TupleUtil.java
View File

@ -27,6 +27,7 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
@ -34,7 +35,6 @@ import java.util.stream.Stream;
class TupleUtil {
private static final byte nil = 0x00;
private static final byte[] nil_rep = new byte[] {nil, (byte)0xFF};
private static final BigInteger[] size_limits;
private static final Charset UTF8;
private static final IterableComparator iterableComparator;
@ -52,6 +52,15 @@ class TupleUtil {
private static final byte UUID_CODE = 0x30;
private static final byte VERSIONSTAMP_CODE = 0x33;
private static final byte[] NULL_ARR = new byte[] {nil};
private static final byte[] NULL_ESCAPED_ARR = new byte[] {nil, (byte)0xFF};
private static final byte[] BYTES_ARR = new byte[]{0x01};
private static final byte[] STRING_ARR = new byte[]{0x02};
private static final byte[] NESTED_ARR = new byte[]{0x05};
private static final byte[] FALSE_ARR = new byte[]{0x26};
private static final byte[] TRUE_ARR = new byte[]{0x27};
private static final byte[] VERSIONSTAMP_ARR = new byte[]{0x33};
static {
size_limits = new BigInteger[9];
for(int i = 0; i < 9; i++) {
@ -72,12 +81,12 @@ class TupleUtil {
}
static class EncodeResult {
final int totalLength;
final int versionPos;
final byte[] data;
EncodeResult(int versionPos, byte[] data) {
EncodeResult(int totalLength, int versionPos) {
this.totalLength = totalLength;
this.versionPos = versionPos;
this.data = data;
}
}
@ -148,65 +157,70 @@ class TupleUtil {
throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName());
}
static EncodeResult encode(Object t, boolean nested) {
static EncodeResult encode(Object t, boolean nested, List<byte[]> encoded) {
if(t == null)
if (nested)
return new EncodeResult(-1, new byte[]{nil, (byte) 0xff});
else
return new EncodeResult(-1, new byte[]{nil});
if (nested) {
encoded.add(NULL_ESCAPED_ARR);
return new EncodeResult(NULL_ESCAPED_ARR.length,-1);
}
else {
encoded.add(NULL_ARR);
return new EncodeResult(NULL_ARR.length, -1);
}
if(t instanceof byte[])
return encode((byte[])t);
return encode((byte[]) t, encoded);
if(t instanceof String)
return encode((String)t);
return encode((String)t, encoded);
if(t instanceof BigInteger)
return encode((BigInteger)t);
return encode((BigInteger)t, encoded);
if(t instanceof Float)
return encode((Float)t);
return encode((Float)t, encoded);
if(t instanceof Double)
return encode((Double)t);
return encode((Double)t, encoded);
if(t instanceof Boolean)
return encode((Boolean)t);
return encode((Boolean)t, encoded);
if(t instanceof UUID)
return encode((UUID)t);
return encode((UUID)t, encoded);
if(t instanceof Number)
return encode(((Number)t).longValue());
return encode(((Number)t).longValue(), encoded);
if(t instanceof Versionstamp)
return encode((Versionstamp)t);
return encode((Versionstamp)t, encoded);
if(t instanceof List<?>)
return encode((List<?>)t);
return encode((List<?>)t, encoded);
if(t instanceof Tuple)
return encode(((Tuple)t).getItems());
return encode(((Tuple)t).getItems(), encoded);
throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName());
}
static EncodeResult encode(Object t) {
return encode(t, false);
static EncodeResult encode(Object t, List<byte[]> encoded) {
return encode(t, false, encoded);
}
static EncodeResult encode(byte[] bytes) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {BYTES_CODE});
list.add(ByteArrayUtil.replace(bytes, new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
static EncodeResult encode(byte[] bytes, List<byte[]> encoded) {
encoded.add(BYTES_ARR);
byte[] escaped = ByteArrayUtil.replace(bytes, NULL_ARR, NULL_ESCAPED_ARR);
encoded.add(escaped);
encoded.add(new byte[] {nil});
//System.out.println("Joining bytes...");
return new EncodeResult(-1, ByteArrayUtil.join(null, list));
return new EncodeResult(2 + escaped.length,-1);
}
static EncodeResult encode(String s) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {STRING_CODE});
list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
static EncodeResult encode(String s, List<byte[]> encoded) {
encoded.add(STRING_ARR);
byte[] escaped = ByteArrayUtil.replace(s.getBytes(UTF8), NULL_ARR, NULL_ESCAPED_ARR);
encoded.add(escaped);
encoded.add(NULL_ARR);
//System.out.println("Joining string...");
return new EncodeResult(-1, ByteArrayUtil.join(null, list));
return new EncodeResult(2 + escaped.length, -1);
}
static EncodeResult encode(BigInteger i) {
static EncodeResult encode(BigInteger i, List<byte[]> encoded) {
//System.out.println("Encoding integral " + i);
if(i.equals(BigInteger.ZERO)) {
return new EncodeResult(-1, new byte[] { INT_ZERO_CODE });
encoded.add(new byte[]{INT_ZERO_CODE});
return new EncodeResult(1,-1);
}
byte[] bytes = i.toByteArray();
if(i.compareTo(BigInteger.ZERO) > 0) {
@ -219,7 +233,8 @@ class TupleUtil {
result[0] = POS_INT_END;
result[1] = (byte)(length);
System.arraycopy(bytes, bytes.length - length, result, 2, length);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
int n = ByteArrayUtil.bisectLeft(size_limits, i);
assert n <= size_limits.length;
@ -228,7 +243,8 @@ class TupleUtil {
byte[] result = new byte[n+1];
result[0] = (byte)(INT_ZERO_CODE + n);
System.arraycopy(bytes, bytes.length - n, result, 1, n);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
if(i.negate().compareTo(size_limits[size_limits.length-1]) > 0) {
int length = byteLength(i.negate().toByteArray());
@ -246,7 +262,8 @@ class TupleUtil {
Arrays.fill(result, 2, result.length - adjusted.length, (byte)0x00);
System.arraycopy(adjusted, 0, result, result.length - adjusted.length, adjusted.length);
}
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
int n = ByteArrayUtil.bisectLeft(size_limits, i.negate());
@ -257,70 +274,71 @@ class TupleUtil {
byte[] result = new byte[n+1];
result[0] = (byte)(20 - n);
System.arraycopy(adjustedBytes, adjustedBytes.length - n, result, 1, n);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Integer i) {
return encode(i.longValue());
static EncodeResult encode(Integer i, List<byte[]> encoded) {
return encode(i.longValue(), encoded);
}
static EncodeResult encode(long i) {
return encode(BigInteger.valueOf(i));
static EncodeResult encode(long i, List<byte[]> encoded) {
return encode(BigInteger.valueOf(i), encoded);
}
static EncodeResult encode(Float f) {
static EncodeResult encode(Float f, List<byte[]> encoded) {
byte[] result = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array();
floatingPointCoding(result, 1, true);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Double d) {
static EncodeResult encode(Double d, List<byte[]> encoded) {
byte[] result = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array();
floatingPointCoding(result, 1, true);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Boolean b) {
static EncodeResult encode(Boolean b, List<byte[]> encoded) {
if (b) {
return new EncodeResult(-1, new byte[] {TRUE_CODE});
encoded.add(TRUE_ARR);
} else {
return new EncodeResult(-1, new byte[] {FALSE_CODE});
encoded.add(FALSE_ARR);
}
return new EncodeResult(1, -1);
}
static EncodeResult encode(UUID uuid) {
static EncodeResult encode(UUID uuid, List<byte[]> encoded) {
byte[] result = ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits())
.array();
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Versionstamp v) {
byte[] result = ByteBuffer.allocate(Versionstamp.LENGTH + 1)
.put(VERSIONSTAMP_CODE)
.put(v.getBytes())
.array();
return new EncodeResult((v.isComplete() ? -1 : 1), result);
static EncodeResult encode(Versionstamp v, List<byte[]> encoded) {
encoded.add(VERSIONSTAMP_ARR);
encoded.add(v.getBytes());
return new EncodeResult(1 + Versionstamp.LENGTH, (v.isComplete() ? -1 : 1));
}
static EncodeResult encode(List<?> value) {
List<byte[]> parts = new LinkedList<byte[]>();
static EncodeResult encode(List<?> value, List<byte[]> encoded) {
int lenSoFar = 0;
int versionPos = -1;
parts.add(new byte[]{NESTED_CODE});
encoded.add(NESTED_ARR);
for(Object t : value) {
EncodeResult childResult = encode(t, true);
EncodeResult childResult = encode(t, true, encoded);
if(childResult.versionPos > 0) {
if(versionPos > 0) {
throw new IllegalArgumentException("Multiple incomplete Versionstamps included in Tuple");
}
versionPos = lenSoFar + childResult.versionPos;
}
lenSoFar += childResult.data.length;
parts.add(childResult.data);
lenSoFar += childResult.totalLength;
}
parts.add(new byte[]{0x00});
return new EncodeResult((versionPos < 0 ? -1 : versionPos + 1), ByteArrayUtil.join(null, parts));
encoded.add(NULL_ARR);
return new EncodeResult(lenSoFar + 2, (versionPos < 0 ? -1 : versionPos + 1));
}
static DecodeResult decode(byte[] rep, int pos, int last) {
@ -335,14 +353,14 @@ class TupleUtil {
if(code == BYTES_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of byte string: " + end);
byte[] range = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
byte[] range = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil });
//System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'");
return new DecodeResult(end + 1, range);
}
if(code == STRING_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of UTF8 string: " + end);
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil });
String str = new String(stringBytes, UTF8);
//System.out.println(" -> UTF8 string contents: '" + str + "'");
return new DecodeResult(end + 1, str);
@ -437,6 +455,18 @@ class TupleUtil {
throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos);
}
static int compareSignedBigEndian(byte[] arr1, byte[] arr2) {
if(arr1[0] < 0 && arr2[0] < 0) {
return -1 * ByteArrayUtil.compareUnsigned(arr1, arr2);
} else if(arr1[0] < 0) {
return -1;
} else if(arr2[0] < 0) {
return 1;
} else {
return ByteArrayUtil.compareUnsigned(arr1, arr2);
}
}
static int compareItems(Object item1, Object item2) {
int code1 = TupleUtil.getCodeFor(item1);
int code2 = TupleUtil.getCodeFor(item2);
@ -473,25 +503,28 @@ class TupleUtil {
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// sorting correctly.
byte[] encoded1 = encode((Double)item1).data;
byte[] encoded2 = encode((Double)item2).data;
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
byte[] dBytes1 = ByteBuffer.allocate(8).putDouble((Double)item1).array();
byte[] dBytes2 = ByteBuffer.allocate(8).putDouble((Double)item2).array();
return compareSignedBigEndian(dBytes1, dBytes2);
}
if(code1 == FLOAT_CODE) {
// This is done for the same reason that double comparison is done
// that way.
byte[] encoded1 = encode((Float)item1).data;
byte[] encoded2 = encode((Float)item2).data;
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
byte[] fBytes1 = ByteBuffer.allocate(4).putFloat((Float)item1).array();
byte[] fBytes2 = ByteBuffer.allocate(4).putFloat((Float)item2).array();
return compareSignedBigEndian(fBytes1, fBytes2);
}
if(code1 == FALSE_CODE) {
return Boolean.compare((Boolean)item1, (Boolean)item2);
}
if(code1 == UUID_CODE) {
// Java UUID.compareTo is signed.
byte[] encoded1 = encode((UUID)item1).data;
byte[] encoded2 = encode((UUID)item2).data;
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
// Java UUID.compareTo is signed, so we have to used the unsigned methods.
UUID uuid1 = (UUID)item1;
UUID uuid2 = (UUID)item2;
int cmp1 = Long.compareUnsigned(uuid1.getMostSignificantBits(), uuid2.getMostSignificantBits());
if(cmp1 != 0)
return cmp1;
return Long.compareUnsigned(uuid1.getLeastSignificantBits(), uuid2.getLeastSignificantBits());
}
if(code1 == VERSIONSTAMP_CODE) {
return ((Versionstamp)item1).compareTo((Versionstamp)item2);
@ -514,29 +547,48 @@ class TupleUtil {
return items;
}
static EncodeResult pack(List<Object> items, byte[] prefix) {
if(items.size() == 0)
return new EncodeResult(-1, new byte[0]);
List<byte[]> parts = new ArrayList<>(items.size() + (prefix == null ? 0 : 1));
static EncodeResult encodeAll(List<Object> items, byte[] prefix, List<byte[]> encoded) {
if(prefix != null) {
parts.add(prefix);
encoded.add(prefix);
}
int lenSoFar = (prefix == null) ? 0 : prefix.length;
int versionPos = -1;
for(Object t : items) {
EncodeResult result = encode(t);
EncodeResult result = encode(t, encoded);
if(result.versionPos > 0) {
if(versionPos > 0) {
throw new IllegalArgumentException("Multiple incomplete Versionstamps included in Tuple");
}
versionPos = result.versionPos + lenSoFar;
}
lenSoFar += result.data.length;
parts.add(result.data);
lenSoFar += result.totalLength;
}
//System.out.println("Joining whole tuple...");
return new EncodeResult(versionPos, ByteArrayUtil.join(null, parts));
return new EncodeResult(lenSoFar, versionPos);
}
static byte[] pack(List<Object> items, byte[] prefix) {
List<byte[]> encoded = new ArrayList<byte[]>(2 * items.size() + (prefix == null ? 0 : 1));
EncodeResult result = encodeAll(items, prefix, encoded);
if(result.versionPos > 0) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
} else {
return ByteArrayUtil.join(null, encoded);
}
}
static byte[] packWithVersionstamp(List<Object> items, byte[] prefix) {
List<byte[]> encoded = new ArrayList<byte[]>(2 * items.size() + (prefix == null ? 1 : 2));
EncodeResult result = encodeAll(items, prefix, encoded);
if(result.versionPos < 0) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
} else {
if(result.versionPos > 0xffff) {
throw new IllegalArgumentException("Tuple has incomplete version at position " + result.versionPos + " which is greater than the maximum " + 0xffff);
}
encoded.add(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array());
return ByteArrayUtil.join(null, encoded);
}
}
static boolean hasIncompleteVersionstamp(Stream<?> items) {
@ -557,7 +609,7 @@ class TupleUtil {
public static void main(String[] args) {
try {
byte[] bytes = encode( 4 ).data;
byte[] bytes = pack(Collections.singletonList(4), null );
assert 4 == (Integer)(decode( bytes, 0, bytes.length ).o);
} catch (Exception e) {
e.printStackTrace();
@ -565,7 +617,7 @@ class TupleUtil {
}
try {
byte[] bytes = encode( "\u021Aest \u0218tring" ).data;
byte[] bytes = pack( Collections.singletonList("\u021Aest \u0218tring"), null );
String string = (String)(decode( bytes, 0, bytes.length ).o);
System.out.println("contents -> " + string);
assert "\u021Aest \u0218tring" == string;
@ -576,10 +628,10 @@ class TupleUtil {
/*Object[] a = new Object[] { "\u0000a", -2, "b\u0001", 12345, ""};
List<Object> o = Arrays.asList(a);
byte[] packed = pack( o );
byte[] packed = pack( o, null );
System.out.println("packed length: " + packed.length);
o = unpack( packed );
System.out.println("unpacked elements: " + packed);
o = unpack( packed, 0, packed.length );
System.out.println("unpacked elements: " + o);
for(Object obj : o)
System.out.println(" -> type: " + obj.getClass().getName());*/
}

15
bindings/java/src/main/com/apple/foundationdb/tuple/Tuple.java
View File

@ -299,11 +299,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @return a serialized representation of this {@code Tuple} prepended by the {@code prefix}.
*/
public byte[] pack(byte[] prefix) {
TupleUtil.EncodeResult encoded = TupleUtil.pack(elements, prefix);
if(encoded.versionPos > 0) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
}
return encoded.data;
return TupleUtil.pack(elements, prefix);
}
/**
@ -337,14 +333,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
* @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple}
*/
public byte[] packWithVersionstamp(byte[] prefix) {
TupleUtil.EncodeResult encoded = TupleUtil.pack(elements, prefix);
if(encoded.versionPos < 0) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
}
return ByteBuffer.allocate(encoded.data.length + 2).order(ByteOrder.LITTLE_ENDIAN)
.put(encoded.data)
.putShort((short)encoded.versionPos)
.array();
return TupleUtil.packWithVersionstamp(elements, prefix);
}
/**

241
bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java
View File

@ -26,13 +26,13 @@ import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
class TupleUtil {
private static final byte nil = 0x00;
private static final byte[] nil_rep = new byte[] {nil, (byte)0xFF};
private static final BigInteger[] size_limits;
private static final Charset UTF8;
private static final IterableComparator iterableComparator;
@ -50,6 +50,15 @@ class TupleUtil {
private static final byte UUID_CODE = 0x30;
private static final byte VERSIONSTAMP_CODE = 0x33;
private static final byte[] NULL_ARR = new byte[] {nil};
private static final byte[] NULL_ESCAPED_ARR = new byte[] {nil, (byte)0xFF};
private static final byte[] BYTES_ARR = new byte[]{0x01};
private static final byte[] STRING_ARR = new byte[]{0x02};
private static final byte[] NESTED_ARR = new byte[]{0x05};
private static final byte[] FALSE_ARR = new byte[]{0x26};
private static final byte[] TRUE_ARR = new byte[]{0x27};
private static final byte[] VERSIONSTAMP_ARR = new byte[]{0x33};
static {
size_limits = new BigInteger[9];
for(int i = 0; i < 9; i++) {
@ -70,12 +79,12 @@ class TupleUtil {
}
static class EncodeResult {
final int totalLength;
final int versionPos;
final byte[] data;
EncodeResult(int versionPos, byte[] data) {
EncodeResult(int totalLength, int versionPos) {
this.totalLength = totalLength;
this.versionPos = versionPos;
this.data = data;
}
}
@ -152,65 +161,70 @@ class TupleUtil {
throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName());
}
static EncodeResult encode(Object t, boolean nested) {
static EncodeResult encode(Object t, boolean nested, List<byte[]> encoded) {
if(t == null)
if (nested)
return new EncodeResult(-1, new byte[]{nil, (byte) 0xff});
else
return new EncodeResult(-1, new byte[]{nil});
if (nested) {
encoded.add(NULL_ESCAPED_ARR);
return new EncodeResult(NULL_ESCAPED_ARR.length,-1);
}
else {
encoded.add(NULL_ARR);
return new EncodeResult(NULL_ARR.length, -1);
}
if(t instanceof byte[])
return encode((byte[])t);
return encode((byte[]) t, encoded);
if(t instanceof String)
return encode((String)t);
return encode((String)t, encoded);
if(t instanceof BigInteger)
return encode((BigInteger)t);
return encode((BigInteger)t, encoded);
if(t instanceof Float)
return encode((Float)t);
return encode((Float)t, encoded);
if(t instanceof Double)
return encode((Double)t);
return encode((Double)t, encoded);
if(t instanceof Boolean)
return encode((Boolean)t);
return encode((Boolean)t, encoded);
if(t instanceof UUID)
return encode((UUID)t);
return encode((UUID)t, encoded);
if(t instanceof Number)
return encode(((Number)t).longValue());
return encode(((Number)t).longValue(), encoded);
if(t instanceof Versionstamp)
return encode((Versionstamp)t);
return encode((Versionstamp)t, encoded);
if(t instanceof List<?>)
return encode((List<?>)t);
return encode((List<?>)t, encoded);
if(t instanceof Tuple)
return encode(((Tuple)t).getItems());
return encode(((Tuple)t).getItems(), encoded);
throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName());
}
static EncodeResult encode(Object t) {
return encode(t, false);
static EncodeResult encode(Object t, List<byte[]> encoded) {
return encode(t, false, encoded);
}
static EncodeResult encode(byte[] bytes) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {BYTES_CODE});
list.add(ByteArrayUtil.replace(bytes, new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
static EncodeResult encode(byte[] bytes, List<byte[]> encoded) {
encoded.add(BYTES_ARR);
byte[] escaped = ByteArrayUtil.replace(bytes, NULL_ARR, NULL_ESCAPED_ARR);
encoded.add(escaped);
encoded.add(new byte[] {nil});
//System.out.println("Joining bytes...");
return new EncodeResult(-1, ByteArrayUtil.join(null, list));
return new EncodeResult(2 + escaped.length,-1);
}
static EncodeResult encode(String s) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {STRING_CODE});
list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
static EncodeResult encode(String s, List<byte[]> encoded) {
encoded.add(STRING_ARR);
byte[] escaped = ByteArrayUtil.replace(s.getBytes(UTF8), NULL_ARR, NULL_ESCAPED_ARR);
encoded.add(escaped);
encoded.add(NULL_ARR);
//System.out.println("Joining string...");
return new EncodeResult(-1, ByteArrayUtil.join(null, list));
return new EncodeResult(2 + escaped.length, -1);
}
static EncodeResult encode(BigInteger i) {
static EncodeResult encode(BigInteger i, List<byte[]> encoded) {
//System.out.println("Encoding integral " + i);
if(i.equals(BigInteger.ZERO)) {
return new EncodeResult(-1, new byte[] { INT_ZERO_CODE });
encoded.add(new byte[]{INT_ZERO_CODE});
return new EncodeResult(1,-1);
}
byte[] bytes = i.toByteArray();
if(i.compareTo(BigInteger.ZERO) > 0) {
@ -223,7 +237,8 @@ class TupleUtil {
result[0] = POS_INT_END;
result[1] = (byte)(length);
System.arraycopy(bytes, bytes.length - length, result, 2, length);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
int n = ByteArrayUtil.bisectLeft(size_limits, i);
assert n <= size_limits.length;
@ -232,7 +247,8 @@ class TupleUtil {
byte[] result = new byte[n+1];
result[0] = (byte)(INT_ZERO_CODE + n);
System.arraycopy(bytes, bytes.length - n, result, 1, n);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
if(i.negate().compareTo(size_limits[size_limits.length-1]) > 0) {
int length = byteLength(i.negate().toByteArray());
@ -250,7 +266,8 @@ class TupleUtil {
Arrays.fill(result, 2, result.length - adjusted.length, (byte)0x00);
System.arraycopy(adjusted, 0, result, result.length - adjusted.length, adjusted.length);
}
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
int n = ByteArrayUtil.bisectLeft(size_limits, i.negate());
@ -261,70 +278,71 @@ class TupleUtil {
byte[] result = new byte[n+1];
result[0] = (byte)(20 - n);
System.arraycopy(adjustedBytes, adjustedBytes.length - n, result, 1, n);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Integer i) {
return encode(i.longValue());
static EncodeResult encode(Integer i, List<byte[]> encoded) {
return encode(i.longValue(), encoded);
}
static EncodeResult encode(long i) {
return encode(BigInteger.valueOf(i));
static EncodeResult encode(long i, List<byte[]> encoded) {
return encode(BigInteger.valueOf(i), encoded);
}
static EncodeResult encode(Float f) {
static EncodeResult encode(Float f, List<byte[]> encoded) {
byte[] result = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array();
floatingPointCoding(result, 1, true);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Double d) {
static EncodeResult encode(Double d, List<byte[]> encoded) {
byte[] result = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array();
floatingPointCoding(result, 1, true);
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Boolean b) {
static EncodeResult encode(Boolean b, List<byte[]> encoded) {
if (b) {
return new EncodeResult(-1, new byte[] {TRUE_CODE});
encoded.add(TRUE_ARR);
} else {
return new EncodeResult(-1, new byte[] {FALSE_CODE});
encoded.add(FALSE_ARR);
}
return new EncodeResult(1, -1);
}
static EncodeResult encode(UUID uuid) {
static EncodeResult encode(UUID uuid, List<byte[]> encoded) {
byte[] result = ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits())
.array();
return new EncodeResult(-1, result);
encoded.add(result);
return new EncodeResult(result.length, -1);
}
static EncodeResult encode(Versionstamp v) {
byte[] result = ByteBuffer.allocate(Versionstamp.LENGTH + 1)
.put(VERSIONSTAMP_CODE)
.put(v.getBytes())
.array();
return new EncodeResult((v.isComplete() ? -1 : 1), result);
static EncodeResult encode(Versionstamp v, List<byte[]> encoded) {
encoded.add(VERSIONSTAMP_ARR);
encoded.add(v.getBytes());
return new EncodeResult(1 + Versionstamp.LENGTH, (v.isComplete() ? -1 : 1));
}
static EncodeResult encode(List<?> value) {
List<byte[]> parts = new LinkedList<byte[]>();
static EncodeResult encode(List<?> value, List<byte[]> encoded) {
int lenSoFar = 0;
int versionPos = -1;
parts.add(new byte[]{NESTED_CODE});
encoded.add(NESTED_ARR);
for(Object t : value) {
EncodeResult childResult = encode(t, true);
EncodeResult childResult = encode(t, true, encoded);
if(childResult.versionPos > 0) {
if(versionPos > 0) {
throw new IllegalArgumentException("Multiple incomplete Versionstamps included in Tuple");
}
versionPos = lenSoFar + childResult.versionPos;
}
lenSoFar += childResult.data.length;
parts.add(childResult.data);
lenSoFar += childResult.totalLength;
}
parts.add(new byte[]{0x00});
return new EncodeResult((versionPos < 0 ? -1 : versionPos + 1), ByteArrayUtil.join(null, parts));
encoded.add(NULL_ARR);
return new EncodeResult(lenSoFar + 2, (versionPos < 0 ? -1 : versionPos + 1));
}
static DecodeResult decode(byte[] rep, int pos, int last) {
@ -339,14 +357,14 @@ class TupleUtil {
if(code == BYTES_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of byte string: " + end);
byte[] range = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
byte[] range = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil });
//System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'");
return new DecodeResult(end + 1, range);
}
if(code == STRING_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of UTF8 string: " + end);
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil });
String str = new String(stringBytes, UTF8);
//System.out.println(" -> UTF8 string contents: '" + str + "'");
return new DecodeResult(end + 1, str);
@ -441,6 +459,18 @@ class TupleUtil {
throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos);
}
static int compareSignedBigEndian(byte[] arr1, byte[] arr2) {
if(arr1[0] < 0 && arr2[0] < 0) {
return -1 * ByteArrayUtil.compareUnsigned(arr1, arr2);
} else if(arr1[0] < 0) {
return -1;
} else if(arr2[0] < 0) {
return 1;
} else {
return ByteArrayUtil.compareUnsigned(arr1, arr2);
}
}
static int compareItems(Object item1, Object item2) {
int code1 = TupleUtil.getCodeFor(item1);
int code2 = TupleUtil.getCodeFor(item2);
@ -477,16 +507,16 @@ class TupleUtil {
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// sorting correctly.
byte[] encoded1 = encode((Double)item1).data;
byte[] encoded2 = encode((Double)item2).data;
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
byte[] dBytes1 = ByteBuffer.allocate(8).putDouble((Double)item1).array();
byte[] dBytes2 = ByteBuffer.allocate(8).putDouble((Double)item2).array();
return compareSignedBigEndian(dBytes1, dBytes2);
}
if(code1 == FLOAT_CODE) {
// This is done for the same reason that double comparison is done
// that way.
byte[] encoded1 = encode((Float)item1).data;
byte[] encoded2 = encode((Float)item2).data;
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
byte[] fBytes1 = ByteBuffer.allocate(4).putFloat((Float)item1).array();
byte[] fBytes2 = ByteBuffer.allocate(4).putFloat((Float)item2).array();
return compareSignedBigEndian(fBytes1, fBytes2);
}
if(code1 == FALSE_CODE) {
boolean b1 = (Boolean)item1;
@ -495,8 +525,16 @@ class TupleUtil {
}
if(code1 == UUID_CODE) {
// Java UUID.compareTo is signed.
byte[] encoded1 = encode((UUID)item1).data;
byte[] encoded2 = encode((UUID)item2).data;
UUID uuid1 = (UUID)item1;
UUID uuid2 = (UUID)item2;
byte[] encoded1 = ByteBuffer.allocate(16).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid1.getMostSignificantBits())
.putLong(uuid1.getLeastSignificantBits())
.array();
byte[] encoded2 = ByteBuffer.allocate(16).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid2.getMostSignificantBits())
.putLong(uuid2.getLeastSignificantBits())
.array();
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == VERSIONSTAMP_CODE) {
@ -520,29 +558,48 @@ class TupleUtil {
return items;
}
static EncodeResult pack(List<Object> items, byte[] prefix) {
if(items.size() == 0)
return new EncodeResult(-1, new byte[0]);
List<byte[]> parts = new ArrayList<byte[]>(items.size() + (prefix == null ? 0 : 1));
static EncodeResult encodeAll(List<Object> items, byte[] prefix, List<byte[]> encoded) {
if(prefix != null) {
parts.add(prefix);
encoded.add(prefix);
}
int lenSoFar = (prefix == null) ? 0 : prefix.length;
int versionPos = -1;
for(Object t : items) {
EncodeResult result = encode(t);
EncodeResult result = encode(t, encoded);
if(result.versionPos > 0) {
if(versionPos > 0) {
throw new IllegalArgumentException("Multiple incomplete Versionstamps included in Tuple");
}
versionPos = result.versionPos + lenSoFar;
}
lenSoFar += result.data.length;
parts.add(result.data);
lenSoFar += result.totalLength;
}
//System.out.println("Joining whole tuple...");
return new EncodeResult(versionPos, ByteArrayUtil.join(null, parts));
return new EncodeResult(lenSoFar, versionPos);
}
static byte[] pack(List<Object> items, byte[] prefix) {
List<byte[]> encoded = new ArrayList<byte[]>(2 * items.size() + (prefix == null ? 0 : 1));
EncodeResult result = encodeAll(items, prefix, encoded);
if(result.versionPos > 0) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple pack");
} else {
return ByteArrayUtil.join(null, encoded);
}
}
static byte[] packWithVersionstamp(List<Object> items, byte[] prefix) {
List<byte[]> encoded = new ArrayList<byte[]>(2 * items.size() + (prefix == null ? 1 : 2));
EncodeResult result = encodeAll(items, prefix, encoded);
if(result.versionPos < 0) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple pack with versionstamp");
} else {
if(result.versionPos > 0xffff) {
throw new IllegalArgumentException("Tuple has incomplete version at position " + result.versionPos + " which is greater than the maximum " + 0xffff);
}
encoded.add(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array());
return ByteArrayUtil.join(null, encoded);
}
}
static boolean hasIncompleteVersionstamp(Iterable<?> items) {
@ -567,7 +624,7 @@ class TupleUtil {
public static void main(String[] args) {
try {
byte[] bytes = encode( 4 ).data;
byte[] bytes = pack( Collections.singletonList((Object)4), null );
assert 4 == (Integer)(decode( bytes, 0, bytes.length ).o);
} catch (Exception e) {
e.printStackTrace();
@ -575,7 +632,7 @@ class TupleUtil {
}
try {
byte[] bytes = encode( "\u021Aest \u0218tring" ).data;
byte[] bytes = pack( Collections.singletonList((Object)"\u021Aest \u0218tring"), null );
String string = (String)(decode( bytes, 0, bytes.length ).o);
System.out.println("contents -> " + string);
assert "\u021Aest \u0218tring" == string;
@ -586,10 +643,10 @@ class TupleUtil {
/*Object[] a = new Object[] { "\u0000a", -2, "b\u0001", 12345, ""};
List<Object> o = Arrays.asList(a);
byte[] packed = pack( o ).data;
byte[] packed = pack( o, null );
System.out.println("packed length: " + packed.length);
o = unpack( packed );
System.out.println("unpacked elements: " + packed);
o = unpack( packed, 0, packed.length );
System.out.println("unpacked elements: " + o);
for(Object obj : o)
System.out.println(" -> type: " + obj.getClass().getName());*/
}