wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

various Java tuple performance tweaks 

These include:

* Memoizing packed representations within Tuples
* Using longs instead of BigIntegers if possible
* As much as possible sticking to manipulating primitive types when using floats/doubles
This commit is contained in:
Alec Grieser 2019-02-24 20:52:28 -08:00
parent e6ce0ebd27
commit e9771364d7
No known key found for this signature in database
GPG Key ID: CAF63551C60D3462
3 changed files with 252 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/java/src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java
View File

@ -229,8 +229,7 @@ public class ByteArrayUtil {
int n = Arrays.binarySearch(arr, i); int n = Arrays.binarySearch(arr, i);
if(n >= 0) if(n >= 0)
return n; return n;
int ip = (n + 1) * -1; return (n + 1) * -1;
return ip;
} }
/** /**

14
bindings/java/src/main/com/apple/foundationdb/tuple/Tuple.java
View File

@ -824,9 +824,12 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
} }
/** /**
* Get the number of bytes in the packed representation of this {@code Tuple}. * Get the number of bytes in the packed representation of this {@code Tuple}. Note that at the
* moment, this number is calculated by packing the {@code Tuple} and looking at its size. This method
* will memoize the result, however, so asking the same {@code Tuple} for its size multiple times
* is a fast operation.
* *
* @return * @return the number of bytes in the packed representation of this {@code Tuple}
*/ */
public int getPackedSize() { public int getPackedSize() {
byte[] p = packMaybeVersionstamp(null); byte[] p = packMaybeVersionstamp(null);
@ -847,7 +850,12 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*/ */
@Override @Override
public int compareTo(Tuple t) { public int compareTo(Tuple t) {
return comparator.compare(elements, t.elements); if(packed != null && t.packed != null) {
return ByteArrayUtil.compareUnsigned(packed, t.packed);
}
else {
return comparator.compare(elements, t.elements);
}
} }
/** /**

446
bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java
View File

@ -36,8 +36,10 @@ import com.apple.foundationdb.FDB;
class TupleUtil { class TupleUtil {
private static final byte nil = 0x00; private static final byte nil = 0x00;
private static final BigInteger[] size_limits; private static final BigInteger[] BIG_INT_SIZE_LIMITS;
private static final Charset UTF8; private static final Charset UTF8;
private static final BigInteger LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE);
private static final BigInteger LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE);
private static final IterableComparator iterableComparator; private static final IterableComparator iterableComparator;
private static final byte BYTES_CODE = 0x01; private static final byte BYTES_CODE = 0x01;
@ -55,27 +57,28 @@ class TupleUtil {
private static final byte[] NULL_ARR = new byte[] {nil}; private static final byte[] NULL_ARR = new byte[] {nil};
private static final byte[] NULL_ESCAPED_ARR = new byte[] {nil, (byte)0xFF}; private static final byte[] NULL_ESCAPED_ARR = new byte[] {nil, (byte)0xFF};
private static final byte[] BYTES_ARR = new byte[]{0x01}; private static final byte[] BYTES_ARR = new byte[]{BYTES_CODE};
private static final byte[] STRING_ARR = new byte[]{0x02}; private static final byte[] STRING_ARR = new byte[]{STRING_CODE};
private static final byte[] NESTED_ARR = new byte[]{0x05}; private static final byte[] NESTED_ARR = new byte[]{NESTED_CODE};
private static final byte[] FALSE_ARR = new byte[]{0x26}; private static final byte[] INT_ZERO_ARR = new byte[]{INT_ZERO_CODE};
private static final byte[] TRUE_ARR = new byte[]{0x27}; private static final byte[] FALSE_ARR = new byte[]{FALSE_CODE};
private static final byte[] VERSIONSTAMP_ARR = new byte[]{0x33}; private static final byte[] TRUE_ARR = new byte[]{TRUE_CODE};
private static final byte[] VERSIONSTAMP_ARR = new byte[]{VERSIONSTAMP_CODE};
static { static {
size_limits = new BigInteger[9]; BIG_INT_SIZE_LIMITS = new BigInteger[9];
for(int i = 0; i < 9; i++) { for(int i = 0; i < BIG_INT_SIZE_LIMITS.length; i++) {
size_limits[i] = (BigInteger.ONE).shiftLeft(i * 8).subtract(BigInteger.ONE); BIG_INT_SIZE_LIMITS[i] = (BigInteger.ONE).shiftLeft(i * 8).subtract(BigInteger.ONE);
} }
UTF8 = Charset.forName("UTF-8"); UTF8 = Charset.forName("UTF-8");
iterableComparator = new IterableComparator(); iterableComparator = new IterableComparator();
} }
static class DecodeResult { static class DecodeState {
final List<Object> values; final List<Object> values;
int end; int end;
DecodeResult() { DecodeState() {
values = new ArrayList<>(); values = new ArrayList<>();
end = 0; end = 0;
} }
@ -86,18 +89,18 @@ class TupleUtil {
} }
} }
static class EncodeResult { static class EncodeState {
final List<byte[]> encodedValues; final List<byte[]> encodedValues;
int totalLength; int totalLength;
int versionPos; int versionPos;
EncodeResult(int capacity) { EncodeState(int capacity) {
this.encodedValues = new ArrayList<>(capacity); this.encodedValues = new ArrayList<>(capacity);
totalLength = 0; totalLength = 0;
versionPos = -1; versionPos = -1;
} }
EncodeResult add(byte[] encoded, int versionPos) { EncodeState add(byte[] encoded, int versionPos) {
if(versionPos >= 0 && this.versionPos >= 0) { if(versionPos >= 0 && this.versionPos >= 0) {
throw new IllegalArgumentException("Multiple incomplete Versionstamps included in Tuple"); throw new IllegalArgumentException("Multiple incomplete Versionstamps included in Tuple");
} }
@ -107,7 +110,7 @@ class TupleUtil {
return this; return this;
} }
EncodeResult add(byte[] encoded) { EncodeState add(byte[] encoded) {
encodedValues.add(encoded); encodedValues.add(encoded);
totalLength += encoded.length; totalLength += encoded.length;
return this; return this;
@ -122,37 +125,37 @@ class TupleUtil {
return 0; return 0;
} }
/** // These four functions are for adjusting the encoding of floating point numbers so
* Takes the Big-Endian byte representation of a floating point number and adjusts // that when their byte representation is written out in big-endian order, unsigned
* it so that it sorts correctly. For encoding, if the sign bit is 1 (the number // lexicographic byte comparison orders the values in the same way as the semantic
* is negative), then we need to flip all of the bits; otherwise, just flip the // ordering of the values. This means flipping all bits for negative values and flipping
* sign bit. For decoding, if the sign bit is 0 (the number is negative), then // only the most-significant bit (i.e., the sign bit as all values in Java are signed)
* we also need to flip all of the bits; otherwise, just flip the sign bit. // in the case that the number is positive. For these purposes, 0.0 is positive and -0.0
* This will mutate in place the given array. // is negative.
*
* @param bytes Big-Endian IEEE encoding of a floating point number
* @param start the (zero-indexed) first byte in the array to mutate
* @param encode <code>true</code> if we encoding the float and <code>false</code> if we are decoding
* @return the encoded {@code byte[]}
*/
static byte[] floatingPointCoding(byte[] bytes, int start, boolean encode) {
if(encode && (bytes[start] & (byte)0x80) != (byte)0x00) {
for(int i = start; i < bytes.length; i++) {
bytes[i] = (byte) (bytes[i] ^ 0xff);
}
} else if(!encode && (bytes[start] & (byte)0x80) != (byte)0x80) {
for(int i = start; i < bytes.length; i++) {
bytes[i] = (byte) (bytes[i] ^ 0xff);
}
} else {
bytes[start] = (byte) (0x80 ^ bytes[start]);
}
return bytes; static int encodeFloatBits(float f) {
int intBits = Float.floatToRawIntBits(f);
return (intBits < 0) ? (~intBits) : (intBits ^ Integer.MIN_VALUE);
} }
static byte[] join(List<byte[]> items) { static long encodeDoubleBits(double d) {
return ByteArrayUtil.join(null, items); long longBits = Double.doubleToRawLongBits(d);
return (longBits < 0L) ? (~longBits) : (longBits ^ Long.MIN_VALUE);
}
static float decodeFloatBits(int i) {
int origBits = (i >= 0) ? (~i) : (i ^ Integer.MIN_VALUE);
return Float.intBitsToFloat(origBits);
}
static double decodeDoubleBits(long l) {
long origBits = (l >= 0) ? (~l) : (l ^ Long.MIN_VALUE);
return Double.longBitsToDouble(origBits);
}
// Get the number of bytes in the representation of a long.
static int byteCount(long i) {
return (Long.SIZE + 7 - Long.numberOfLeadingZeros(i >= 0 ? i : -i)) / 8;
} }
private static void adjustVersionPosition300(byte[] packed, int delta) { private static void adjustVersionPosition300(byte[] packed, int delta) {
@ -215,64 +218,64 @@ class TupleUtil {
throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName()); throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName());
} }
static void encode(EncodeResult result, Object t, boolean nested) { static void encode(EncodeState state, Object t, boolean nested) {
if(t == null) { if(t == null) {
if(nested) { if(nested) {
result.add(NULL_ESCAPED_ARR); state.add(NULL_ESCAPED_ARR);
} }
else { else {
result.add(NULL_ARR); state.add(NULL_ARR);
} }
} }
else if(t instanceof byte[]) else if(t instanceof byte[])
encode(result, (byte[]) t); encode(state, (byte[]) t);
else if(t instanceof String) else if(t instanceof String)
encode(result, (String)t); encode(state, (String)t);
else if(t instanceof BigInteger)
encode(result, (BigInteger)t);
else if(t instanceof Float) else if(t instanceof Float)
encode(result, (Float)t); encode(state, (Float)t);
else if(t instanceof Double) else if(t instanceof Double)
encode(result, (Double)t); encode(state, (Double)t);
else if(t instanceof Boolean) else if(t instanceof Boolean)
encode(result, (Boolean)t); encode(state, (Boolean)t);
else if(t instanceof UUID) else if(t instanceof UUID)
encode(result, (UUID)t); encode(state, (UUID)t);
else if(t instanceof BigInteger)
encode(state, (BigInteger)t);
else if(t instanceof Number) else if(t instanceof Number)
encode(result, ((Number)t).longValue()); encode(state, ((Number)t).longValue());
else if(t instanceof Versionstamp) else if(t instanceof Versionstamp)
encode(result, (Versionstamp)t); encode(state, (Versionstamp)t);
else if(t instanceof List<?>) else if(t instanceof List<?>)
encode(result, (List<?>)t); encode(state, (List<?>)t);
else if(t instanceof Tuple) else if(t instanceof Tuple)
encode(result, ((Tuple)t).getItems()); encode(state, ((Tuple)t).getItems());
else else
throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName()); throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName());
} }
static void encode(EncodeResult result, Object t) { static void encode(EncodeState state, Object t) {
encode(result, t, false); encode(state, t, false);
} }
static void encode(EncodeResult result, byte[] bytes) { static void encode(EncodeState state, byte[] bytes) {
byte[] escaped = ByteArrayUtil.replace(bytes, NULL_ARR, NULL_ESCAPED_ARR); byte[] escaped = ByteArrayUtil.replace(bytes, NULL_ARR, NULL_ESCAPED_ARR);
result.add(BYTES_ARR).add(escaped).add(NULL_ARR); state.add(BYTES_ARR).add(escaped).add(NULL_ARR);
} }
static void encode(EncodeResult result, String s) { static void encode(EncodeState state, String s) {
byte[] escaped = ByteArrayUtil.replace(s.getBytes(UTF8), NULL_ARR, NULL_ESCAPED_ARR); byte[] escaped = ByteArrayUtil.replace(s.getBytes(UTF8), NULL_ARR, NULL_ESCAPED_ARR);
result.add(STRING_ARR).add(escaped).add(NULL_ARR); state.add(STRING_ARR).add(escaped).add(NULL_ARR);
} }
static void encode(EncodeResult result, BigInteger i) { static void encode(EncodeState state, BigInteger i) {
//System.out.println("Encoding integral " + i); //System.out.println("Encoding integral " + i);
if(i.equals(BigInteger.ZERO)) { if(i.equals(BigInteger.ZERO)) {
result.add(new byte[]{INT_ZERO_CODE}); state.add(INT_ZERO_ARR);
return; return;
} }
byte[] bytes = i.toByteArray(); byte[] bytes = i.toByteArray();
if(i.compareTo(BigInteger.ZERO) > 0) { if(i.compareTo(BigInteger.ZERO) > 0) {
if(i.compareTo(size_limits[size_limits.length-1]) > 0) { if(i.compareTo(BIG_INT_SIZE_LIMITS[BIG_INT_SIZE_LIMITS.length-1]) > 0) {
int length = byteLength(bytes); int length = byteLength(bytes);
if(length > 0xff) { if(length > 0xff) {
throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)"); throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)");
@ -281,21 +284,20 @@ class TupleUtil {
intBytes[0] = POS_INT_END; intBytes[0] = POS_INT_END;
intBytes[1] = (byte)(length); intBytes[1] = (byte)(length);
System.arraycopy(bytes, bytes.length - length, intBytes, 2, length); System.arraycopy(bytes, bytes.length - length, intBytes, 2, length);
result.add(intBytes); state.add(intBytes);
} }
else { else {
int n = ByteArrayUtil.bisectLeft(size_limits, i); int n = ByteArrayUtil.bisectLeft(BIG_INT_SIZE_LIMITS, i);
assert n <= size_limits.length; assert n <= BIG_INT_SIZE_LIMITS.length;
//byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array();
//System.out.println(" -- integral has 'n' of " + n + " and output bytes of " + bytes.length); //System.out.println(" -- integral has 'n' of " + n + " and output bytes of " + bytes.length);
byte[] intBytes = new byte[n + 1]; byte[] intBytes = new byte[n + 1];
intBytes[0] = (byte) (INT_ZERO_CODE + n); intBytes[0] = (byte) (INT_ZERO_CODE + n);
System.arraycopy(bytes, bytes.length - n, intBytes, 1, n); System.arraycopy(bytes, bytes.length - n, intBytes, 1, n);
result.add(intBytes); state.add(intBytes);
} }
} }
else { else {
if(i.negate().compareTo(size_limits[size_limits.length - 1]) > 0) { if(i.negate().compareTo(BIG_INT_SIZE_LIMITS[BIG_INT_SIZE_LIMITS.length - 1]) > 0) {
int length = byteLength(i.negate().toByteArray()); int length = byteLength(i.negate().toByteArray());
if (length > 0xff) { if (length > 0xff) {
throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)"); throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)");
@ -311,92 +313,109 @@ class TupleUtil {
Arrays.fill(intBytes, 2, intBytes.length - adjusted.length, (byte) 0x00); Arrays.fill(intBytes, 2, intBytes.length - adjusted.length, (byte) 0x00);
System.arraycopy(adjusted, 0, intBytes, intBytes.length - adjusted.length, adjusted.length); System.arraycopy(adjusted, 0, intBytes, intBytes.length - adjusted.length, adjusted.length);
} }
result.add(intBytes); state.add(intBytes);
} }
else { else {
int n = ByteArrayUtil.bisectLeft(size_limits, i.negate()); int n = ByteArrayUtil.bisectLeft(BIG_INT_SIZE_LIMITS, i.negate());
assert n >= 0 && n < size_limits.length; // can we do this? it seems to be required for the following statement assert n >= 0 && n < BIG_INT_SIZE_LIMITS.length; // can we do this? it seems to be required for the following statement
long maxv = size_limits[n].add(i).longValue(); long maxv = BIG_INT_SIZE_LIMITS[n].add(i).longValue();
byte[] adjustedBytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array(); byte[] adjustedBytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array();
byte[] intBytes = new byte[n + 1]; byte[] intBytes = new byte[n + 1];
intBytes[0] = (byte) (20 - n); intBytes[0] = (byte) (INT_ZERO_CODE - n);
System.arraycopy(adjustedBytes, adjustedBytes.length - n, intBytes, 1, n); System.arraycopy(adjustedBytes, adjustedBytes.length - n, intBytes, 1, n);
result.add(intBytes); state.add(intBytes);
} }
} }
} }
static void encode(EncodeResult result, Integer i) { static void encode(EncodeState state, long i) {
encode(result, i.longValue()); if(i == 0L) {
state.add(INT_ZERO_ARR);
return;
}
int n = byteCount(i);
byte[] intBytes = new byte[n + 1];
// First byte encodes number of bytes (as difference from INT_ZERO_CODE)
intBytes[0] = (byte)(INT_ZERO_CODE + (i >= 0 ? n : -n));
// For positive integers, copy the bytes in big-endian order excluding leading 0x00 bytes.
// For negative integers, copy the bytes of the one's complement representation excluding
// the leading 0xff bytes. As Java stores negative values in two's complement, we subtract 1
// from negative values.
long val = Long.reverseBytes((i >= 0) ? i : (i - 1)) >> (Long.SIZE - 8 * n);
for(int x = 1; x < intBytes.length; x++) {
intBytes[x] = (byte)(val & 0xff);
val >>= 8;
}
state.add(intBytes);
} }
static void encode(EncodeResult result, long i) { static void encode(EncodeState state, Float f) {
encode(result, BigInteger.valueOf(i)); byte[] floatBytes = ByteBuffer.allocate(1 + Float.BYTES).order(ByteOrder.BIG_ENDIAN)
.put(FLOAT_CODE)
.putInt(encodeFloatBits(f))
.array();
state.add(floatBytes);
} }
static void encode(EncodeResult result, Float f) { static void encode(EncodeState state, Double d) {
byte[] floatBytes = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array(); byte[] doubleBytes = ByteBuffer.allocate(1 + Double.BYTES).order(ByteOrder.BIG_ENDIAN)
floatingPointCoding(floatBytes, 1, true); .put(DOUBLE_CODE)
result.add(floatBytes); .putLong(encodeDoubleBits(d))
.array();
state.add(doubleBytes);
} }
static void encode(EncodeResult result, Double d) { static void encode(EncodeState state, Boolean b) {
byte[] doubleBytes = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array();
floatingPointCoding(doubleBytes, 1, true);
result.add(doubleBytes);
}
static void encode(EncodeResult result, Boolean b) {
if(b) { if(b) {
result.add(TRUE_ARR); state.add(TRUE_ARR);
} }
else { else {
result.add(FALSE_ARR); state.add(FALSE_ARR);
} }
} }
static void encode(EncodeResult result, UUID uuid) { static void encode(EncodeState state, UUID uuid) {
byte[] uuidBytes = ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN) byte[] uuidBytes = ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits()) .putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits())
.array(); .array();
result.add(uuidBytes); state.add(uuidBytes);
} }
static void encode(EncodeResult result, Versionstamp v) { static void encode(EncodeState state, Versionstamp v) {
result.add(VERSIONSTAMP_ARR); state.add(VERSIONSTAMP_ARR);
if(v.isComplete()) { if(v.isComplete()) {
result.add(v.getBytes()); state.add(v.getBytes());
} }
else { else {
result.add(v.getBytes(), result.totalLength); state.add(v.getBytes(), state.totalLength);
} }
} }
static void encode(EncodeResult result, List<?> value) { static void encode(EncodeState state, List<?> value) {
result.add(NESTED_ARR); state.add(NESTED_ARR);
for(Object t : value) { for(Object t : value) {
encode(result, t, true); encode(state, t, true);
} }
result.add(NULL_ARR); state.add(NULL_ARR);
} }
static void decode(DecodeResult result, byte[] rep, int pos, int last) { static void decode(DecodeState state, byte[] rep, int pos, int last) {
//System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos); //System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos);
// SOMEDAY: codes over 127 will be a problem with the signed Java byte mess // SOMEDAY: codes over 127 will be a problem with the signed Java byte mess
int code = rep[pos]; int code = rep[pos];
int start = pos + 1; int start = pos + 1;
if(code == nil) { if(code == nil) {
result.add(null, start); state.add(null, start);
} }
else if(code == BYTES_CODE) { else if(code == BYTES_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last); int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of byte string: " + end); //System.out.println("End of byte string: " + end);
byte[] range = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil }); byte[] range = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil });
//System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'"); //System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'");
result.add(range, end + 1); state.add(range, end + 1);
} }
else if(code == STRING_CODE) { else if(code == STRING_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last); int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
@ -404,78 +423,91 @@ class TupleUtil {
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil }); byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, NULL_ESCAPED_ARR, new byte[] { nil });
String str = new String(stringBytes, UTF8); String str = new String(stringBytes, UTF8);
//System.out.println(" -> UTF8 string contents: '" + str + "'"); //System.out.println(" -> UTF8 string contents: '" + str + "'");
result.add(str, end + 1); state.add(str, end + 1);
} }
else if(code == FLOAT_CODE) { else if(code == FLOAT_CODE) {
byte[] resBytes = Arrays.copyOfRange(rep, start, start+4); int rawFloatBits = ByteBuffer.wrap(rep, start, Float.BYTES).getInt();
floatingPointCoding(resBytes, 0, false); float res = decodeFloatBits(rawFloatBits);
float res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getFloat(); state.add(res, start + Float.BYTES);
result.add(res, start + Float.BYTES);
} }
else if(code == DOUBLE_CODE) { else if(code == DOUBLE_CODE) {
byte[] resBytes = Arrays.copyOfRange(rep, start, start+8); long rawDoubleBits = ByteBuffer.wrap(rep, start, Double.BYTES).getLong();
floatingPointCoding(resBytes, 0, false); double res = decodeDoubleBits(rawDoubleBits);
double res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getDouble(); state.add(res, start + Double.BYTES);
result.add(res, start + Double.BYTES);
} }
else if(code == FALSE_CODE) { else if(code == FALSE_CODE) {
result.add(false, start); state.add(false, start);
} }
else if(code == TRUE_CODE) { else if(code == TRUE_CODE) {
result.add(true, start); state.add(true, start);
} }
else if(code == UUID_CODE) { else if(code == UUID_CODE) {
ByteBuffer bb = ByteBuffer.wrap(rep, start, 16).order(ByteOrder.BIG_ENDIAN); ByteBuffer bb = ByteBuffer.wrap(rep, start, 16).order(ByteOrder.BIG_ENDIAN);
long msb = bb.getLong(); long msb = bb.getLong();
long lsb = bb.getLong(); long lsb = bb.getLong();
result.add(new UUID(msb, lsb), start + 16); state.add(new UUID(msb, lsb), start + 16);
} }
else if(code == POS_INT_END) { else if(code == POS_INT_END) {
int n = rep[start] & 0xff; int n = rep[start] & 0xff;
BigInteger res = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1))); BigInteger res = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1)));
result.add(res, start + n + 1); state.add(res, start + n + 1);
} }
else if(code == NEG_INT_START) { else if(code == NEG_INT_START) {
int n = (rep[start] ^ 0xff) & 0xff; int n = (rep[start] ^ 0xff) & 0xff;
BigInteger origValue = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1))); BigInteger origValue = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1)));
BigInteger offset = BigInteger.ONE.shiftLeft(n*8).subtract(BigInteger.ONE); BigInteger offset = BigInteger.ONE.shiftLeft(n*8).subtract(BigInteger.ONE);
result.add(origValue.subtract(offset), start + n + 1); state.add(origValue.subtract(offset), start + n + 1);
} }
else if(code > NEG_INT_START && code < POS_INT_END) { else if(code > NEG_INT_START && code < POS_INT_END) {
// decode a long // decode a long
byte[] longBytes = new byte[9]; boolean positive = code >= INT_ZERO_CODE;
boolean upper = code >= INT_ZERO_CODE; int n = positive ? code - INT_ZERO_CODE : INT_ZERO_CODE - code;
int n = upper ? code - 20 : 20 - code;
int end = start + n; int end = start + n;
if(rep.length < end) { if(rep.length < end) {
throw new RuntimeException("Invalid tuple (possible truncation)"); throw new RuntimeException("Invalid tuple (possible truncation)");
} }
System.arraycopy(rep, start, longBytes, longBytes.length-n, n); if(positive && (n < Long.BYTES || rep[start] > 0)) {
if (!upper) long res = 0L;
for(int i=longBytes.length-n; i<longBytes.length; i++) for(int i = start; i < end; i++) {
longBytes[i] = (byte)(longBytes[i] ^ 0xff); res = (res << 8) + (rep[i] & 0xff);
}
state.add(res, end);
}
else if(!positive && (n < Long.BYTES || rep[start] < 0)) {
long res = ~0L;
for(int i = start; i < end; i++) {
res = (res << 8) + (rep[i] & 0xff);
}
state.add(res + 1, end);
}
else {
byte[] longBytes = new byte[9];
System.arraycopy(rep, start, longBytes, longBytes.length-n, n);
if (!positive)
for(int i=longBytes.length-n; i<longBytes.length; i++)
longBytes[i] = (byte)(longBytes[i] ^ 0xff);
BigInteger val = new BigInteger(longBytes); BigInteger val = new BigInteger(longBytes);
if (!upper) val = val.negate(); if (!positive) val = val.negate();
// Convert to long if in range -- otherwise, leave as BigInteger. // Convert to long if in range -- otherwise, leave as BigInteger.
if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0|| if (val.compareTo(LONG_MIN_VALUE) >= 0 && val.compareTo(LONG_MAX_VALUE) <= 0) {
val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0) { state.add(val.longValue(), end);
// This can occur if the thing can be represented with 8 bytes but not } else {
// the right sign information. // This can occur if the thing can be represented with 8 bytes but not
result.add(val, end); // the right sign information.
} else { state.add(val, end);
result.add(val.longValue(), end); }
} }
} }
else if(code == VERSIONSTAMP_CODE) { else if(code == VERSIONSTAMP_CODE) {
Versionstamp val = Versionstamp.fromBytes(Arrays.copyOfRange(rep, start, start + Versionstamp.LENGTH)); Versionstamp val = Versionstamp.fromBytes(Arrays.copyOfRange(rep, start, start + Versionstamp.LENGTH));
result.add(val, start + Versionstamp.LENGTH); state.add(val, start + Versionstamp.LENGTH);
} }
else if(code == NESTED_CODE) { else if(code == NESTED_CODE) {
DecodeResult subResult = new DecodeResult(); DecodeState subResult = new DecodeState();
int endPos = start; int endPos = start;
while(endPos < rep.length) { while(endPos < rep.length) {
if(rep[endPos] == nil) { if(rep[endPos] == nil) {
@ -491,25 +523,13 @@ class TupleUtil {
endPos = subResult.end; endPos = subResult.end;
} }
} }
result.add(subResult.values, endPos); state.add(subResult.values, endPos);
} }
else { else {
throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos); throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos);
} }
} }
static int compareSignedBigEndian(byte[] arr1, byte[] arr2) {
if(arr1[0] < 0 && arr2[0] < 0) {
return -1 * ByteArrayUtil.compareUnsigned(arr1, arr2);
} else if(arr1[0] < 0) {
return -1;
} else if(arr2[0] < 0) {
return 1;
} else {
return ByteArrayUtil.compareUnsigned(arr1, arr2);
}
}
static int compareItems(Object item1, Object item2) { static int compareItems(Object item1, Object item2) {
int code1 = TupleUtil.getCodeFor(item1); int code1 = TupleUtil.getCodeFor(item1);
int code2 = TupleUtil.getCodeFor(item2); int code2 = TupleUtil.getCodeFor(item2);
@ -529,33 +549,39 @@ class TupleUtil {
return ByteArrayUtil.compareUnsigned(((String)item1).getBytes(UTF8), ((String)item2).getBytes(UTF8)); return ByteArrayUtil.compareUnsigned(((String)item1).getBytes(UTF8), ((String)item2).getBytes(UTF8));
} }
if(code1 == INT_ZERO_CODE) { if(code1 == INT_ZERO_CODE) {
BigInteger bi1; if(item1 instanceof Long && item2 instanceof Long) {
if(item1 instanceof BigInteger) { // This should be the common case, so it's probably worth including as a way out.
bi1 = (BigInteger)item1; return Long.compare((Long)item1, (Long)item2);
} else {
bi1 = BigInteger.valueOf(((Number)item1).longValue());
} }
BigInteger bi2; else {
if(item2 instanceof BigInteger) { BigInteger bi1;
bi2 = (BigInteger)item2; if (item1 instanceof BigInteger) {
} else { bi1 = (BigInteger) item1;
bi2 = BigInteger.valueOf(((Number)item2).longValue()); } else {
bi1 = BigInteger.valueOf(((Number) item1).longValue());
}
BigInteger bi2;
if (item2 instanceof BigInteger) {
bi2 = (BigInteger) item2;
} else {
bi2 = BigInteger.valueOf(((Number) item2).longValue());
}
return bi1.compareTo(bi2);
} }
return bi1.compareTo(bi2);
}
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// sorting correctly.
byte[] dBytes1 = ByteBuffer.allocate(8).putDouble((Double)item1).array();
byte[] dBytes2 = ByteBuffer.allocate(8).putDouble((Double)item2).array();
return compareSignedBigEndian(dBytes1, dBytes2);
} }
if(code1 == FLOAT_CODE) { if(code1 == FLOAT_CODE) {
// This is done for the same reason that double comparison is done // This is done for the same reason that double comparison is done
// that way. // that way.
byte[] fBytes1 = ByteBuffer.allocate(4).putFloat((Float)item1).array(); int fbits1 = encodeFloatBits((Float)item1);
byte[] fBytes2 = ByteBuffer.allocate(4).putFloat((Float)item2).array(); int fbits2 = encodeFloatBits((Float)item2);
return compareSignedBigEndian(fBytes1, fBytes2); return Integer.compareUnsigned(fbits1, fbits2);
}
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// sorting correctly.
long dbits1 = encodeDoubleBits((Double)item1);
long dbits2 = encodeDoubleBits((Double)item2);
return Long.compareUnsigned(dbits1, dbits2);
} }
if(code1 == FALSE_CODE) { if(code1 == FALSE_CODE) {
return Boolean.compare((Boolean)item1, (Boolean)item2); return Boolean.compare((Boolean)item1, (Boolean)item2);
@ -579,51 +605,53 @@ class TupleUtil {
} }
static List<Object> unpack(byte[] bytes, int start, int length) { static List<Object> unpack(byte[] bytes, int start, int length) {
DecodeResult decodeResult = new DecodeResult(); DecodeState decodeState = new DecodeState();
int pos = start; int pos = start;
int end = start + length; int end = start + length;
while(pos < end) { while(pos < end) {
decode(decodeResult, bytes, pos, end); decode(decodeState, bytes, pos, end);
pos = decodeResult.end; pos = decodeState.end;
} }
return decodeResult.values; return decodeState.values;
} }
static void encodeAll(EncodeResult result, List<Object> items, byte[] prefix) { static void encodeAll(EncodeState state, List<Object> items, byte[] prefix) {
if(prefix != null) { if(prefix != null) {
result.add(prefix); state.add(prefix);
} }
for(Object t : items) { for(Object t : items) {
encode(result, t); encode(state, t);
} }
//System.out.println("Joining whole tuple..."); //System.out.println("Joining whole tuple...");
} }
static byte[] pack(List<Object> items, byte[] prefix) { static byte[] pack(List<Object> items, byte[] prefix) {
EncodeResult result = new EncodeResult(2 * items.size() + (prefix == null ? 0 : 1)); EncodeState state = new EncodeState(2 * items.size() + (prefix == null ? 0 : 1));
encodeAll(result, items, prefix); encodeAll(state, items, prefix);
if(result.versionPos >= 0) { if(state.versionPos >= 0) {
throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple packInternal"); throw new IllegalArgumentException("Incomplete Versionstamp included in vanilla tuple packInternal");
} else { }
return ByteArrayUtil.join(null, result.encodedValues); else {
return ByteArrayUtil.join(null, state.encodedValues);
} }
} }
static byte[] packWithVersionstamp(List<Object> items, byte[] prefix) { static byte[] packWithVersionstamp(List<Object> items, byte[] prefix) {
EncodeResult result = new EncodeResult(2 * items.size() + (prefix == null ? 1 : 2)); EncodeState state = new EncodeState(2 * items.size() + (prefix == null ? 1 : 2));
encodeAll(result, items, prefix); encodeAll(state, items, prefix);
if(result.versionPos < 0) { if(state.versionPos < 0) {
throw new IllegalArgumentException("No incomplete Versionstamp included in tuple packInternal with versionstamp"); throw new IllegalArgumentException("No incomplete Versionstamp included in tuple packInternal with versionstamp");
} else { }
if(result.versionPos > 0xffff) { else {
throw new IllegalArgumentException("Tuple has incomplete version at position " + result.versionPos + " which is greater than the maximum " + 0xffff); if(state.versionPos > 0xffff) {
throw new IllegalArgumentException("Tuple has incomplete version at position " + state.versionPos + " which is greater than the maximum " + 0xffff);
} }
if (FDB.instance().getAPIVersion() < 520) { if (FDB.instance().getAPIVersion() < 520) {
result.add(ByteBuffer.allocate(Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array()); state.add(ByteBuffer.allocate(Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).putShort((short)state.versionPos).array());
} else { } else {
result.add(ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(result.versionPos).array()); state.add(ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(state.versionPos).array());
} }
return ByteArrayUtil.join(null, result.encodedValues); return ByteArrayUtil.join(null, state.encodedValues);
} }
} }
@ -631,13 +659,17 @@ class TupleUtil {
return items.anyMatch(item -> { return items.anyMatch(item -> {
if(item == null) { if(item == null) {
return false; return false;
} else if(item instanceof Versionstamp) { }
else if(item instanceof Versionstamp) {
return !((Versionstamp) item).isComplete(); return !((Versionstamp) item).isComplete();
} else if(item instanceof Tuple) { }
else if(item instanceof Tuple) {
return hasIncompleteVersionstamp(((Tuple) item).stream()); return hasIncompleteVersionstamp(((Tuple) item).stream());
} else if(item instanceof Collection<?>) { }
else if(item instanceof Collection<?>) {
return hasIncompleteVersionstamp(((Collection) item).stream()); return hasIncompleteVersionstamp(((Collection) item).stream());
} else { }
else {
return false; return false;
} }
}); });
@ -646,23 +678,25 @@ class TupleUtil {
public static void main(String[] args) { public static void main(String[] args) {
try { try {
byte[] bytes = pack(Collections.singletonList(4), null); byte[] bytes = pack(Collections.singletonList(4), null);
DecodeResult result = new DecodeResult(); DecodeState result = new DecodeState();
decode(result, bytes, 0, bytes.length); decode(result, bytes, 0, bytes.length);
int val = (int)result.values.get(0); int val = (int)result.values.get(0);
assert 4 == val; assert 4 == val;
} catch (Exception e) { }
catch(Exception e) {
e.printStackTrace(); e.printStackTrace();
System.out.println("Error " + e.getMessage()); System.out.println("Error " + e.getMessage());
} }
try { try {
byte[] bytes = pack(Collections.singletonList("\u021Aest \u0218tring"), null); byte[] bytes = pack(Collections.singletonList("\u021Aest \u0218tring"), null);
DecodeResult result = new DecodeResult(); DecodeState result = new DecodeState();
decode(result, bytes, 0, bytes.length); decode(result, bytes, 0, bytes.length);
String string = (String)result.values.get(0); String string = (String)result.values.get(0);
System.out.println("contents -> " + string); System.out.println("contents -> " + string);
assert "\u021Aest \u0218tring".equals(string); assert "\u021Aest \u0218tring".equals(string);
} catch (Exception e) { }
catch(Exception e) {
e.printStackTrace(); e.printStackTrace();
System.out.println("Error " + e.getMessage()); System.out.println("Error " + e.getMessage());
} }