wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release-5.0' into bindings-tuple-improvements 

# Conflicts:
#	bindings/java/src-completable/main/com/apple/apple/foundationdbdb/tuple/Tuple.java
This commit is contained in:
A.J. Beamon 2017-05-26 12:33:33 -07:00
parent 93509133ad
commit fc468f682b
41 changed files with 3680 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
bindings/bindingtester/known_testers.py
View File

@ -56,13 +56,13 @@ _java_completable_cmd = 'java -ea -cp %s:%s com.apple.cie.foundationdb.test.' %
testers = {
'python' : Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION),
'python3' : Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION),
'node' : Tester('node', _absolute_path('nodejs/tests/tester.js'), 53, 23, MAX_API_VERSION),
'node' : Tester('node', _absolute_path('nodejs/tests/tester.js'), 53, 500, MAX_API_VERSION),
'streamline' : Tester('streamline', _absolute_path('nodejs/tests/streamline_tester._js'), 53, 23, MAX_API_VERSION),
'ruby' : Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 64, 23, MAX_API_VERSION),
'java' : Tester('java', _java_cmd + 'StackTester', 63, 500, MAX_API_VERSION),
'java_async' : Tester('java', _java_cmd + 'AsyncStackTester', 63, 500, MAX_API_VERSION),
'java_completable' : Tester('java', _java_completable_cmd + 'StackTester', 63, 500, MAX_API_VERSION),
'java_completable_async' : Tester('java', _java_completable_cmd + 'AsyncStackTester', 63, 500, MAX_API_VERSION),
'java' : Tester('java', _java_cmd + 'StackTester', 2040, 500, MAX_API_VERSION),
'java_async' : Tester('java', _java_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION),
'java_completable' : Tester('java', _java_completable_cmd + 'StackTester', 2040, 500, MAX_API_VERSION),
'java_completable_async' : Tester('java', _java_completable_cmd + 'AsyncStackTester', 2040, 500, MAX_API_VERSION),
'go' : Tester('go', _absolute_path('go/bin/_stacktester'), 63, 200, MAX_API_VERSION),
'flow' : Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 200, MAX_API_VERSION),
}

34
bindings/bindingtester/spec/bindingApiTester.txt
View File

@ -306,6 +306,40 @@ TUPLE_RANGE
structure) to the tuple range method. Pushes the begin and end elements of
the returned range onto the stack.
TUPLE_SORT
Pops the top item off of the stack as N. Pops the next N items off of the
stack as packed tuples (i.e., byte strings), unpacks them, sorts the tuples,
repacks them into byte strings, and then pushes these packed tuples onto
the stack so that the final top of the stack now has the greatest
element. If the binding has some kind of tuple comparison function, it should
use that to sort. Otherwise, it should sort them lexicographically by
their byte representation. The choice of function should not affect final sort order.
ENCODE_FLOAT
Pops the top item off of the stack. This will be a byte-string of length 4
containing the IEEE 754 encoding of a float in big-endian order.
This is then converted into a float and pushed onto the stack.
ENCODE_DOUBLE
Pops the top item off of the stack. This will be a byte-string of length 8
containing the IEEE 754 encoding of a double in big-endian order.
This is then converted into a double and pushed onto the stack.
DECODE_FLOAT
Pops the top item off of the stack. This will be a single-precision float.
This is converted into a (4 byte) byte-string of its IEEE 754 representation
in big-endian order, and pushed onto the stack.
DECODE_DOUBLE
Pops the top item off of the stack. This will be a double-precision float.
This is converted into a (8 byte) byte-string its IEEE 754 representation
in big-endian order, and pushed onto the stack.
Thread Operations
-----------------

50
bindings/bindingtester/tests/api.py
View File

@ -18,9 +18,12 @@
# limitations under the License.
#
import ctypes
import random
import struct
import fdb
import fdb.tuple
from bindingtester import FDB_API_VERSION
from bindingtester.tests import Test, Instruction, InstructionSet, ResultSpecification
@ -51,7 +54,7 @@ class ApiTest(Test):
self.generated_keys = []
self.outstanding_ops = []
self.random = test_util.RandomGenerator(args.max_int_bits)
self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version)
def add_stack_items(self, num):
self.stack_size += num
@ -144,7 +147,7 @@ class ApiTest(Test):
mutations += ['VERSIONSTAMP']
versions = ['GET_READ_VERSION', 'SET_READ_VERSION', 'GET_COMMITTED_VERSION']
snapshot_versions = ['GET_READ_VERSION_SNAPSHOT']
tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'SUB']
tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT']
resets = ['ON_ERROR', 'RESET', 'CANCEL']
read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY']
write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT']
@ -179,6 +182,7 @@ class ApiTest(Test):
for i in range(args.num_ops):
op = random.choice(op_choices)
index = len(instructions)
read_performed = False
#print 'Adding instruction %s at %d' % (op, index)
@ -218,6 +222,7 @@ class ApiTest(Test):
instructions.append(op)
self.add_strings(1)
self.can_set_version = False
read_performed = True
elif op == 'GET_KEY' or op == 'GET_KEY_SNAPSHOT' or op == 'GET_KEY_DATABASE':
if op.endswith('_DATABASE') or self.can_use_key_selectors:
@ -230,6 +235,7 @@ class ApiTest(Test):
#Don't add key here because we may be outside of our prefix
self.add_strings(1)
self.can_set_version = False
read_performed = True
elif op == 'GET_RANGE' or op == 'GET_RANGE_SNAPSHOT' or op == 'GET_RANGE_DATABASE':
self.ensure_key(instructions, 2)
@ -245,6 +251,7 @@ class ApiTest(Test):
self.add_stack_items(1)
self.can_set_version = False
read_performed = True
elif op == 'GET_RANGE_STARTS_WITH' or op == 'GET_RANGE_STARTS_WITH_SNAPSHOT' or op == 'GET_RANGE_STARTS_WITH_DATABASE':
#TODO: not tested well
@ -260,6 +267,7 @@ class ApiTest(Test):
self.add_stack_items(1)
self.can_set_version = False
read_performed = True
elif op == 'GET_RANGE_SELECTOR' or op == 'GET_RANGE_SELECTOR_SNAPSHOT' or op == 'GET_RANGE_SELECTOR_DATABASE':
if op.endswith('_DATABASE') or self.can_use_key_selectors:
@ -279,6 +287,7 @@ class ApiTest(Test):
self.add_stack_items(1)
self.can_set_version = False
read_performed = True
elif op == 'GET_READ_VERSION' or op == 'GET_READ_VERSION_SNAPSHOT':
instructions.append(op)
@ -409,6 +418,15 @@ class ApiTest(Test):
instructions.append(op)
self.add_strings(len(tup))
elif op == 'TUPLE_SORT':
tups = self.random.random_tuple_list(10, 30)
for tup in tups:
instructions.push_args(len(tup), *tup)
instructions.append('TUPLE_PACK')
instructions.push_args(len(tups))
instructions.append(op)
self.add_strings(len(tups))
#Use SUB to test if integers are correctly unpacked
elif op == 'SUB':
a = self.random.random_int() / 2
@ -422,10 +440,36 @@ class ApiTest(Test):
instructions.append('TUPLE_PACK')
self.add_stack_items(1)
elif op == 'ENCODE_FLOAT':
f = self.random.random_float(8)
f_bytes = struct.pack('>f', f)
instructions.push_args(f_bytes)
instructions.append(op)
self.add_stack_items(1)
elif op == 'ENCODE_DOUBLE':
d = self.random.random_float(11)
d_bytes = struct.pack('>d', d)
instructions.push_args(d_bytes)
instructions.append(op)
self.add_stack_items(1)
elif op == 'DECODE_FLOAT':
f = self.random.random_float(8)
instructions.push_args(fdb.tuple.SingleFloat(f))
instructions.append(op)
self.add_strings(1)
elif op == 'DECODE_DOUBLE':
d = self.random.random_float(11)
instructions.push_args(d)
instructions.append(op)
self.add_strings(1)
else:
assert False
if op in reads or op in snapshot_reads:
if read_performed and op not in database_reads:
self.outstanding_ops.append((self.stack_size, len(instructions)-1))
if args.concurrency == 1 and (op in database_reads or op in database_mutations):

2
bindings/bindingtester/tests/directory.py
View File

@ -65,7 +65,7 @@ class DirectoryTest(Test):
def setup(self, args):
self.dir_index = 0
self.random = test_util.RandomGenerator(args.max_int_bits)
self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version)
def generate(self, args, thread_number):
instructions = InstructionSet()

2
bindings/bindingtester/tests/directory_hca.py
View File

@ -38,7 +38,7 @@ class DirectoryHcaTest(Test):
self.next_path = 1
def setup(self, args):
self.random = test_util.RandomGenerator(args.max_int_bits)
self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version)
self.transactions = ['tr%d' % i for i in range(3)] # SOMEDAY: parameterize this number?
self.barrier_num = 0

66
bindings/bindingtester/tests/test_util.py
View File

@ -19,15 +19,21 @@
#
import random
import uuid
import unicodedata
import ctypes
import math
import fdb
import fdb.tuple
from bindingtester import util
from bindingtester import FDB_API_VERSION
class RandomGenerator(object):
def __init__(self, max_int_bits=64):
def __init__(self, max_int_bits=64, api_version=FDB_API_VERSION):
self.max_int_bits = max_int_bits
self.api_version = api_version
def random_unicode_str(self, length):
return u''.join(self.random_unicode_char() for i in range(0, length))
@ -42,12 +48,23 @@ class RandomGenerator(object):
#util.get_logger().debug('generating int (%d): %d - %s' % (num_bits, num, repr(fdb.tuple.pack((num,)))))
return num
def random_float(self, exp_bits):
if random.random() < 0.05:
# Choose a special value.
return random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')])
else:
# Choose a value from all over the range of acceptable floats for this precision.
sign = -1 if random.random() < 0.5 else 1
exponent = random.randint(-(1 << (exp_bits-1))-10, (1 << (exp_bits-1) - 1))
mantissa = random.random()
return sign * math.pow(2, exponent) * mantissa
def random_tuple(self, max_size):
size = random.randint(1, max_size)
tup = []
for i in range(size):
choice = random.randint(0, 3)
choice = random.randint(0, 8)
if choice == 0:
tup.append(self.random_int())
elif choice == 1:
@ -56,11 +73,51 @@ class RandomGenerator(object):
tup.append(self.random_string(random.randint(0, 100)))
elif choice == 3:
tup.append(self.random_unicode_str(random.randint(0, 100)))
elif choice == 4:
tup.append(uuid.uuid4())
elif choice == 5:
b = random.random() < 0.5
if self.api_version < 500:
tup.append(int(b))
else:
tup.append(b)
elif choice == 6:
tup.append(fdb.tuple.SingleFloat(self.random_float(8)))
elif choice == 7:
tup.append(self.random_float(11))
elif choice == 8:
length = random.randint(0, max_size - size)
if length == 0:
tup.append(())
else:
tup.append(self.random_tuple(length))
else:
assert false
return tuple(tup)
def random_tuple_list(self, max_size, max_list_size):
size = random.randint(1, max_list_size)
tuples = []
for i in range(size):
to_add = self.random_tuple(max_size)
tuples.append(to_add)
if len(to_add) > 1 and random.random() < 0.25:
# Add a smaller one to test prefixes.
smaller_size = random.randint(1, len(to_add))
tuples.append(to_add[:smaller_size])
else:
non_empty = filter(lambda (i,x): (isinstance(x, list) or isinstance(x, tuple)) and len(x) > 0, enumerate(to_add))
if len(non_empty) > 0 and random.random() < 0.25:
# Add a smaller list to test prefixes of nested structures.
idx, choice = random.choice(non_empty)
smaller_size = random.randint(0, len(to_add[idx]))
tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx+1:])
random.shuffle(tuples)
return tuples
def random_range_params(self):
if random.random() < 0.75:
limit = random.randint(1, 1e3)
@ -87,6 +144,11 @@ class RandomGenerator(object):
def random_unicode_char(self):
while True:
if random.random() < 0.05:
# Choose one of these special character sequences.
specials = [u'\U0001f4a9', u'\U0001f63c', u'\U0001f3f3\ufe0f\u200d\U0001f308', u'\U0001f1f5\U0001f1f2', u'\uf8ff',
u'\U0002a2b2', u'\u05e9\u05dc\u05d5\u05dd']
return random.choice(specials)
c = random.randint(0, 0xffff)
if unicodedata.category(unichr(c))[0] in 'LMNPSZ':
return unichr(c)

2
bindings/flow/DirectoryLayer.actor.cpp
View File

@ -26,7 +26,7 @@ namespace FDB {
const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = LiteralStringRef("hca");
const StringRef DirectoryLayer::LAYER_KEY = LiteralStringRef("layer");
const StringRef DirectoryLayer::VERSION_KEY = LiteralStringRef("version");
const uint64_t DirectoryLayer::SUB_DIR_KEY = 0;
const int64_t DirectoryLayer::SUB_DIR_KEY = 0;
const uint32_t DirectoryLayer::VERSION[3] = {1, 0, 0};

2
bindings/flow/DirectoryLayer.h
View File

@ -58,7 +58,7 @@ namespace FDB {
static const StringRef HIGH_CONTENTION_KEY;
static const StringRef LAYER_KEY;
static const StringRef VERSION_KEY;
static const uint64_t SUB_DIR_KEY;
static const int64_t SUB_DIR_KEY;
static const uint32_t VERSION[3];
static const StringRef DEFAULT_NODE_SUBSPACE_PREFIX;

2
bindings/flow/HighContentionAllocator.h
View File

@ -28,7 +28,7 @@
namespace FDB {
class HighContentionAllocator {
public:
HighContentionAllocator(Subspace subspace) : counters(subspace.get(0)), recent(subspace.get(1)) {}
HighContentionAllocator(Subspace subspace) : counters(subspace.get((int64_t)0)), recent(subspace.get((int64_t)1)) {}
Future<Standalone<StringRef>> allocate(Reference<Transaction> const& tr) const;
static int64_t windowSize(int64_t start);

12
bindings/flow/Subspace.h
View File

@ -49,6 +49,12 @@ namespace FDB {
return pack(t);
}
Key packNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return pack(t);
}
Key pack(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);
@ -65,6 +71,12 @@ namespace FDB {
return get(t);
}
Subspace getNested(Tuple const& item) const {
Tuple t;
t.appendNested(item);
return get(t);
}
Subspace get(StringRef const& item, bool utf8=false) const {
Tuple t;
t.append(item, utf8);

428
bindings/flow/Tuple.cpp
View File

@ -19,9 +19,50 @@
*/
#include "Tuple.h"
#include <boost/static_assert.hpp>
namespace FDB {
// The floating point operations depend on this using the IEEE 754 standard.
BOOST_STATIC_ASSERT(std::numeric_limits<float>::is_iec559);
BOOST_STATIC_ASSERT(std::numeric_limits<double>::is_iec559);
const size_t Uuid::SIZE = 16;
const uint8_t Tuple::NULL_CODE = 0x00;
const uint8_t Tuple::BYTES_CODE = 0x01;
const uint8_t Tuple::STRING_CODE = 0x02;
const uint8_t Tuple::NESTED_CODE = 0x05;
const uint8_t Tuple::INT_ZERO_CODE = 0x14;
const uint8_t Tuple::POS_INT_END = 0x1c;
const uint8_t Tuple::NEG_INT_START = 0x0c;
const uint8_t Tuple::FLOAT_CODE = 0x20;
const uint8_t Tuple::DOUBLE_CODE = 0x21;
const uint8_t Tuple::FALSE_CODE = 0x26;
const uint8_t Tuple::TRUE_CODE = 0x27;
const uint8_t Tuple::UUID_CODE = 0x30;
static float bigEndianFloat(float orig) {
int32_t big = *(int32_t*)&orig;
big = bigEndian32(big);
return *(float*)&big;
}
static double bigEndianDouble(double orig) {
int64_t big = *(int64_t*)&orig;
big = bigEndian64(big);
return *(double*)&big;
}
static size_t find_string_terminator(const StringRef data, size_t offset) {
size_t i = offset;
while (i < data.size() - 1 && !(data[i] == (uint8_t)'\x00' && data[i+1] != (uint8_t)'\xff')) {
i += (data[i] == '\x00' ? 2 : 1);
}
return i;
}
static size_t find_string_terminator(const Standalone<VectorRef<unsigned char> > data, size_t offset ) {
size_t i = offset;
while (i < data.size() - 1 && !(data[i] == '\x00' && data[i+1] != (uint8_t)'\xff')) {
i += (data[i] == '\x00' ? 2 : 1);
@ -30,26 +71,74 @@ namespace FDB {
return i;
}
// If encoding and the sign bit is 1 (the number is negative), flip all the bits.
// If decoding and the sign bit is 0 (the number is negative), flip all the bits.
// Otherwise, the number is positive, so flip the sign bit.
static void adjust_floating_point(uint8_t *bytes, size_t size, bool encode) {
if((encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x00)) || (!encode && ((uint8_t)(bytes[0] & 0x80) != (uint8_t)0x80))) {
for(size_t i = 0; i < size; i++) {
bytes[i] ^= (uint8_t)0xff;
}
} else {
bytes[0] ^= (uint8_t)0x80;
}
}
Tuple::Tuple(StringRef const& str) {
data.append(data.arena(), str.begin(), str.size());
size_t i = 0;
int depth = 0;
while(i < data.size()) {
offsets.push_back(i);
if(depth == 0) offsets.push_back(i);
if(data[i] == '\x01' || data[i] == '\x02') {
if(depth > 0 && data[i] == NULL_CODE) {
if(i + 1 < data.size() && data[i+1] == 0xff) {
// NULL value.
i += 2;
} else {
// Nested terminator.
i += 1;
depth -= 1;
}
}
else if(data[i] == BYTES_CODE || data[i] == STRING_CODE) {
i = find_string_terminator(str, i+1) + 1;
}
else if(data[i] >= '\x0c' && data[i] <= '\x1c') {
i += abs(data[i] - '\x14') + 1;
else if(data[i] >= NEG_INT_START && data[i] <= POS_INT_END) {
i += abs(data[i] - INT_ZERO_CODE) + 1;
}
else if(data[i] == '\x00') {
else if(data[i] == NULL_CODE || data[i] == TRUE_CODE || data[i] == FALSE_CODE) {
i += 1;
}
else if(data[i] == UUID_CODE) {
i += Uuid::SIZE + 1;
}
else if(data[i] == FLOAT_CODE) {
i += sizeof(float) + 1;
}
else if(data[i] == DOUBLE_CODE) {
i += sizeof(double) + 1;
}
else if(data[i] == NESTED_CODE) {
i += 1;
depth += 1;
}
else {
throw invalid_tuple_data_type();
}
}
if(depth != 0) {
throw invalid_tuple_data_type();
}
}
// Note: this is destructive of the original offsets, so should only
// be used once we are done.
Tuple::Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets) {
this->data = data;
this->offsets = std::move(offsets);
}
Tuple Tuple::unpack(StringRef const& str) {
@ -69,7 +158,7 @@ namespace FDB {
Tuple& Tuple::append(StringRef const& str, bool utf8) {
offsets.push_back(data.size());
const uint8_t utfChar = uint8_t(utf8 ? '\x02' : '\x01');
const uint8_t utfChar = utf8 ? STRING_CODE : BYTES_CODE;
data.append(data.arena(), &utfChar, 1);
size_t lastPos = 0;
@ -88,6 +177,10 @@ namespace FDB {
return *this;
}
Tuple& Tuple::append( int32_t value ) {
return append((int64_t)value);
}
Tuple& Tuple::append( int64_t value ) {
uint64_t swap = value;
bool neg = false;
@ -103,19 +196,82 @@ namespace FDB {
for ( int i = 0; i < 8; i++ ) {
if ( ((uint8_t*)&swap)[i] != (neg ? 255 : 0) ) {
data.push_back( data.arena(), (uint8_t)(20 + (8-i) * (neg ? -1 : 1)) );
data.push_back( data.arena(), (uint8_t)(INT_ZERO_CODE + (8-i) * (neg ? -1 : 1)) );
data.append( data.arena(), ((const uint8_t *)&swap) + i, 8 - i );
return *this;
}
}
data.push_back( data.arena(), (uint8_t)'\x14' );
data.push_back( data.arena(), INT_ZERO_CODE );
return *this;
}
Tuple& Tuple::append( bool value ) {
offsets.push_back( data.size() );
if(value) {
data.push_back( data.arena(), TRUE_CODE );
} else {
data.push_back( data.arena(), FALSE_CODE );
}
return *this;
}
Tuple& Tuple::append( float value ) {
offsets.push_back( data.size() );
float swap = bigEndianFloat(value);
uint8_t *bytes = (uint8_t*)&swap;
adjust_floating_point(bytes, sizeof(float), true);
data.push_back( data.arena(), FLOAT_CODE );
data.append( data.arena(), bytes, sizeof(float) );
return *this;
}
Tuple& Tuple::append( double value ) {
offsets.push_back( data.size() );
double swap = value;
swap = bigEndianDouble(swap);
uint8_t *bytes = (uint8_t*)&swap;
adjust_floating_point(bytes, sizeof(double), true);
data.push_back( data.arena(), DOUBLE_CODE );
data.append( data.arena(), bytes, sizeof(double) );
return *this;
}
Tuple& Tuple::append( Uuid value ) {
offsets.push_back( data.size() );
data.push_back( data.arena(), UUID_CODE );
data.append( data.arena(), value.getData().begin(), Uuid::SIZE );
return *this;
}
Tuple& Tuple::appendNested( Tuple const& value ) {
offsets.push_back( data.size() );
data.push_back( data.arena(), NESTED_CODE );
for(size_t i = 0; i < value.size(); i++) {
size_t offset = value.offsets[i];
size_t next_offset = (i+1 < value.offsets.size() ? value.offsets[i+1] : value.data.size());
ASSERT(offset < value.data.size());
ASSERT(next_offset <= value.data.size());
uint8_t code = value.data[offset];
if(code == NULL_CODE) {
data.push_back( data.arena(), NULL_CODE );
data.push_back( data.arena(), 0xff );
} else {
data.append( data.arena(), value.data.begin() + offset, next_offset - offset);
}
}
data.push_back( data.arena(), (uint8_t)'\x00');
return *this;
}
Tuple& Tuple::appendNull() {
offsets.push_back(data.size());
data.push_back(data.arena(), (uint8_t)'\x00');
data.push_back(data.arena(), NULL_CODE);
return *this;
}
@ -126,18 +282,33 @@ namespace FDB {
uint8_t code = data[offsets[index]];
if(code == '\x00') {
if(code == NULL_CODE) {
return ElementType::NULL_TYPE;
}
else if(code == '\x01') {
else if(code == BYTES_CODE) {
return ElementType::BYTES;
}
else if(code == '\x02') {
else if(code == STRING_CODE) {
return ElementType::UTF8;
}
else if(code >= '\x0c' && code <= '\x1c') {
else if(code == NESTED_CODE) {
return ElementType::NESTED;
}
else if(code >= NEG_INT_START && code <= POS_INT_END) {
return ElementType::INT;
}
else if(code == FLOAT_CODE) {
return ElementType::FLOAT;
}
else if(code == DOUBLE_CODE) {
return ElementType::DOUBLE;
}
else if(code == FALSE_CODE || code == TRUE_CODE) {
return ElementType::BOOL;
}
else if(code == UUID_CODE) {
return ElementType::UUID;
}
else {
throw invalid_tuple_data_type();
}
@ -149,7 +320,7 @@ namespace FDB {
}
uint8_t code = data[offsets[index]];
if(code != '\x01' && code != '\x02') {
if(code != BYTES_CODE && code != STRING_CODE) {
throw invalid_tuple_data_type();
}
@ -194,11 +365,11 @@ namespace FDB {
ASSERT(offsets[index] < data.size());
uint8_t code = data[offsets[index]];
if(code < '\x0c' || code > '\x1c') {
if(code < NEG_INT_START || code > POS_INT_END) {
throw invalid_tuple_data_type();
}
int8_t len = code - '\x14';
int8_t len = code - INT_ZERO_CODE;
if ( len < 0 ) {
len = -len;
@ -217,6 +388,160 @@ namespace FDB {
return swap;
}
bool Tuple::getBool(size_t index) const {
if(index >= offsets.size()) {
throw invalid_tuple_index();
}
ASSERT(offsets[index] < data.size());
uint8_t code = data[offsets[index]];
if(code == FALSE_CODE) {
return false;
} else if(code == TRUE_CODE) {
return true;
} else {
throw invalid_tuple_data_type();
}
}
float Tuple::getFloat(size_t index) const {
if(index >= offsets.size()) {
throw invalid_tuple_index();
}
ASSERT(offsets[index] < data.size());
uint8_t code = data[offsets[index]];
if(code != FLOAT_CODE) {
throw invalid_tuple_data_type();
}
float swap;
uint8_t* bytes = (uint8_t*)&swap;
ASSERT(offsets[index] + 1 + sizeof(float) <= data.size());
swap = *(float*)(data.begin() + offsets[index] + 1);
adjust_floating_point( bytes, sizeof(float), false );
return bigEndianFloat(swap);
}
double Tuple::getDouble(size_t index) const {
if(index >= offsets.size()) {
throw invalid_tuple_index();
}
ASSERT(offsets[index] < data.size());
uint8_t code = data[offsets[index]];
if(code != DOUBLE_CODE) {
throw invalid_tuple_data_type();
}
double swap;
uint8_t* bytes = (uint8_t*)&swap;
ASSERT(offsets[index] + 1 + sizeof(double) <= data.size());
swap = *(double*)(data.begin() + offsets[index] + 1);
adjust_floating_point( bytes, sizeof(double), false );
return bigEndianDouble(swap);
}
Uuid Tuple::getUuid(size_t index) const {
if(index >= offsets.size()) {
throw invalid_tuple_index();
}
size_t offset = offsets[index];
ASSERT(offset < data.size());
uint8_t code = data[offset];
if(code != UUID_CODE) {
throw invalid_tuple_data_type();
}
ASSERT(offset + Uuid::SIZE + 1 <= data.size());
StringRef uuidData(data.begin() + offset + 1, Uuid::SIZE);
return Uuid(uuidData);
}
Tuple Tuple::getNested(size_t index) const {
if(index >= offsets.size()) {
throw invalid_tuple_index();
}
size_t offset = offsets[index];
ASSERT(offset < data.size());
uint8_t code = data[offset];
if(code != NESTED_CODE) {
throw invalid_tuple_data_type();
}
size_t next_offset = (index + 1 < offsets.size() ? offsets[index+1] : data.size());
ASSERT(next_offset <= data.size());
ASSERT(data[next_offset - 1] == (uint8_t)0x00);
Standalone<VectorRef<uint8_t>> dest;
dest.reserve(dest.arena(), next_offset - offset);
std::vector<size_t> dest_offsets;
size_t i = offset + 1;
int depth = 0;
while(i < next_offset - 1) {
if (depth == 0) dest_offsets.push_back(dest.size());
uint8_t code = data[i];
dest.push_back(dest.arena(), code); // Copy over the type code.
if(code == NULL_CODE) {
if(depth > 0) {
if(i + 1 < next_offset - 1 && data[i+1] == 0xff) {
// Null with a tuple nested in the nested tuple.
dest.push_back(dest.arena(), 0xff);
i += 2;
} else {
// Nested terminator.
depth -= 1;
i += 1;
}
} else {
// A null object within the nested tuple.
ASSERT(i + 1 < next_offset - 1);
ASSERT(data[i+1] == 0xff);
i += 2;
}
}
else if(code == BYTES_CODE || code == STRING_CODE) {
size_t next_i = find_string_terminator(data, i+1) + 1;
ASSERT(next_i <= next_offset - 1);
size_t length = next_i - i - 1;
dest.append(dest.arena(), data.begin() + i + 1, length);
i = next_i;
}
else if(code >= NEG_INT_START && code <= POS_INT_END) {
size_t int_size = abs(code - INT_ZERO_CODE);
ASSERT(i + int_size <= next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, int_size);
i += int_size + 1;
}
else if(code == TRUE_CODE || code == FALSE_CODE) {
i += 1;
}
else if(code == UUID_CODE) {
ASSERT(i + 1 + Uuid::SIZE <= next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, Uuid::SIZE);
i += Uuid::SIZE + 1;
}
else if(code == FLOAT_CODE) {
ASSERT(i + 1 + sizeof(float) <= next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(float));
i += sizeof(float) + 1;
}
else if(code == DOUBLE_CODE) {
ASSERT(i + 1 + sizeof(double) <= next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(double));
i += sizeof(double) + 1;
}
else if(code == NESTED_CODE) {
i += 1;
depth += 1;
}
else {
throw invalid_tuple_data_type();
}
}
// The item may shrink because of escaped nulls that are unespaced.
return Tuple(dest, dest_offsets);
}
KeyRange Tuple::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;
@ -245,4 +570,75 @@ namespace FDB {
size_t endPos = end < offsets.size() ? offsets[end] : data.size();
return Tuple(StringRef(data.begin() + offsets[start], endPos - offsets[start]));
}
// Comparisons
int compare(Standalone<VectorRef<unsigned char> > const& v1, Standalone<VectorRef<unsigned char> > const& v2) {
size_t i = 0;
while(i < v1.size() && i < v2.size()) {
if(v1[i] < v2[i]) {
return -1;
} else if(v1[i] > v2[i]) {
return 1;
}
i += 1;
}
if(i < v1.size()) {
return 1;
}
if(i < v2.size()) {
return -1;
}
return 0;
}
bool Tuple::operator==(Tuple const& other) const {
return compare(data, other.data) == 0;
}
bool Tuple::operator!=(Tuple const& other) const {
return compare(data, other.data) != 0;
}
bool Tuple::operator<(Tuple const& other) const {
return compare(data, other.data) < 0;
}
bool Tuple::operator<=(Tuple const& other) const {
return compare(data, other.data) <= 0;
}
bool Tuple::operator>(Tuple const& other) const {
return compare(data, other.data) > 0;
}
bool Tuple::operator>=(Tuple const& other) const {
return compare(data, other.data) >= 0;
}
// UUID implementation
Uuid::Uuid(const StringRef& data) {
if (data.size() != Uuid::SIZE) {
throw invalid_uuid_size();
}
this->data = data;
}
StringRef Uuid::getData() const {
return data;
}
bool Uuid::operator==(Uuid const& other) const {
return data == other.data;
}
bool Uuid::operator!=(Uuid const& other) const {
return data != other.data;
}
bool Uuid::operator<(Uuid const& other) const {
return data < other.data;
}
bool Uuid::operator<=(Uuid const& other) const {
return data <= other.data;
}
bool Uuid::operator>(Uuid const& other) const {
return data > other.data;
}
bool Uuid::operator>=(Uuid const& other) const {
return data >= other.data;
}
}

53
bindings/flow/Tuple.h
View File

@ -26,6 +26,24 @@
#include "bindings/flow/fdb_flow.h"
namespace FDB {
struct Uuid {
const static size_t SIZE;
Uuid(StringRef const& data);
StringRef getData() const;
// Comparisons
bool operator==(Uuid const& other) const;
bool operator!=(Uuid const& other) const;
bool operator<(Uuid const& other) const;
bool operator<=(Uuid const& other) const;
bool operator>(Uuid const& other) const;
bool operator>=(Uuid const& other) const;
private:
Standalone<StringRef> data;
};
struct Tuple {
Tuple() {}
@ -33,7 +51,13 @@ namespace FDB {
Tuple& append(Tuple const& tuple);
Tuple& append(StringRef const& str, bool utf8=false);
Tuple& append(int32_t);
Tuple& append(int64_t);
Tuple& append(bool);
Tuple& append(float);
Tuple& append(double);
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
StringRef pack() const { return StringRef(data.begin(), data.size()); }
@ -43,7 +67,7 @@ namespace FDB {
return append(t);
}
enum ElementType { NULL_TYPE, INT, BYTES, UTF8 };
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
@ -51,13 +75,40 @@ namespace FDB {
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
double getDouble(size_t index) const;
Uuid getUuid(size_t index) const;
Tuple getNested(size_t index) const;
KeyRange range(Tuple const& tuple = Tuple()) const;
Tuple subTuple(size_t beginIndex, size_t endIndex = std::numeric_limits<size_t>::max()) const;
// Comparisons
bool operator==(Tuple const& other) const;
bool operator!=(Tuple const& other) const;
bool operator<(Tuple const& other) const;
bool operator<=(Tuple const& other) const;
bool operator>(Tuple const& other) const;
bool operator>=(Tuple const& other) const;
private:
static const uint8_t NULL_CODE;
static const uint8_t BYTES_CODE;
static const uint8_t STRING_CODE;
static const uint8_t NESTED_CODE;
static const uint8_t INT_ZERO_CODE;
static const uint8_t POS_INT_END;
static const uint8_t NEG_INT_START;
static const uint8_t FLOAT_CODE;
static const uint8_t DOUBLE_CODE;
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);
Standalone<VectorRef<uint8_t>> data;
std::vector<size_t> offsets;
};

10
bindings/flow/tester/DirectoryTester.actor.cpp
View File

@ -405,7 +405,7 @@ struct DirectoryExistsFunc : InstructionFunc {
logOp(format("exists %s: %d", pathToString(combinePaths(directory->getPath(), path)).c_str(), result));
}
data->stack.push(Tuple().append(result ? 1 : 0).pack());
data->stack.push(Tuple().append((int64_t)(result ? 1 : 0)).pack());
return Void();
}
};
@ -468,7 +468,7 @@ struct DirectoryContainsFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple key = wait(data->stack.waitAndPop());
bool result = data->directoryData.subspace()->contains(key.getString(0));
data->stack.push(Tuple().append(result ? 1 : 0).pack());
data->stack.push(Tuple().append((int64_t)(result ? 1 : 0)).pack());
return Void();
}
@ -500,7 +500,7 @@ struct DirectoryLogSubspaceFunc : InstructionFunc {
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
Tuple prefix = wait(data->stack.waitAndPop());
Tuple tuple;
tuple.append(data->directoryData.directoryListIndex);
tuple.append((int64_t)data->directoryData.directoryListIndex);
instruction->tr->set(Subspace(tuple, prefix.getString(0)).key(), data->directoryData.subspace()->key());
return Void();
@ -526,7 +526,7 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
}
}
Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0));
Subspace logSubspace(Tuple().append((int64_t)(data->directoryData.directoryListIndex)), prefix.getString(0));
Tuple pathTuple;
for(auto &p : directory->getPath()) {
@ -535,7 +535,7 @@ struct DirectoryLogDirectoryFunc : InstructionFunc {
instruction->tr->set(logSubspace.pack(LiteralStringRef("path"), true), pathTuple.pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("layer"), true), Tuple().append(directory->getLayer()).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append(exists ? 1 : 0).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("exists"), true), Tuple().append((int64_t)(exists ? 1 : 0)).pack());
instruction->tr->set(logSubspace.pack(LiteralStringRef("children"), true), childrenTuple.pack());
return Void();

188
bindings/flow/tester/Tester.actor.cpp
View File

@ -24,7 +24,6 @@
#include "bindings/flow/FDBLoanerTypes.h"
#include "Tester.actor.h"
#ifdef __linux__
#include <string.h>
#endif
@ -102,6 +101,22 @@ std::string tupleToString(Tuple const& tuple) {
else if(type == Tuple::INT) {
str += format("%ld", tuple.getInt(i));
}
else if(type == Tuple::FLOAT) {
str += format("%f", tuple.getFloat(i));
}
else if(type == Tuple::DOUBLE) {
str += format("%f", tuple.getDouble(i));
}
else if(type == Tuple::BOOL) {
str += tuple.getBool(i) ? "true" : "false";
}
else if(type == Tuple::UUID) {
Uuid u = tuple.getUuid(i);
str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8));
}
else if(type == Tuple::NESTED) {
str += tupleToString(tuple.getNested(i));
}
else {
ASSERT(false);
}
@ -377,8 +392,8 @@ struct LogStackFunc : InstructionFunc {
try {
for(auto it : entries) {
Tuple tk;
tk.append(it.first);
tk.append(it.second.index);
tk.append((int64_t)it.first);
tk.append((int64_t)it.second.index);
state Standalone<StringRef> pk = tk.pack().withPrefix(prefix);
Standalone<StringRef> pv = wait(it.second.value);
tr->set(pk, pv.substr(0, std::min(pv.size(), 40000)));
@ -1027,6 +1042,21 @@ struct TuplePackFunc : InstructionFunc {
else if(type == Tuple::UTF8) {
tuple.append(itemTuple.getString(0), true);
}
else if(type == Tuple::FLOAT) {
tuple << itemTuple.getFloat(0);
}
else if(type == Tuple::DOUBLE) {
tuple << itemTuple.getDouble(0);
}
else if(type == Tuple::BOOL) {
tuple << itemTuple.getBool(0);
}
else if(type == Tuple::UUID) {
tuple << itemTuple.getUuid(0);
}
else if(type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0));
}
else {
ASSERT(false);
}
@ -1083,7 +1113,7 @@ struct TupleRangeFunc : InstructionFunc {
return Void();
state Tuple tuple;
state int i = 0;
state size_t i = 0;
for (; i < items1.size(); ++i) {
Standalone<StringRef> str = wait(items1[i].value);
Tuple itemTuple = Tuple::unpack(str);
@ -1101,6 +1131,21 @@ struct TupleRangeFunc : InstructionFunc {
else if(type == Tuple::UTF8) {
tuple.append(itemTuple.getString(0), true);
}
else if(type == Tuple::FLOAT) {
tuple << itemTuple.getFloat(0);
}
else if(type == Tuple::DOUBLE) {
tuple << itemTuple.getDouble(0);
}
else if(type == Tuple::BOOL) {
tuple << itemTuple.getBool(0);
}
else if(type == Tuple::UUID) {
tuple << itemTuple.getUuid(0);
}
else if(type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0));
}
else {
ASSERT(false);
}
@ -1120,6 +1165,141 @@ struct TupleRangeFunc : InstructionFunc {
const char* TupleRangeFunc::name = "TUPLE_RANGE";
REGISTER_INSTRUCTION_FUNC(TupleRangeFunc);
// TUPLE_SORT
struct TupleSortFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
state std::vector<StackItem> items = data->stack.pop();
if (items.size() != 1)
return Void();
Standalone<StringRef> s1 = wait(items[0].value);
state int64_t count = Tuple::unpack(s1).getInt(0);
state std::vector<StackItem> items1 = data->stack.pop(count);
if (items1.size() != count)
return Void();
state std::vector<Tuple> tuples;
state size_t i = 0;
for(; i < items1.size(); i++) {
Standalone<StringRef> value = wait(items1[i].value);
tuples.push_back(Tuple::unpack(value));
}
std::sort(tuples.begin(), tuples.end());
for(Tuple const& t : tuples) {
data->stack.push(t.pack());
}
return Void();
}
};
const char* TupleSortFunc::name = "TUPLE_SORT";
REGISTER_INSTRUCTION_FUNC(TupleSortFunc);
// ENCODE_FLOAT
struct EncodeFloatFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
std::vector<StackItem> items = data->stack.pop();
if (items.size() != 1)
return Void();
Standalone<StringRef> s1 = wait(items[0].value);
Standalone<StringRef> fBytes = Tuple::unpack(s1).getString(0);
ASSERT(fBytes.size() == 4);
int32_t intVal = *(int32_t*)fBytes.begin();
intVal = bigEndian32(intVal);
float fVal = *(float*)&intVal;
Tuple t;
t.append(fVal);
data->stack.push(t.pack());
return Void();
}
};
const char* EncodeFloatFunc::name = "ENCODE_FLOAT";
REGISTER_INSTRUCTION_FUNC(EncodeFloatFunc);
// ENCODE_DOUBLE
struct EncodeDoubleFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
std::vector<StackItem> items = data->stack.pop();
if (items.size() != 1)
return Void();
Standalone<StringRef> s1 = wait(items[0].value);
Standalone<StringRef> dBytes = Tuple::unpack(s1).getString(0);
ASSERT(dBytes.size() == 8);
int64_t intVal = *(int64_t*)dBytes.begin();
intVal = bigEndian64(intVal);
double dVal = *(double*)&intVal;
Tuple t;
t.append(dVal);
data->stack.push(t.pack());
return Void();
}
};
const char* EncodeDoubleFunc::name = "ENCODE_DOUBLE";
REGISTER_INSTRUCTION_FUNC(EncodeDoubleFunc);
// DECODE_FLOAT
struct DecodeFloatFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
std::vector<StackItem> items = data->stack.pop();
if (items.size() != 1)
return Void();
Standalone<StringRef> s1 = wait(items[0].value);
float fVal = Tuple::unpack(s1).getFloat(0);
int32_t intVal = *(int32_t*)&fVal;
intVal = bigEndian32(intVal);
Tuple t;
t.append(StringRef((uint8_t*)&intVal, 4), false);
data->stack.push(t.pack());
return Void();
}
};
const char* DecodeFloatFunc::name = "DECODE_FLOAT";
REGISTER_INSTRUCTION_FUNC(DecodeFloatFunc);
// DECODE_DOUBLE
struct DecodeDoubleFunc : InstructionFunc {
static const char* name;
ACTOR static Future<Void> call(Reference<FlowTesterData> data, Reference<InstructionData> instruction) {
std::vector<StackItem> items = data->stack.pop();
if (items.size() != 1)
return Void();
Standalone<StringRef> s1 = wait(items[0].value);
double dVal = Tuple::unpack(s1).getDouble(0);
int64_t intVal = *(int64_t*)&dVal;
intVal = bigEndian64(intVal);
Tuple t;
t.append(StringRef((uint8_t*)&intVal, 8), false);
data->stack.push(t.pack());
return Void();
}
};
const char* DecodeDoubleFunc::name = "DECODE_DOUBLE";
REGISTER_INSTRUCTION_FUNC(DecodeDoubleFunc);
// Thread Operations
// START_THREAD
struct StartThreadFunc : InstructionFunc {

103
bindings/go/src/_stacktester/stacktester.go
View File

@ -22,11 +22,14 @@ package main
import (
"bytes"
"encoding/binary"
"encoding/hex"
"fdb"
"fdb/tuple"
"log"
"fmt"
"os"
"sort"
"strings"
"sync"
"runtime"
@ -40,6 +43,21 @@ const verbose bool = false
var trMap = map[string]fdb.Transaction {}
var trMapLock = sync.RWMutex{}
// Make tuples sortable by byte-order
type byBytes []tuple.Tuple
func (b byBytes) Len() int {
return len(b)
}
func (b byBytes) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}
func (b byBytes) Less(i, j int) bool {
return bytes.Compare(b[i].Pack(), b[j].Pack()) < 0
}
func int64ToBool(i int64) bool {
switch i {
case 0:
@ -83,7 +101,10 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) {
ret, sm.stack = sm.stack[len(sm.stack) - 1], sm.stack[:len(sm.stack) - 1]
switch el := ret.item.(type) {
case int64, []byte, string:
case []byte:
ret.item = el
case int64, string, bool, tuple.UUID, float32, float64, tuple.Tuple:
ret.item = el
case fdb.Key:
ret.item = []byte(el)
case fdb.FutureNil:
@ -145,6 +166,40 @@ func (sm *StackMachine) store(idx int, item interface{}) {
sm.stack = append(sm.stack, stackEntry{item, idx})
}
func tupleToString(t tuple.Tuple) string {
var buffer bytes.Buffer
buffer.WriteByte('(')
for i, el := range t {
if i > 0 {
buffer.WriteString(", ")
}
switch el := el.(type) {
case int64:
buffer.WriteString(fmt.Sprintf("%d", el))
case []byte:
buffer.WriteString(fmt.Sprintf("%+q", string(el)))
case string:
buffer.WriteString(fmt.Sprintf("%+q", el))
case bool:
buffer.WriteString(fmt.Sprintf("%t", el))
case tuple.UUID:
buffer.WriteString(hex.EncodeToString(el[:]))
case float32:
buffer.WriteString(fmt.Sprintf("%f", el))
case float64:
buffer.WriteString(fmt.Sprintf("%f", el))
case nil:
buffer.WriteString("nil")
case tuple.Tuple:
buffer.WriteString(tupleToString(el))
default:
log.Fatalf("Don't know how to stringify tuple elemement %v %T\n", el, el)
}
}
buffer.WriteByte(')')
return buffer.String()
}
func (sm *StackMachine) dumpStack() {
for i := len(sm.stack) - 1; i >= 0; i-- {
fmt.Printf(" %d.", sm.stack[i].idx)
@ -164,6 +219,16 @@ func (sm *StackMachine) dumpStack() {
fmt.Printf(" %+q", string(el))
case string:
fmt.Printf(" %+q", el)
case bool:
fmt.Printf(" %t", el)
case tuple.Tuple:
fmt.Printf(" %s", tupleToString(el))
case tuple.UUID:
fmt.Printf(" %s", hex.EncodeToString(el[:]))
case float32:
fmt.Printf(" %f", el)
case float64:
fmt.Printf(" %f", el)
case nil:
fmt.Printf(" nil")
default:
@ -561,6 +626,39 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) {
for _, el := range(t) {
sm.store(idx, []byte(tuple.Tuple{el}.Pack()))
}
case op == "TUPLE_SORT":
count := sm.waitAndPop().item.(int64)
tuples := make([]tuple.Tuple, count)
for i := 0; i < int(count); i++ {
tuples[i], e = tuple.Unpack(fdb.Key(sm.waitAndPop().item.([]byte)))
if e != nil {
panic(e)
}
}
sort.Sort(byBytes(tuples))
for _, t := range tuples {
sm.store(idx, t.Pack())
}
case op == "ENCODE_FLOAT":
val_bytes := sm.waitAndPop().item.([]byte)
var val float32
binary.Read(bytes.NewBuffer(val_bytes), binary.BigEndian, &val)
sm.store(idx, val)
case op == "ENCODE_DOUBLE":
val_bytes := sm.waitAndPop().item.([]byte)
var val float64
binary.Read(bytes.NewBuffer(val_bytes), binary.BigEndian, &val)
sm.store(idx, val)
case op == "DECODE_FLOAT":
val := sm.waitAndPop().item.(float32)
var ibuf bytes.Buffer
binary.Write(&ibuf, binary.BigEndian, val)
sm.store(idx, ibuf.Bytes())
case op == "DECODE_DOUBLE":
val := sm.waitAndPop().item.(float64)
var ibuf bytes.Buffer
binary.Write(&ibuf, binary.BigEndian, val)
sm.store(idx, ibuf.Bytes())
case op == "TUPLE_RANGE":
var t tuple.Tuple
count := sm.waitAndPop().item.(int64)
@ -618,7 +716,8 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) {
case op == "ATOMIC_OP":
opname := strings.Replace(strings.Title(strings.Replace(strings.ToLower(sm.waitAndPop().item.(string)), "_", " ", -1)), " ", "", -1)
key := fdb.Key(sm.waitAndPop().item.([]byte))
value := sm.waitAndPop().item.([]byte)
ival := sm.waitAndPop().item
value := ival.([]byte)
sm.executeMutation(t, func (tr fdb.Transaction) (interface{}, error) {
reflect.ValueOf(tr).MethodByName(opname).Call([]reflect.Value{reflect.ValueOf(key), reflect.ValueOf(value)})
return nil, nil

221
bindings/go/src/fdb/tuple/tuple.go
View File

@ -47,7 +47,7 @@ import (
// result in a runtime panic).
//
// The valid types for TupleElement are []byte (or fdb.KeyConvertible), string,
// int64 (or int), and nil.
// int64 (or int), float, double, bool, UUID, Tuple, and nil.
type TupleElement interface{}
// Tuple is a slice of objects that can be encoded as FoundationDB tuples. If
@ -59,6 +59,28 @@ type TupleElement interface{}
// packing T (modulo type normalization to []byte and int64).
type Tuple []TupleElement
// UUID wraps a basic byte array as a UUID. We do not provide any special
// methods for accessing or generating the UUID, but as Go does not provide
// a built-in UUID type, this simple wrapper allows for other libraries
// to write the output of their UUID type as a 16-byte array into
// an instance of this type.
type UUID [16]byte
// Type codes: These prefix the different elements in a packed Tuple
// to indicate what type they are.
const nilCode = 0x00
const bytesCode = 0x01
const stringCode = 0x02
const nestedCode = 0x05
const intZeroCode = 0x14
const posIntEnd = 0x1c
const negIntStart = 0x0c
const floatCode = 0x20
const doubleCode = 0x21
const falseCode = 0x26
const trueCode = 0x27
const uuidCode = 0x30
var sizeLimits = []uint64{
1 << (0 * 8) - 1,
1 << (1 * 8) - 1,
@ -71,6 +93,18 @@ var sizeLimits = []uint64{
1 << (8 * 8) - 1,
}
func adjustFloatBytes(b []byte, encode bool) {
if (encode && b[0] & 0x80 != 0x00) || (!encode && b[0] & 0x80 == 0x00) {
// Negative numbers: flip all of the bytes.
for i := 0; i < len(b); i++ {
b[i] = b[i] ^ 0xff
}
} else {
// Positive number: flip just the sign bit.
b[0] = b[0] ^ 0x80
}
}
func encodeBytes(buf *bytes.Buffer, code byte, b []byte) {
buf.WriteByte(code)
buf.Write(bytes.Replace(b, []byte{0x00}, []byte{0x00, 0xFF}, -1))
@ -97,7 +131,7 @@ func encodeInt(buf *bytes.Buffer, i int64) {
switch {
case i > 0:
n = bisectLeft(uint64(i))
buf.WriteByte(byte(0x14+n))
buf.WriteByte(byte(intZeroCode+n))
binary.Write(&ibuf, binary.BigEndian, i)
case i < 0:
n = bisectLeft(uint64(-i))
@ -108,35 +142,86 @@ func encodeInt(buf *bytes.Buffer, i int64) {
buf.Write(ibuf.Bytes()[8-n:])
}
func encodeFloat(buf *bytes.Buffer, f float32) {
var ibuf bytes.Buffer
binary.Write(&ibuf, binary.BigEndian, f)
buf.WriteByte(floatCode)
out := ibuf.Bytes()
adjustFloatBytes(out, true)
buf.Write(out)
}
func encodeDouble(buf *bytes.Buffer, d float64) {
var ibuf bytes.Buffer
binary.Write(&ibuf, binary.BigEndian, d)
buf.WriteByte(doubleCode)
out := ibuf.Bytes()
adjustFloatBytes(out, true)
buf.Write(out)
}
func encodeUUID(buf *bytes.Buffer, u UUID) {
buf.WriteByte(uuidCode)
buf.Write(u[:])
}
func encodeTuple(buf *bytes.Buffer, t Tuple, nested bool) {
if nested {
buf.WriteByte(nestedCode)
}
for i, e := range(t) {
switch e := e.(type) {
case Tuple:
encodeTuple(buf, e, true)
case nil:
buf.WriteByte(nilCode)
if nested {
buf.WriteByte(0xff)
}
case int64:
encodeInt(buf, e)
case int:
encodeInt(buf, int64(e))
case []byte:
encodeBytes(buf, bytesCode, e)
case fdb.KeyConvertible:
encodeBytes(buf, bytesCode, []byte(e.FDBKey()))
case string:
encodeBytes(buf, stringCode, []byte(e))
case float32:
encodeFloat(buf, e)
case float64:
encodeDouble(buf, e)
case bool:
if e {
buf.WriteByte(trueCode)
} else {
buf.WriteByte(falseCode)
}
case UUID:
encodeUUID(buf, e)
default:
panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i]))
}
}
if nested {
buf.WriteByte(0x00)
}
}
// Pack returns a new byte slice encoding the provided tuple. Pack will panic if
// the tuple contains an element of any type other than []byte,
// fdb.KeyConvertible, string, int64, int or nil.
// fdb.KeyConvertible, string, int64, int, float32, float64, bool, tuple.UUID,
// nil, or a Tuple with elements of valid types.
//
// Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to
// call Pack when using a Tuple with a FoundationDB API function that requires a
// key.
func (t Tuple) Pack() []byte {
buf := new(bytes.Buffer)
for i, e := range(t) {
switch e := e.(type) {
case nil:
buf.WriteByte(0x00)
case int64:
encodeInt(buf, e)
case int:
encodeInt(buf, int64(e))
case []byte:
encodeBytes(buf, 0x01, e)
case fdb.KeyConvertible:
encodeBytes(buf, 0x01, []byte(e.FDBKey()))
case string:
encodeBytes(buf, 0x02, []byte(e))
default:
panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i]))
}
}
encodeTuple(buf, t, false)
return buf.Bytes()
}
@ -168,13 +253,13 @@ func decodeString(b []byte) (string, int) {
}
func decodeInt(b []byte) (int64, int) {
if b[0] == 0x14 {
if b[0] == intZeroCode {
return 0, 1
}
var neg bool
n := int(b[0]) - 20
n := int(b[0]) - intZeroCode
if n < 0 {
n = -n
neg = true
@ -194,9 +279,31 @@ func decodeInt(b []byte) (int64, int) {
return ret, n+1
}
// Unpack returns the tuple encoded by the provided byte slice, or an error if
// the key does not correctly encode a FoundationDB tuple.
func Unpack(b []byte) (Tuple, error) {
func decodeFloat(b []byte) (float32, int) {
bp := make([]byte, 4)
copy(bp, b[1:])
adjustFloatBytes(bp, false)
var ret float32
binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret)
return ret, 5
}
func decodeDouble(b []byte) (float64, int) {
bp := make([]byte, 8)
copy(bp, b[1:])
adjustFloatBytes(bp, false)
var ret float64
binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret)
return ret, 9
}
func decodeUUID(b []byte) (UUID, int) {
var u UUID
copy(u[:], b[1:])
return u, 17
}
func decodeTuple(b []byte, nested bool) (Tuple, int, error) {
var t Tuple
var i int
@ -206,24 +313,66 @@ func Unpack(b []byte) (Tuple, error) {
var off int
switch {
case b[i] == 0x00:
el = nil
off = 1
case b[i] == 0x01:
case b[i] == nilCode:
if !nested {
el = nil
off = 1
} else if i + 1 < len(b) && b[i+1] == 0xff {
el = nil
off = 2
} else {
return t, i+1, nil
}
case b[i] == bytesCode:
el, off = decodeBytes(b[i:])
case b[i] == 0x02:
case b[i] == stringCode:
el, off = decodeString(b[i:])
case 0x0c <= b[i] && b[i] <= 0x1c:
case negIntStart <= b[i] && b[i] <= posIntEnd:
el, off = decodeInt(b[i:])
case b[i] == floatCode:
if i + 5 > len(b) {
return nil, i, fmt.Errorf("insufficient bytes to decode float starting at position %d of byte array for tuple", i)
}
el, off = decodeFloat(b[i:])
case b[i] == doubleCode:
if i + 9 > len(b) {
return nil, i, fmt.Errorf("insufficient bytes to decode double starting at position %d of byte array for tuple", i)
}
el, off = decodeDouble(b[i:])
case b[i] == trueCode:
el = true
off = 1
case b[i] == falseCode:
el = false
off = 1
case b[i] == uuidCode:
if i + 17 > len(b) {
return nil, i, fmt.Errorf("insufficient bytes to decode UUID starting at position %d of byte array for tuple", i)
}
el, off = decodeUUID(b[i:])
case b[i] == nestedCode:
var err error
el, off, err = decodeTuple(b[i+1:], true)
if err != nil {
return nil, i, err
}
off += 1
default:
return nil, fmt.Errorf("unable to decode tuple element with unknown typecode %02x", b[i])
return nil, i, fmt.Errorf("unable to decode tuple element with unknown typecode %02x", b[i])
}
t = append(t, el)
i += off
}
return t, nil
return t, i, nil
}
// Unpack returns the tuple encoded by the provided byte slice, or an error if
// the key does not correctly encode a FoundationDB tuple.
func Unpack(b []byte) (Tuple, error) {
t, _, err := decodeTuple(b, false)
return t, err
}
// FDBKey returns the packed representation of a Tuple, and allows Tuple to

2
bindings/java/src-completable/main/com/apple/cie/foundationdb/FDBDatabase.java
View File

@ -32,7 +32,7 @@ import com.apple.cie.foundationdb.async.AsyncUtil;
class FDBDatabase extends DefaultDisposableImpl implements Database, Disposable, OptionConsumer {
private DatabaseOptions options;
private Executor executor;
private final Executor executor;
protected FDBDatabase(long cPtr, Executor executor) {
super(cPtr);

107
bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/IterableComparator.java Normal file
View File

@ -0,0 +1,107 @@
/*
* IterableComparator.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.cie.foundationdb.tuple;
import java.util.Comparator;
import java.util.Iterator;
/**
* A {@link Tuple}-compatible {@link Comparator} that will sort {@link Iterable}s
* in a manner that is consistent with the byte-representation of {@link Tuple}s.
* In particular, if one has two {@link Tuple}s, {@code tuple1} and {@code tuple2},
* it is the case that:
*
* <pre>
* {@code
* tuple1.compareTo(tuple2)
* == new IterableComparator().compare(tuple1, tuple2)
* == new IterableComparator().compare(tuple1.getItems(), tuple2.getItems()),
* == ByteArrayUtil.compareUnsigned(tuple1.pack(), tuple2.pack())}
* </pre>
*
* <p>
* The individual elements of the {@link Iterable} must be of a type that can
* be serialized by a {@link Tuple}. For items of identical types, they will be
* sorted in a way that is consistent with their natural ordering with a few
* caveats:
* </p>
*
* <ul>
* <li>
* For floating point types, negative NaN values are sorted before all regular values, and
* positive NaN values are sorted after all regular values.
* </li>
* <li>
* Single-precision floating point numbers are sorted before
* all double-precision floating point numbers.
* </li>
* <li>
* UUIDs are sorted by their <i>unsigned</i> Big-Endian byte representation
* rather than their signed byte representation (which is the behavior of
* {@link java.util.UUID#compareTo(java.util.UUID) UUID.compareTo()}).
* </li>
* <li>Strings are sorted explicitly by their UTF-8 byte representation</li>
* <li>Nested {@link Tuple}s and {@link java.util.List List}s are sorted element-wise.</li>
* </ul>
*/
public class IterableComparator implements Comparator<Iterable<?>> {
/**
* Creates a new {@code IterableComparator}. This {@link Comparator} has
* no internal state.
*/
public IterableComparator() {}
/**
* Compare two {@link Iterable}s in a way consistent with their
* byte representation. This is done element-wise and is consistent
* with a number of other ways of sorting {@link Tuple}s. This will
* raise an {@link IllegalArgumentException} if any of the items
* of either {@link Iterable} cannot be serialized by a {@link Tuple}.
*
* @param iterable1 the first {@link Iterable} of items
* @param iterable2 the second {@link Iterable} of items
* @return a negative number if the first iterable would sort before the second
* when serialized, a positive number if the opposite is true, and zero
* if the two are equal
*/
@Override
public int compare(Iterable<?> iterable1, Iterable<?> iterable2) {
Iterator<?> i1 = iterable1.iterator();
Iterator<?> i2 = iterable2.iterator();
while(i1.hasNext() && i2.hasNext()) {
int itemComp = TupleUtil.compareItems(i1.next(), i2.next());
if(itemComp != 0) {
return itemComp;
}
}
if(i1.hasNext()) {
// iterable2 is a prefix of iterable1.
return 1;
}
if(i2.hasNext()) {
// iterable1 is a prefix of iterable2.
return -1;
}
return 0;
}
}

311
bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/Tuple.java
View File

@ -20,12 +20,15 @@
package com.apple.cie.foundationdb.tuple;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -39,26 +42,34 @@ import com.apple.cie.foundationdb.Range;
* ideal for building a variety of higher-level data models.<br>
* <h3>Types</h3>
* A {@code Tuple} can
* contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, and {@code null}. All
* {@code Number}s will be converted to a {@code long} integral value, so all
* floating point information will be lost and their range will be constrained to the range
* [{@code 2^63-1}, {@code -2^63}]. Note that for numbers outside this range the way that Java
* contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, {@link UUID}s,
* {@code boolean}s, {@link List}s, other {@code Tuple}s, and {@code null}.
* {@link Float} and {@link Double} instances will be serialized as single- and double-precision
* numbers respectively, and {@link BigInteger}s within the range [{@code -2^2040+1},
* {@code 2^2040-1}] are serialized without loss of precision (those outside the range
* will raise an {@link IllegalArgumentException}). All other {@code Number}s will be converted to
* a {@code long} integral value, so the range will be constrained to
* [{@code -2^63}, {@code 2^63-1}]. Note that for numbers outside this range the way that Java
* truncates integral values may yield unexpected results.<br>
* <h3>{@code null} values</h3>
* The FoundationDB tuple specification has a special type-code for {@code None}; {@code nil}; or,
* as Java would understand it, {@code null}.
* The behavior of the layer in the presence of {@code null} varies by type with the intention
* of matching expected behavior in Java. {@code byte[]} and {@link String}s can be {@code null},
* where integral numbers (i.e. {@code long}s) cannot.
* This means that the typed getters ({@link #getBytes(int) getBytes()} and {@link #getString(int) getString()})
* of matching expected behavior in Java. {@code byte[]}, {@link String}s, {@link UUID}s, and
* nested {@link List}s and {@code Tuple}s can be {@code null},
* whereas numbers (e.g., {@code long}s and {@code double}s) and booleans cannot.
* This means that the typed getters ({@link #getBytes(int) getBytes()}, {@link #getString(int) getString()}),
* {@link #getUUID(int) getUUID()}, {@link #getNestedTuple(int) getNestedTuple()}, and {@link #getNestedList(int) getNestedList()})
* will return {@code null} if the entry at that location was {@code null} and the typed adds
* ({@link #add(byte[])} and {@link #add(String)}) will accept {@code null}. The
* {@link #getLong(int) typed get for integers}, however, will throw a {@code NullPointerException} if
* {@link #getLong(int) typed get for integers} and other typed getters, however, will throw a {@link NullPointerException} if
* the entry in the {@code Tuple} was {@code null} at that position.<br>
* <br>
* This class is not thread safe.
*/
public class Tuple implements Comparable<Tuple>, Iterable<Object> {
private static IterableComparator comparator = new IterableComparator();
private List<Object> elements;
private Tuple(List<? extends Object> elements, Object newItem) {
@ -72,12 +83,12 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a copy of this {@code Tuple} with an appended last element. The parameter
* is untyped but only {@link String}, {@code byte[]}, {@link Number}s, and {@code null} are allowed.
* All {@code Number}s are converted to a 8 byte integral value, so all floating point
* information is lost.
* is untyped but only {@link String}, {@code byte[]}, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, and {@code null} are allowed. If an object of
* another type is passed, then an {@link IllegalArgumentException} is thrown.
*
* @param o the object to append. Must be {@link String}, {@code byte[]},
* {@link Number}s, or {@code null}.
* {@link Number}s, {@link UUID}, {@link List}, {@link Boolean}, or {@code null}.
*
* @return a newly created {@code Tuple}
*/
@ -85,6 +96,10 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
if(o != null &&
!(o instanceof String) &&
!(o instanceof byte[]) &&
!(o instanceof UUID) &&
!(o instanceof List<?>) &&
!(o instanceof Tuple) &&
!(o instanceof Boolean) &&
!(o instanceof Number)) {
throw new IllegalArgumentException("Parameter type (" + o.getClass().getName() + ") not recognized");
}
@ -124,6 +139,92 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return new Tuple(this.elements, b);
}
/**
* Creates a copy of this {@code Tuple} with a {@code boolean} appended as the last element.
*
* @param b the {@code boolean} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(boolean b) {
return new Tuple(this.elements, b);
}
/**
* Creates a copy of this {@code Tuple} with a {@link UUID} appended as the last element.
*
* @param uuid the {@link UUID} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(UUID uuid) {
return new Tuple(this.elements, uuid);
}
/**
* Creates a copy of this {@code Tuple} with a {@link BigInteger} appended as the last element.
* As {@link Tuple}s cannot contain {@code null} numeric types, a {@link NullPointerException}
* is raised if a {@code null} argument is passed.
*
* @param bi the {@link BigInteger} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(BigInteger bi) {
if(bi == null) {
throw new NullPointerException("Number types in Tuple cannot be null");
}
return new Tuple(this.elements, bi);
}
/**
* Creates a copy of this {@code Tuple} with a {@code float} appended as the last element.
*
* @param f the {@code float} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(float f) {
return new Tuple(this.elements, f);
}
/**
* Creates a copy of this {@code Tuple} with a {@code double} appended as the last element.
*
* @param d the {@code double} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(double d) {
return new Tuple(this.elements, d);
}
/**
* Creates a copy of this {@code Tuple} with an {@link List} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}).
* This adds the list as a single elemented nested within the outer {@code Tuple}.
*
* @param l the {@link List} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(List<? extends Object> l) {
return new Tuple(this.elements, l);
}
/**
* Creates a copy of this {@code Tuple} with a {@code Tuple} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(Tuple) Tuple.addAll}).
* This adds the list as a single elemented nested within the outer {@code Tuple}.
*
* @param t the {@code Tuple} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(Tuple t) {
return new Tuple(this.elements, t);
}
/**
* Creates a copy of this {@code Tuple} with a {@code byte} array appended as the last element.
*
@ -320,6 +421,138 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return (String)o;
}
/**
* Gets an indexed item as a {@link BigInteger}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the tuple element is not of
* a {@code Number} type. If the underlying type is a floating point value, this
* will lead to a loss of precision. The element at the index may not be {@code null}.
*
* @param index the location of the element to return
*
* @return the item at {@code index} as a {@link BigInteger}
*/
public BigInteger getBigInteger(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Number types in Tuples may not be null");
if(o instanceof BigInteger) {
return (BigInteger)o;
} else {
return BigInteger.valueOf(((Number)o).longValue());
}
}
/**
* Gets an indexed item as a {@code float}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a number type.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@code float}
*/
public float getFloat(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Number types in Tuples may not be null");
return ((Number)o).floatValue();
}
/**
* Gets an indexed item as a {@code double}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a number type.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@code double}
*/
public double getDouble(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Number types in Tuples may not be null");
return ((Number)o).doubleValue();
}
/**
* Gets an indexed item as a {@code boolean}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@code Boolean}.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@code boolean}
*/
public boolean getBoolean(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Boolean type in Tuples may not be null");
return (Boolean)o;
}
/**
* Gets an indexed item as a {@link UUID}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@code UUID}.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@link UUID}
*/
public UUID getUUID(int index) {
Object o = this.elements.get(index);
if(o == null)
return null;
return (UUID)o;
}
/**
* Gets an indexed item as a {@link List}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@link List}
* or {@code Tuple}. The element at the index may be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@link List}
*/
public List<Object> getNestedList(int index) {
Object o = this.elements.get(index);
if(o == null) {
return null;
} else if(o instanceof Tuple) {
return ((Tuple)o).getItems();
} else if(o instanceof List<?>) {
List<Object> ret = new LinkedList<Object>();
ret.addAll((List<? extends Object>)o);
return ret;
} else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to list");
}
}
/**
* Gets an indexed item as a {@link Tuple}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@link List}
* or {@code Tuple}. The element at the index may be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@link List}
*/
public Tuple getNestedTuple(int index) {
Object o = this.elements.get(index);
if(o == null) {
return null;
} else if(o instanceof Tuple) {
return (Tuple)o;
} else if(o instanceof List<?>) {
return Tuple.fromItems((List<? extends Object>)o);
} else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to tuple");
}
}
/**
* Gets an indexed item without forcing a type.
*
@ -400,7 +633,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*/
@Override
public int compareTo(Tuple t) {
return ByteArrayUtil.compareUnsigned(this.pack(), t.pack());
return comparator.compare(elements, t.elements);
}
/**
@ -473,7 +706,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a new {@code Tuple} from a variable number of elements. The elements
* must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s.
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s.
*
* @param items the elements from which to create the {@code Tuple}.
*
@ -490,7 +724,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Efficiently creates a new {@code Tuple} from a list of objects. The elements
* must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s.
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s.
*
* @param items the elements from which to create the {@code Tuple}.
*
@ -518,7 +753,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a new {@code Tuple} from a variable number of elements. The elements
* must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s.
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s.
*
* @param items the elements from which to create the {@code Tuple}.
*
@ -544,13 +780,27 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
t = t.add(Long.MIN_VALUE + 1);
t = t.add(Long.MIN_VALUE);
t = t.add("foo");
t = t.addObject(null);
t = t.add(false);
t = t.add(true);
t = t.add(3.14159);
t = t.add(3.14159f);
t = t.add(java.util.UUID.randomUUID());
t = t.add(t.getItems());
t = t.add(t);
t = t.add(new BigInteger("100000000000000000000000000000000000000000000"));
t = t.add(new BigInteger("-100000000000000000000000000000000000000000000"));
byte[] bytes = t.pack();
System.out.println("Packed: " + ByteArrayUtil.printable(bytes));
List<Object> items = Tuple.fromBytes(bytes).getItems();
for(Object obj : items) {
System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj);
if (obj != null)
System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj);
else
System.out.println(" -> type: (null): null");
}
t = Tuple.fromStream(t.stream().map(item -> {
if(item instanceof String) {
return ((String)item).toUpperCase();
@ -559,6 +809,33 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
}
}));
System.out.println("Upper cased: " + t);
Tuple t2 = Tuple.fromBytes(bytes);
System.out.println("t2.getLong(0): " + t2.getLong(0));
System.out.println("t2.getBigInteger(1): " + t2.getBigInteger(1));
System.out.println("t2.getString(9): " + t2.getString(9));
System.out.println("t2.get(10): " + t2.get(10));
System.out.println("t2.getBoolean(11): " + t2.getBoolean(11));
System.out.println("t2.getBoolean(12): " + t2.getBoolean(12));
System.out.println("t2.getDouble(13): " + t2.getDouble(13));
System.out.println("t2.getFloat(13): " + t2.getFloat(13));
System.out.println("t2.getLong(13): " + t2.getLong(13));
System.out.println("t2.getBigInteger(13): " + t2.getBigInteger(13));
System.out.println("t2.getDouble(14): " + t2.getDouble(14));
System.out.println("t2.getFloat(14): " + t2.getFloat(14));
System.out.println("t2.getLong(14): " + t2.getLong(14));
System.out.println("t2.getBigInteger(14): " + t2.getBigInteger(14));
System.out.println("t2.getNestedList(17): " + t2.getNestedList(17));
System.out.println("t2.getNestedTuple(17): " + t2.getNestedTuple(17));
System.out.println("(2*(Long.MAX_VALUE+1),) = " + ByteArrayUtil.printable(Tuple.from(
BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE).shiftLeft(1)
).pack()));
System.out.println("(2*Long.MIN_VALUE,) = " + ByteArrayUtil.printable(Tuple.from(
BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2"))
).pack()));
System.out.println("2*Long.MIN_VALUE = " + Tuple.fromBytes(Tuple.from(
BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2"))).pack()).getBigInteger(0));
}
private static Tuple createTuple(int items) {

354
bindings/java/src-completable/main/com/apple/cie/foundationdb/tuple/TupleUtil.java
View File

@ -28,12 +28,26 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
class TupleUtil {
private static final byte nil = 0x0;
private static final byte nil = 0x00;
private static final byte[] nil_rep = new byte[] {nil, (byte)0xFF};
private static final BigInteger[] size_limits;
private static final Charset UTF8;
private static final IterableComparator iterableComparator;
private static final byte BYTES_CODE = 0x01;
private static final byte STRING_CODE = 0x02;
private static final byte NESTED_CODE = 0x05;
private static final byte INT_ZERO_CODE = 0x14;
private static final byte POS_INT_END = 0x1d;
private static final byte NEG_INT_START = 0x0b;
private static final byte FLOAT_CODE = 0x20;
private static final byte DOUBLE_CODE = 0x21;
private static final byte FALSE_CODE = 0x26;
private static final byte TRUE_CODE = 0x27;
private static final byte UUID_CODE = 0x30;
static {
size_limits = new BigInteger[9];
@ -41,6 +55,7 @@ class TupleUtil {
size_limits[i] = (BigInteger.ONE).shiftLeft(i * 8).subtract(BigInteger.ONE);
}
UTF8 = Charset.forName("UTF-8");
iterableComparator = new IterableComparator();
}
static class DecodeResult {
@ -53,27 +68,109 @@ class TupleUtil {
}
}
static int byteLength(byte[] bytes) {
for(int i = 0; i < bytes.length; i++) {
if(bytes[i] == 0x00) continue;
return bytes.length - i;
}
return 0;
}
/**
* Takes the Big-Endian byte representation of a floating point number and adjusts
* it so that it sorts correctly. For encoding, if the sign bit is 1 (the number
* is negative), then we need to flip all of the bits; otherwise, just flip the
* sign bit. For decoding, if the sign bit is 0 (the number is negative), then
* we also need to flip all of the bits; otherwise, just flip the sign bit.
* This will mutate in place the given array.
*
* @param bytes Big-Endian IEEE encoding of a floating point number
* @param start the (zero-indexed) first byte in the array to mutate
* @param encode <code>true</code> if we encoding the float and <code>false</code> if we are decoding
* @return the encoded {@code byte[]}
*/
static byte[] floatingPointCoding(byte[] bytes, int start, boolean encode) {
if(encode && (bytes[start] & (byte)0x80) != (byte)0x00) {
for(int i = start; i < bytes.length; i++) {
bytes[i] = (byte) (bytes[i] ^ 0xff);
}
} else if(!encode && (bytes[start] & (byte)0x80) != (byte)0x80) {
for(int i = start; i < bytes.length; i++) {
bytes[i] = (byte) (bytes[i] ^ 0xff);
}
} else {
bytes[start] = (byte) (0x80 ^ bytes[start]);
}
return bytes;
}
public static byte[] join(List<byte[]> items) {
return ByteArrayUtil.join(null, items);
}
static byte[] encode(Object t) {
static int getCodeFor(Object o) {
if(o == null)
return nil;
if(o instanceof byte[])
return BYTES_CODE;
if(o instanceof String)
return STRING_CODE;
if(o instanceof Float)
return FLOAT_CODE;
if(o instanceof Double)
return DOUBLE_CODE;
if(o instanceof Boolean)
return FALSE_CODE;
if(o instanceof UUID)
return UUID_CODE;
if(o instanceof Number)
return INT_ZERO_CODE;
if(o instanceof List<?>)
return NESTED_CODE;
if(o instanceof Tuple)
return NESTED_CODE;
throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName());
}
static byte[] encode(Object t, boolean nested) {
if(t == null)
return new byte[] {nil};
if (nested)
return new byte[]{nil, (byte) 0xff};
else
return new byte[]{nil};
if(t instanceof byte[])
return encode((byte[])t);
if(t instanceof String)
return encode((String)t);
if(t instanceof BigInteger)
return encode((BigInteger)t);
if(t instanceof Float)
return encode((Float)t);
if(t instanceof Double)
return encode((Double)t);
if(t instanceof Boolean)
return encode((Boolean)t);
if(t instanceof UUID)
return encode((UUID)t);
if(t instanceof Number)
return encode(((Number)t).longValue());
if(t instanceof List<?>)
return encode((List<?>)t);
if(t instanceof Tuple)
return encode(((Tuple)t).getItems());
throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName());
}
static byte[] encode(Object t) {
return encode(t, false);
}
static byte[] encode(byte[] bytes) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {0x1});
list.add(ByteArrayUtil.replace(bytes, new byte[] {0x0}, nil_rep));
list.add(new byte[] {0x0});
list.add(new byte[] {BYTES_CODE});
list.add(ByteArrayUtil.replace(bytes, new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
//System.out.println("Joining bytes...");
return ByteArrayUtil.join(null, list);
@ -81,39 +178,68 @@ class TupleUtil {
static byte[] encode(String s) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {0x2});
list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {0x0}, nil_rep));
list.add(new byte[] {0x0});
list.add(new byte[] {STRING_CODE});
list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
//System.out.println("Joining string...");
return ByteArrayUtil.join(null, list);
}
static byte[] encode(long i) {
static byte[] encode(BigInteger i) {
//System.out.println("Encoding integral " + i);
if(i == 0) {
return new byte[] { 20 };
if(i.equals(BigInteger.ZERO)) {
return new byte[] { INT_ZERO_CODE };
}
if(i > 0) {
int n = ByteArrayUtil.bisectLeft(size_limits, BigInteger.valueOf(i));
byte[] bytes = i.toByteArray();
if(i.compareTo(BigInteger.ZERO) > 0) {
if(i.compareTo(size_limits[size_limits.length-1]) > 0) {
int length = byteLength(bytes);
if(length > 0xff) {
throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)");
}
byte[] result = new byte[length + 2];
result[0] = POS_INT_END;
result[1] = (byte)(length);
System.arraycopy(bytes, bytes.length - length, result, 2, length);
return result;
}
int n = ByteArrayUtil.bisectLeft(size_limits, i);
assert n <= size_limits.length;
byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array();
//byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array();
//System.out.println(" -- integral has 'n' of " + n + " and output bytes of " + bytes.length);
byte[] result = new byte[n+1];
result[0] = (byte)(20 + n);
result[0] = (byte)(INT_ZERO_CODE + n);
System.arraycopy(bytes, bytes.length - n, result, 1, n);
return result;
}
BigInteger bI = BigInteger.valueOf(i);
int n = ByteArrayUtil.bisectLeft(size_limits, bI.negate());
if(i.negate().compareTo(size_limits[size_limits.length-1]) > 0) {
int length = byteLength(i.negate().toByteArray());
if(length > 0xff) {
throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)");
}
BigInteger offset = BigInteger.ONE.shiftLeft(length*8).subtract(BigInteger.ONE);
byte[] adjusted = i.add(offset).toByteArray();
byte[] result = new byte[length + 2];
result[0] = NEG_INT_START;
result[1] = (byte)(length ^ 0xff);
if(adjusted.length >= length) {
System.arraycopy(adjusted, adjusted.length - length, result, 2, length);
} else {
Arrays.fill(result, 2, result.length - adjusted.length, (byte)0x00);
System.arraycopy(adjusted, 0, result, result.length - adjusted.length, adjusted.length);
}
return result;
}
int n = ByteArrayUtil.bisectLeft(size_limits, i.negate());
assert n >= 0 && n < size_limits.length; // can we do this? it seems to be required for the following statement
long maxv = size_limits[n].add(bI).longValue();
byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array();
long maxv = size_limits[n].add(i).longValue();
byte[] adjustedBytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array();
byte[] result = new byte[n+1];
result[0] = (byte)(20 - n);
System.arraycopy(bytes, bytes.length - n, result, 1, n);
System.arraycopy(adjustedBytes, adjustedBytes.length - n, result, 1, n);
return result;
}
@ -121,23 +247,63 @@ class TupleUtil {
return encode(i.longValue());
}
static byte[] encode(long i) {
return encode(BigInteger.valueOf(i));
}
static byte[] encode(Float f) {
byte[] result = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array();
floatingPointCoding(result, 1, true);
return result;
}
static byte[] encode(Double d) {
byte[] result = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array();
floatingPointCoding(result, 1, true);
return result;
}
static byte[] encode(Boolean b) {
if (b) {
return new byte[] {TRUE_CODE};
} else {
return new byte[] {FALSE_CODE};
}
}
static byte[] encode(UUID uuid) {
return ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits())
.array();
}
static byte[] encode(List<?> value) {
List<byte[]> parts = new LinkedList<byte[]>();
parts.add(new byte[]{NESTED_CODE});
for(Object t : value) {
parts.add(encode(t, true));
}
parts.add(new byte[]{0x00});
return ByteArrayUtil.join(null, parts);
}
static DecodeResult decode(byte[] rep, int pos, int last) {
//System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos);
// SOMEDAY: codes over 127 will be a problem with the signed Java byte mess
int code = rep[pos];
int start = pos + 1;
if(code == 0x0) {
if(code == nil) {
return new DecodeResult(start, null);
}
if(code == 0x1) {
if(code == BYTES_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of byte string: " + end);
byte[] range = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
//System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'");
return new DecodeResult(end + 1, range);
}
if(code == 0x2) {
if(code == STRING_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of UTF8 string: " + end);
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
@ -145,11 +311,44 @@ class TupleUtil {
//System.out.println(" -> UTF8 string contents: '" + str + "'");
return new DecodeResult(end + 1, str);
}
if(code >=12 && code <=28) {
if(code == FLOAT_CODE) {
byte[] resBytes = Arrays.copyOfRange(rep, start, start+4);
floatingPointCoding(resBytes, 0, false);
float res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getFloat();
return new DecodeResult(start + 4, res);
}
if(code == DOUBLE_CODE) {
byte[] resBytes = Arrays.copyOfRange(rep, start, start+8);
floatingPointCoding(resBytes, 0, false);
double res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getDouble();
return new DecodeResult(start + 8, res);
}
if(code == FALSE_CODE) {
return new DecodeResult(start, false);
}
if(code == TRUE_CODE) {
return new DecodeResult(start, true);
}
if(code == UUID_CODE) {
ByteBuffer bb = ByteBuffer.wrap(rep, start, 16).order(ByteOrder.BIG_ENDIAN);
long msb = bb.getLong();
long lsb = bb.getLong();
return new DecodeResult(start + 16, new UUID(msb, lsb));
}
if(code == POS_INT_END) {
int n = rep[start] & 0xff;
return new DecodeResult(start + n + 1, new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1))));
}
if(code == NEG_INT_START) {
int n = (rep[start] ^ 0xff) & 0xff;
BigInteger origValue = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1)));
BigInteger offset = BigInteger.ONE.shiftLeft(n*8).subtract(BigInteger.ONE);
return new DecodeResult(start + n + 1, origValue.subtract(offset));
}
if(code > NEG_INT_START && code < POS_INT_END) {
// decode a long
byte[] longBytes = new byte[9];
Arrays.fill(longBytes, (byte)0);
boolean upper = code >= 20;
boolean upper = code >= INT_ZERO_CODE;
int n = upper ? code - 20 : 20 - code;
int end = start + n;
@ -157,23 +356,108 @@ class TupleUtil {
throw new RuntimeException("Invalid tuple (possible truncation)");
}
System.arraycopy(rep, start, longBytes, 9-n, n);
System.arraycopy(rep, start, longBytes, longBytes.length-n, n);
if (!upper)
for(int i=9-n; i<9; i++)
longBytes[i] = (byte)~longBytes[i];
for(int i=longBytes.length-n; i<longBytes.length; i++)
longBytes[i] = (byte)(longBytes[i] ^ 0xff);
BigInteger val = new BigInteger(longBytes);
if (!upper) val = val.negate();
if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0 ||
val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0)
throw new RuntimeException("Value out of range for type long.");
// Convert to long if in range -- otherwise, leave as BigInteger.
if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0||
val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0) {
// This can occur if the thing can be represented with 8 bytes but not
// the right sign information.
return new DecodeResult(end, val);
}
return new DecodeResult(end, val.longValue());
}
if(code == NESTED_CODE) {
List<Object> items = new LinkedList<Object>();
int endPos = start;
while(endPos < rep.length) {
if(rep[endPos] == nil) {
if(endPos + 1 < rep.length && rep[endPos+1] == (byte)0xff) {
items.add(null);
endPos += 2;
} else {
endPos += 1;
break;
}
} else {
DecodeResult subResult = decode(rep, endPos, last);
items.add(subResult.o);
endPos = subResult.end;
}
}
return new DecodeResult(endPos, items);
}
throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos);
}
static int compareItems(Object item1, Object item2) {
int code1 = TupleUtil.getCodeFor(item1);
int code2 = TupleUtil.getCodeFor(item2);
if(code1 != code2) {
return Integer.compare(code1, code2);
}
if(code1 == nil) {
// All null's are equal. (Some may be more equal than others.)
return 0;
}
if(code1 == BYTES_CODE) {
return ByteArrayUtil.compareUnsigned((byte[])item1, (byte[])item2);
}
if(code1 == STRING_CODE) {
return ByteArrayUtil.compareUnsigned(((String)item1).getBytes(UTF8), ((String)item2).getBytes(UTF8));
}
if(code1 == INT_ZERO_CODE) {
BigInteger bi1;
if(item1 instanceof BigInteger) {
bi1 = (BigInteger)item1;
} else {
bi1 = BigInteger.valueOf(((Number)item1).longValue());
}
BigInteger bi2;
if(item2 instanceof BigInteger) {
bi2 = (BigInteger)item2;
} else {
bi2 = BigInteger.valueOf(((Number)item2).longValue());
}
return bi1.compareTo(bi2);
}
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// sorting correctly.
byte[] encoded1 = encode((Double)item1);
byte[] encoded2 = encode((Double)item2);
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == FLOAT_CODE) {
// This is done for the same reason that double comparison is done
// that way.
byte[] encoded1 = encode((Float)item1);
byte[] encoded2 = encode((Float)item2);
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == FALSE_CODE) {
return Boolean.compare((Boolean)item1, (Boolean)item2);
}
if(code1 == UUID_CODE) {
// Java UUID.compareTo is signed.
byte[] encoded1 = encode((UUID)item1);
byte[] encoded2 = encode((UUID)item2);
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == NESTED_CODE) {
return iterableComparator.compare((Iterable<?>)item1, (Iterable<?>)item2);
}
throw new IllegalArgumentException("Unknown tuple data type: " + item1.getClass());
}
static List<Object> unpack(byte[] bytes, int start, int length) {
List<Object> items = new LinkedList<Object>();
int pos = start;
@ -188,7 +472,7 @@ class TupleUtil {
static byte[] pack(List<Object> items) {
if(items.size() == 0)
return new byte[0];
return new byte[0];
List<byte[]> parts = new ArrayList<byte[]>(items.size());
for(Object t : items) {

72
bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AsyncStackTester.java
View File

@ -20,8 +20,11 @@
package com.apple.cie.foundationdb.test;
import java.math.BigInteger;
import java.util.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
@ -435,8 +438,9 @@ public class AsyncStackTester {
return inst.popParams(2).thenApplyAsync(new Function<List<Object>, Void>() {
@Override
public Void apply(List<Object> params) {
long result = StackUtils.getNumber(params.get(0)).longValue() -
StackUtils.getNumber(params.get(1)).longValue();
BigInteger result = StackUtils.getBigInteger(params.get(0)).subtract(
StackUtils.getBigInteger(params.get(1))
);
inst.push(result);
return null;
}
@ -508,6 +512,70 @@ public class AsyncStackTester {
}
});
}
else if(op == StackOperation.TUPLE_SORT) {
return inst.popParam().thenComposeAsync(new Function<Object, CompletableFuture<Void>>() {
@Override
public CompletableFuture<Void> apply(Object param) {
final int listSize = StackUtils.getInt(param);
return inst.popParams(listSize).thenApply(new Function<List<Object>, Void>() {
@Override
public Void apply(List<Object> rawElements) {
List<Tuple> tuples = new ArrayList(listSize);
for(Object o : rawElements) {
tuples.add(Tuple.fromBytes((byte[])o));
}
Collections.sort(tuples);
for(Tuple t : tuples) {
inst.push(t.pack());
}
return null;
}
});
}
});
}
else if (op == StackOperation.ENCODE_FLOAT) {
return inst.popParam().thenApply(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
byte[] fBytes = (byte[])param;
float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat();
inst.push(value);
return null;
}
});
}
else if (op == StackOperation.ENCODE_DOUBLE) {
return inst.popParam().thenApply(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
byte[] dBytes = (byte[])param;
double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble();
inst.push(value);
return null;
}
});
}
else if (op == StackOperation.DECODE_FLOAT) {
return inst.popParam().thenApply(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
float value = ((Number)param).floatValue();
inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array());
return null;
}
});
}
else if (op == StackOperation.DECODE_DOUBLE) {
return inst.popParam().thenApply(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
double value = ((Number)param).doubleValue();
inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array());
return null;
}
});
}
else if(op == StackOperation.UNIT_TESTS) {
inst.context.db.options().setLocationCacheSize(100001);
return inst.context.db.runAsync(tr -> {

5
bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackOperation.java
View File

@ -62,6 +62,11 @@ enum StackOperation {
TUPLE_PACK,
TUPLE_UNPACK,
TUPLE_RANGE,
TUPLE_SORT,
ENCODE_FLOAT,
ENCODE_DOUBLE,
DECODE_FLOAT,
DECODE_DOUBLE,
UNIT_TESTS, /* Possibly unimplemented */
LOG_STACK

42
bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackTester.java
View File

@ -20,8 +20,11 @@
package com.apple.cie.foundationdb.test;
import java.math.BigInteger;
import java.util.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.function.Function;
@ -312,8 +315,9 @@ public class StackTester {
}
else if(op == StackOperation.SUB) {
List<Object> params = inst.popParams(2).join();
long result = StackUtils.getNumber(params.get(0)).longValue() - StackUtils.getNumber(params.get(1)).longValue();
inst.push(result);
BigInteger a = StackUtils.getBigInteger(params.get(0));
BigInteger b = StackUtils.getBigInteger(params.get(1));
inst.push(a.subtract(b));
}
else if(op == StackOperation.CONCAT) {
List<Object> params = inst.popParams(2).join();
@ -352,6 +356,40 @@ public class StackTester {
inst.push(range.begin);
inst.push(range.end);
}
else if (op == StackOperation.TUPLE_SORT) {
int listSize = StackUtils.getInt(inst.popParam().join());
List<Object> rawElements = inst.popParams(listSize).join();
List<Tuple> tuples = new ArrayList<Tuple>(listSize);
for(Object o : rawElements) {
tuples.add(Tuple.fromBytes((byte[])o));
}
Collections.sort(tuples);
for(Tuple t : tuples) {
inst.push(t.pack());
}
}
else if (op == StackOperation.ENCODE_FLOAT) {
Object param = inst.popParam().join();
byte[] fBytes = (byte[])param;
float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat();
inst.push(value);
}
else if (op == StackOperation.ENCODE_DOUBLE) {
Object param = inst.popParam().join();
byte[] dBytes = (byte[])param;
double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble();
inst.push(value);
}
else if (op == StackOperation.DECODE_FLOAT) {
Object param = inst.popParam().join();
float value = ((Number)param).floatValue();
inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array());
}
else if (op == StackOperation.DECODE_DOUBLE) {
Object param = inst.popParam().join();
double value = ((Number)param).doubleValue();
inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array());
}
else if(op == StackOperation.UNIT_TESTS) {
try {
inst.context.db.options().setLocationCacheSize(100001);

9
bindings/java/src-completable/test/com/apple/cie/foundationdb/test/StackUtils.java
View File

@ -20,6 +20,7 @@
package com.apple.cie.foundationdb.test;
import java.math.BigInteger;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
@ -92,6 +93,14 @@ public class StackUtils {
return ((Number)object);
}
static BigInteger getBigInteger(Object object) {
if (object instanceof BigInteger) {
return (BigInteger)object;
} else {
return BigInteger.valueOf(((Number)object).longValue());
}
}
static boolean getBoolean(Object o) {
return getBoolean(o, null);
}

107
bindings/java/src/main/com/apple/cie/foundationdb/tuple/IterableComparator.java Normal file
View File

@ -0,0 +1,107 @@
/*
* IterableComparator.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.apple.cie.foundationdb.tuple;
import java.util.Comparator;
import java.util.Iterator;
/**
* A {@link Tuple}-compatible {@link Comparator} that will sort {@link Iterable}s
* in a manner that is consistent with the byte-representation of {@link Tuple}s.
* In particular, if one has two {@link Tuple}s, {@code tuple1} and {@code tuple2},
* it is the case that:
*
* <pre>
* {@code
* tuple1.compareTo(tuple2)
* == new IterableComparator().compare(tuple1, tuple2)
* == new IterableComparator().compare(tuple1.getItems(), tuple2.getItems()),
* == ByteArrayUtil.compareUnsigned(tuple1.pack(), tuple2.pack())}
* </pre>
*
* <p>
* The individual elements of the {@link Iterable} must be of a type that can
* be serialized by a {@link Tuple}. For items of identical types, they will be
* sorted in a way that is consistent with their natural ordering with a few
* caveats:
* </p>
*
* <ul>
* <li>
* For floating point types, negative NaN values are sorted before all regular values, and
* positive NaN values are sorted after all regular values.
* </li>
* <li>
* Single-precision floating point numbers are sorted before
* all double-precision floating point numbers.
* </li>
* <li>
* UUIDs are sorted by their <i>unsigned</i> Big-Endian byte representation
* rather than their signed byte representation (which is the behavior of
* {@link java.util.UUID#compareTo(java.util.UUID) UUID.compareTo()}).
* </li>
* <li>Strings are sorted explicitly by their UTF-8 byte representation</li>
* <li>Nested {@link Tuple}s and {@link java.util.List List}s are sorted element-wise.</li>
* </ul>
*/
public class IterableComparator implements Comparator<Iterable<?>> {
/**
* Creates a new {@code IterableComparator}. This {@link Comparator} has
* no internal state.
*/
public IterableComparator() {}
/**
* Compare two {@link Iterable}s in a way consistent with their
* byte representation. This is done element-wise and is consistent
* with a number of other ways of sorting {@link Tuple}s. This will
* raise an {@link IllegalArgumentException} if any of the items
* of either {@link Iterable} cannot be serialized by a {@link Tuple}.
*
* @param iterable1 the first {@link Iterable} of items
* @param iterable2 the second {@link Iterable} of items
* @return a negative number if the first iterable would sort before the second
* when serialized, a positive number if the opposite is true, and zero
* if the two are equal
*/
@Override
public int compare(Iterable<?> iterable1, Iterable<?> iterable2) {
Iterator<?> i1 = iterable1.iterator();
Iterator<?> i2 = iterable2.iterator();
while(i1.hasNext() && i2.hasNext()) {
int itemComp = TupleUtil.compareItems(i1.next(), i2.next());
if(itemComp != 0) {
return itemComp;
}
}
if(i1.hasNext()) {
// iterable2 is a prefix of iterable1.
return 1;
}
if(i2.hasNext()) {
// iterable1 is a prefix of iterable2.
return -1;
}
return 0;
}
}

309
bindings/java/src/main/com/apple/cie/foundationdb/tuple/Tuple.java
View File

@ -20,12 +20,15 @@
package com.apple.cie.foundationdb.tuple;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import com.apple.cie.foundationdb.Range;
@ -37,26 +40,34 @@ import com.apple.cie.foundationdb.Range;
* ideal for building a variety of higher-level data models.<br>
* <h3>Types</h3>
* A {@code Tuple} can
* contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, and {@code null}. All
* {@code Number}s will be converted to a {@code long} integral value, so all
* floating point information will be lost and their range will be constrained to the range
* [{@code 2^63-1}, {@code -2^63}]. Note that for numbers outside this range the way that Java
* contain byte arrays ({@code byte[]}), {@link String}s, {@link Number}s, {@link UUID}s,
* {@code boolean}s, {@link List}s, other {@code Tuple}s, and {@code null}.
* {@link Float} and {@link Double} instances will be serialized as single- and double-precision
* numbers respectively, and {@link BigInteger}s within the range [{@code -2^2040+1},
* {@code 2^2040-1}] are serialized without loss of precision (those outside the range
* will raise an {@link IllegalArgumentException}). All other {@code Number}s will be converted to
* a {@code long} integral value, so the range will be constrained to
* [{@code -2^63}, {@code 2^63-1}]. Note that for numbers outside this range the way that Java
* truncates integral values may yield unexpected results.<br>
* <h3>{@code null} values</h3>
* The FoundationDB tuple specification has a special type-code for {@code None}; {@code nil}; or,
* as Java would understand it, {@code null}.
* The behavior of the layer in the presence of {@code null} varies by type with the intention
* of matching expected behavior in Java. {@code byte[]} and {@link String}s can be {@code null},
* where integral numbers (i.e. {@code long}s) cannot.
* This means that the typed getters ({@link #getBytes(int) getBytes()} and {@link #getString(int) getString()})
* of matching expected behavior in Java. {@code byte[]}, {@link String}s, {@link UUID}s, and
* nested {@link List}s and {@code Tuple}s can be {@code null},
* whereas numbers (e.g., {@code long}s and {@code double}s) and booleans cannot.
* This means that the typed getters ({@link #getBytes(int) getBytes()}, {@link #getString(int) getString()}),
* {@link #getUUID(int) getUUID()}, {@link #getNestedTuple(int) getNestedTuple()}, and {@link #getNestedList(int) getNestedList()})
* will return {@code null} if the entry at that location was {@code null} and the typed adds
* ({@link #add(byte[])} and {@link #add(String)}) will accept {@code null}. The
* {@link #getLong(int) typed get for integers}, however, will throw a {@code NullPointerException} if
* {@link #getLong(int) typed get for integers} and other typed getters, however, will throw a {@link NullPointerException} if
* the entry in the {@code Tuple} was {@code null} at that position.<br>
* <br>
* This class is not thread safe.
*/
public class Tuple implements Comparable<Tuple>, Iterable<Object> {
private static IterableComparator comparator = new IterableComparator();
private List<Object> elements;
private Tuple(List<? extends Object> elements, Object newItem) {
@ -70,12 +81,12 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a copy of this {@code Tuple} with an appended last element. The parameter
* is untyped but only {@link String}, {@code byte[]}, {@link Number}s, and {@code null} are allowed.
* All {@code Number}s are converted to a 8 byte integral value, so all floating point
* information is lost.
* is untyped but only {@link String}, {@code byte[]}, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, and {@code null} are allowed. If an object of
* another type is passed, then an {@link IllegalArgumentException} is thrown.
*
* @param o the object to append. Must be {@link String}, {@code byte[]},
* {@link Number}s, or {@code null}.
* {@link Number}s, {@link UUID}, {@link List}, {@link Boolean}, or {@code null}.
*
* @return a newly created {@code Tuple}
*/
@ -83,6 +94,10 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
if(o != null &&
!(o instanceof String) &&
!(o instanceof byte[]) &&
!(o instanceof UUID) &&
!(o instanceof List<?>) &&
!(o instanceof Tuple) &&
!(o instanceof Boolean) &&
!(o instanceof Number)) {
throw new IllegalArgumentException("Parameter type (" + o.getClass().getName() + ") not recognized");
}
@ -122,6 +137,92 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return new Tuple(this.elements, b);
}
/**
* Creates a copy of this {@code Tuple} with a {@code boolean} appended as the last element.
*
* @param b the {@code boolean} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(boolean b) {
return new Tuple(this.elements, b);
}
/**
* Creates a copy of this {@code Tuple} with a {@link UUID} appended as the last element.
*
* @param uuid the {@link UUID} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(UUID uuid) {
return new Tuple(this.elements, uuid);
}
/**
* Creates a copy of this {@code Tuple} with a {@link BigInteger} appended as the last element.
* As {@link Tuple}s cannot contain {@code null} numeric types, a {@link NullPointerException}
* is raised if a {@code null} argument is passed.
*
* @param bi the {@link BigInteger} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(BigInteger bi) {
if(bi == null) {
throw new NullPointerException("Number types in Tuple cannot be null");
}
return new Tuple(this.elements, bi);
}
/**
* Creates a copy of this {@code Tuple} with a {@code float} appended as the last element.
*
* @param f the {@code float} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(float f) {
return new Tuple(this.elements, f);
}
/**
* Creates a copy of this {@code Tuple} with a {@code double} appended as the last element.
*
* @param d the {@code double} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(double d) {
return new Tuple(this.elements, d);
}
/**
* Creates a copy of this {@code Tuple} with an {@link List} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(List) Tuple.addAll}).
* This adds the list as a single elemented nested within the outer {@code Tuple}.
*
* @param l the {@link List} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(List<? extends Object> l) {
return new Tuple(this.elements, l);
}
/**
* Creates a copy of this {@code Tuple} with a {@code Tuple} appended as the last element.
* This does not add the elements individually (for that, use {@link Tuple#addAll(Tuple) Tuple.addAll}).
* This adds the list as a single elemented nested within the outer {@code Tuple}.
*
* @param t the {@code Tuple} to append
*
* @return a newly created {@code Tuple}
*/
public Tuple add(Tuple t) {
return new Tuple(this.elements, t);
}
/**
* Creates a copy of this {@code Tuple} with a {@code byte} array appended as the last element.
*
@ -309,6 +410,138 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
return (String)o;
}
/**
* Gets an indexed item as a {@link BigInteger}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the tuple element is not of
* a {@code Number} type. If the underlying type is a floating point value, this
* will lead to a loss of precision. The element at the index may not be {@code null}.
*
* @param index the location of the element to return
*
* @return the item at {@code index} as a {@link BigInteger}
*/
public BigInteger getBigInteger(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Number types in Tuples may not be null");
if(o instanceof BigInteger) {
return (BigInteger)o;
} else {
return BigInteger.valueOf(((Number)o).longValue());
}
}
/**
* Gets an indexed item as a {@code float}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a number type.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@code float}
*/
public float getFloat(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Number types in Tuples may not be null");
return ((Number)o).floatValue();
}
/**
* Gets an indexed item as a {@code double}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a number type.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@code double}
*/
public double getDouble(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Number types in Tuples may not be null");
return ((Number)o).doubleValue();
}
/**
* Gets an indexed item as a {@code boolean}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@code Boolean}.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@code boolean}
*/
public boolean getBoolean(int index) {
Object o = this.elements.get(index);
if(o == null)
throw new NullPointerException("Boolean type in Tuples may not be null");
return (Boolean)o;
}
/**
* Gets an indexed item as a {@link UUID}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@code UUID}.
* The element at the index may not be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@link UUID}
*/
public UUID getUUID(int index) {
Object o = this.elements.get(index);
if(o == null)
return null;
return (UUID)o;
}
/**
* Gets an indexed item as a {@link List}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@link List}
* or {@code Tuple}. The element at the index may be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@link List}
*/
public List<Object> getNestedList(int index) {
Object o = this.elements.get(index);
if(o == null) {
return null;
} else if(o instanceof Tuple) {
return ((Tuple)o).getItems();
} else if(o instanceof List<?>) {
List<Object> ret = new LinkedList<Object>();
ret.addAll((List<? extends Object>)o);
return ret;
} else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to list");
}
}
/**
* Gets an indexed item as a {@link Tuple}. This function will not do type conversion
* and so will throw a {@code ClassCastException} if the element is not a {@link List}
* or {@code Tuple}. The element at the index may be {@code null}.
*
* @param index the location of the item to return
*
* @return the item at {@code index} as a {@link List}
*/
public Tuple getNestedTuple(int index) {
Object o = this.elements.get(index);
if(o == null) {
return null;
} else if(o instanceof Tuple) {
return (Tuple)o;
} else if(o instanceof List<?>) {
return Tuple.fromItems((List<? extends Object>)o);
} else {
throw new ClassCastException("Cannot convert item of type " + o.getClass() + " to tuple");
}
}
/**
* Gets an indexed item without forcing a type.
*
@ -389,7 +622,7 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
*/
@Override
public int compareTo(Tuple t) {
return ByteArrayUtil.compareUnsigned(this.pack(), t.pack());
return comparator.compare(elements, t.elements);
}
/**
@ -462,7 +695,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a new {@code Tuple} from a variable number of elements. The elements
* must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s.
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s.
*
* @param items the elements from which to create the {@code Tuple}.
*
@ -479,7 +713,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Efficiently creates a new {@code Tuple} from a list of objects. The elements
* must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s.
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s.
*
* @param items the elements from which to create the {@code Tuple}.
*
@ -492,7 +727,8 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
/**
* Creates a new {@code Tuple} from a variable number of elements. The elements
* must follow the type guidelines from {@link Tuple#addObject(Object) add}, and so
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, or {@code null}s.
* can only be {@link String}s, {@code byte[]}s, {@link Number}s, {@link UUID}s,
* {@link Boolean}s, {@link List}s, {@code Tuple}s, or {@code null}s.
*
* @param items the elements from which to create the {@code Tuple}.
*
@ -518,12 +754,51 @@ public class Tuple implements Comparable<Tuple>, Iterable<Object> {
t = t.add(Long.MIN_VALUE + 1);
t = t.add(Long.MIN_VALUE);
t = t.add("foo");
t = t.addObject(null);
t = t.add(false);
t = t.add(true);
t = t.add(3.14159);
t = t.add(3.14159f);
t = t.add(java.util.UUID.randomUUID());
t = t.add(t.getItems());
t = t.add(t);
t = t.add(new BigInteger("100000000000000000000000000000000000000000000"));
t = t.add(new BigInteger("-100000000000000000000000000000000000000000000"));
byte[] bytes = t.pack();
System.out.println("Packed: " + ByteArrayUtil.printable(bytes));
List<Object> items = Tuple.fromBytes(bytes).getItems();
for(Object obj : items) {
System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj);
if (obj != null)
System.out.println(" -> type: (" + obj.getClass().getName() + "): " + obj);
else
System.out.println(" -> type: (null): null");
}
Tuple t2 = Tuple.fromBytes(bytes);
System.out.println("t2.getLong(0): " + t2.getLong(0));
System.out.println("t2.getBigInteger(1): " + t2.getBigInteger(1));
System.out.println("t2.getString(9): " + t2.getString(9));
System.out.println("t2.get(10): " + t2.get(10));
System.out.println("t2.getBoolean(11): " + t2.getBoolean(11));
System.out.println("t2.getBoolean(12): " + t2.getBoolean(12));
System.out.println("t2.getDouble(13): " + t2.getDouble(13));
System.out.println("t2.getFloat(13): " + t2.getFloat(13));
System.out.println("t2.getLong(13): " + t2.getLong(13));
System.out.println("t2.getBigInteger(13): " + t2.getBigInteger(13));
System.out.println("t2.getDouble(14): " + t2.getDouble(14));
System.out.println("t2.getFloat(14): " + t2.getFloat(14));
System.out.println("t2.getLong(14): " + t2.getLong(14));
System.out.println("t2.getBigInteger(14): " + t2.getBigInteger(14));
System.out.println("t2.getNestedList(17): " + t2.getNestedList(17));
System.out.println("t2.getNestedTuple(17): " + t2.getNestedTuple(17));
System.out.println("(2*(Long.MAX_VALUE+1),) = " + ByteArrayUtil.printable(Tuple.from(
BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE).shiftLeft(1)
).pack()));
System.out.println("(2*Long.MIN_VALUE,) = " + ByteArrayUtil.printable(Tuple.from(
BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2"))
).pack()));
System.out.println("2*Long.MIN_VALUE = " + Tuple.fromBytes(Tuple.from(
BigInteger.valueOf(Long.MIN_VALUE).multiply(new BigInteger("2"))).pack()).getBigInteger(0));
}
private static Tuple createTuple(int items) {

355
bindings/java/src/main/com/apple/cie/foundationdb/tuple/TupleUtil.java
View File

@ -28,12 +28,26 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
class TupleUtil {
private static final byte nil = 0x0;
private static final byte nil = 0x00;
private static final byte[] nil_rep = new byte[] {nil, (byte)0xFF};
private static final BigInteger[] size_limits;
private static final Charset UTF8;
private static final IterableComparator iterableComparator;
private static final byte BYTES_CODE = 0x01;
private static final byte STRING_CODE = 0x02;
private static final byte NESTED_CODE = 0x05;
private static final byte INT_ZERO_CODE = 0x14;
private static final byte POS_INT_END = 0x1d;
private static final byte NEG_INT_START = 0x0b;
private static final byte FLOAT_CODE = 0x20;
private static final byte DOUBLE_CODE = 0x21;
private static final byte FALSE_CODE = 0x26;
private static final byte TRUE_CODE = 0x27;
private static final byte UUID_CODE = 0x30;
static {
size_limits = new BigInteger[9];
@ -41,6 +55,7 @@ class TupleUtil {
size_limits[i] = (BigInteger.ONE).shiftLeft(i * 8).subtract(BigInteger.ONE);
}
UTF8 = Charset.forName("UTF-8");
iterableComparator = new IterableComparator();
}
static class DecodeResult {
@ -53,27 +68,110 @@ class TupleUtil {
}
}
static int byteLength(byte[] bytes) {
for(int i = 0; i < bytes.length; i++) {
if(bytes[i] == 0x00) continue;
return bytes.length - i;
}
return 0;
}
/**
* Takes the Big-Endian byte representation of a floating point number and adjusts
* it so that it sorts correctly. For encoding, if the sign bit is 1 (the number
* is negative), then we need to flip all of the bits; otherwise, just flip the
* sign bit. For decoding, if the sign bit is 0 (the number is negative), then
* we also need to flip all of the bits; otherwise, just flip the sign bit.
* This will mutate in place the given array.
*
* @param bytes Big-Endian IEEE encoding of a floating point number
* @param start the (zero-indexed) first byte in the array to mutate
* @param encode <code>true</code> if we encoding the float and <code>false</code> if we are decoding
* @return the encoded {@code byte[]}
*/
static byte[] floatingPointCoding(byte[] bytes, int start, boolean encode) {
if(encode && (bytes[start] & (byte)0x80) != (byte)0x00) {
for(int i = start; i < bytes.length; i++) {
bytes[i] = (byte) (bytes[i] ^ 0xff);
}
} else if(!encode && (bytes[start] & (byte)0x80) != (byte)0x80) {
for(int i = start; i < bytes.length; i++) {
bytes[i] = (byte) (bytes[i] ^ 0xff);
}
} else {
bytes[start] = (byte) (0x80 ^ bytes[start]);
}
return bytes;
}
public static byte[] join(List<byte[]> items) {
return ByteArrayUtil.join(null, items);
}
static byte[] encode(Object t) {
static int getCodeFor(Object o) {
if(o == null)
return nil;
if(o instanceof byte[])
return BYTES_CODE;
if(o instanceof String)
return STRING_CODE;
if(o instanceof Float)
return FLOAT_CODE;
if(o instanceof Double)
return DOUBLE_CODE;
if(o instanceof Boolean)
return FALSE_CODE;
if(o instanceof UUID)
return UUID_CODE;
if(o instanceof Number)
return INT_ZERO_CODE;
if(o instanceof List<?>)
return NESTED_CODE;
if(o instanceof Tuple)
return NESTED_CODE;
throw new IllegalArgumentException("Unsupported data type: " + o.getClass().getName());
}
static byte[] encode(Object t, boolean nested) {
if(t == null)
return new byte[] {nil};
if (nested)
return new byte[]{nil, (byte) 0xff};
else
return new byte[]{nil};
if(t instanceof byte[])
return encode((byte[])t);
if(t instanceof String)
return encode((String)t);
if(t instanceof BigInteger)
return encode((BigInteger)t);
if(t instanceof Float)
return encode((Float)t);
if(t instanceof Double)
return encode((Double)t);
if(t instanceof Boolean)
return encode((Boolean)t);
if(t instanceof UUID)
return encode((UUID)t);
if(t instanceof Number)
return encode(((Number)t).longValue());
if(t instanceof List<?>)
return encode((List<?>)t);
if(t instanceof Tuple)
return encode(((Tuple)t).getItems());
throw new IllegalArgumentException("Unsupported data type: " + t.getClass().getName());
}
static byte[] encode(Object t) {
return encode(t, false);
}
static byte[] encode(byte[] bytes) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {0x1});
list.add(ByteArrayUtil.replace(bytes, new byte[] {0x0}, nil_rep));
list.add(new byte[] {0x0});
list.add(new byte[] {BYTES_CODE});
list.add(ByteArrayUtil.replace(bytes, new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
//System.out.println("Joining bytes...");
return ByteArrayUtil.join(null, list);
@ -81,39 +179,68 @@ class TupleUtil {
static byte[] encode(String s) {
List<byte[]> list = new ArrayList<byte[]>(3);
list.add(new byte[] {0x2});
list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {0x0}, nil_rep));
list.add(new byte[] {0x0});
list.add(new byte[] {STRING_CODE});
list.add(ByteArrayUtil.replace(s.getBytes(UTF8), new byte[] {nil}, nil_rep));
list.add(new byte[] {nil});
//System.out.println("Joining string...");
return ByteArrayUtil.join(null, list);
}
static byte[] encode(long i) {
static byte[] encode(BigInteger i) {
//System.out.println("Encoding integral " + i);
if(i == 0) {
return new byte[] { 20 };
if(i.equals(BigInteger.ZERO)) {
return new byte[] { INT_ZERO_CODE };
}
if(i > 0) {
int n = ByteArrayUtil.bisectLeft(size_limits, BigInteger.valueOf(i));
byte[] bytes = i.toByteArray();
if(i.compareTo(BigInteger.ZERO) > 0) {
if(i.compareTo(size_limits[size_limits.length-1]) > 0) {
int length = byteLength(bytes);
if(length > 0xff) {
throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)");
}
byte[] result = new byte[length + 2];
result[0] = POS_INT_END;
result[1] = (byte)(length);
System.arraycopy(bytes, bytes.length - length, result, 2, length);
return result;
}
int n = ByteArrayUtil.bisectLeft(size_limits, i);
assert n <= size_limits.length;
byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array();
//byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(i).array();
//System.out.println(" -- integral has 'n' of " + n + " and output bytes of " + bytes.length);
byte[] result = new byte[n+1];
result[0] = (byte)(20 + n);
result[0] = (byte)(INT_ZERO_CODE + n);
System.arraycopy(bytes, bytes.length - n, result, 1, n);
return result;
}
BigInteger bI = BigInteger.valueOf(i);
int n = ByteArrayUtil.bisectLeft(size_limits, bI.negate());
if(i.negate().compareTo(size_limits[size_limits.length-1]) > 0) {
int length = byteLength(i.negate().toByteArray());
if(length > 0xff) {
throw new IllegalArgumentException("BigInteger magnitude is too large (more than 255 bytes)");
}
BigInteger offset = BigInteger.ONE.shiftLeft(length*8).subtract(BigInteger.ONE);
byte[] adjusted = i.add(offset).toByteArray();
byte[] result = new byte[length + 2];
result[0] = NEG_INT_START;
result[1] = (byte)(length ^ 0xff);
if(adjusted.length >= length) {
System.arraycopy(adjusted, adjusted.length - length, result, 2, length);
} else {
Arrays.fill(result, 2, result.length - adjusted.length, (byte)0x00);
System.arraycopy(adjusted, 0, result, result.length - adjusted.length, adjusted.length);
}
return result;
}
int n = ByteArrayUtil.bisectLeft(size_limits, i.negate());
assert n >= 0 && n < size_limits.length; // can we do this? it seems to be required for the following statement
long maxv = size_limits[n].add(bI).longValue();
byte[] bytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array();
long maxv = size_limits[n].add(i).longValue();
byte[] adjustedBytes = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(maxv).array();
byte[] result = new byte[n+1];
result[0] = (byte)(20 - n);
System.arraycopy(bytes, bytes.length - n, result, 1, n);
System.arraycopy(adjustedBytes, adjustedBytes.length - n, result, 1, n);
return result;
}
@ -121,23 +248,63 @@ class TupleUtil {
return encode(i.longValue());
}
static byte[] encode(long i) {
return encode(BigInteger.valueOf(i));
}
static byte[] encode(Float f) {
byte[] result = ByteBuffer.allocate(5).order(ByteOrder.BIG_ENDIAN).put(FLOAT_CODE).putFloat(f).array();
floatingPointCoding(result, 1, true);
return result;
}
static byte[] encode(Double d) {
byte[] result = ByteBuffer.allocate(9).order(ByteOrder.BIG_ENDIAN).put(DOUBLE_CODE).putDouble(d).array();
floatingPointCoding(result, 1, true);
return result;
}
static byte[] encode(Boolean b) {
if (b) {
return new byte[] {TRUE_CODE};
} else {
return new byte[] {FALSE_CODE};
}
}
static byte[] encode(UUID uuid) {
return ByteBuffer.allocate(17).put(UUID_CODE).order(ByteOrder.BIG_ENDIAN)
.putLong(uuid.getMostSignificantBits()).putLong(uuid.getLeastSignificantBits())
.array();
}
static byte[] encode(List<?> value) {
List<byte[]> parts = new LinkedList<byte[]>();
parts.add(new byte[]{NESTED_CODE});
for(Object t : value) {
parts.add(encode(t, true));
}
parts.add(new byte[]{0x00});
return ByteArrayUtil.join(null, parts);
}
static DecodeResult decode(byte[] rep, int pos, int last) {
//System.out.println("Decoding '" + ArrayUtils.printable(rep) + "' at " + pos);
// SOMEDAY: codes over 127 will be a problem with the signed Java byte mess
int code = rep[pos];
int start = pos + 1;
if(code == 0x0) {
if(code == nil) {
return new DecodeResult(start, null);
}
if(code == 0x1) {
if(code == BYTES_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of byte string: " + end);
byte[] range = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
//System.out.println(" -> byte string contents: '" + ArrayUtils.printable(range) + "'");
return new DecodeResult(end + 1, range);
}
if(code == 0x2) {
if(code == STRING_CODE) {
int end = ByteArrayUtil.findTerminator(rep, (byte)0x0, (byte)0xff, start, last);
//System.out.println("End of UTF8 string: " + end);
byte[] stringBytes = ByteArrayUtil.replace(rep, start, end - start, nil_rep, new byte[] { nil });
@ -145,11 +312,44 @@ class TupleUtil {
//System.out.println(" -> UTF8 string contents: '" + str + "'");
return new DecodeResult(end + 1, str);
}
if(code >=12 && code <=28) {
if(code == FLOAT_CODE) {
byte[] resBytes = Arrays.copyOfRange(rep, start, start+4);
floatingPointCoding(resBytes, 0, false);
float res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getFloat();
return new DecodeResult(start + 4, res);
}
if(code == DOUBLE_CODE) {
byte[] resBytes = Arrays.copyOfRange(rep, start, start+8);
floatingPointCoding(resBytes, 0, false);
double res = ByteBuffer.wrap(resBytes).order(ByteOrder.BIG_ENDIAN).getDouble();
return new DecodeResult(start + 8, res);
}
if(code == FALSE_CODE) {
return new DecodeResult(start, false);
}
if(code == TRUE_CODE) {
return new DecodeResult(start, true);
}
if(code == UUID_CODE) {
ByteBuffer bb = ByteBuffer.wrap(rep, start, 16).order(ByteOrder.BIG_ENDIAN);
long msb = bb.getLong();
long lsb = bb.getLong();
return new DecodeResult(start + 16, new UUID(msb, lsb));
}
if(code == POS_INT_END) {
int n = rep[start] & 0xff;
return new DecodeResult(start + n + 1, new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1))));
}
if(code == NEG_INT_START) {
int n = (rep[start] ^ 0xff) & 0xff;
BigInteger origValue = new BigInteger(ByteArrayUtil.join(new byte[]{0x00}, Arrays.copyOfRange(rep, start+1, start+n+1)));
BigInteger offset = BigInteger.ONE.shiftLeft(n*8).subtract(BigInteger.ONE);
return new DecodeResult(start + n + 1, origValue.subtract(offset));
}
if(code > NEG_INT_START && code < POS_INT_END) {
// decode a long
byte[] longBytes = new byte[9];
Arrays.fill(longBytes, (byte)0);
boolean upper = code >= 20;
boolean upper = code >= INT_ZERO_CODE;
int n = upper ? code - 20 : 20 - code;
int end = start + n;
@ -157,23 +357,108 @@ class TupleUtil {
throw new RuntimeException("Invalid tuple (possible truncation)");
}
System.arraycopy(rep, start, longBytes, 9-n, n);
System.arraycopy(rep, start, longBytes, longBytes.length-n, n);
if (!upper)
for(int i=9-n; i<9; i++)
longBytes[i] = (byte)~longBytes[i];
for(int i=longBytes.length-n; i<longBytes.length; i++)
longBytes[i] = (byte)(longBytes[i] ^ 0xff);
BigInteger val = new BigInteger(longBytes);
if (!upper) val = val.negate();
if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0 ||
val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0)
throw new RuntimeException("Value out of range for type long.");
// Convert to long if in range -- otherwise, leave as BigInteger.
if (val.compareTo(BigInteger.valueOf(Long.MIN_VALUE))<0||
val.compareTo(BigInteger.valueOf(Long.MAX_VALUE))>0) {
// This can occur if the thing can be represented with 8 bytes but not
// the right sign information.
return new DecodeResult(end, val);
}
return new DecodeResult(end, val.longValue());
}
if(code == NESTED_CODE) {
List<Object> items = new LinkedList<Object>();
int endPos = start;
while(endPos < rep.length) {
if(rep[endPos] == nil) {
if(endPos + 1 < rep.length && rep[endPos+1] == (byte)0xff) {
items.add(null);
endPos += 2;
} else {
endPos += 1;
break;
}
} else {
DecodeResult subResult = decode(rep, endPos, last);
items.add(subResult.o);
endPos = subResult.end;
}
}
return new DecodeResult(endPos, items);
}
throw new IllegalArgumentException("Unknown tuple data type " + code + " at index " + pos);
}
static int compareItems(Object item1, Object item2) {
int code1 = TupleUtil.getCodeFor(item1);
int code2 = TupleUtil.getCodeFor(item2);
if(code1 != code2) {
return Integer.compare(code1, code2);
}
if(code1 == nil) {
// All null's are equal. (Some may be more equal than others.)
return 0;
}
if(code1 == BYTES_CODE) {
return ByteArrayUtil.compareUnsigned((byte[])item1, (byte[])item2);
}
if(code1 == STRING_CODE) {
return ByteArrayUtil.compareUnsigned(((String)item1).getBytes(UTF8), ((String)item2).getBytes(UTF8));
}
if(code1 == INT_ZERO_CODE) {
BigInteger bi1;
if(item1 instanceof BigInteger) {
bi1 = (BigInteger)item1;
} else {
bi1 = BigInteger.valueOf(((Number)item1).longValue());
}
BigInteger bi2;
if(item2 instanceof BigInteger) {
bi2 = (BigInteger)item2;
} else {
bi2 = BigInteger.valueOf(((Number)item2).longValue());
}
return bi1.compareTo(bi2);
}
if(code1 == DOUBLE_CODE) {
// This is done over vanilla double comparison basically to handle NaN
// sorting correctly.
byte[] encoded1 = encode((Double)item1);
byte[] encoded2 = encode((Double)item2);
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == FLOAT_CODE) {
// This is done for the same reason that double comparison is done
// that way.
byte[] encoded1 = encode((Float)item1);
byte[] encoded2 = encode((Float)item2);
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == FALSE_CODE) {
return Boolean.compare((Boolean)item1, (Boolean)item2);
}
if(code1 == UUID_CODE) {
// Java UUID.compareTo is signed.
byte[] encoded1 = encode((UUID)item1);
byte[] encoded2 = encode((UUID)item2);
return ByteArrayUtil.compareUnsigned(encoded1, encoded2);
}
if(code1 == NESTED_CODE) {
return iterableComparator.compare((Iterable<?>)item1, (Iterable<?>)item2);
}
throw new IllegalArgumentException("Unknown tuple data type: " + item1.getClass());
}
static List<Object> unpack(byte[] bytes, int start, int length) {
List<Object> items = new LinkedList<Object>();
int pos = start;
@ -188,7 +473,7 @@ class TupleUtil {
static byte[] pack(List<Object> items) {
if(items.size() == 0)
return new byte[0];
return new byte[0];
List<byte[]> parts = new ArrayList<byte[]>(items.size());
for(Object t : items) {

72
bindings/java/src/test/com/apple/cie/foundationdb/test/AsyncStackTester.java
View File

@ -20,6 +20,9 @@
package com.apple.cie.foundationdb.test;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.*;
import com.apple.cie.foundationdb.Cluster;
@ -471,8 +474,9 @@ public class AsyncStackTester {
return inst.popParams(2).flatMap(new Function<List<Object>, Future<Void>>() {
@Override
public Future<Void> apply(List<Object> params) {
long result = StackUtils.getNumber(params.get(0)).longValue() -
StackUtils.getNumber(params.get(1)).longValue();
BigInteger result = StackUtils.getBigInteger(params.get(0)).subtract(
StackUtils.getBigInteger(params.get(1))
);
inst.push(result);
return ReadyFuture.DONE;
}
@ -526,6 +530,28 @@ public class AsyncStackTester {
}
});
}
else if(op == StackOperation.TUPLE_SORT) {
return inst.popParam().flatMap(new Function<Object, Future<Void>>() {
@Override
public Future<Void> apply(Object param) {
final int listSize = StackUtils.getInt(param);
return inst.popParams(listSize).flatMap(new Function<List<Object>, Future<Void>>() {
@Override
public Future<Void> apply(List<Object> rawElements) {
List<Tuple> tuples = new ArrayList(listSize);
for(Object o : rawElements) {
tuples.add(Tuple.fromBytes((byte[])o));
}
Collections.sort(tuples);
for(Tuple t : tuples) {
inst.push(t.pack());
}
return ReadyFuture.DONE;
}
});
}
});
}
else if(op == StackOperation.TUPLE_RANGE) {
return inst.popParam().flatMap(new Function<Object, Future<Void>>() {
@Override
@ -544,6 +570,48 @@ public class AsyncStackTester {
}
});
}
else if (op == StackOperation.ENCODE_FLOAT) {
return inst.popParam().map(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
byte[] fBytes = (byte[])param;
float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat();
inst.push(value);
return null;
}
});
}
else if (op == StackOperation.ENCODE_DOUBLE) {
return inst.popParam().map(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
byte[] dBytes = (byte[])param;
double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble();
inst.push(value);
return null;
}
});
}
else if (op == StackOperation.DECODE_FLOAT) {
return inst.popParam().map(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
float value = ((Number)param).floatValue();
inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array());
return null;
}
});
}
else if (op == StackOperation.DECODE_DOUBLE) {
return inst.popParam().map(new Function<Object, Void>() {
@Override
public Void apply(Object param) {
double value = ((Number)param).doubleValue();
inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array());
return null;
}
});
}
else if(op == StackOperation.UNIT_TESTS) {
inst.context.db.options().setLocationCacheSize(100001);

5
bindings/java/src/test/com/apple/cie/foundationdb/test/StackOperation.java
View File

@ -62,6 +62,11 @@ enum StackOperation {
TUPLE_PACK,
TUPLE_UNPACK,
TUPLE_RANGE,
TUPLE_SORT,
ENCODE_FLOAT,
ENCODE_DOUBLE,
DECODE_FLOAT,
DECODE_DOUBLE,
UNIT_TESTS, /* Possibly unimplemented */
LOG_STACK

44
bindings/java/src/test/com/apple/cie/foundationdb/test/StackTester.java
View File

@ -20,6 +20,9 @@
package com.apple.cie.foundationdb.test;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.*;
import com.apple.cie.foundationdb.Database;
@ -341,8 +344,9 @@ public class StackTester {
}
else if(op == StackOperation.SUB) {
List<Object> params = inst.popParams(2).get();
long result = StackUtils.getNumber(params.get(0)).longValue() - StackUtils.getNumber(params.get(1)).longValue();
inst.push(result);
BigInteger a = StackUtils.getBigInteger(params.get(0));
BigInteger b = StackUtils.getBigInteger(params.get(1));
inst.push(a.subtract(b));
}
else if(op == StackOperation.CONCAT) {
List<Object> params = inst.popParams(2).get();
@ -372,6 +376,18 @@ public class StackTester {
inst.push(itemBytes);
}
}
else if (op == StackOperation.TUPLE_SORT) {
int listSize = StackUtils.getInt(inst.popParam().get());
List<Object> rawElements = inst.popParams(listSize).get();
List<Tuple> tuples = new ArrayList<Tuple>(listSize);
for(Object o : rawElements) {
tuples.add(Tuple.fromBytes((byte[])o));
}
Collections.sort(tuples);
for(Tuple t : tuples) {
inst.push(t.pack());
}
}
else if(op == StackOperation.TUPLE_RANGE) {
List<Object> params = inst.popParams(1).get();
int tupleSize = StackUtils.getInt(params, 0);
@ -381,6 +397,28 @@ public class StackTester {
inst.push(range.begin);
inst.push(range.end);
}
else if (op == StackOperation.ENCODE_FLOAT) {
Object param = inst.popParam().get();
byte[] fBytes = (byte[])param;
float value = ByteBuffer.wrap(fBytes).order(ByteOrder.BIG_ENDIAN).getFloat();
inst.push(value);
}
else if (op == StackOperation.ENCODE_DOUBLE) {
Object param = inst.popParam().get();
byte[] dBytes = (byte[])param;
double value = ByteBuffer.wrap(dBytes).order(ByteOrder.BIG_ENDIAN).getDouble();
inst.push(value);
}
else if (op == StackOperation.DECODE_FLOAT) {
Object param = inst.popParam().get();
float value = ((Number)param).floatValue();
inst.push(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putFloat(value).array());
}
else if (op == StackOperation.DECODE_DOUBLE) {
Object param = inst.popParam().get();
double value = ((Number)param).doubleValue();
inst.push(ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putDouble(value).array());
}
else if(op == StackOperation.UNIT_TESTS) {
try {
inst.context.db.options().setLocationCacheSize(100001);
@ -566,7 +604,7 @@ public class StackTester {
private static void logStack(final Database db, final Map<Integer, StackEntry> entries, final byte[] prefix) {
db.run(new Function<Transaction, Void>() {
@Override
@Override
public Void apply(Transaction tr) {
for(Map.Entry<Integer, StackEntry> it : entries.entrySet()) {
byte[] pk = ByteArrayUtil.join(prefix, Tuple.from(it.getKey(), it.getValue().idx).pack());

9
bindings/java/src/test/com/apple/cie/foundationdb/test/StackUtils.java
View File

@ -20,6 +20,7 @@
package com.apple.cie.foundationdb.test;
import java.math.BigInteger;
import java.util.List;
import com.apple.cie.foundationdb.FDBException;
@ -92,6 +93,14 @@ public class StackUtils {
return ((Number)object);
}
static BigInteger getBigInteger(Object object) {
if (object instanceof BigInteger) {
return (BigInteger)object;
} else {
return BigInteger.valueOf(((Number)object).longValue());
}
}
static boolean getBoolean(Object o) {
return getBoolean(o, null);
}

4
bindings/nodejs/lib/fdb.js
View File

@ -41,8 +41,8 @@ module.exports = {
apiVersion: function(version) {
if(selectedApiVersion.value && version !== selectedApiVersion.value)
throw new Error('Cannot select multiple different FDB API versions');
if(version < 14)
throw new RangeError('FDB API versions before 14 are not supported');
if(version < 500)
throw new RangeError('FDB API versions before 500 are not supported');
if(version > 500)
throw new RangeError('Latest known FDB API version is 500');

253
bindings/nodejs/lib/tuple.js
View File

@ -24,6 +24,7 @@
var assert = require('assert');
var buffer = require('./bufferConversion');
var fdbUtil = require('./fdbUtil');
var FDBError = require('./error');
var sizeLimits = new Array(8);
@ -43,6 +44,91 @@ var minInt = -Math.pow(2, 53);
var nullByte = new Buffer('00', 'hex');
var BYTES_CODE = 0x01;
var STRING_CODE = 0x02;
var NESTED_CODE = 0x05;
var INT_ZERO_CODE = 0x14;
var POS_INT_END = 0x1d;
var NEG_INT_END = 0x0b;
var FLOAT_CODE = 0x20;
var DOUBLE_CODE = 0x21;
var FALSE_CODE = 0x26;
var TRUE_CODE = 0x27;
var UUID_CODE = 0x30;
function validateData(data, length) {
if(!(data instanceof Buffer) && !(data instanceof Array)) {
throw new TypeError('Data for FDB tuple type not array or buffer.');
} else if(data.length != length) {
throw new RangeError('Data for FDB tuple type has length ' + data.length + ' instead of expected length ' + length);
}
}
// If encoding and sign bit is 1 (negative), flip all of the bits. Otherwise, just flip sign.
// If decoding and sign bit is 0 (negative), flip all of the bits. Otherwise, just flip sign.
function adjustFloat(data, start, encode) {
if((encode && (data[start] & 0x80) === 0x80) || (!encode && (data[start] & 0x80) === 0x00)) {
for(var i = start; i < data.length; i++) {
data[i] = data[i] ^ 0xff;
}
} else {
data[start] = data[start] ^ 0x80;
}
}
function Float(value) {
this.value = value;
this.toBytes = function () {
if (this.rawData !== undefined) {
return this.rawData;
} else {
var buf = new Buffer(4);
buf.writeFloatBE(this.value, 0);
return buf;
}
};
}
Float.fromBytes = function (buf) {
validateData(buf, 4);
var f = new Float(buf.readFloatBE(0));
if(isNaN(f.value)) {
f.rawData = buf;
}
return f;
}
function Double(value) {
this.value = value;
this.toBytes = function () {
if (this.rawData !== undefined) {
return this.rawData;
} else {
var buf = new Buffer(8);
buf.writeDoubleBE(this.value, 0);
return buf;
}
};
}
Double.fromBytes = function (buf) {
validateData(buf, 8);
var d = new Double(buf.readDoubleBE(0));
if(isNaN(d.value)) {
d.rawData = buf;
}
return d;
}
function UUID(data) {
if (data.length != 16) {
// There's a special code for this, so we check it first and throw the error if appropriate.
throw new FDBError("invalid_uuid_size", 2268);
}
validateData(data, 16);
this.data = new Buffer(data);
}
function findNullBytes(buf, pos, searchForTerminators) {
var nullBytes = [];
@ -88,7 +174,7 @@ function encode(item, buf, pos) {
var nullBytes = findNullBytes(item, 0);
encodedString = new Buffer(2 + item.length + nullBytes.length);
encodedString[0] = unicode ? 2 : 1;
encodedString[0] = unicode ? STRING_CODE : BYTES_CODE;
var srcPos = 0;
var targetPos = 1;
@ -106,7 +192,7 @@ function encode(item, buf, pos) {
}
//64-bit integer
else if(item % 1 === 0) {
else if((typeof item === 'number' || item instanceof Number) && item % 1 === 0) {
var negative = item < 0;
var posItem = Math.abs(item);
@ -119,7 +205,7 @@ function encode(item, buf, pos) {
if(item > maxInt || item < minInt)
throw new RangeError('Cannot pack signed integer larger than 54 bits');
var prefix = negative ? 20 - length : 20 + length;
var prefix = negative ? INT_ZERO_CODE - length : INT_ZERO_CODE + length;
var outBuf = new Buffer(length+1);
outBuf[0] = prefix;
@ -137,6 +223,74 @@ function encode(item, buf, pos) {
return outBuf;
}
// Floats
else if(item instanceof Float) {
var outBuf = new Buffer(5);
outBuf[0] = FLOAT_CODE;
if (isNaN(item.value) && item.rawData !== undefined) {
item.rawData.copy(outBuf, 1, 0, 4);
} else {
outBuf.writeFloatBE(item.value, 1);
}
adjustFloat(outBuf, 1, true);
return outBuf;
}
// Doubles
else if(item instanceof Double) {
var outBuf = new Buffer(9);
outBuf[0] = DOUBLE_CODE;
if (isNaN(item.value) && item.rawData !== undefined) {
item.rawData.copy(outBuf, 1, 0, 8);
} else {
outBuf.writeDoubleBE(item.value, 1);
}
adjustFloat(outBuf, 1, true);
return outBuf;
}
// UUIDs
else if(item instanceof UUID) {
var outBuf = new Buffer(17);
outBuf[0] = UUID_CODE;
item.data.copy(outBuf, 1);
return outBuf;
}
// booleans
else if(item instanceof Boolean || typeof item === 'boolean') {
var outBuf = new Buffer(1);
var boolItem;
if(item instanceof Boolean) {
boolItem = item.valueOf();
} else {
boolItem = item;
}
if(boolItem) {
outBuf[0] = TRUE_CODE;
} else {
outBuf[0] = FALSE_CODE;
}
return outBuf;
}
// nested tuples
else if(item instanceof Array) {
var totalLength = 2;
var outArr = [new Buffer('05', 'hex')];
for(var i = 0; i < item.length; ++i) {
if(item[i] === null) {
outArr.push(new Buffer('00ff', 'hex'));
totalLength += 2;
} else {
outArr.push(encode(item[i]));
totalLength += outArr[i+1].length;
}
}
outArr.push(new Buffer('00', 'hex'))
return Buffer.concat(outArr, totalLength);
}
else
throw new TypeError('Packed element must either be a string, a buffer, an integer, or null');
}
@ -181,21 +335,27 @@ function decodeNumber(buf, offset, bytes) {
return num;
}
function decode(buf, pos) {
function decode(buf, pos, nested) {
if(typeof nested === 'undefined') nested = false;
var code = buf[pos];
var value;
if(code === 0) {
value = null;
pos++;
if(nested) {
pos += 2;
} else {
pos++;
}
}
else if(code === 1 || code === 2) {
else if(code === BYTES_CODE || code === STRING_CODE) {
var nullBytes = findNullBytes(buf, pos+1, true);
var start = pos+1;
var end = nullBytes[nullBytes.length-1];
if(code === 2 && nullBytes.length === 1) {
if(code === STRING_CODE && nullBytes.length === 1) {
value = buf.toString('utf8', start, end);
}
else {
@ -211,22 +371,60 @@ function decode(buf, pos) {
}
}
if(code === 2)
if(code === STRING_CODE)
value = value.toString('utf8');
}
pos = end + 1;
}
else if(Math.abs(code-20) <= 7) {
if(code === 20)
else if(Math.abs(code-INT_ZERO_CODE) <= 7) {
if(code === INT_ZERO_CODE)
value = 0;
else
value = decodeNumber(buf, pos+1, code-20);
value = decodeNumber(buf, pos+1, code-INT_ZERO_CODE);
pos += Math.abs(20-code) + 1;
pos += Math.abs(INT_ZERO_CODE-code) + 1;
}
else if(Math.abs(code-20) <= 8)
else if(Math.abs(code-INT_ZERO_CODE) <= 8)
throw new RangeError('Cannot unpack signed integers larger than 54 bits');
else if(code === FLOAT_CODE) {
var valBuf = new Buffer(4);
buf.copy(valBuf, 0, pos+1, pos+5);
adjustFloat(valBuf, 0, false);
value = Float.fromBytes(valBuf);
pos += 5;
}
else if(code === DOUBLE_CODE) {
var valBuf = new Buffer(8);
buf.copy(valBuf, 0, pos+1, pos+9);
adjustFloat(valBuf, 0, false);
value = Double.fromBytes(valBuf);
pos += 9;
}
else if(code === UUID_CODE) {
var valBuf = new Buffer(16);
buf.copy(valBuf, 0, pos+1, pos+17);
value = new UUID(valBuf);
pos += 17;
}
else if(code === FALSE_CODE) {
pos++;
value = false;
}
else if(code === TRUE_CODE) {
pos++;
value = true;
}
else if(code === NESTED_CODE) {
pos++;
value = []
while (buf[pos] != 0 || pos+1 < buf.length && buf[pos+1] === 0xff) {
var nestedVal = decode(buf, pos, true)
pos = nestedVal.pos
value.push(nestedVal.value)
}
pos++;
}
else
throw new TypeError('Unknown data type in DB: ' + buf + ' at ' + pos);
@ -252,4 +450,31 @@ function range(arr) {
return { begin: Buffer.concat([packed, nullByte]), end: Buffer.concat([packed, new Buffer('ff', 'hex')]) };
}
module.exports = {pack: pack, unpack: unpack, range: range};
function compare(arr1, arr2) {
// NOTE: There is built-in comparison function included in 0.11.13 that we might want to switch to.
var buf1 = pack(arr1);
var buf2 = pack(arr2);
var pos = 0;
while(pos < buf1.length && pos < buf2.length) {
if(buf1[pos] != buf2[pos]) {
if(buf1[pos] < buf2[pos]) {
return -1;
} else {
return 1;
}
}
pos += 1;
}
// The two arrays begin with a common prefix.
if(buf1.length < buf2.length) {
return -1;
} else if(buf1.length == buf2.length) {
return 0;
} else {
return 1;
}
}
module.exports = {pack: pack, unpack: unpack, range: range, compare: compare, Float: Float, Double: Double, UUID: UUID};

38
bindings/nodejs/tests/tester.js
View File

@ -620,12 +620,50 @@ function processOperation(context, inst, cb) {
});
})(promiseCb);
}
else if(inst.op === 'TUPLE_SORT') {
inst.pop()
.then(function(numParams) {
return inst.pop({count: numParams})
.then(function(params) {
var tuples = [];
for(var i = 0; i < params.length; ++i)
tuples.push(fdb.tuple.unpack(params[i]));
tuples.sort(fdb.tuple.compare);
for(var i = 0; i < tuples.length; ++i)
inst.push(fdb.tuple.pack(tuples[i]));
});
})(promiseCb);
}
else if(inst.op === 'SUB') {
inst.pop({count: 2})
.then(function(params) {
inst.push(params[0] - params[1]);
})(promiseCb);
}
else if(inst.op === 'ENCODE_FLOAT') {
inst.pop()
.then(function(fBytes) {
inst.push(fdb.tuple.Float.fromBytes(fBytes));
})(promiseCb);
}
else if(inst.op === 'ENCODE_DOUBLE') {
inst.pop()
.then(function(dBytes) {
inst.push(fdb.tuple.Double.fromBytes(dBytes));
})(promiseCb);
}
else if(inst.op === 'DECODE_FLOAT') {
inst.pop()
.then(function(fVal) {
inst.push(fVal.toBytes());
})(promiseCb);
}
else if(inst.op === 'DECODE_DOUBLE') {
inst.pop()
.then(function(dVal) {
inst.push(dVal.toBytes());
})(promiseCb);
}
else if(inst.op === 'CONCAT') {
inst.pop({count: 2})
.then(function(params) {

40
bindings/nodejs/tests/tuple_test.js
View File

@ -35,6 +35,44 @@ catch(err) {
console.log(fdb.tuple.pack([0xff * 0xff]));
console.log(fdb.tuple.pack([0xffffffff + 100 ]));
console.log(fdb.buffer.printable(fdb.tuple.pack(['begin', [true, null, false], 'end'])))
console.log(fdb.tuple.unpack(fdb.buffer.fromByteLiteral('\x1a\xff\xff\xff\xff\xff\xff')));
console.log(fdb.tuple.unpack(fdb.tuple.pack(['TEST', 'herp', 1, -10, 393493, '\u0000abc', 0xffffffff + 100])));
console.log(fdb.tuple.unpack(fdb.tuple.pack(['TEST', 'herp', 1, -10, 393493, '\u0000abc', 0xffffffff + 100, true, false, [new Boolean(true), null, new Boolean(false), 0, 'asdf'], null])));
console.log(fdb.buffer.printable(fdb.tuple.pack([[[[['three']]], 'two'], 'one'])))
console.log(fdb.tuple.range(['TEST', 1]));
console.log(fdb.buffer.printable(fdb.tuple.pack([fdb.tuple.Float.fromBytes(new Buffer('402df854', 'hex')), fdb.tuple.Double.fromBytes(new Buffer('4005BF0A8B145769', 'hex')), new fdb.tuple.UUID(new Buffer('deadc0deba5eba115ca1ab1edeadc0de', 'hex'))])))
console.log(fdb.tuple.unpack(fdb.tuple.pack([fdb.tuple.Float.fromBytes(new Buffer('2734236f', 'hex'))])))
tuples = [
[1,2],
[1],
[2],
[true],
[false],
[1,true],
[1,false],
[1, []],
[1, [null]],
[1, [0]],
[1, [1]],
[1, [0,1,2]],
[null],
[]
];
tuples.sort(fdb.tuple.compare);
console.log(tuples);
tuples = [
[fdb.tuple.Float.fromBytes(new Buffer('2734236f', 'hex'))], // A really small value.
[fdb.tuple.Float.fromBytes(new Buffer('80000000', 'hex'))], // -0.0
[new fdb.tuple.Float(0.0)],
[new fdb.tuple.Float(3.14)],
[new fdb.tuple.Float(-3.14)],
[new fdb.tuple.Float(2.7182818)],
[new fdb.tuple.Float(-2.7182818)],
[fdb.tuple.Float.fromBytes(new Buffer('7f800000', 'hex'))], // Infinity
[fdb.tuple.Float.fromBytes(new Buffer('7fffffff', 'hex'))], // NaN
[fdb.tuple.Float.fromBytes(new Buffer('ffffffff', 'hex'))], // -NaN
];
tuples.sort(fdb.tuple.compare);
console.log(tuples);

248
bindings/python/fdb/tuple.py
View File

@ -20,13 +20,30 @@
# FoundationDB Python API
import struct, math
import ctypes, uuid, struct, math
from bisect import bisect_left
from fdb import six
import fdb
_size_limits = tuple( (1 << (i*8))-1 for i in range(9) )
# Define type codes:
NULL_CODE = 0x00
BYTES_CODE = 0x01
STRING_CODE = 0x02
NESTED_CODE = 0x05
INT_ZERO_CODE = 0x14
POS_INT_END = 0x1d
NEG_INT_START = 0x0b
FLOAT_CODE = 0x20
DOUBLE_CODE = 0x21
FALSE_CODE = 0x26
TRUE_CODE = 0x27
UUID_CODE = 0x30
# Reserved: Codes 0x03, 0x04, 0x23, and 0x24 are reserved for historical reasons.
def _find_terminator( v, pos ):
# Finds the start of the next terminator [\x00]![\xff] or the end of v
while True:
@ -37,76 +54,183 @@ def _find_terminator( v, pos ):
return pos
pos += 2
# If encoding and sign bit is 1 (negative), flip all of the bits. Otherwise, just flip sign.
# If decoding and sign bit is 0 (negative), flip all of the bits. Otherwise, just flip sign.
def _float_adjust( v, encode ):
if encode and six.indexbytes(v, 0) & 0x80 != 0x00:
return b''.join(map(lambda x: six.int2byte(x ^ 0xff), six.iterbytes(v)))
elif not encode and six.indexbytes(v, 0) & 0x80 != 0x80:
return b''.join(map(lambda x: six.int2byte(x ^ 0xff), six.iterbytes(v)))
else:
return six.int2byte(six.indexbytes(v, 0) ^ 0x80) + v[1:]
class SingleFloat(object):
def __init__(self, value):
if isinstance(value, float):
# Restrict to the first 4 bytes (essentially)
self.value = ctypes.c_float(value).value
elif isinstance(value, ctypes.c_float):
self.value = value.value
elif isinstance(value, six.integertypes):
self.value = ctypes.c_float(value).value
else:
raise ValueError("Incompatible type for single-precision float: " + repr(value))
# Comparisons
def __eq__(self, other):
if isinstance(other, SingleFloat):
return _compare_floats(self.value, other.value) == 0
else:
return False
def __ne__(self, other):
return not (self == other)
def __lt__(self, other):
return _compare_floats(self.value, other.value) < 0
def __le__(self, other):
return _compare_floats(self.value, other.value) <= 0
def __gt__(self, other):
return not (self <= other)
def __ge__(self, other):
return not (self < other)
def __str__(self):
return str(self.value)
def __repr__(self):
return "SingleFloat(" + str(self.value) + ")"
def __hash__(self):
# Left-circulate the child hash to make hash(self) != hash(self.value)
v_hash = hash(self.value)
if v_hash >= 0:
return (v_hash >> 16) + ((v_hash & 0xFFFF) << 16)
else:
return ((v_hash >> 16) + 1) - ((abs(v_hash) & 0xFFFF) << 16)
def __nonzero__(self):
return bool(self.value)
def _decode(v, pos):
code = six.indexbytes(v, pos)
if code == 0:
if code == NULL_CODE:
return None, pos+1
elif code == 1:
elif code == BYTES_CODE:
end = _find_terminator(v, pos+1)
return v[pos+1:end].replace(b"\x00\xFF", b"\x00"), end+1
elif code == 2:
elif code == STRING_CODE:
end = _find_terminator(v, pos+1)
return v[pos+1:end].replace(b"\x00\xFF", b"\x00").decode("utf-8"), end+1
elif code >= 20 and code <= 28:
elif code >= INT_ZERO_CODE and code < POS_INT_END:
n = code - 20
end = pos + 1 + n
return struct.unpack(">Q", b'\x00'*(8-n) + v[pos+1:end])[0], end
elif code >= 12 and code < 20:
elif code > NEG_INT_START and code < INT_ZERO_CODE:
n = 20 - code
end = pos + 1 + n
return struct.unpack(">Q", b'\x00'*(8-n) + v[pos+1:end])[0]-_size_limits[n], end
elif code == 29: # 0x1d; Positive 9-255 byte integer
elif code == POS_INT_END: # 0x1d; Positive 9-255 byte integer
length = six.indexbytes(v, pos+1)
val = 0
for i in _range(length):
val = val << 8
val += six.indexbytes(v, pos+2+i)
return val, pos+2+length
elif code == 11: # 0x0b; Negative 9-255 byte integer
elif code == NEG_INT_START: # 0x0b; Negative 9-255 byte integer
length = six.indexbytes(v, pos+1)^0xff
val = 0
for i in _range(length):
val = val << 8
val += six.indexbytes(v, pos+2+i)
return val - (1<<(length*8)) + 1, pos+2+length
elif code == FLOAT_CODE:
return SingleFloat(struct.unpack(">f", _float_adjust(v[pos+1:pos+5], False))[0]), pos+5
elif code == DOUBLE_CODE:
return struct.unpack(">d", _float_adjust(v[pos+1:pos+9], False))[0], pos+9
elif code == UUID_CODE:
return uuid.UUID(bytes=v[pos+1:pos+17]), pos+17
elif code == FALSE_CODE:
if hasattr(fdb, "_version") and fdb._version < 500:
raise ValueError("Invalid API version " + str(fdb._version) + " for boolean types")
return False, pos+1
elif code == TRUE_CODE:
if hasattr(fdb, "_version") and fdb._version < 500:
raise ValueError("Invalid API version " + str(fdb._version) + " for boolean types")
return True, pos+1
elif code == NESTED_CODE:
ret = []
end_pos = pos+1
while end_pos < len(v):
if six.indexbytes(v, end_pos) == 0x00:
if end_pos+1 < len(v) and six.indexbytes(v, end_pos+1) == 0xff:
ret.append(None)
end_pos += 2
else:
break
else:
val, end_pos = _decode(v, end_pos)
ret.append(val)
return tuple(ret), end_pos+1
else:
raise ValueError("Unknown data type in DB: " + repr(v))
def _encode(value):
def _encode(value, nested=False):
# returns [code][data] (code != 0xFF)
# encoded values are self-terminating
# sorting need to work too!
if value == None: # ==, not is, because some fdb.impl.Value are equal to None
return b'\x00'
if nested:
return b''.join([six.int2byte(NULL_CODE), six.int2byte(0xff)])
else:
return b''.join([six.int2byte(NULL_CODE)])
elif isinstance(value, bytes): # also gets non-None fdb.impl.Value
return b'\x01' + value.replace(b'\x00', b'\x00\xFF') + b'\x00'
return six.int2byte(BYTES_CODE) + value.replace(b'\x00', b'\x00\xFF') + b'\x00'
elif isinstance(value, six.text_type):
return b'\x02' + value.encode('utf-8').replace(b'\x00', b'\x00\xFF') + b'\x00'
elif isinstance(value, six.integer_types):
return six.int2byte(STRING_CODE) + value.encode('utf-8').replace(b'\x00', b'\x00\xFF') + b'\x00'
elif isinstance(value, six.integer_types) and (not isinstance(value, bool) or (hasattr(fdb, '_version') and fdb._version < 500)):
if value == 0:
return b'\x14'
return b''.join([six.int2byte(INT_ZERO_CODE)])
elif value > 0:
if value >= _size_limits[-1]:
length = (value.bit_length()+7)//8
data = [b'\x1d', six.int2byte(length)]
data = [six.int2byte(POS_INT_END), six.int2byte(length)]
for i in _range(length-1,-1,-1):
data.append(six.int2byte( (value>>(8*i))&0xff ))
return b''.join(data)
n = bisect_left( _size_limits, value )
return six.int2byte(20 + n) + struct.pack( ">Q", value )[-n:]
return six.int2byte(INT_ZERO_CODE + n) + struct.pack( ">Q", value )[-n:]
else:
if -value >= _size_limits[-1]:
length = (value.bit_length()+7)//8
value += (1<<(length*8)) - 1
data = [b'\x0b', six.int2byte(length^0xff)]
data = [six.int2byte(NEG_INT_START), six.int2byte(length^0xff)]
for i in _range(length-1,-1,-1):
data.append(six.int2byte( (value>>(8*i))&0xff ))
return b''.join(data)
n = bisect_left( _size_limits, -value )
maxv = _size_limits[n]
return six.int2byte(20 - n) + struct.pack( ">Q", maxv+value)[-n:]
return six.int2byte(INT_ZERO_CODE - n) + struct.pack( ">Q", maxv+value)[-n:]
elif isinstance(value, ctypes.c_float) or isinstance(value, SingleFloat):
return six.int2byte(FLOAT_CODE) + _float_adjust(struct.pack(">f", value.value), True)
elif isinstance(value, ctypes.c_double):
return six.int2byte(DOUBLE_CODE) + _float_adjust(struct.pack(">d", value.value), True)
elif isinstance(value, float):
return six.int2byte(DOUBLE_CODE) + _float_adjust(struct.pack(">d", value), True)
elif isinstance(value, uuid.UUID):
return six.int2byte(UUID_CODE) + value.bytes
elif isinstance(value, bool):
if value:
return b''.join([six.int2byte(TRUE_CODE)])
else:
return b''.join([six.int2byte(FALSE_CODE)])
elif isinstance(value, tuple) or isinstance(value, list):
return b''.join([six.int2byte(NESTED_CODE)] + list(map(lambda x: _encode(x, True), value)) + [six.int2byte(0x00)])
else:
raise ValueError("Unsupported data type: " + str(type(value)))
@ -140,3 +264,91 @@ def range(t):
return slice(
p+b'\x00',
p+b'\xff')
def _code_for(value):
if value == None:
return NULL_CODE
elif isinstance(value, bytes):
return BYTES_CODE
elif isinstance(value, six.text_type):
return STRING_CODE
elif (not hasattr(fdb, '_version') or fdb._version >= 500) and isinstance(value, bool):
return FALSE_CODE
elif isinstance(value, six.integer_types):
return INT_ZERO_CODE
elif isinstance(value, ctypes.c_float) or isinstance(value, SingleFloat):
return FLOAT_CODE
elif isinstance(value, ctypes.c_double) or isinstance(value, float):
return DOUBLE_CODE
elif isinstance(value, uuid.UUID):
return UUID_CODE
elif isinstance(value, tuple) or isinstance(value, list):
return NESTED_CODE
else:
raise ValueError("Unsupported data type: " + str(type(value)))
def _compare_floats(f1, f2):
sign1 = int(math.copysign(1, f1))
sign2 = int(math.copysign(1, f2))
# This business with signs is to deal with negative zero, NaN, and infinity.
if sign1 < sign2:
# f1 is negative and f2 is positive.
return -1
elif sign1 > sign2:
# f1 is positive and f2 is negative.
return 1
if not math.isnan(f1) and not math.isnan(f2):
return -1 if f1 < f2 else 0 if f1 == f2 else 1
# There are enough edge cases that bit comparison is safer.
bytes1 = struct.pack(">d", f1)
bytes2 = struct.pack(">d", f2)
return sign1*(-1 if bytes1 < bytes2 else 0 if bytes1 == bytes2 else 1)
def _compare_values(value1, value2):
code1 = _code_for(value1)
code2 = _code_for(value2)
if code1 < code2:
return -1
elif code1 > code2:
return 1
# Compatible types.
if code1 == NULL_CODE:
return 0
elif code1 == STRING_CODE:
encoded1 = value1.encode('utf-8')
encoded2 = value2.encode('utf-8')
return -1 if encoded1 < encoded2 else 0 if encoded1 == encoded2 else 1
elif code1 == FLOAT_CODE:
f1 = value1 if isinstance(value1, SingleFloat) else SingleFloat(value1.value)
f2 = value2 if isinstance(value2, SingleFloat) else SingleFloat(value2.value)
return -1 if f1 < f2 else 0 if f1 == f2 else 1
elif code1 == DOUBLE_CODE:
d1 = value1.value if isinstance(value1, ctypes.c_double) else value1
d2 = value2.value if isinstance(value2, ctypes.c_double) else value2
return _compare_floats(d1, d2)
elif code1 == NESTED_CODE:
return compare(value1, value2)
else:
# Booleans, UUIDs, and integers can just use standard comparison.
return -1 if value1 < value2 else 0 if value1 == value2 else 1
# compare element by element and return -1 if t1 < t2 or 1 if t1 > t2 or 0 if t1 == t2
def compare(t1, t2):
i = 0
while i < len(t1) and i < len(t2):
c = _compare_values(t1[i], t2[i])
if c != 0:
return c
i += 1
if i < len(t1):
return 1
elif i < len(t2):
return -1
else:
return 0

28
bindings/python/tests/tester.py
View File

@ -20,8 +20,10 @@
#
import ctypes
import sys
import os
import struct
import threading
import time
import random
@ -454,12 +456,38 @@ class Tester:
elif inst.op == six.u("TUPLE_UNPACK"):
for i in fdb.tuple.unpack( inst.pop() ):
inst.push(fdb.tuple.pack((i,)))
elif inst.op == six.u("TUPLE_SORT"):
count = inst.pop()
items = inst.pop(count)
unpacked = map(fdb.tuple.unpack, items)
if six.PY3:
sorted_items = sorted(unpacked, key=fdb.tuple.pack)
else:
sorted_items = sorted(unpacked, cmp=fdb.tuple.compare)
for item in sorted_items:
inst.push(fdb.tuple.pack(item))
elif inst.op == six.u("TUPLE_RANGE"):
count = inst.pop()
items = inst.pop(count)
r = fdb.tuple.range( tuple(items) )
inst.push(r.start)
inst.push(r.stop)
elif inst.op == six.u("ENCODE_FLOAT"):
f_bytes = inst.pop()
f = struct.unpack(">f", f_bytes)[0]
inst.push(fdb.tuple.SingleFloat(f))
elif inst.op == six.u("ENCODE_DOUBLE"):
d_bytes = inst.pop()
d = struct.unpack(">d", d_bytes)[0]
inst.push(d)
elif inst.op == six.u("DECODE_FLOAT"):
f = inst.pop()
f_bytes = struct.pack(">f", f.value)
inst.push(f_bytes)
elif inst.op == six.u("DECODE_DOUBLE"):
d = inst.pop()
d_bytes = struct.pack(">d", d)
inst.push(d_bytes)
elif inst.op == six.u("START_THREAD"):
t = Tester( self.db, inst.pop() )
thr = threading.Thread(target=t.run)

131
bindings/python/tests/tuple_tests.py
View File

@ -20,54 +20,107 @@
#
import sys, random
import ctypes, sys, random, struct, unicodedata, math, uuid
_range = range
from fdb.tuple import pack, unpack, range
from fdb.tuple import pack, unpack, range, compare, SingleFloat
from fdb import six
from fdb.six import u
def randomUnicode():
while True:
c = random.randint(0, 0xffff)
if unicodedata.category(unichr(c))[0] in 'LMNPSZ':
return unichr(c)
def randomElement():
r = random.randint(0,4)
r = random.randint(0,9)
if r == 0:
chars = [b'\x00', b'\x01', b'a', b'7', b'\xfe', b'\ff']
return b''.join([random.choice(chars) for c in _range(random.randint(0, 5))])
if random.random() < 0.5:
chars = [b'\x00', b'\x01', b'a', b'7', b'\xfe', b'\ff']
return b''.join([random.choice(chars) for c in _range(random.randint(0, 5))])
else:
return b''.join([six.int2byte(random.randint(0,255)) for _ in _range(random.randint(0,10))])
elif r == 1:
chars = [u('\x00'), u('\x01'), u('a'), u('7'), u('\xfe'), u('\ff'), u('\u0000'), u('\u0001'), u('\uffff'), u('\uff00')]
return u('').join([random.choice(chars) for c in _range(random.randint(0, 10))])
if random.random() < 0.5:
chars = [u('\x00'), u('\x01'), u('a'), u('7'), u('\xfe'), u('\ff'), u('\u0000'), u('\u0001'), u('\uffff'), u('\uff00'), u('\U0001f4a9')]
return u('').join([random.choice(chars) for c in _range(random.randint(0, 10))])
else:
return u('').join([randomUnicode() for _ in _range(random.randint(0, 10))])
elif r == 2:
return random.choice([-1, 1]) * min(2**random.randint(0, 2040) + random.randint(-10, 10), 2**2040 - 1)
elif r == 3:
return random.choice([-1, 1]) * 2**random.randint(0, 64) + random.randint(-10, 10)
elif r == 4:
return None
elif r == 5:
ret = random.choice([float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan')])
if random.random() < 0.5:
return SingleFloat(ret)
else:
return ret
elif r == 6:
is_double = random.random() < 0.5
byte_str = b''.join([six.int2byte(random.randint(0,255)) for _ in _range(8 if is_double else 4)])
if is_double:
return struct.unpack(">d", byte_str)[0]
else:
return SingleFloat(struct.unpack(">f", byte_str)[0])
elif r == 7:
return random.random() < 0.5
elif r == 8:
return uuid.uuid4()
elif r == 9:
return [randomElement() for _ in _range(random.randint(0,5))]
def randomTuple():
return tuple( randomElement() for x in _range(random.randint(0,4)) )
def isprefix(a,b):
return tupleorder(a) == tupleorder(b[:len(a)])
return compare(a, b[:len(a)]) == 0
def torder(x):
if x == None:
return 0
elif type(x) == type(b''):
return 1
elif isinstance(x, six.text_type):
return 2
elif isinstance(x, six.integer_types):
return 3
raise Exception("Unknown type")
def tupleorder(t):
return tuple( (torder(e),e) for e in t )
def find_bad_sort(a, b):
for x1 in a:
for x2 in b:
if compare(x1, x2) < 0 and pack(x1) >= pack(x2):
return (x1, x2)
return None
def equalEnough(t1, t2):
if len(t1) != len(t2): return False
for i in _range(len(t1)):
e1 = t1[i]
e2 = t2[i]
if isinstance(e1, SingleFloat):
if not isinstance(e2, SingleFloat): return False
return ctypes.c_float(e1.value).value == ctypes.c_float(e2.value).value
elif isinstance(e1, list) or isinstance(e2, tuple):
if not (isinstance(e2, list) or isinstance(e2, tuple)): return False
if not equalEnough(e1, e2): return False
else:
if e1 != e2: return False
return True
def tupleTest(N=10000):
someTuples = [ randomTuple() for i in _range(N) ]
a = sorted(someTuples, key=tupleorder)
a = sorted(someTuples, cmp=compare)
b = sorted(someTuples, key=pack)
assert a == b
if not a == b:
problem = find_bad_sort(a, b)
if problem:
print("Bad sort:\n %s\n %s" % (problem[0], problem[1]))
print("Bytes:\n %s\n %s" % (repr(pack(problem[0])), repr(pack(problem[1]))))
#print("Tuple order:\n %s\n %s" % (tupleorder(problem[0]), tupleorder(problem[1])))
return False
else:
print("Sorts unequal but every pair correct")
return False
print("Sort %d OK" % N)
@ -75,26 +128,36 @@ def tupleTest(N=10000):
t = randomTuple()
t2 = t + (randomElement(),)
t3 = randomTuple()
try:
assert(unpack(pack(t)) == t)
r = range(t)
assert not (r.start <= pack(t) < r.stop)
assert (r.start <= pack(t2) < r.stop)
if not compare(unpack(pack(t)), t) == 0:
print("unpack . pack /= identity:\n Orig: %s\n Bytes: %s\n New: %s" % (t, repr(pack(t)), unpack(pack(t))))
return False
if not isprefix(t, t3):
assert not (r.start <= pack(t3) <= r.stop)
r = range(t)
if r.start <= pack(t) < r.stop:
print("element within own range:\n Tuple: %s\n Bytes: %s\n Start: %s\n Stop: %s" % (t, repr(pack(t)), repr(r.start), repr(r.stop)))
if not r.start <= pack(t2) < r.stop:
print("prefixed element not in range:\n Tuple: %s\n Bytes: %s\n Prefixed: %s\n Bytes: %s" % (t, repr(pack(t)), t2, repr(pack(t2))))
return False
assert (tupleorder(t) < tupleorder(t3)) == (pack(t) < pack(t3))
except:
print (repr(t), repr(t2), repr(t3))
raise
if not isprefix(t, t3):
if r.start <= pack(t3) <= r.stop:
print("non-prefixed element in range:\n Tuple: %s\n Bytes: %s\n Other: %s\n Bytes: %s" % (t, repr(pack(t)), t3, repr(pack(t3))))
return False
if (compare(t, t3) < 0) != (pack(t) < pack(t3)):
print("Bad comparison:\n Tuple: %s\n Bytes: %s\n Other: %s\n Bytes: %s" % (t, repr(pack(t)), t3, repr(pack(t3))))
return False
if not pack(t) < pack(t2):
print("Prefix not before prefixed:\n Tuple: %s\n Bytes: %s\n Other: %s\n Bytes: %s" % (t, repr(pack(t)), t2, repr(pack(t2))))
return False
print ("Tuple check %d OK" % N)
return True
# test:
# a = ('\x00a', -2, 'b\x01', 12345, '')
# assert(a==fdbtuple.unpack(fdbtuple.pack(a)))
if __name__=='__main__':
tupleTest()
assert tupleTest(10000)

204
bindings/ruby/lib/fdbtuple.rb
View File

@ -29,6 +29,60 @@ module FDB
module Tuple
@@size_limits = (0..8).map {|x| (1 << (x*8)) - 1}
# Type codes
@@NULL_CODE = 0x00
@@BYTES_CODE = 0x01
@@STRING_CODE = 0x02
@@NESTED_CODE = 0x05
@@INT_ZERO_CODE = 0x14
@@POS_INT_END = 0x1c
@@NEG_INT_START = 0x0c
@@FLOAT_CODE = 0x20
@@DOUBLE_CODE = 0x21
@@FALSE_CODE = 0x26
@@TRUE_CODE = 0x27
@@UUID_CODE = 0x30
class UUID
def initialize(data)
if data.length != 16
raise Error.new(2268) # invalid_uuid_size
end
@data=data.slice(0,16)
end
def data
@data
end
def <=> (other)
self.data <=> other.data
end
def to_s
self.data.each_byte.map { |b| b.to_s(16) } .join
end
end
class SingleFloat
def initialize(value)
if value.kind_of? Float
@value=value
elsif value.kind_of? Integer
@value=value.to_f
else
raise ArgumentError, "Invalid value type for SingleFloat: " + value.class.name
end
@value=value
end
def value
@value
end
def <=> (other)
Tuple._compare_floats(self.value, other.value, false)
end
def to_s
self.value.to_s
end
end
def self.find_terminator(v, pos)
while true
pos = v.index("\x00", pos)
@ -42,24 +96,62 @@ module FDB
end
private_class_method :find_terminator
def self.float_adjust(v, pos, length, encode)
if (encode and v[pos].ord & 0x80 != 0x00) or (not encode and v[pos].ord & 0x80 == 0x00)
v.slice(pos, length).chars.map { |b| (b.ord ^ 0xff).chr } .join
else
ret = v.slice(pos, length)
ret[0] = (ret[0].ord ^ 0x80).chr
ret
end
end
private_class_method :float_adjust
def self.decode(v, pos)
code = v.getbyte(pos)
if code == 0
if code == @@NULL_CODE
[nil, pos+1]
elsif code == 1
elsif code == @@BYTES_CODE
epos = find_terminator(v, pos+1)
[v.slice(pos+1, epos-pos-1).gsub("\x00\xFF", "\x00"), epos+1]
elsif code == 2
elsif code == @@STRING_CODE
epos = find_terminator(v, pos+1)
[v.slice(pos+1, epos-pos-1).gsub("\x00\xFF", "\x00").force_encoding("UTF-8"), epos+1]
elsif code >= 20 && code <= 28
n = code - 20
elsif code >= @@INT_ZERO_CODE && code <= @@POS_INT_END
n = code - @@INT_ZERO_CODE
[("\x00" * (8-n) + v.slice(pos+1, n)).unpack("Q>")[0], pos+n+1]
elsif code >= 12 and code < 20
n = 20 - code
elsif code >= @@NEG_INT_START and code < @@INT_ZERO_CODE
n = @@INT_ZERO_CODE - code
[("\x00" * (8-n) + v.slice(pos+1, n)).unpack("Q>")[0]-@@size_limits[n], pos+n+1]
elsif code == @@FALSE_CODE
[false, pos+1]
elsif code == @@TRUE_CODE
[true, pos+1]
elsif code == @@FLOAT_CODE
[SingleFloat.new(float_adjust(v, pos+1, 4, false).unpack("g")[0]), pos+5]
elsif code == @@DOUBLE_CODE
[float_adjust(v, pos+1, 8, false).unpack("G")[0], pos+9]
elsif code == @@UUID_CODE
[UUID.new(v.slice(pos+1, 16)), pos+17]
elsif code == @@NESTED_CODE
epos = pos+1
nested = []
while epos < v.length
if v.getbyte(epos) == @@NULL_CODE
if epos+1 < v.length and v.getbyte(epos+1) == 0xFF
nested << nil
epos += 2
else
break
end
else
r, epos = decode(v, epos)
nested << r
end
end
[nested, epos+1]
else
raise "Unknown data type in DB: " + v
raise "Unknown data type in DB: " + code.ord.to_s
end
end
private_class_method :decode
@ -74,21 +166,25 @@ module FDB
end
private_class_method :bisect_left
def self.encode(v)
if !v
"\x00"
def self.encode(v, nested=false)
if v.nil?
if nested
"\x00\xFF"
else
@@NULL_CODE.chr
end
elsif v.kind_of? String
if v.encoding == Encoding::BINARY || v.encoding == Encoding::ASCII
1.chr + v.gsub("\x00", "\x00\xFF") + 0.chr
@@BYTES_CODE.chr + v.gsub("\x00", "\x00\xFF") + 0.chr
elsif v.encoding == Encoding::UTF_8
2.chr + v.dup.force_encoding("BINARY").gsub("\x00", "\x00\xFF") + 0.chr
@@STRING_CODE.chr + v.dup.force_encoding("BINARY").gsub("\x00", "\x00\xFF") + 0.chr
else
raise ArgumentError, "unsupported encoding #{v.encoding.name}"
end
elsif v.kind_of? Integer
raise RangeError, "value outside inclusive range -2**64+1 to 2**64-1" if v < -2**64+1 || v > 2**64-1
if v == 0
20.chr
@@INT_ZERO_CODE.chr
elsif v > 0
n = bisect_left( @@size_limits, v )
(20+n).chr + [v].pack("Q>").slice(8-n, n)
@ -96,6 +192,18 @@ module FDB
n = bisect_left( @@size_limits, -v )
(20-n).chr + [@@size_limits[n]+v].pack("Q>").slice(8-n, n)
end
elsif v.kind_of? TrueClass
@@TRUE_CODE.chr
elsif v.kind_of? FalseClass
@@FALSE_CODE.chr
elsif v.kind_of? SingleFloat
@@FLOAT_CODE.chr + float_adjust([v.value].pack("g"), 0, 4, true)
elsif v.kind_of? Float
@@DOUBLE_CODE.chr + float_adjust([v].pack("G"), 0, 8, true)
elsif v.kind_of? UUID
@@UUID_CODE.chr + v.data
elsif v.kind_of? Array
@@NESTED_CODE.chr + (v.map { |el| encode(el, true).force_encoding("BINARY") }).join + 0.chr
else
raise ArgumentError, "unsupported type #{v.class}"
end
@ -127,5 +235,73 @@ module FDB
p = pack(tuple)
[p+"\x00", p+"\xFF"]
end
def self._code_for(v)
if v.nil?
@@NULL_CODE
elsif v.kind_of? String
if v.encoding == Encoding::BINARY || v.encoding == Encoding::ASCII
@@BYTES_CODE
elsif v.encoding == Encoding::UTF_8
@@STRING_CODE
else
raise ArgumentError, "unsupported encoding #{v.encoding.name}"
end
elsif v.kind_of? Integer
@@INT_ZERO_CODE
elsif v.kind_of? TrueClass
@@TRUE_CODE
elsif v.kind_of? FalseClass
@@FALSE_CODE
elsif v.kind_of? SingleFloat
@@FLOAT_CODE
elsif v.kind_of? Float
@@DOUBLE_CODE
elsif v.kind_of? UUID
@@UUID_CODE
elsif v.kind_of? Array
@@NESTED_CODE
else
raise ArgumentError, "unsupported type #{v.class}"
end
end
def self._compare_floats(f1, f2, is_double)
# This converts the floats to their byte representation and then
# does the comparison. Why?
# 1) NaN comparison - Ruby doesn't really do this
# 2) -0.0 == 0.0 in Ruby but not in our representation
# It would be better to just take the floats and compare them, but
# this way handles the edge cases correctly.
b1 = float_adjust([f1].pack(is_double ? ">G" : ">g"), 0, (is_double ? 8 : 4), true)
b2 = float_adjust([f2].pack(is_double ? ">G" : ">g"), 0, (is_double ? 8 : 4), true)
b1 <=> b2
end
def self._compare_elems(v1, v2)
c1 = _code_for(v1)
c2 = _code_for(v2)
return c1 <=> c2 unless c1 == c2
if c1 == @@NULL_CODE
0
elsif c1 == @@DOUBLE_CODE
_compare_floats(v1, v2, true)
elsif c1 == @@NESTED_CODE
compare(v1, v2) # recurse
else
v1 <=> v2
end
end
def self.compare(tuple1, tuple2)
i = 0
while i < tuple1.length && i < tuple2.length
c = self._compare_elems(tuple1[i], tuple2[i])
return c unless c == 0
i += 1
end
tuple1.length <=> tuple2.length
end
end
end

21
bindings/ruby/tests/tester.rb
View File

@ -393,6 +393,27 @@ class Tester
(FDB::Tuple.range arr).each do |x|
inst.push(x)
end
when "TUPLE_SORT"
arr = []
inst.wait_and_pop.times do |i|
arr.push(FDB::Tuple.unpack inst.wait_and_pop)
end
arr.sort! { |t1, t2| FDB::Tuple.compare(t1, t2) }
arr.each do |x|
inst.push( FDB::Tuple.pack x)
end
when "ENCODE_FLOAT"
bytes = inst.wait_and_pop
inst.push(FDB::Tuple::SingleFloat.new(bytes.unpack("g")[0]))
when "ENCODE_DOUBLE"
bytes = inst.wait_and_pop
inst.push(bytes.unpack("G")[0])
when "DECODE_FLOAT"
f_val = inst.wait_and_pop
inst.push([f_val.value].pack("g"))
when "DECODE_DOUBLE"
d_val = inst.wait_and_pop
inst.push([d_val].pack("G"))
when "START_THREAD"
t = Tester.new( @db, inst.wait_and_pop )
thr = Thread.new do

1
flow/error_definitions.h
View File

@ -157,6 +157,7 @@ ERROR( directory_prefix_not_empty, 2264, "The database has keys stored at the pr
ERROR( directory_prefix_in_use, 2265, "The directory layer already has a conflicting prefix" )
ERROR( invalid_destination_directory, 2266, "The target directory is invalid" )
ERROR( cannot_modify_root_directory, 2267, "The root directory cannot be modified" )
ERROR( invalid_uuid_size, 2268, "UUID is not sixteen bytes");
// 2300 - backup and restore errors
ERROR( backup_error, 2300, "Backup error")