From 2be1cabc72eaeadeb63829393990973d210c4dd9 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 24 Oct 2018 15:50:07 -0700 Subject: [PATCH 01/13] Support for big integers in go and ruby tuple layers --- bindings/bindingtester/known_testers.py | 4 +- bindings/go/src/_stacktester/stacktester.go | 23 +++- bindings/go/src/fdb/tuple/tuple.go | 120 ++++++++++++++++++-- bindings/ruby/lib/fdbimpl.rb | 2 +- bindings/ruby/lib/fdbtuple.rb | 53 +++++++-- documentation/sphinx/source/api-ruby.rst | 4 +- 6 files changed, 179 insertions(+), 27 deletions(-) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index e80cc1f0c8..7606a23c3e 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -59,9 +59,9 @@ _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % ( testers = { 'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), 'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), - 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 64, 23, MAX_API_VERSION), + 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION), 'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), 'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), - 'go': Tester('go', _absolute_path('go/build/bin/_stacktester'), 63, 200, MAX_API_VERSION), + 'go': Tester('go', _absolute_path('go/build/bin/_stacktester'), 2040, 200, MAX_API_VERSION), 'flow': Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION, directory_snapshot_ops_enabled=False), } diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index c5cbd8852f..56ccb1667d 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -28,6 +28,7 @@ import ( "github.com/apple/foundationdb/bindings/go/src/fdb" "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" "log" + "math/big" "os" "reflect" "runtime" @@ -103,7 +104,7 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) { switch el := ret.item.(type) { case []byte: ret.item = el - case int64, string, bool, tuple.UUID, float32, float64, tuple.Tuple: + case int64, *big.Int, string, bool, tuple.UUID, float32, float64, tuple.Tuple: ret.item = el case fdb.Key: ret.item = []byte(el) @@ -176,6 +177,8 @@ func tupleToString(t tuple.Tuple) string { switch el := el.(type) { case int64: buffer.WriteString(fmt.Sprintf("%d", el)) + case *big.Int: + buffer.WriteString(fmt.Sprintf("%s", el)) case []byte: buffer.WriteString(fmt.Sprintf("%+q", string(el))) case string: @@ -207,6 +210,8 @@ func (sm *StackMachine) dumpStack() { switch el := el.(type) { case int64: fmt.Printf(" %d", el) + case *big.Int: + fmt.Printf(" %s", el) case fdb.FutureNil: fmt.Printf(" FutureNil") case fdb.FutureByteSlice: @@ -490,7 +495,21 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { case op == "POP": sm.stack = sm.stack[:len(sm.stack)-1] case op == "SUB": - sm.store(idx, sm.waitAndPop().item.(int64)-sm.waitAndPop().item.(int64)) + var x, y *big.Int + switch x1 := sm.waitAndPop().item.(type) { + case *big.Int: + x = x1 + case int64: + x = big.NewInt(x1) + } + switch y1 := sm.waitAndPop().item.(type) { + case *big.Int: + y = y1 + case int64: + y = big.NewInt(y1) + } + + sm.store(idx, x.Sub(x, y)) case op == "CONCAT": str1 := sm.waitAndPop().item str2 := sm.waitAndPop().item diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 2bea679d1a..be3b5e52d3 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -30,9 +30,10 @@ // (https://apple.github.io/foundationdb/data-modeling.html#tuples). // // FoundationDB tuples can currently encode byte and unicode strings, integers, -// floats, doubles, booleans, UUIDs, tuples, and NULL values. In Go these are -// represented as []byte (or fdb.KeyConvertible), string, int64 (or int), -// float32, float64, bool, UUID, Tuple, and nil. +// large integers, floats, doubles, booleans, UUIDs, tuples, and NULL values. +// In Go these are represented as []byte (or fdb.KeyConvertible), string, int64 +// (or int, uint, uint64), *big.Int (or big.Int, uint64), float32, float64, bool, +// UUID, Tuple, and nil. package tuple import ( @@ -40,6 +41,7 @@ import ( "encoding/binary" "fmt" "math" + "math/big" "github.com/apple/foundationdb/bindings/go/src/fdb" ) @@ -50,7 +52,8 @@ import ( // result in a runtime panic). // // The valid types for TupleElement are []byte (or fdb.KeyConvertible), string, -// int64 (or int), float, double, bool, UUID, Tuple, and nil. +// int64 (or int, uint, uint64), *big.Int (or big.Int, uint64), float, double, +// bool, UUID, Tuple, and nil. type TupleElement interface{} // Tuple is a slice of objects that can be encoded as FoundationDB tuples. If @@ -76,8 +79,8 @@ const bytesCode = 0x01 const stringCode = 0x02 const nestedCode = 0x05 const intZeroCode = 0x14 -const posIntEnd = 0x1c -const negIntStart = 0x0c +const posIntEnd = 0x1d +const negIntStart = 0x0b const floatCode = 0x20 const doubleCode = 0x21 const falseCode = 0x26 @@ -150,7 +153,7 @@ func (p *packer) encodeBytes(code byte, b []byte) { func (p *packer) encodeInt(i int64) { if i == 0 { - p.putByte(0x14) + p.putByte(intZeroCode) return } @@ -164,7 +167,7 @@ func (p *packer) encodeInt(i int64) { binary.BigEndian.PutUint64(scratch[:], uint64(i)) case i < 0: n = bisectLeft(uint64(-i)) - p.putByte(byte(0x14 - n)) + p.putByte(byte(intZeroCode - n)) offsetEncoded := int64(sizeLimits[n]) + i binary.BigEndian.PutUint64(scratch[:], uint64(offsetEncoded)) } @@ -172,6 +175,59 @@ func (p *packer) encodeInt(i int64) { p.putBytes(scratch[8-n:]) } +func (p *packer) encodeBigInt(i *big.Int) { + if i.Cmp(big.NewInt(math.MaxInt64)) <= 0 && i.Cmp(big.NewInt(math.MinInt64)) >= 0 { + p.encodeInt(i.Int64()) + return + } + + length := len(i.Bytes()) + if length > 0xff { + panic(fmt.Sprintf("Integer magnitude is too large (more than 255 bytes)")) + } + + if i.Sign() > 0 { + intBytes := i.Bytes() + if length > 8 { + p.putByte(byte(posIntEnd)) + p.putByte(byte(len(intBytes))) + } else { + p.putByte(byte(intZeroCode + length)) + } + + p.putBytes(intBytes) + } else { + add := new(big.Int).Lsh(big.NewInt(1), uint(length*8)) + add.Sub(add, big.NewInt(1)) + transformed := new(big.Int) + transformed.Add(i, add) + + intBytes := transformed.Bytes() + if length > 8 { + p.putByte(byte(negIntStart)) + p.putByte(byte(length ^ 0xff)) + } else { + p.putByte(byte(intZeroCode - length)) + } + + for i := len(intBytes); i < length; i++ { + p.putByte(0x00) + } + + p.putBytes(intBytes) + } +} + +func (p *packer) encodeUint(i uint64) { + if i > math.MaxInt64 { + val := new(big.Int) + val.SetUint64(i) + p.encodeBigInt(val) + } else { + p.encodeInt(int64(i)) + } +} + func (p *packer) encodeFloat(f float32) { var scratch [4]byte binary.BigEndian.PutUint32(scratch[:], math.Float32bits(f)) @@ -211,8 +267,16 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { } case int64: p.encodeInt(e) + case uint64: + p.encodeUint(e) case int: p.encodeInt(int64(e)) + case uint: + p.encodeUint(uint64(e)) + case *big.Int: + p.encodeBigInt(e) + case big.Int: + p.encodeBigInt(&e) case []byte: p.encodeBytes(bytesCode, e) case fdb.KeyConvertible: @@ -243,8 +307,10 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // Pack returns a new byte slice encoding the provided tuple. Pack will panic if // the tuple contains an element of any type other than []byte, -// fdb.KeyConvertible, string, int64, int, float32, float64, bool, tuple.UUID, -// nil, or a Tuple with elements of valid types. +// fdb.KeyConvertible, string, int64, int, uint64, uint, *big.Int, big.Int, float32, +// float64, bool, tuple.UUID, nil, or a Tuple with elements of valid types. It will +// also panic if an integer is specified with a value outside the range +// [-2**2040+1, 2**2040-1] // // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to // call Pack when using a Tuple with a FoundationDB API function that requires a @@ -309,6 +375,36 @@ func decodeInt(b []byte) (int64, int) { return ret, n + 1 } +func decodeBigInt(b []byte) (*big.Int, int) { + val := new(big.Int) + offset := 1 + var length int + + if b[0] == negIntStart || b[0] == posIntEnd { + length = int(b[1]) + if b[0] == negIntStart { + length ^= 0xff + } + + offset += 1 + } else { + length = int(b[0]) - intZeroCode + if length < 0 { + length = -length + } + } + + val.SetBytes(b[offset : length+offset]) + + if b[0] < intZeroCode { + sub := new(big.Int).Lsh(big.NewInt(1), uint(length)*8) + sub.Sub(sub, big.NewInt(1)) + val.Sub(val, sub) + } + + return val, length + offset +} + func decodeFloat(b []byte) (float32, int) { bp := make([]byte, 4) copy(bp, b[1:]) @@ -357,8 +453,10 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { el, off = decodeBytes(b[i:]) case b[i] == stringCode: el, off = decodeString(b[i:]) - case negIntStart <= b[i] && b[i] <= posIntEnd: + case negIntStart+1 < b[i] && b[i] < posIntEnd-1: el, off = decodeInt(b[i:]) + case negIntStart <= b[i] && b[i] <= posIntEnd: + el, off = decodeBigInt(b[i:]) case b[i] == floatCode: if i+5 > len(b) { return nil, i, fmt.Errorf("insufficient bytes to decode float starting at position %d of byte array for tuple", i) diff --git a/bindings/ruby/lib/fdbimpl.rb b/bindings/ruby/lib/fdbimpl.rb index 565bc915b9..52603e4d5b 100644 --- a/bindings/ruby/lib/fdbimpl.rb +++ b/bindings/ruby/lib/fdbimpl.rb @@ -170,7 +170,7 @@ module FDB Proc.new do || @setfunc.call(v[0], nil) end when String then Proc.new do |opt=nil| @setfunc.call(v[0], (opt.nil? ? opt : opt.encode('UTF-8')) ) end - when Fixnum then + when Integer then Proc.new do |opt| @setfunc.call(v[0], [opt].pack("q<")) end else raise ArgumentError, "Don't know how to set options of type #{v[2].class}" diff --git a/bindings/ruby/lib/fdbtuple.rb b/bindings/ruby/lib/fdbtuple.rb index 71b2402693..fc0b954841 100644 --- a/bindings/ruby/lib/fdbtuple.rb +++ b/bindings/ruby/lib/fdbtuple.rb @@ -35,8 +35,8 @@ module FDB @@STRING_CODE = 0x02 @@NESTED_CODE = 0x05 @@INT_ZERO_CODE = 0x14 - @@POS_INT_END = 0x1c - @@NEG_INT_START = 0x0c + @@POS_INT_END = 0x1d + @@NEG_INT_START = 0x0b @@FLOAT_CODE = 0x20 @@DOUBLE_CODE = 0x21 @@FALSE_CODE = 0x26 @@ -117,12 +117,28 @@ module FDB elsif code == @@STRING_CODE epos = find_terminator(v, pos+1) [v.slice(pos+1, epos-pos-1).gsub("\x00\xFF", "\x00").force_encoding("UTF-8"), epos+1] - elsif code >= @@INT_ZERO_CODE && code <= @@POS_INT_END + elsif code >= @@INT_ZERO_CODE && code < @@POS_INT_END n = code - @@INT_ZERO_CODE [("\x00" * (8-n) + v.slice(pos+1, n)).unpack("Q>")[0], pos+n+1] - elsif code >= @@NEG_INT_START and code < @@INT_ZERO_CODE + elsif code > @@NEG_INT_START and code < @@INT_ZERO_CODE n = @@INT_ZERO_CODE - code [("\x00" * (8-n) + v.slice(pos+1, n)).unpack("Q>")[0]-@@size_limits[n], pos+n+1] + elsif code == @@POS_INT_END + length = v.getbyte(pos+1) + val = 0 + length.times do |i| + val = val << 8 + val += v.getbyte(pos+2+i) + end + [val, pos+length+2] + elsif code == @@NEG_INT_START + length = v.getbyte(pos+1) ^ 0xff + val = 0 + length.times do |i| + val = val << 8 + val += v.getbyte(pos+2+i) + end + [val - (1 << (length*8)) + 1, pos+length+2] elsif code == @@FALSE_CODE [false, pos+1] elsif code == @@TRUE_CODE @@ -182,15 +198,34 @@ module FDB raise ArgumentError, "unsupported encoding #{v.encoding.name}" end elsif v.kind_of? Integer - raise RangeError, "value outside inclusive range -2**64+1 to 2**64-1" if v < -2**64+1 || v > 2**64-1 + raise RangeError, "Integer magnitude is too large (more than 255 bytes)" if v < -2**2040+1 || v > 2**2040-1 if v == 0 @@INT_ZERO_CODE.chr elsif v > 0 - n = bisect_left( @@size_limits, v ) - (20+n).chr + [v].pack("Q>").slice(8-n, n) + if v >= @@size_limits[-1] + length = (v.bit_length + 7) / 8 + result = @@POS_INT_END.chr + length.chr + length.times do |i| + result << ((v >> (8 * (length-i-1))) & 0xff) + end + result + else + n = bisect_left( @@size_limits, v ) + (@@INT_ZERO_CODE+n).chr + [v].pack("Q>").slice(8-n, n) + end else - n = bisect_left( @@size_limits, -v ) - (20-n).chr + [@@size_limits[n]+v].pack("Q>").slice(8-n, n) + if -v >= @@size_limits[-1] + length = (v.bit_length + 7) / 8 + v += (1 << (length * 8)) - 1 + result = @@NEG_INT_START.chr + (length ^ 0xff).chr + length.times do |i| + result << ((v >> (8 * (length-i-1))) & 0xff) + end + result + else + n = bisect_left( @@size_limits, -v ) + (@@INT_ZERO_CODE-n).chr + [@@size_limits[n]+v].pack("Q>").slice(8-n, n) + end end elsif v.kind_of? TrueClass @@TRUE_CODE.chr diff --git a/documentation/sphinx/source/api-ruby.rst b/documentation/sphinx/source/api-ruby.rst index 34f777e2d4..0898b03732 100644 --- a/documentation/sphinx/source/api-ruby.rst +++ b/documentation/sphinx/source/api-ruby.rst @@ -983,8 +983,8 @@ In the FoundationDB Ruby API, a tuple is an :class:`Enumerable` of elements of t | Unicode string | Any value ``v`` where ``v.kind_of? String == true`` and ``v.encoding`` is | ``String`` with encoding ``Encoding::UTF_8`` | | | ``Encoding::UTF_8`` | | +----------------------+-----------------------------------------------------------------------------+------------------------------------------------------------------------------+ -| 64-bit signed integer| Any value ``v`` where ``v.kind_of? Integer == true`` and ``-2**64+1 <= v <= | ``Fixnum`` or ``Bignum`` (depending on the magnitude of the value) | -| | 2**64-1`` | | +| Integer | Any value ``v`` where ``v.kind_of? Integer == true`` and | ``Integer`` | +| | ``-2**2040+1 <= v <= 2**2040-1`` | | +----------------------+-----------------------------------------------------------------------------+------------------------------------------------------------------------------+ | Floating point number| Any value ``v`` where ``v.kind_of? FDB::Tuple::SingleFloat`` where | :class:`FDB::Tuple::SingleFloat` | | (single-precision) | ``v.value.kind_of? Float`` and ``v.value`` fits inside an IEEE 754 32-bit | | From c6e9d2ff12f119e9d839ad87de4733376af101cd Mon Sep 17 00:00:00 2001 From: Justin Lowery Date: Thu, 24 May 2018 09:47:12 -0400 Subject: [PATCH 02/13] add uint64 encoding and decoding --- bindings/go/src/fdb/tuple/tuple.go | 41 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 2bea679d1a..c6ee47325d 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -150,7 +150,7 @@ func (p *packer) encodeBytes(code byte, b []byte) { func (p *packer) encodeInt(i int64) { if i == 0 { - p.putByte(0x14) + p.putByte(intZeroCode) return } @@ -164,7 +164,7 @@ func (p *packer) encodeInt(i int64) { binary.BigEndian.PutUint64(scratch[:], uint64(i)) case i < 0: n = bisectLeft(uint64(-i)) - p.putByte(byte(0x14 - n)) + p.putByte(byte(intZeroCode - n)) offsetEncoded := int64(sizeLimits[n]) + i binary.BigEndian.PutUint64(scratch[:], uint64(offsetEncoded)) } @@ -172,6 +172,21 @@ func (p *packer) encodeInt(i int64) { p.putBytes(scratch[8-n:]) } +func (p *packer) encodeUint(i uint64) { + if i == 0 { + p.putByte(intZeroCode) + return + } + + n := bisectLeft(i) + var scratch [8]byte + + p.putByte(byte(intZeroCode + n)) + binary.BigEndian.PutUint64(scratch[:], i) + + p.putBytes(scratch[8-n:]) +} + func (p *packer) encodeFloat(f float32) { var scratch [4]byte binary.BigEndian.PutUint32(scratch[:], math.Float32bits(f)) @@ -209,10 +224,14 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { if nested { p.putByte(0xff) } - case int64: - p.encodeInt(e) case int: p.encodeInt(int64(e)) + case uint: + p.encodeUint(uint64(e)) + case int64: + p.encodeInt(e) + case uint64: + p.encodeUint(e) case []byte: p.encodeBytes(bytesCode, e) case fdb.KeyConvertible: @@ -282,9 +301,9 @@ func decodeString(b []byte) (string, int) { return string(bp), idx } -func decodeInt(b []byte) (int64, int) { +func decodeInt(b []byte) (interface{}, int) { if b[0] == intZeroCode { - return 0, 1 + return uint64(0), 1 } var neg bool @@ -298,14 +317,14 @@ func decodeInt(b []byte) (int64, int) { bp := make([]byte, 8) copy(bp[8-n:], b[1:n+1]) - var ret int64 - - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) - if neg { - ret -= int64(sizeLimits[n]) + var ret int64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) + return ret - int64(sizeLimits[n]), n + 1 } + var ret uint64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) return ret, n + 1 } From 9909c1c959d64ae1a38c6eb8c5e9e2b341885187 Mon Sep 17 00:00:00 2001 From: Justin Lowery Date: Thu, 24 May 2018 10:22:33 -0400 Subject: [PATCH 03/13] add uint64 and uint types to the package documentation comments --- bindings/go/src/fdb/tuple/tuple.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index c6ee47325d..e6b82aecb4 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -31,8 +31,8 @@ // // FoundationDB tuples can currently encode byte and unicode strings, integers, // floats, doubles, booleans, UUIDs, tuples, and NULL values. In Go these are -// represented as []byte (or fdb.KeyConvertible), string, int64 (or int), -// float32, float64, bool, UUID, Tuple, and nil. +// represented as []byte (or fdb.KeyConvertible), string, uint64 (or uint), +// int64 (or int), float32, float64, bool, UUID, Tuple, and nil. package tuple import ( @@ -50,7 +50,7 @@ import ( // result in a runtime panic). // // The valid types for TupleElement are []byte (or fdb.KeyConvertible), string, -// int64 (or int), float, double, bool, UUID, Tuple, and nil. +// uint64 (or uint), int64 (or int), float, double, bool, UUID, Tuple, and nil. type TupleElement interface{} // Tuple is a slice of objects that can be encoded as FoundationDB tuples. If @@ -59,7 +59,7 @@ type TupleElement interface{} // // Given a Tuple T containing objects only of these types, then T will be // identical to the Tuple returned by unpacking the byte slice obtained by -// packing T (modulo type normalization to []byte and int64). +// packing T (modulo type normalization to []byte, uint64, and int64). type Tuple []TupleElement // UUID wraps a basic byte array as a UUID. We do not provide any special @@ -262,8 +262,8 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // Pack returns a new byte slice encoding the provided tuple. Pack will panic if // the tuple contains an element of any type other than []byte, -// fdb.KeyConvertible, string, int64, int, float32, float64, bool, tuple.UUID, -// nil, or a Tuple with elements of valid types. +// fdb.KeyConvertible, string, uint64, uint, int64, int, float32, float64, bool, +// tuple.UUID, nil, or a Tuple with elements of valid types. // // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to // call Pack when using a Tuple with a FoundationDB API function that requires a From 0b00b4d99b205800781bce8d9779c9efd477a587 Mon Sep 17 00:00:00 2001 From: Justin Lowery Date: Thu, 24 May 2018 21:47:27 -0400 Subject: [PATCH 04/13] keep original behavior related to decoding integers to int64, only using uint64 when needed --- bindings/go/src/fdb/tuple/tuple.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index e6b82aecb4..b425aadeed 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -303,7 +303,7 @@ func decodeString(b []byte) (string, int) { func decodeInt(b []byte) (interface{}, int) { if b[0] == intZeroCode { - return uint64(0), 1 + return int64(0), 1 } var neg bool @@ -318,14 +318,20 @@ func decodeInt(b []byte) (interface{}, int) { copy(bp[8-n:], b[1:n+1]) if neg { - var ret int64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) - return ret - int64(sizeLimits[n]), n + 1 + var retInt int64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) + return retInt - int64(sizeLimits[n]), n + 1 } - var ret uint64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) - return ret, n + 1 + var retInt int64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) + if retInt > 0 { + return retInt, n + 1 + } + + var retUint uint64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retUint) + return retUint, n + 1 } func decodeFloat(b []byte) (float32, int) { From 36b3818aef1874b70b527c82390bd07bebdfb974 Mon Sep 17 00:00:00 2001 From: Justin Lowery Date: Fri, 25 May 2018 19:19:11 -0400 Subject: [PATCH 05/13] remove redundant calls to NewBuffer that use the bp slice. remove redundant declaration of retInt --- bindings/go/src/fdb/tuple/tuple.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index b425aadeed..f8078d7490 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -316,21 +316,21 @@ func decodeInt(b []byte) (interface{}, int) { bp := make([]byte, 8) copy(bp[8-n:], b[1:n+1]) + buf := bytes.NewBuffer(bp) + var retInt int64 if neg { - var retInt int64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) + binary.Read(buf, binary.BigEndian, &retInt) return retInt - int64(sizeLimits[n]), n + 1 } - var retInt int64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) + binary.Read(buf, binary.BigEndian, &retInt) if retInt > 0 { return retInt, n + 1 } var retUint uint64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retUint) + binary.Read(buf, binary.BigEndian, &retUint) return retUint, n + 1 } From f112ba4f0766a6f3630a2f485455b400057f38c2 Mon Sep 17 00:00:00 2001 From: Justin Lowery Date: Mon, 28 May 2018 02:37:20 -0400 Subject: [PATCH 06/13] restore NewBuffer calls, as buffer was emptied on Read calls when the value was a uint64 --- bindings/go/src/fdb/tuple/tuple.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index f8078d7490..c0a05baa6f 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -316,21 +316,20 @@ func decodeInt(b []byte) (interface{}, int) { bp := make([]byte, 8) copy(bp[8-n:], b[1:n+1]) - buf := bytes.NewBuffer(bp) var retInt int64 if neg { - binary.Read(buf, binary.BigEndian, &retInt) + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) return retInt - int64(sizeLimits[n]), n + 1 } - binary.Read(buf, binary.BigEndian, &retInt) + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) if retInt > 0 { return retInt, n + 1 } var retUint uint64 - binary.Read(buf, binary.BigEndian, &retUint) + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retUint) return retUint, n + 1 } From bee8d12b24cdd608a7eb919d370488fe05924b41 Mon Sep 17 00:00:00 2001 From: Justin Lowery Date: Tue, 29 May 2018 17:04:53 -0400 Subject: [PATCH 07/13] remove multiple binary.Read calls and use a single type conversion instead --- bindings/go/src/fdb/tuple/tuple.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index c0a05baa6f..2209937ba1 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -317,20 +317,18 @@ func decodeInt(b []byte) (interface{}, int) { bp := make([]byte, 8) copy(bp[8-n:], b[1:n+1]) - var retInt int64 + var ret int64 + binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) + if neg { - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) - return retInt - int64(sizeLimits[n]), n + 1 + return ret - int64(sizeLimits[n]), n + 1 } - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retInt) - if retInt > 0 { - return retInt, n + 1 + if ret > 0 { + return ret, n + 1 } - var retUint uint64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &retUint) - return retUint, n + 1 + return uint64(ret), n + 1 } func decodeFloat(b []byte) (float32, int) { From 377f5e2c48f454b6c98d3ed2295f62b568876ee0 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Tue, 30 Oct 2018 15:06:47 -0700 Subject: [PATCH 08/13] Add release notes and fix formatting --- bindings/go/src/fdb/tuple/tuple.go | 5 ++--- documentation/sphinx/source/release-notes.rst | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index af9b8eeb43..ad4a94f2ff 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -32,7 +32,7 @@ // FoundationDB tuples can currently encode byte and unicode strings, integers, // large integers, floats, doubles, booleans, UUIDs, tuples, and NULL values. // In Go these are represented as []byte (or fdb.KeyConvertible), string, int64 -// (or int, uint, uint64), *big.Int (or big.Int), float32, float64, bool, +// (or int, uint, uint64), *big.Int (or big.Int), float32, float64, bool, // UUID, Tuple, and nil. package tuple @@ -166,7 +166,6 @@ func (p *packer) encodeUint(i uint64) { p.putBytes(scratch[8-n:]) } - func (p *packer) encodeInt(i int64) { if i >= 0 { p.encodeUint(uint64(i)) @@ -302,7 +301,7 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // the tuple contains an element of any type other than []byte, // fdb.KeyConvertible, string, int64, int, uint64, uint, *big.Int, big.Int, float32, // float64, bool, tuple.UUID, nil, or a Tuple with elements of valid types. It will -// also panic if an integer is specified with a value outside the range +// also panic if an integer is specified with a value outside the range // [-2**2040+1, 2**2040-1] // // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 0dd29acf5e..862807707a 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -93,7 +93,8 @@ Bindings * Java: the `Versionstamp::getUserVersion() `_ method did not handle user versions greater than ``0x00FF`` due to operator precedence errors. [6.0.11] `(Issue #761) `_ * Python: bindings didn't work with Python 3.7 because of the new `async` keyword. [6.0.13] `(Issue #830) `_ * Go: `PrefixRange` didn't correctly return an error if it failed to generate the range. [6.0.15] `(PR #878) `_ - +* Go: Add Tuple layer support for `uint`, `uint64`, and `*big.Int` integers up to 255 bytes. Integer values will be decoded into the first of `int64`, `uint64`, or `*big.Int` in which they fit. [6.0.15] +* Ruby: Add Tuple layer support for integers up to 255 bytes. [6.0.15] Other Changes ------------- From 23d5382ceacdb40dffdd617b25913f8ce15c8c6b Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 31 Oct 2018 09:19:13 -0700 Subject: [PATCH 09/13] Add uint64 handling to golang stack tester tuples --- bindings/go/src/_stacktester/stacktester.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 56ccb1667d..aff3059c22 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -104,7 +104,7 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) { switch el := ret.item.(type) { case []byte: ret.item = el - case int64, *big.Int, string, bool, tuple.UUID, float32, float64, tuple.Tuple: + case int64, uint64, *big.Int, string, bool, tuple.UUID, float32, float64, tuple.Tuple: ret.item = el case fdb.Key: ret.item = []byte(el) @@ -175,7 +175,7 @@ func tupleToString(t tuple.Tuple) string { buffer.WriteString(", ") } switch el := el.(type) { - case int64: + case int64, uint64: buffer.WriteString(fmt.Sprintf("%d", el)) case *big.Int: buffer.WriteString(fmt.Sprintf("%s", el)) @@ -187,9 +187,7 @@ func tupleToString(t tuple.Tuple) string { buffer.WriteString(fmt.Sprintf("%t", el)) case tuple.UUID: buffer.WriteString(hex.EncodeToString(el[:])) - case float32: - buffer.WriteString(fmt.Sprintf("%f", el)) - case float64: + case float32, float64: buffer.WriteString(fmt.Sprintf("%f", el)) case nil: buffer.WriteString("nil") @@ -208,7 +206,7 @@ func (sm *StackMachine) dumpStack() { fmt.Printf(" %d.", sm.stack[i].idx) el := sm.stack[i].item switch el := el.(type) { - case int64: + case int64, uint64: fmt.Printf(" %d", el) case *big.Int: fmt.Printf(" %s", el) @@ -230,9 +228,7 @@ func (sm *StackMachine) dumpStack() { fmt.Printf(" %s", tupleToString(el)) case tuple.UUID: fmt.Printf(" %s", hex.EncodeToString(el[:])) - case float32: - fmt.Printf(" %f", el) - case float64: + case float32, float64: fmt.Printf(" %f", el) case nil: fmt.Printf(" nil") @@ -501,12 +497,18 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { x = x1 case int64: x = big.NewInt(x1) + case uint64: + x = new(big.Int) + x.SetUint64(x1) } switch y1 := sm.waitAndPop().item.(type) { case *big.Int: y = y1 case int64: y = big.NewInt(y1) + case uint64: + y = new(big.Int) + y.SetUint64(y1) } sm.store(idx, x.Sub(x, y)) From 5035c51d15eab16360069dda2b6299896c715b3a Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 31 Oct 2018 14:09:23 -0700 Subject: [PATCH 10/13] Some fixes to the type chosen for decoding integers in Go. Fix to ruby encoding at byte boundaries. Add execute bit to ruby tester. --- bindings/go/src/fdb/tuple/tuple.go | 21 ++++++++++++++------- bindings/ruby/lib/fdbtuple.rb | 6 +++--- bindings/ruby/tests/tester.rb | 0 3 files changed, 17 insertions(+), 10 deletions(-) mode change 100644 => 100755 bindings/ruby/tests/tester.rb diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index ad4a94f2ff..51cd6df5a4 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -99,6 +99,8 @@ var sizeLimits = []uint64{ 1<<(8*8) - 1, } +var minInt64BigInt = big.NewInt(math.MinInt64) + func bisectLeft(u uint64) int { var n int for sizeLimits[n] < u { @@ -188,7 +190,7 @@ func (p *packer) encodeBigInt(i *big.Int) { panic(fmt.Sprintf("Integer magnitude is too large (more than 255 bytes)")) } - if i.Sign() > 0 { + if i.Sign() >= 0 { intBytes := i.Bytes() if length > 8 { p.putByte(byte(posIntEnd)) @@ -370,7 +372,7 @@ func decodeInt(b []byte) (interface{}, int) { return uint64(ret), n + 1 } -func decodeBigInt(b []byte) (*big.Int, int) { +func decodeBigInt(b []byte) (interface{}, int) { val := new(big.Int) offset := 1 var length int @@ -383,10 +385,8 @@ func decodeBigInt(b []byte) (*big.Int, int) { offset += 1 } else { - length = int(b[0]) - intZeroCode - if length < 0 { - length = -length - } + // Must be a negative 8 byte integer + length = 8 } val.SetBytes(b[offset : length+offset]) @@ -397,6 +397,11 @@ func decodeBigInt(b []byte) (*big.Int, int) { val.Sub(val, sub) } + // This is the only value that fits in an int64 or uint64 that is decoded with this function + if val.Cmp(minInt64BigInt) == 0 { + return val.Int64(), length + offset + } + return val, length + offset } @@ -448,7 +453,9 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { el, off = decodeBytes(b[i:]) case b[i] == stringCode: el, off = decodeString(b[i:]) - case negIntStart+1 < b[i] && b[i] < posIntEnd-1: + case negIntStart+1 < b[i] && b[i] < posIntEnd: + el, off = decodeInt(b[i:]) + case negIntStart+1 == b[i] && (b[i+1] & 0x80 != 0): el, off = decodeInt(b[i:]) case negIntStart <= b[i] && b[i] <= posIntEnd: el, off = decodeBigInt(b[i:]) diff --git a/bindings/ruby/lib/fdbtuple.rb b/bindings/ruby/lib/fdbtuple.rb index fc0b954841..6475a8e0a2 100644 --- a/bindings/ruby/lib/fdbtuple.rb +++ b/bindings/ruby/lib/fdbtuple.rb @@ -202,7 +202,7 @@ module FDB if v == 0 @@INT_ZERO_CODE.chr elsif v > 0 - if v >= @@size_limits[-1] + if v > @@size_limits[-1] length = (v.bit_length + 7) / 8 result = @@POS_INT_END.chr + length.chr length.times do |i| @@ -214,8 +214,8 @@ module FDB (@@INT_ZERO_CODE+n).chr + [v].pack("Q>").slice(8-n, n) end else - if -v >= @@size_limits[-1] - length = (v.bit_length + 7) / 8 + if -v > @@size_limits[-1] + length = ((-v).bit_length + 7) / 8 v += (1 << (length * 8)) - 1 result = @@NEG_INT_START.chr + (length ^ 0xff).chr length.times do |i| diff --git a/bindings/ruby/tests/tester.rb b/bindings/ruby/tests/tester.rb old mode 100644 new mode 100755 From 3c8347e724cd95c781731cf5972d9e52b4574412 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 13 Nov 2018 11:40:13 -0800 Subject: [PATCH 11/13] add comments for some tricky int-packing code --- bindings/go/src/fdb/tuple/tuple.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 51cd6df5a4..352c63381c 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -214,6 +214,10 @@ func (p *packer) encodeBigInt(i *big.Int) { p.putByte(byte(intZeroCode - length)) } + // For large negative numbers whose absolute value begins with 0xff bytes, + // the transformed bytes may begin with 0x00 bytes. However, intBytes + // will only contain the non-zero suffix, so this loop is needed to make + // the value written be the correct length for i := len(intBytes); i < length; i++ { p.putByte(0x00) } @@ -369,6 +373,11 @@ func decodeInt(b []byte) (interface{}, int) { return ret, n + 1 } + // The encoded value claimed to be positive yet when put in an int64 + // produced a negative value. This means that the number must be a positive + // 64-bit value that uses the most significant bit. This can be fit in a + // uint64, so return that. Note that this is the *only* time we return + // a uint64. return uint64(ret), n + 1 } @@ -455,7 +464,7 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { el, off = decodeString(b[i:]) case negIntStart+1 < b[i] && b[i] < posIntEnd: el, off = decodeInt(b[i:]) - case negIntStart+1 == b[i] && (b[i+1] & 0x80 != 0): + case negIntStart+1 == b[i] && (b[i+1]&0x80 != 0): el, off = decodeInt(b[i:]) case negIntStart <= b[i] && b[i] <= posIntEnd: el, off = decodeBigInt(b[i:]) From 6ab59b2ec2f2d401be0c344b42d1397758d7e8d2 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 13 Nov 2018 11:47:28 -0800 Subject: [PATCH 12/13] update PR number to (hopefully) final PR for bigint feature --- documentation/sphinx/source/release-notes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 54b651bb1c..d42367c638 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -100,8 +100,8 @@ Bindings * Java: the `Versionstamp::getUserVersion() `_ method did not handle user versions greater than ``0x00FF`` due to operator precedence errors. [6.0.11] `(Issue #761) `_ * Python: bindings didn't work with Python 3.7 because of the new `async` keyword. [6.0.13] `(Issue #830) `_ * Go: `PrefixRange` didn't correctly return an error if it failed to generate the range. [6.0.15] `(PR #878) `_ -* Go: Add Tuple layer support for `uint`, `uint64`, and `*big.Int` integers up to 255 bytes. Integer values will be decoded into the first of `int64`, `uint64`, or `*big.Int` in which they fit. `(PR #889) `_ [6.0.15] -* Ruby: Add Tuple layer support for integers up to 255 bytes. `(PR #889) `_ [6.0.15] +* Go: Add Tuple layer support for `uint`, `uint64`, and `*big.Int` integers up to 255 bytes. Integer values will be decoded into the first of `int64`, `uint64`, or `*big.Int` in which they fit. `(PR #915) `_ [6.0.15] +* Ruby: Add Tuple layer support for integers up to 255 bytes. `(PR #915) `_ [6.0.15] * Python: bindings didn't work with Python 3.7 because of the new ``async`` keyword. [6.0.13] `(Issue #830) `_ * Go: ``PrefixRange`` didn't correctly return an error if it failed to generate the range. [6.0.15] `(PR #878) `_ From 8424fc57ec5deeb62c326243b8446508a51b8037 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Tue, 13 Nov 2018 11:55:54 -0800 Subject: [PATCH 13/13] add period to comment --- bindings/go/src/fdb/tuple/tuple.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 352c63381c..afd959420f 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -217,7 +217,7 @@ func (p *packer) encodeBigInt(i *big.Int) { // For large negative numbers whose absolute value begins with 0xff bytes, // the transformed bytes may begin with 0x00 bytes. However, intBytes // will only contain the non-zero suffix, so this loop is needed to make - // the value written be the correct length + // the value written be the correct length. for i := len(intBytes); i < length; i++ { p.putByte(0x00) }