From 7ac098dc0d79a17f74e01710519987586698dafe Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Mon, 25 Feb 2019 18:39:14 -0500 Subject: [PATCH 01/10] Add Versionstamp support to the Go Tuple layer --- bindings/bindingtester/known_testers.py | 2 +- bindings/go/src/_stacktester/stacktester.go | 26 ++++- bindings/go/src/fdb/tuple/tuple.go | 117 +++++++++++++++++++- 3 files changed, 139 insertions(+), 6 deletions(-) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index 8abe8c5741..fee09f5adf 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -62,6 +62,6 @@ testers = { 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION), 'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), 'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), - 'go': Tester('go', _absolute_path('go/build/bin/_stacktester'), 2040, 200, MAX_API_VERSION), + 'go': Tester('go', _absolute_path('go/build/bin/_stacktester'), 2040, 200, MAX_API_VERSION, types=ALL_TYPES), 'flow': Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION, directory_snapshot_ops_enabled=False), } diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index f76641629e..151f1e766c 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -25,8 +25,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/apple/foundationdb/bindings/go/src/fdb" - "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" "log" "math/big" "os" @@ -37,6 +35,9 @@ import ( "strings" "sync" "time" + + "github.com/apple/foundationdb/bindings/go/src/fdb" + "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" ) const verbose bool = false @@ -661,6 +662,25 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { t = append(t, sm.waitAndPop().item) } sm.store(idx, []byte(t.Pack())) + case op == "TUPLE_PACK_VERSIONSTAMP": + var t tuple.Tuple + count := sm.waitAndPop().item.(int64) + for i := 0; i < int(count); i++ { + t = append(t, sm.waitAndPop().item) + } + + incomplete, err := t.HasIncompleteVersionstamp() + if incomplete == false { + sm.store(idx, []byte("ERROR: NONE")) + } else { + if err != nil { + sm.store(idx, []byte("ERROR: MULTIPLE")) + } else { + packed := t.Pack() + sm.store(idx, "OK") + sm.store(idx, packed) + } + } case op == "TUPLE_UNPACK": t, e := tuple.Unpack(fdb.Key(sm.waitAndPop().item.([]byte))) if e != nil { @@ -893,7 +913,7 @@ func main() { log.Fatal("API version not equal to value selected") } - db, e = fdb.OpenDatabase(clusterFile) + db, e = fdb.Open(clusterFile, []byte("DB")) if e != nil { log.Fatal(e) } diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index afd959420f..0e0cc732bd 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -39,6 +39,7 @@ package tuple import ( "bytes" "encoding/binary" + "errors" "fmt" "math" "math/big" @@ -72,6 +73,37 @@ type Tuple []TupleElement // an instance of this type. type UUID [16]byte +// Versionstamp . +type Versionstamp struct { + TransactionVersion [10]byte + UserVersion uint16 +} + +var incompleteTransactionVersion = [10]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} + +const versionstampLength = 13 + +// IncompleteVersionstamp . +func IncompleteVersionstamp(userVersion uint16) Versionstamp { + return Versionstamp{ + TransactionVersion: incompleteTransactionVersion, + UserVersion: userVersion, + } +} + +// Bytes . +func (v Versionstamp) Bytes() []byte { + var scratch [12]byte + + copy(scratch[:], v.TransactionVersion[:]) + + binary.BigEndian.PutUint16(scratch[10:], v.UserVersion) + + fmt.Println(scratch) + + return scratch[:] +} + // Type codes: These prefix the different elements in a packed Tuple // to indicate what type they are. const nilCode = 0x00 @@ -86,6 +118,7 @@ const doubleCode = 0x21 const falseCode = 0x26 const trueCode = 0x27 const uuidCode = 0x30 +const versionstampCode = 0x33 var sizeLimits = []uint64{ 1<<(0*8) - 1, @@ -122,7 +155,15 @@ func adjustFloatBytes(b []byte, encode bool) { } type packer struct { - buf []byte + versionstampPos int32 + buf []byte +} + +func newPacker() *packer { + return &packer{ + versionstampPos: -1, + buf: make([]byte, 0, 64), + } } func (p *packer) putByte(b byte) { @@ -249,6 +290,18 @@ func (p *packer) encodeUUID(u UUID) { p.putBytes(u[:]) } +func (p *packer) encodeVersionstamp(v Versionstamp) { + p.putByte(versionstampCode) + + if p.versionstampPos != 0 && v.TransactionVersion == incompleteTransactionVersion { + panic(fmt.Sprintf("Tuple can only contain one unbound versionstamp")) + } else { + p.versionstampPos = int32(len(p.buf)) + } + + p.putBytes(v.Bytes()) +} + func (p *packer) encodeTuple(t Tuple, nested bool) { if nested { p.putByte(nestedCode) @@ -293,6 +346,8 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { } case UUID: p.encodeUUID(e) + case Versionstamp: + p.encodeVersionstamp(e) default: panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i])) } @@ -314,11 +369,50 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // call Pack when using a Tuple with a FoundationDB API function that requires a // key. func (t Tuple) Pack() []byte { - p := packer{buf: make([]byte, 0, 64)} + p := newPacker() p.encodeTuple(t, false) return p.buf } +// PackWithVersionstamp packs the specified tuple into a key for versionstamp operations +func (t Tuple) PackWithVersionstamp() ([]byte, error) { + hasVersionstamp, err := t.HasIncompleteVersionstamp() + if err != nil { + return nil, err + } + + p := newPacker() + p.encodeTuple(t, false) + + if hasVersionstamp { + var scratch [4]byte + binary.LittleEndian.PutUint32(scratch[:], uint32(p.versionstampPos)) + p.putBytes(scratch[:]) + } + + return p.buf, nil +} + +// HasIncompleteVersionstamp determines if there is at least one incomplete versionstamp in a tuple +func (t Tuple) HasIncompleteVersionstamp() (bool, error) { + incompleteCount := 0 + for _, el := range t { + switch e := el.(type) { + case Versionstamp: + if e.TransactionVersion == incompleteTransactionVersion { + incompleteCount++ + } + } + } + + var err error + if incompleteCount > 1 { + err = errors.New("Tuple can only contain one unbound versionstamp") + } + + return incompleteCount == 1, err +} + func findTerminator(b []byte) int { bp := b var length int @@ -438,6 +532,20 @@ func decodeUUID(b []byte) (UUID, int) { return u, 17 } +func decodeVersionstamp(b []byte) (Versionstamp, int) { + var transactionVersion [10]byte + var userVersion uint16 + + copy(transactionVersion[:], b[1:11]) + + userVersion = binary.BigEndian.Uint16(b[11:]) + + return Versionstamp{ + TransactionVersion: transactionVersion, + UserVersion: userVersion, + }, versionstampLength +} + func decodeTuple(b []byte, nested bool) (Tuple, int, error) { var t Tuple @@ -489,6 +597,11 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { return nil, i, fmt.Errorf("insufficient bytes to decode UUID starting at position %d of byte array for tuple", i) } el, off = decodeUUID(b[i:]) + case b[i] == versionstampCode: + if i+versionstampLength > len(b) { + return nil, i, fmt.Errorf("insufficient bytes to decode Versionstamp starting at position %d of byte array for tuple", i) + } + el, off = decodeVersionstamp(b[i:]) case b[i] == nestedCode: var err error el, off, err = decodeTuple(b[i+1:], true) From b2f26224b9a46a6f96a3bcccd6a7bbdee66ac198 Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Mon, 25 Feb 2019 18:41:57 -0500 Subject: [PATCH 02/10] Revert unintentional change back to old API --- bindings/go/src/_stacktester/stacktester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 151f1e766c..6a1100ffe7 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -913,7 +913,7 @@ func main() { log.Fatal("API version not equal to value selected") } - db, e = fdb.Open(clusterFile, []byte("DB")) + db, e = fdb.OpenDatabase(clusterFile) if e != nil { log.Fatal(e) } From 292bb6ab0f564a4649f20b2e4d60ab5e918758ea Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Mon, 25 Feb 2019 18:57:28 -0500 Subject: [PATCH 03/10] Make `versionstampLength` constant equal Versionstamp actual length. --- bindings/go/src/fdb/tuple/tuple.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 0e0cc732bd..2c30705ba0 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -81,7 +81,7 @@ type Versionstamp struct { var incompleteTransactionVersion = [10]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} -const versionstampLength = 13 +const versionstampLength = 12 // IncompleteVersionstamp . func IncompleteVersionstamp(userVersion uint16) Versionstamp { @@ -543,7 +543,7 @@ func decodeVersionstamp(b []byte) (Versionstamp, int) { return Versionstamp{ TransactionVersion: transactionVersion, UserVersion: userVersion, - }, versionstampLength + }, versionstampLength + 1 } func decodeTuple(b []byte, nested bool) (Tuple, int, error) { @@ -598,7 +598,7 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { } el, off = decodeUUID(b[i:]) case b[i] == versionstampCode: - if i+versionstampLength > len(b) { + if i+versionstampLength+1 > len(b) { return nil, i, fmt.Errorf("insufficient bytes to decode Versionstamp starting at position %d of byte array for tuple", i) } el, off = decodeVersionstamp(b[i:]) From 4dd04862c7037a5ba246eab3ea0aa4a40f644942 Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Mon, 25 Feb 2019 19:05:45 -0500 Subject: [PATCH 04/10] Flatten if statements --- bindings/go/src/_stacktester/stacktester.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 6a1100ffe7..8ef2c3d78f 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -672,14 +672,12 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { incomplete, err := t.HasIncompleteVersionstamp() if incomplete == false { sm.store(idx, []byte("ERROR: NONE")) + } else if err != nil { + sm.store(idx, []byte("ERROR: MULTIPLE")) } else { - if err != nil { - sm.store(idx, []byte("ERROR: MULTIPLE")) - } else { - packed := t.Pack() - sm.store(idx, "OK") - sm.store(idx, packed) - } + packed := t.Pack() + sm.store(idx, "OK") + sm.store(idx, packed) } case op == "TUPLE_UNPACK": t, e := tuple.Unpack(fdb.Key(sm.waitAndPop().item.([]byte))) From 05d347e194d0de66f635b61b72477ead4b75c7da Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Mon, 25 Feb 2019 19:08:29 -0500 Subject: [PATCH 05/10] Push byte slice instead of string onto the stack --- bindings/go/src/_stacktester/stacktester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 8ef2c3d78f..b2012546e7 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -676,7 +676,7 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { sm.store(idx, []byte("ERROR: MULTIPLE")) } else { packed := t.Pack() - sm.store(idx, "OK") + sm.store(idx, []byte("OK")) sm.store(idx, packed) } case op == "TUPLE_UNPACK": From 2fbc7522e4c75168b156eccdc4f5e75bc99ed237 Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Wed, 6 Mar 2019 18:25:02 -0500 Subject: [PATCH 06/10] review feedback: fix bindingtester test, add comments to versionstamp and other structures, handle nested tuples, handle prefix []byte in PackWithVersionstamp --- bindings/go/src/_stacktester/stacktester.go | 17 +++-- bindings/go/src/fdb/tuple/tuple.go | 84 ++++++++++++++++----- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index b2012546e7..0c925af503 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -105,7 +105,7 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) { switch el := ret.item.(type) { case []byte: ret.item = el - case int64, uint64, *big.Int, string, bool, tuple.UUID, float32, float64, tuple.Tuple: + case int64, uint64, *big.Int, string, bool, tuple.UUID, float32, float64, tuple.Tuple, tuple.Versionstamp: ret.item = el case fdb.Key: ret.item = []byte(el) @@ -662,20 +662,21 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { t = append(t, sm.waitAndPop().item) } sm.store(idx, []byte(t.Pack())) - case op == "TUPLE_PACK_VERSIONSTAMP": + case op == "TUPLE_PACK_WITH_VERSIONSTAMP": var t tuple.Tuple - count := sm.waitAndPop().item.(int64) - for i := 0; i < int(count); i++ { + + prefix := sm.waitAndPop().item.([]byte) + c := sm.waitAndPop().item.(int64) + for i := 0; i < int(c); i++ { t = append(t, sm.waitAndPop().item) } - incomplete, err := t.HasIncompleteVersionstamp() - if incomplete == false { + packed, err := t.PackWithVersionstamp(prefix) + if err != nil && strings.Contains(err.Error(), "No incomplete") { sm.store(idx, []byte("ERROR: NONE")) } else if err != nil { sm.store(idx, []byte("ERROR: MULTIPLE")) } else { - packed := t.Pack() sm.store(idx, []byte("OK")) sm.store(idx, packed) } @@ -911,7 +912,7 @@ func main() { log.Fatal("API version not equal to value selected") } - db, e = fdb.OpenDatabase(clusterFile) + db, e = fdb.Open(clusterFile, []byte("DB")) if e != nil { log.Fatal(e) } diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 2c30705ba0..d3f289e2eb 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -73,7 +73,10 @@ type Tuple []TupleElement // an instance of this type. type UUID [16]byte -// Versionstamp . +// Versionstamp is struct for a FoundationDB verionstamp. Versionstamps are +// 12 bytes long composed of a 10 byte transaction version and a 2 byte user +// version. The transaction version is filled in at commit time and the user +// version is provided by your layer during a transaction. type Versionstamp struct { TransactionVersion [10]byte UserVersion uint16 @@ -83,7 +86,8 @@ var incompleteTransactionVersion = [10]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, const versionstampLength = 12 -// IncompleteVersionstamp . +// IncompleteVersionstamp is the constructor you should use to make +// an incomplete versionstamp to use in a tuple. func IncompleteVersionstamp(userVersion uint16) Versionstamp { return Versionstamp{ TransactionVersion: incompleteTransactionVersion, @@ -91,16 +95,14 @@ func IncompleteVersionstamp(userVersion uint16) Versionstamp { } } -// Bytes . +// Bytes converts a Versionstamp struct to a byte slice for encoding in a tuple. func (v Versionstamp) Bytes() []byte { - var scratch [12]byte + var scratch [versionstampLength]byte copy(scratch[:], v.TransactionVersion[:]) binary.BigEndian.PutUint16(scratch[10:], v.UserVersion) - fmt.Println(scratch) - return scratch[:] } @@ -293,8 +295,8 @@ func (p *packer) encodeUUID(u UUID) { func (p *packer) encodeVersionstamp(v Versionstamp) { p.putByte(versionstampCode) - if p.versionstampPos != 0 && v.TransactionVersion == incompleteTransactionVersion { - panic(fmt.Sprintf("Tuple can only contain one unbound versionstamp")) + if p.versionstampPos != -1 && v.TransactionVersion == incompleteTransactionVersion { + panic(fmt.Sprintf("Tuple can only contain one incomplete versionstamp")) } else { p.versionstampPos = int32(len(p.buf)) } @@ -368,49 +370,93 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to // call Pack when using a Tuple with a FoundationDB API function that requires a // key. +// +// This method will panic if it contains an incomplete Versionstamp. Use +// PackWithVersionstamp instead. +// func (t Tuple) Pack() []byte { p := newPacker() p.encodeTuple(t, false) return p.buf } -// PackWithVersionstamp packs the specified tuple into a key for versionstamp operations -func (t Tuple) PackWithVersionstamp() ([]byte, error) { +// PackWithVersionstamp packs the specified tuple into a key for versionstamp +// operations. See Pack for more information. This function will return an error +// if you attempt to pack a tuple with more than one versionstamp. This function will +// return an error if you attempt to pack a tuple with a versionstamp position larger +// than an uint16 on apiVersion < 520. +func (t Tuple) PackWithVersionstamp(prefix []byte) ([]byte, error) { hasVersionstamp, err := t.HasIncompleteVersionstamp() if err != nil { return nil, err } + if hasVersionstamp == false { + return nil, errors.New("No incomplete versionstamp included in tuple pack with versionstamp") + } + p := newPacker() + + prefixLength := int32(0) + if prefix != nil { + prefixLength = int32(len(prefix)) + p.putBytes(prefix) + } + p.encodeTuple(t, false) if hasVersionstamp { var scratch [4]byte - binary.LittleEndian.PutUint32(scratch[:], uint32(p.versionstampPos)) - p.putBytes(scratch[:]) + var offsetIndex int + + apiVersion := fdb.MustGetAPIVersion() + if apiVersion < 520 { + if p.versionstampPos > math.MaxUint16 { + return nil, errors.New("Versionstamp position too large") + } + + offsetIndex = 1 + binary.LittleEndian.PutUint16(scratch[:], uint16(prefixLength+p.versionstampPos)) + } else { + offsetIndex = 3 + binary.LittleEndian.PutUint32(scratch[:], uint32(prefixLength+p.versionstampPos)) + } + + p.putBytes(scratch[0:offsetIndex]) } return p.buf, nil } -// HasIncompleteVersionstamp determines if there is at least one incomplete versionstamp in a tuple +// HasIncompleteVersionstamp determines if there is at least one incomplete +// versionstamp in a tuple. This function will return an error this tuple has +// more than one versionstamp. func (t Tuple) HasIncompleteVersionstamp() (bool, error) { + incompleteCount := t.countIncompleteVersionstamps() + + var err error + if incompleteCount > 1 { + err = errors.New("Tuple can only contain one incomplete versionstamp") + } + + return incompleteCount == 1, err +} + +func (t Tuple) countIncompleteVersionstamps() int { incompleteCount := 0 + for _, el := range t { switch e := el.(type) { case Versionstamp: if e.TransactionVersion == incompleteTransactionVersion { incompleteCount++ } + case Tuple: + incompleteCount += e.countIncompleteVersionstamps() } } - var err error - if incompleteCount > 1 { - err = errors.New("Tuple can only contain one unbound versionstamp") - } - - return incompleteCount == 1, err + return incompleteCount } func findTerminator(b []byte) int { From 77f7c0721f2ad93f5d4f03a7da49220987fc3c84 Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Wed, 6 Mar 2019 18:27:43 -0500 Subject: [PATCH 07/10] revent this again because my environment is dumb --- bindings/go/src/_stacktester/stacktester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 0c925af503..d2dc1bcbe3 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -912,7 +912,7 @@ func main() { log.Fatal("API version not equal to value selected") } - db, e = fdb.Open(clusterFile, []byte("DB")) + db, e = fdb.OpenDatabase(clusterFile) if e != nil { log.Fatal(e) } From 806655675381eccb67ae5a6c2acb69aabd84083a Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Sat, 9 Mar 2019 10:48:22 -0500 Subject: [PATCH 08/10] address review comments and bugs after running binding tester compared to python bindings --- bindings/go/src/fdb/tuple/tuple.go | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index d3f289e2eb..5cc1990cc5 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -76,7 +76,7 @@ type UUID [16]byte // Versionstamp is struct for a FoundationDB verionstamp. Versionstamps are // 12 bytes long composed of a 10 byte transaction version and a 2 byte user // version. The transaction version is filled in at commit time and the user -// version is provided by your layer during a transaction. +// version is provided by the application to order results within a transaction. type Versionstamp struct { TransactionVersion [10]byte UserVersion uint16 @@ -295,9 +295,12 @@ func (p *packer) encodeUUID(u UUID) { func (p *packer) encodeVersionstamp(v Versionstamp) { p.putByte(versionstampCode) - if p.versionstampPos != -1 && v.TransactionVersion == incompleteTransactionVersion { - panic(fmt.Sprintf("Tuple can only contain one incomplete versionstamp")) - } else { + isIncomplete := v.TransactionVersion == incompleteTransactionVersion + if isIncomplete { + if p.versionstampPos != -1 { + panic(fmt.Sprintf("Tuple can only contain one incomplete versionstamp")) + } + p.versionstampPos = int32(len(p.buf)) } @@ -363,9 +366,9 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // Pack returns a new byte slice encoding the provided tuple. Pack will panic if // the tuple contains an element of any type other than []byte, // fdb.KeyConvertible, string, int64, int, uint64, uint, *big.Int, big.Int, float32, -// float64, bool, tuple.UUID, nil, or a Tuple with elements of valid types. It will -// also panic if an integer is specified with a value outside the range -// [-2**2040+1, 2**2040-1] +// float64, bool, tuple.UUID, tuple.Versionstamp, nil, or a Tuple with elements of +// valid types. It will also panic if an integer is specified with a value outside +// the range [-2**2040+1, 2**2040-1] // // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to // call Pack when using a Tuple with a FoundationDB API function that requires a @@ -384,22 +387,25 @@ func (t Tuple) Pack() []byte { // operations. See Pack for more information. This function will return an error // if you attempt to pack a tuple with more than one versionstamp. This function will // return an error if you attempt to pack a tuple with a versionstamp position larger -// than an uint16 on apiVersion < 520. +// than an uint16 if the API version is less than 520. func (t Tuple) PackWithVersionstamp(prefix []byte) ([]byte, error) { hasVersionstamp, err := t.HasIncompleteVersionstamp() if err != nil { return nil, err } + apiVersion, err := fdb.GetAPIVersion() + if err != nil { + return nil, err + } + if hasVersionstamp == false { return nil, errors.New("No incomplete versionstamp included in tuple pack with versionstamp") } p := newPacker() - prefixLength := int32(0) if prefix != nil { - prefixLength = int32(len(prefix)) p.putBytes(prefix) } @@ -408,18 +414,16 @@ func (t Tuple) PackWithVersionstamp(prefix []byte) ([]byte, error) { if hasVersionstamp { var scratch [4]byte var offsetIndex int - - apiVersion := fdb.MustGetAPIVersion() if apiVersion < 520 { if p.versionstampPos > math.MaxUint16 { return nil, errors.New("Versionstamp position too large") } - offsetIndex = 1 - binary.LittleEndian.PutUint16(scratch[:], uint16(prefixLength+p.versionstampPos)) + offsetIndex = 2 + binary.LittleEndian.PutUint16(scratch[:], uint16(p.versionstampPos)) } else { - offsetIndex = 3 - binary.LittleEndian.PutUint32(scratch[:], uint32(prefixLength+p.versionstampPos)) + offsetIndex = 4 + binary.LittleEndian.PutUint32(scratch[:], uint32(p.versionstampPos)) } p.putBytes(scratch[0:offsetIndex]) @@ -439,7 +443,7 @@ func (t Tuple) HasIncompleteVersionstamp() (bool, error) { err = errors.New("Tuple can only contain one incomplete versionstamp") } - return incompleteCount == 1, err + return incompleteCount >= 1, err } func (t Tuple) countIncompleteVersionstamps() int { From 92167fd03f4d78b064328939adc73b5f91697c86 Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Sat, 9 Mar 2019 11:11:22 -0500 Subject: [PATCH 09/10] handle incomplete versionstamp attempting to be packed into vanilla tuple --- bindings/go/src/fdb/tuple/tuple.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index 5cc1990cc5..a37ce5f3e8 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -307,7 +307,7 @@ func (p *packer) encodeVersionstamp(v Versionstamp) { p.putBytes(v.Bytes()) } -func (p *packer) encodeTuple(t Tuple, nested bool) { +func (p *packer) encodeTuple(t Tuple, nested bool, versionstamps bool) { if nested { p.putByte(nestedCode) } @@ -315,7 +315,7 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { for i, e := range t { switch e := e.(type) { case Tuple: - p.encodeTuple(e, true) + p.encodeTuple(e, true, versionstamps) case nil: p.putByte(nilCode) if nested { @@ -352,6 +352,10 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { case UUID: p.encodeUUID(e) case Versionstamp: + if versionstamps == false && e.TransactionVersion == incompleteTransactionVersion { + panic(fmt.Sprintf("Incomplete Versionstamp included in vanilla tuple pack")) + } + p.encodeVersionstamp(e) default: panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i])) @@ -379,7 +383,7 @@ func (p *packer) encodeTuple(t Tuple, nested bool) { // func (t Tuple) Pack() []byte { p := newPacker() - p.encodeTuple(t, false) + p.encodeTuple(t, false, false) return p.buf } @@ -409,7 +413,7 @@ func (t Tuple) PackWithVersionstamp(prefix []byte) ([]byte, error) { p.putBytes(prefix) } - p.encodeTuple(t, false) + p.encodeTuple(t, false, true) if hasVersionstamp { var scratch [4]byte From d90da27ee5f8d345ca4e47cd75eda97223f5218a Mon Sep 17 00:00:00 2001 From: Ryan Worl Date: Mon, 11 Mar 2019 14:33:32 -0400 Subject: [PATCH 10/10] Add Go to list of supported bindings for `set_versionstamped_key` in the Tuple layer. --- documentation/sphinx/source/api-common.rst.inc | 4 ++-- fdbclient/vexillographer/fdb.options | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index 80f26fcd9b..6ce102359f 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -142,10 +142,10 @@ A transaction is not permitted to read any transformed key or value previously set within that transaction, and an attempt to do so will result in an error. .. |atomic-versionstamps-tuple-warning-key| replace:: - At this time, versionstamped keys are not compatible with the Tuple layer except in Java and Python. Note that this implies versionstamped keys may not be used with the Subspace and Directory layers except in those languages. + At this time, versionstamped keys are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped keys may not be used with the Subspace and Directory layers except in those languages. .. |atomic-versionstamps-tuple-warning-value| replace:: - At this time, versionstamped values are not compatible with the Tuple layer except in Java and Python. Note that this implies versionstamped values may not be used with the Subspace and Directory layers except in those languages. + At this time, versionstamped values are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped values may not be used with the Subspace and Directory layers except in those languages. .. |api-version| replace:: 610 diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index d3161632b1..2194139224 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -247,10 +247,10 @@ description is not currently required but encouraged. description="Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database."/>