cloudwego
/
dynamicgo
mirror of https://github.com/cloudwego/dynamicgo.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

fix:(conv/j2p) support skip unknow fields (#52) 

* fix:(conv/j2p) support skip unknow fields

* more test

* fix: clear `visitorUserNode.inskip` when recycle

* upate sonic
This commit is contained in:
Yi Duan 2024-05-24 16:51:03 +08:00 committed by GitHub
parent 707f41b6b8
commit 7d0f9d3e80
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 109 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
conv/j2p/conv_test.go
View File

@ -13,6 +13,7 @@ import (
"testing"
"time"
"github.com/bytedance/sonic/ast"
"github.com/cloudwego/dynamicgo/conv"
"github.com/cloudwego/dynamicgo/internal/util_test"
"github.com/cloudwego/dynamicgo/proto"
@ -48,11 +49,11 @@ const (
)
func TestConvJSON2Protobf(t *testing.T) {
buildExampleJSONData()
// buildExampleJSONData()
desc := getExampleDesc()
data := getExampleData()
pdata, _ := ioutil.ReadFile(util_test.MustGitPath(exampleProtoPath))
fmt.Println(pdata)
// pdata, _ := ioutil.ReadFile(util_test.MustGitPath(exampleProtoPath))
// fmt.Println(pdata)
cv := NewBinaryConv(conv.Options{})
ctx := context.Background()
// get protobuf-encode bytes
@ -245,6 +246,66 @@ func getExampleInt2Float() *proto.TypeDescriptor {
return (*svc).LookupMethodByName("Int2FloatMethod").Output()
}
func TestUnknownFields(t *testing.T) {
t.Run("disallow", func(t *testing.T) {
desc := getExampleDesc()
data := getExampleData()
root := ast.NewRaw(string(data))
root.SetAny("UNKNOWN", 1)
data, _ = root.MarshalJSON()
cv := NewBinaryConv(conv.Options{
DisallowUnknownField: true,
})
ctx := context.Background()
_, err := cv.Do(ctx, desc, data)
require.Error(t, err)
})
t.Run("allow", func(t *testing.T) {
desc := getExampleDesc()
data := getExampleData()
root := ast.NewRaw(string(data))
root.Set("UNKNOWN-null", ast.NewNull())
root.SetAny("UNKNOWN-num", 1)
root.SetAny("UNKNOWN-bool", true)
root.SetAny("UNKNOWN-string", "a")
root.Set("UNKNOWN-object", ast.NewRaw(`{"1":1}`))
root.Set("UNKNOWN-array", ast.NewRaw(`[1]`))
base := root.Get("InnerBase2")
base.SetAny("UNKNOWN-sub", 1)
data, _ = root.MarshalJSON()
println(string(data))
cv := NewBinaryConv(conv.Options{
DisallowUnknownField: false,
})
ctx := context.Background()
out, err := cv.Do(ctx, desc, data)
require.NoError(t, err)
exp := &example2.ExampleReq{}
// unmarshal target struct
err = json.Unmarshal(data, exp)
require.NoError(t, err)
act := &example2.ExampleReq{}
l := 0
// fmt.Print(out)
dataLen := len(out)
// fastRead to get target struct
for l < dataLen {
id, wtyp, tagLen := goprotowire.ConsumeTag(out)
if tagLen < 0 {
t.Fatal("test failed")
}
l += tagLen
out = out[tagLen:]
offset, err := act.FastRead(out, int8(wtyp), int32(id))
require.Nil(t, err)
out = out[offset:]
l += offset
}
require.Nil(t, err)
require.Equal(t, exp, act)
})
}
func TestFloat2Int(t *testing.T) {
t.Run("double2Int", func(t *testing.T) {
desc := getExampleInt2Float()

46
conv/j2p/decode.go
View File

@ -75,6 +75,7 @@ func (self *visitorUserNode) reset() {
}
self.globalFieldDesc = nil
self.opts = nil
self.inskip = false
}
// visitorUserNode is used to store some conditional variables about Protobuf when parsing json
@ -84,10 +85,11 @@ func (self *visitorUserNode) reset() {
// GlobalFieldDesc：After parsing the FieldKey, save the FieldDescriptor
type visitorUserNode struct {
sp uint8
inskip bool
stk []visitorUserNodeStack
p *binary.BinaryProtocol
globalFieldDesc *proto.FieldDescriptor
opts *conv.Options
opts *conv.Options
}
// keep hierarchy when parsing, objStkType represents Message and arrStkType represents List and mapStkType represents Map
@ -167,6 +169,10 @@ func (self *visitorUserNode) incrSP() error {
}
func (self *visitorUserNode) OnNull() error {
if self.inskip {
self.inskip = false
return nil
}
// self.stk[self.sp].val = &visitorUserNull{}
if err := self.incrSP(); err != nil {
return err
@ -175,6 +181,10 @@ func (self *visitorUserNode) OnNull() error {
}
func (self *visitorUserNode) OnBool(v bool) error {
if self.inskip {
self.inskip = false
return nil
}
var err error
if self.globalFieldDesc == nil {
return newError(meta.ErrConvert, "self.globalFieldDescriptor is nil, type Onbool", nil)
@ -191,6 +201,10 @@ func (self *visitorUserNode) OnBool(v bool) error {
// Parse stringType/bytesType
func (self *visitorUserNode) OnString(v string) error {
if self.inskip {
self.inskip = false
return nil
}
var err error
top := self.stk[self.sp].state.fieldDesc
fieldDesc := self.globalFieldDesc
@ -226,6 +240,10 @@ func (self *visitorUserNode) OnString(v string) error {
}
func (self *visitorUserNode) OnInt64(v int64, n json.Number) error {
if self.inskip {
self.inskip = false
return nil
}
var err error
top := self.stk[self.sp]
fieldDesc := self.globalFieldDesc
@ -282,6 +300,10 @@ func (self *visitorUserNode) OnInt64(v int64, n json.Number) error {
}
func (self *visitorUserNode) OnFloat64(v float64, n json.Number) error {
if self.inskip {
self.inskip = false
return nil
}
var err error
top := self.stk[self.sp]
fieldDesc := self.globalFieldDesc
@ -337,6 +359,9 @@ func (self *visitorUserNode) OnFloat64(v float64, n json.Number) error {
// 3. When Field is Message type, encode Tag and PrefixLen, and push FieldDescriptor and PrefixLen to the stack.
// When Field is Map type, only perform pressing stack operation.
func (self *visitorUserNode) OnObjectBegin(capacity int) error {
if self.inskip {
return ast.VisitOPSkip
}
var err error
fieldDesc := self.globalFieldDesc
top := self.stk[self.sp]
@ -425,10 +450,10 @@ func (self *visitorUserNode) OnObjectKey(key string) error {
rootDesc := top.state.msgDesc
fd := rootDesc.ByJSONName(key)
if fd == nil {
if !self.opts.DisallowUnknownField {
if self.opts.DisallowUnknownField {
return newError(meta.ErrUnknownField, fmt.Sprintf("json key '%s' is unknown", key), err)
} else {
// todo
self.inskip = true
return nil
}
}
@ -439,10 +464,10 @@ func (self *visitorUserNode) OnObjectKey(key string) error {
if top.typ == objStkType {
fd := fieldDesc.Message().ByJSONName(key)
if fd == nil {
if !self.opts.DisallowUnknownField {
if self.opts.DisallowUnknownField {
return newError(meta.ErrUnknownField, fmt.Sprintf("json key '%s' is unknown", key), err)
} else {
// todo
self.inskip = true
return nil
}
}
@ -484,6 +509,10 @@ func (self *visitorUserNode) OnObjectKey(key string) error {
// After parsing JSONObject, write back prefixLen of Message
func (self *visitorUserNode) OnObjectEnd() error {
if self.inskip {
self.inskip = false
return nil
}
top := &self.stk[self.sp]
// root layer no need to write tag and len
if top.state.lenPos != -1 {
@ -496,6 +525,9 @@ func (self *visitorUserNode) OnObjectEnd() error {
// 1. If PackedList, Encode ListTag、PrefixLen
// 2. push ListDescriptor、PrefixLen(UnPackedList is -1) into stack
func (self *visitorUserNode) OnArrayBegin(capacity int) error {
if self.inskip {
return ast.VisitOPSkip
}
var err error
curNodeLenPos := -1
if self.globalFieldDesc != nil {
@ -515,6 +547,10 @@ func (self *visitorUserNode) OnArrayBegin(capacity int) error {
// After Parsing JSONArray, writing back PrefixLen If PackedList
func (self *visitorUserNode) OnArrayEnd() error {
if self.inskip {
self.inskip = false
return nil
}
var top *visitorUserNodeStack
top = &self.stk[self.sp]
// case PackedList

2
conv/j2p/impl.go
View File

@ -33,7 +33,7 @@ func (self *BinaryConv) unmarshal(src []byte, out *[]byte, desc *proto.TypeDescr
data, err := vu.decode(src, desc)
freeVisitorUserNodePool(vu)
if err != nil {
return newError(meta.ErrConvert, "sonic decode json bytes failed", err)
return err
}
*out = data
return nil

2
go.mod
View File

@ -4,7 +4,7 @@ go 1.16
require (
github.com/apache/thrift v0.13.0
github.com/bytedance/sonic v1.11.6
github.com/bytedance/sonic v1.11.8-0.20240523084635-eecfc904bfc9
github.com/bytedance/sonic/loader v0.1.1
github.com/cloudwego/base64x v0.1.4
github.com/cloudwego/fastpb v0.0.4

3
go.sum
View File

@ -17,8 +17,9 @@ github.com/bytedance/gopkg v0.0.0-20230728082804-614d0af6619b h1:R6PWoQtxEMpWJPH
github.com/bytedance/gopkg v0.0.0-20230728082804-614d0af6619b/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ=
github.com/bytedance/mockey v1.2.7/go.mod h1:bNrUnI1u7+pAc0TYDgPATM+wF2yzHxmNH+iDXg4AOCU=
github.com/bytedance/sonic v1.11.5/go.mod h1:X2PC2giUdj/Cv2lliWFLk6c/DUQok5rViJSemeB0wDw=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic v1.11.8-0.20240523084635-eecfc904bfc9 h1:l7at7ML0R5vz6blFje6YpsHxTBUjtvOW4rH3WLFaxo4=
github.com/bytedance/sonic v1.11.8-0.20240523084635-eecfc904bfc9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.0/go.mod h1:UmRT+IRTGKz/DAkzcEGzyVqQFJ7H9BqwBO3pm9H/+HY=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=