github.com/bytedance/sonic@v1.11.7-0.20240517092252-d2edb31b167b/internal/decoder/decoder_test.go (about) 1 /* 2 * Copyright 2021 ByteDance Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package decoder 18 19 import ( 20 `encoding/json` 21 `runtime` 22 `runtime/debug` 23 `strings` 24 `sync` 25 `testing` 26 `time` 27 `reflect` 28 29 `github.com/bytedance/sonic/internal/rt` 30 `github.com/davecgh/go-spew/spew` 31 `github.com/stretchr/testify/assert` 32 `github.com/stretchr/testify/require` 33 ) 34 35 func TestMain(m *testing.M) { 36 go func () { 37 if !debugAsyncGC { 38 return 39 } 40 println("Begin GC looping...") 41 for { 42 runtime.GC() 43 debug.FreeOSMemory() 44 } 45 println("stop GC looping!") 46 }() 47 time.Sleep(time.Millisecond) 48 m.Run() 49 } 50 51 func TestGC(t *testing.T) { 52 if debugSyncGC { 53 return 54 } 55 var w interface{} 56 out, err := decode(TwitterJson, &w, true) 57 if err != nil { 58 t.Fatal(err) 59 } 60 if out != len(TwitterJson) { 61 t.Fatal(out) 62 } 63 wg := &sync.WaitGroup{} 64 N := 10000 65 for i:=0; i<N; i++ { 66 wg.Add(1) 67 go func (wg *sync.WaitGroup) { 68 defer wg.Done() 69 var w interface{} 70 out, err := decode(TwitterJson, &w, true) 71 if err != nil { 72 t.Fatal(err) 73 } 74 if out != len(TwitterJson) { 75 t.Fatal(out) 76 } 77 runtime.GC() 78 }(wg) 79 } 80 wg.Wait() 81 } 82 83 var _BindingValue TwitterStruct 84 85 func init() { 86 _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) 87 } 88 89 90 func TestSkipMismatchTypeError(t *testing.T) { 91 t.Run("struct", func(t *testing.T) { 92 println("TestSkipError") 93 type skiptype struct { 94 A int `json:"a"` 95 B string `json:"b"` 96 97 Pass *int `json:"pass"` 98 99 C struct{ 100 101 Pass4 interface{} `json:"pass4"` 102 103 D struct{ 104 E float32 `json:"e"` 105 } `json:"d"` 106 107 Pass2 int `json:"pass2"` 108 109 } `json:"c"` 110 111 E bool `json:"e"` 112 F []int `json:"f"` 113 G map[string]int `json:"g"` 114 H bool `json:"h,string"` 115 116 Pass3 int `json:"pass2"` 117 118 I json.Number `json:"i"` 119 } 120 var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)} 121 var data = `{"a":"","b":1,"c":{"d":true,"pass2":1,"pass4":true},"e":{},"f":"","g":[],"pass":null,"h":"1.0","i":true,"pass3":1}` 122 d := NewDecoder(data) 123 err := d.Decode(obj) 124 err2 := json.Unmarshal([]byte(data), obj2) 125 println(err2.Error()) 126 assert.Equal(t, err2 == nil, err == nil) 127 // assert.Equal(t, len(data), d.i) 128 assert.Equal(t, obj2, obj) 129 if te, ok := err.(*MismatchTypeError); ok { 130 assert.Equal(t, reflect.TypeOf(obj.I), te.Type) 131 assert.Equal(t, strings.Index(data, `"i":t`)+4, te.Pos) 132 println(err.Error()) 133 } else { 134 t.Fatal("invalid error") 135 } 136 }) 137 t.Run("short array", func(t *testing.T) { 138 var obj, obj2 = &[]int{}, &[]int{} 139 var data = `[""]` 140 d := NewDecoder(data) 141 err := d.Decode(obj) 142 err2 := json.Unmarshal([]byte(data), obj2) 143 // println(err2.Error()) 144 assert.Equal(t, err2 == nil, err == nil) 145 // assert.Equal(t, len(data), d.i) 146 assert.Equal(t, obj2, obj) 147 }) 148 149 t.Run("int ", func(t *testing.T) { 150 var obj int = 123 151 var obj2 int = 123 152 var data = `[""]` 153 d := NewDecoder(data) 154 err := d.Decode(&obj) 155 err2 := json.Unmarshal([]byte(data), &obj2) 156 println(err.Error(), obj, obj2) 157 assert.Equal(t, err2 == nil, err == nil) 158 // assert.Equal(t, len(data), d.i) 159 assert.Equal(t, obj2, obj) 160 }) 161 162 t.Run("array", func(t *testing.T) { 163 var obj, obj2 = &[]int{}, &[]int{} 164 var data = `["",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]` 165 d := NewDecoder(data) 166 err := d.Decode(obj) 167 err2 := json.Unmarshal([]byte(data), obj2) 168 // println(err2.Error()) 169 assert.Equal(t, err2 == nil, err == nil) 170 // assert.Equal(t, len(data), d.i) 171 assert.Equal(t, obj2, obj) 172 }) 173 174 t.Run("map", func(t *testing.T) { 175 var obj, obj2 = &map[int]int{}, &map[int]int{} 176 var data = `{"true" : { },"1":1,"2" : true,"3":3}` 177 d := NewDecoder(data) 178 err := d.Decode(obj) 179 err2 := json.Unmarshal([]byte(data), obj2) 180 assert.Equal(t, err2 == nil, err == nil) 181 // assert.Equal(t, len(data), d.i) 182 assert.Equal(t, obj2, obj) 183 }) 184 t.Run("map error", func(t *testing.T) { 185 var obj, obj2 = &map[int]int{}, &map[int]int{} 186 var data = `{"true" : { ],"1":1,"2" : true,"3":3}` 187 d := NewDecoder(data) 188 err := d.Decode(obj) 189 err2 := json.Unmarshal([]byte(data), obj2) 190 println(err.Error()) 191 println(err2.Error()) 192 assert.Equal(t, err2 == nil, err == nil) 193 // assert.Equal(t, len(data), d.i) 194 // assert.Equal(t, obj2, obj) 195 }) 196 } 197 198 func TestDecodeCorrupt(t *testing.T) { 199 var ds = []string{ 200 `{,}`, 201 `{,"a"}`, 202 `{"a":}`, 203 `{"a":1,}`, 204 `{"a":1,"b"}`, 205 `{"a":1,"b":}`, 206 `{,"a":1 "b":2}`, 207 `{"a",:1 "b":2}`, 208 `{"a":,1 "b":2}`, 209 `{"a":1 "b",:2}`, 210 `{"a":1 "b":,2}`, 211 `{"a":1 "b":2,}`, 212 `{"a":1 "b":2}`, 213 `[,]`, 214 `[,1]`, 215 `[1,]`, 216 `[,1,2]`, 217 `[1,2,]`, 218 } 219 for _, d := range ds { 220 var o interface{} 221 _, err := decode(d, &o, false) 222 if err == nil { 223 t.Fatalf("%#v", d) 224 } 225 if !strings.Contains(err.Error(), "invalid char"){ 226 t.Fatal(err.Error()) 227 } 228 } 229 } 230 231 func decode(s string, v interface{}, copy bool) (int, error) { 232 d := NewDecoder(s) 233 if copy { 234 d.CopyString() 235 } 236 err := d.Decode(v) 237 if err != nil { 238 return 0, err 239 } 240 return d.i, err 241 } 242 243 func TestCopyString(t *testing.T) { 244 var data []byte 245 var dc *Decoder 246 var err error 247 data = []byte(`{"A":"0","B":"1"}`) 248 dc = NewDecoder(rt.Mem2Str(data)) 249 dc.UseNumber() 250 dc.CopyString() 251 var obj struct{ 252 A string 253 B string 254 } 255 err = dc.Decode(&obj) 256 if err != nil { 257 t.Fatal(err) 258 } 259 data[6] = '1' 260 if obj.A != "0" { 261 t.Fatal(obj) 262 } 263 data[14] = '0' 264 if obj.B != "1" { 265 t.Fatal(obj) 266 } 267 268 data = []byte(`{"A":"0","B":"1"}`) 269 dc = NewDecoder(rt.Mem2Str(data)) 270 dc.UseNumber() 271 err = dc.Decode(&obj) 272 if err != nil { 273 t.Fatal(err) 274 } 275 data[6] = '1' 276 if obj.A != "1" { 277 t.Fatal(obj) 278 } 279 data[14] = '0' 280 if obj.B != "0" { 281 t.Fatal(obj) 282 } 283 284 data = []byte(`{"A":"0","B":"1"}`) 285 dc = NewDecoder(rt.Mem2Str(data)) 286 dc.UseNumber() 287 dc.CopyString() 288 m := map[string]interface{}{} 289 err = dc.Decode(&m) 290 if err != nil { 291 t.Fatal(err) 292 } 293 data[2] = 'C' 294 data[6] = '1' 295 if m["A"] != "0" { 296 t.Fatal(m) 297 } 298 data[10] = 'D' 299 data[14] = '0' 300 if m["B"] != "1" { 301 t.Fatal(m) 302 } 303 304 data = []byte(`{"A":"0","B":"1"}`) 305 dc = NewDecoder(rt.Mem2Str(data)) 306 dc.UseNumber() 307 m = map[string]interface{}{} 308 err = dc.Decode(&m) 309 if err != nil { 310 t.Fatal(err) 311 } 312 data[6] = '1' 313 if m["A"] != "1" { 314 t.Fatal(m) 315 } 316 data[14] = '0' 317 if m["B"] != "0" { 318 t.Fatal(m) 319 } 320 321 data = []byte(`{"A":"0","B":"1"}`) 322 dc = NewDecoder(rt.Mem2Str(data)) 323 dc.UseNumber() 324 dc.CopyString() 325 var x interface{} 326 err = dc.Decode(&x) 327 if err != nil { 328 t.Fatal(err) 329 } 330 data[2] = 'C' 331 data[6] = '1' 332 m = x.(map[string]interface{}) 333 if m["A"] != "0" { 334 t.Fatal(m) 335 } 336 data[10] = 'D' 337 data[14] = '0' 338 if m["B"] != "1" { 339 t.Fatal(m) 340 } 341 342 data = []byte(`{"A":"0","B":"1"}`) 343 dc = NewDecoder(rt.Mem2Str(data)) 344 dc.UseNumber() 345 var y interface{} 346 err = dc.Decode(&y) 347 if err != nil { 348 t.Fatal(err) 349 } 350 m = y.(map[string]interface{}) 351 data[6] = '1' 352 if m["A"] != "1" { 353 t.Fatal(m) 354 } 355 data[14] = '0' 356 if m["B"] != "0" { 357 t.Fatal(m) 358 } 359 } 360 361 func TestDecoder_Basic(t *testing.T) { 362 var v int 363 pos, err := decode("12345", &v, false) 364 assert.NoError(t, err) 365 assert.Equal(t, 5, pos) 366 assert.Equal(t, 12345, v) 367 } 368 369 func TestDecoder_Generic(t *testing.T) { 370 var v interface{} 371 pos, err := decode(TwitterJson, &v, false) 372 assert.NoError(t, err) 373 assert.Equal(t, len(TwitterJson), pos) 374 } 375 376 func TestDecoder_Binding(t *testing.T) { 377 var v TwitterStruct 378 pos, err := decode(TwitterJson, &v, false) 379 assert.NoError(t, err) 380 assert.Equal(t, len(TwitterJson), pos) 381 assert.Equal(t, _BindingValue, v, 0) 382 spew.Dump(v) 383 } 384 385 func TestDecoder_SetOption(t *testing.T) { 386 var v interface{} 387 d := NewDecoder("123") 388 d.SetOptions(OptionUseInt64) 389 err := d.Decode(&v) 390 assert.NoError(t, err) 391 assert.Equal(t, v, int64(123)) 392 } 393 394 func TestDecoder_MapWithIndirectElement(t *testing.T) { 395 var v map[string]struct { A [129]byte } 396 _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false) 397 if x, ok := err.(SyntaxError); ok { 398 println(x.Description()) 399 } 400 require.NoError(t, err) 401 assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A) 402 } 403 404 func BenchmarkDecoder_Generic_Sonic(b *testing.B) { 405 var w interface{} 406 _, _ = decode(TwitterJson, &w, true) 407 b.SetBytes(int64(len(TwitterJson))) 408 b.ResetTimer() 409 for i := 0; i < b.N; i++ { 410 var v interface{} 411 _, _ = decode(TwitterJson, &v, true) 412 } 413 } 414 415 func BenchmarkDecoder_Generic_Sonic_Fast(b *testing.B) { 416 var w interface{} 417 _, _ = decode(TwitterJson, &w, false) 418 b.SetBytes(int64(len(TwitterJson))) 419 b.ResetTimer() 420 for i := 0; i < b.N; i++ { 421 var v interface{} 422 _, _ = decode(TwitterJson, &v, false) 423 } 424 } 425 426 func BenchmarkDecoder_Generic_StdLib(b *testing.B) { 427 var w interface{} 428 m := []byte(TwitterJson) 429 _ = json.Unmarshal(m, &w) 430 b.SetBytes(int64(len(TwitterJson))) 431 b.ResetTimer() 432 for i := 0; i < b.N; i++ { 433 var v interface{} 434 _ = json.Unmarshal(m, &v) 435 } 436 } 437 438 func BenchmarkDecoder_Binding_Sonic(b *testing.B) { 439 var w TwitterStruct 440 _, _ = decode(TwitterJson, &w, true) 441 b.SetBytes(int64(len(TwitterJson))) 442 b.ResetTimer() 443 for i := 0; i < b.N; i++ { 444 var v TwitterStruct 445 _, _ = decode(TwitterJson, &v, true) 446 } 447 } 448 449 func BenchmarkDecoder_Binding_Sonic_Fast(b *testing.B) { 450 var w TwitterStruct 451 _, _ = decode(TwitterJson, &w, false) 452 b.SetBytes(int64(len(TwitterJson))) 453 b.ResetTimer() 454 for i := 0; i < b.N; i++ { 455 var v TwitterStruct 456 _, _ = decode(TwitterJson, &v, false) 457 } 458 } 459 460 func BenchmarkDecoder_Binding_StdLib(b *testing.B) { 461 var w TwitterStruct 462 m := []byte(TwitterJson) 463 _ = json.Unmarshal(m, &w) 464 b.SetBytes(int64(len(TwitterJson))) 465 b.ResetTimer() 466 for i := 0; i < b.N; i++ { 467 var v TwitterStruct 468 _ = json.Unmarshal(m, &v) 469 } 470 } 471 472 func BenchmarkDecoder_Parallel_Generic_Sonic(b *testing.B) { 473 var w interface{} 474 _, _ = decode(TwitterJson, &w, true) 475 b.SetBytes(int64(len(TwitterJson))) 476 b.ResetTimer() 477 b.RunParallel(func(pb *testing.PB) { 478 for pb.Next() { 479 var v interface{} 480 _, _ = decode(TwitterJson, &v, true) 481 } 482 }) 483 } 484 485 func BenchmarkDecoder_Parallel_Generic_Sonic_Fast(b *testing.B) { 486 var w interface{} 487 _, _ = decode(TwitterJson, &w, false) 488 b.SetBytes(int64(len(TwitterJson))) 489 b.ResetTimer() 490 b.RunParallel(func(pb *testing.PB) { 491 for pb.Next() { 492 var v interface{} 493 _, _ = decode(TwitterJson, &v, false) 494 } 495 }) 496 } 497 498 func BenchmarkDecoder_Parallel_Generic_StdLib(b *testing.B) { 499 var w interface{} 500 m := []byte(TwitterJson) 501 _ = json.Unmarshal(m, &w) 502 b.SetBytes(int64(len(TwitterJson))) 503 b.ResetTimer() 504 b.RunParallel(func(pb *testing.PB) { 505 for pb.Next() { 506 var v interface{} 507 _ = json.Unmarshal(m, &v) 508 } 509 }) 510 } 511 512 func BenchmarkDecoder_Parallel_Binding_Sonic(b *testing.B) { 513 var w TwitterStruct 514 _, _ = decode(TwitterJson, &w, true) 515 b.SetBytes(int64(len(TwitterJson))) 516 b.ResetTimer() 517 b.RunParallel(func(pb *testing.PB) { 518 for pb.Next() { 519 var v TwitterStruct 520 _, _ = decode(TwitterJson, &v, true) 521 } 522 }) 523 } 524 525 func BenchmarkDecoder_Parallel_Binding_Sonic_Fast(b *testing.B) { 526 var w TwitterStruct 527 _, _ = decode(TwitterJson, &w, false) 528 b.SetBytes(int64(len(TwitterJson))) 529 b.ResetTimer() 530 b.RunParallel(func(pb *testing.PB) { 531 for pb.Next() { 532 var v TwitterStruct 533 _, _ = decode(TwitterJson, &v, false) 534 } 535 }) 536 } 537 538 func BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) { 539 var w TwitterStruct 540 m := []byte(TwitterJson) 541 _ = json.Unmarshal(m, &w) 542 b.SetBytes(int64(len(TwitterJson))) 543 b.ResetTimer() 544 b.RunParallel(func(pb *testing.PB) { 545 for pb.Next() { 546 var v TwitterStruct 547 _ = json.Unmarshal(m, &v) 548 } 549 }) 550 } 551 552 func BenchmarkSkip_Sonic(b *testing.B) { 553 var data = rt.Str2Mem(TwitterJson) 554 if ret, _ := Skip(data); ret < 0 { 555 b.Fatal() 556 } 557 b.SetBytes(int64(len(TwitterJson))) 558 b.ResetTimer() 559 for i:=0; i<b.N; i++ { 560 _, _ = Skip(data) 561 } 562 }