github.com/bytedance/sonic@v1.11.7-0.20240517092252-d2edb31b167b/encoder/encoder_test.go (about)

     1  /*
     2   * Copyright 2021 ByteDance Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package encoder
    18  
    19  import (
    20      `bytes`
    21      `encoding`
    22      `encoding/json`
    23      `runtime`
    24      `runtime/debug`
    25      `strconv`
    26      `sync`
    27      `testing`
    28      `time`
    29  
    30      `github.com/bytedance/sonic/internal/rt`
    31      `github.com/stretchr/testify/require`
    32  )
    33  
    34  func TestMain(m *testing.M) {
    35      go func ()  {
    36          if !debugAsyncGC {
    37              return
    38          }
    39          println("Begin GC looping...")
    40          for {
    41              runtime.GC()
    42              debug.FreeOSMemory()
    43          }
    44          println("stop GC looping!")
    45      }()
    46      time.Sleep(time.Millisecond)
    47      m.Run()
    48  }
    49  
    50  func TestGC(t *testing.T) {
    51      if debugSyncGC {
    52          return
    53      }
    54      out, err := Encode(_GenericValue, 0)
    55      if err != nil {
    56          t.Fatal(err)
    57      }
    58      n := len(out)
    59      wg := &sync.WaitGroup{}
    60      N := 10000
    61      for i:=0; i<N; i++ {
    62          wg.Add(1)
    63          go func (wg *sync.WaitGroup, size int)  {
    64              defer wg.Done()
    65              out, err := Encode(_GenericValue, 0)
    66              if err != nil {
    67                  t.Error(err)
    68                  return
    69              }
    70              if len(out) != size {
    71                  t.Error(len(out), size)
    72                  return
    73              }
    74              runtime.GC()
    75          }(wg, n)
    76      }
    77      wg.Wait()
    78  }
    79  
    80  type sample struct {
    81      M map[string]interface{}
    82      S []interface{}
    83      A [0]interface{}
    84      MP *map[string]interface{}
    85      SP *[]interface{}
    86      AP *[0]interface{}
    87  }
    88  
    89  func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
    90      b.Run("true", func (b *testing.B) {
    91          obj := sample{}
    92          _, err := Encode(obj, NoNullSliceOrMap)
    93          if err != nil {
    94              b.Fatal(err)
    95          }
    96          b.ResetTimer()
    97          for i:=0;i<b.N;i++{
    98              _, _ = Encode(obj, NoNullSliceOrMap)
    99          }
   100      })
   101  
   102      b.Run("false", func (b *testing.B) {
   103          obj2 := sample{}
   104          _, err := Encode(obj2, 0)
   105          if err != nil {
   106              b.Fatal(err)
   107          }
   108          for i:=0;i<b.N;i++{
   109              _, _ = Encode(obj2, 0)
   110          }
   111      })
   112  }
   113  
   114  func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
   115      require.Equal(t, exp, fn(arg))
   116  }
   117  
   118  func TestEncoder_String(t *testing.T) {
   119      runEncoderTest(t, Quote, `""`                                                 , "")
   120      runEncoderTest(t, Quote, `"hello, world"`                                     , "hello, world")
   121      runEncoderTest(t, Quote, `"hello啊啊啊aa"`                                    , "hello啊啊啊aa")
   122      runEncoderTest(t, Quote, `"hello\\\"world"`                                   , "hello\\\"world")
   123      runEncoderTest(t, Quote, `"hello\n\tworld"`                                   , "hello\n\tworld")
   124      runEncoderTest(t, Quote, `"hello\u0000\u0001world"`                           , "hello\x00\x01world")
   125      runEncoderTest(t, Quote, `"hello\u0000\u0001world"`                           , "hello\x00\x01world")
   126      runEncoderTest(t, Quote, `"Cartoonist, Illustrator, and T-Shirt connoisseur"` , "Cartoonist, Illustrator, and T-Shirt connoisseur")
   127  }
   128  
   129  type StringStruct struct {
   130      X *int        `json:"x,string,omitempty"`
   131      Y []int       `json:"y"`
   132      Z json.Number `json:"z,string"`
   133      W string      `json:"w,string"`
   134  }
   135  
   136  func TestEncoder_FieldStringize(t *testing.T) {
   137      x := 12345
   138      v := StringStruct{X: &x, Y: []int{1, 2, 3}, Z: "4567456", W: "asdf"}
   139      r, e := Encode(v, 0)
   140      require.NoError(t, e)
   141      println(string(r))
   142  }
   143  
   144  func TestEncodeErrorAndScratchBuf(t *testing.T) {
   145      var obj = map[string]interface{}{
   146          "a": json.RawMessage(" [} "),
   147      }
   148      buf := make([]byte, 0, 10)
   149      _ = EncodeInto(&buf, obj, 0)
   150      if len(buf) < 0 || len(buf) > 10 {
   151          t.Fatal()
   152      }
   153  }
   154  
   155  type MarshalerImpl struct {
   156      X int
   157  }
   158  
   159  func (self *MarshalerImpl) MarshalJSON() ([]byte, error) {
   160      ret := []byte(strconv.Itoa(self.X))
   161      return append(ret, "    "...), nil
   162  }
   163  
   164  type MarshalerStruct struct {
   165      V MarshalerImpl
   166  }
   167  
   168  type MarshalerErrorStruct struct {
   169      V MarshalerImpl
   170  }
   171  
   172  func (self *MarshalerErrorStruct) MarshalJSON() ([]byte, error) {
   173      return []byte(`[""] {`), nil
   174  }
   175  
   176  type RawMessageStruct struct {
   177      X json.RawMessage
   178  }
   179  
   180  type TextMarshalerImpl struct {
   181      X string
   182  }
   183  
   184  func (self *TextMarshalerImpl) MarshalText() ([]byte, error) {
   185      return []byte(self.X), nil
   186  }
   187  
   188  type TextMarshalerImplV struct {
   189      X string
   190  }
   191  
   192  func (self TextMarshalerImplV) MarshalText() ([]byte, error) {
   193      return []byte(self.X), nil
   194  }
   195  
   196  type TextMarshalerStruct struct {
   197      V TextMarshalerImpl
   198  }
   199  
   200  func TestTextMarshalTextKey_SortKeys(t *testing.T) {
   201      v := map[*TextMarshalerImpl]string{
   202          {"b"}: "b",
   203          {"c"}: "c",
   204          {"a"}: "a",
   205      }
   206      ret, err := Encode(v, SortMapKeys)
   207      require.NoError(t, err)
   208      require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
   209  
   210      v2 := map[TextMarshalerImplV]string{
   211          {"b"}: "b",
   212          {"c"}: "c",
   213          {"a"}: "a",
   214      }
   215      ret, err = Encode(v2, SortMapKeys)
   216      require.NoError(t, err)
   217      require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
   218  
   219      v3 := map[encoding.TextMarshaler]string{
   220          TextMarshalerImplV{"b"}: "b",
   221          &TextMarshalerImpl{"c"}: "c",
   222          TextMarshalerImplV{"a"}: "a",
   223      }
   224      ret, err = Encode(v3, SortMapKeys)
   225      require.NoError(t, err)
   226      require.Equal(t, `{"a":"a","b":"b","c":"c"}`, string(ret))
   227  }
   228  
   229  func TestEncoder_EscapeHTML(t *testing.T) {
   230      // test data from libfuzzer
   231      test := []string{
   232          "&&&&&&&&&&&&&&&&&&&&&&&\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2\xe2&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&",
   233          "{\"\"\u2028\x94\xe2\x00\x00\x00\x00\x00\x00\x00\x00\u2028\x80\u2028\x80\u2028\xe2\u2028\x8a\u2028⑀\xa8\x8a\xa8\xe2\u2028\xe2\u2028\xe2\u2028\xe2\u2000\x8d\xe2\u2028\xe2\u2028\xe2\xe2\xa8\"}",
   234      }
   235      for _, s := range(test) {
   236          data := []byte(s)
   237          sdst := HTMLEscape(nil, data)
   238          var dst bytes.Buffer
   239          json.HTMLEscape(&dst, data)
   240          require.Equal(t, string(sdst), dst.String())
   241      }
   242  }
   243  
   244  func TestEncoder_Marshal_EscapeHTML_LargeJson(t *testing.T) {
   245      buf1, err1 := Encode(&_BindingValue, SortMapKeys | EscapeHTML)
   246      require.NoError(t, err1)
   247      buf2, err2 :=json.Marshal(&_BindingValue)
   248      require.NoError(t, err2)
   249      require.Equal(t, buf1, buf2)
   250  }
   251  
   252  var _GenericValue interface{}
   253  var _BindingValue TwitterStruct
   254  
   255  func init() {
   256      _ = json.Unmarshal([]byte(TwitterJson), &_GenericValue)
   257      _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
   258  }
   259  
   260  func TestEncoder_Generic(t *testing.T) {
   261      v, e := Encode(_GenericValue, 0)
   262      require.NoError(t, e)
   263      println(string(v))
   264  }
   265  
   266  func TestEncoder_Binding(t *testing.T) {
   267      v, e := Encode(_BindingValue, 0)
   268      require.NoError(t, e)
   269      println(string(v))
   270  }
   271  
   272  func TestEncoder_MapSortKey(t *testing.T) {
   273      m := map[string]string {
   274          "C": "third",
   275          "D": "forth",
   276          "A": "first",
   277          "F": "sixth",
   278          "E": "fifth",
   279          "B": "second",
   280      }
   281      v, e := Encode(m, SortMapKeys)
   282      require.NoError(t, e)
   283      require.Equal(t, `{"A":"first","B":"second","C":"third","D":"forth","E":"fifth","F":"sixth"}`, string(v))
   284  }
   285  
   286  func BenchmarkEncoder_Generic_Sonic(b *testing.B) {
   287      _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   288      b.SetBytes(int64(len(TwitterJson)))
   289      b.ResetTimer()
   290      for i := 0; i < b.N; i++ {
   291          _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   292      }
   293  }
   294  
   295  func BenchmarkEncoder_Generic_Sonic_Fast(b *testing.B) {
   296      _, _ = Encode(_GenericValue, 0)
   297      b.SetBytes(int64(len(TwitterJson)))
   298      b.ResetTimer()
   299      for i := 0; i < b.N; i++ {
   300          _, _ = Encode(_GenericValue, 0)
   301      }
   302  }
   303  
   304  func BenchmarkEncoder_Generic_StdLib(b *testing.B) {
   305      _, _ = json.Marshal(_GenericValue)
   306      b.SetBytes(int64(len(TwitterJson)))
   307      b.ResetTimer()
   308      for i := 0; i < b.N; i++ {
   309          _, _ = json.Marshal(_GenericValue)
   310      }
   311  }
   312  
   313  func BenchmarkEncoder_Binding_Sonic(b *testing.B) {
   314      _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   315      b.SetBytes(int64(len(TwitterJson)))
   316      b.ResetTimer()
   317      for i := 0; i < b.N; i++ {
   318          _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   319      }
   320  }
   321  
   322  func BenchmarkEncoder_Binding_Sonic_Fast(b *testing.B) {
   323      _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
   324      b.SetBytes(int64(len(TwitterJson)))
   325      b.ResetTimer()
   326      for i := 0; i < b.N; i++ {
   327          _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
   328      }
   329  }
   330  
   331  func BenchmarkEncoder_Binding_StdLib(b *testing.B) {
   332      _, _ = json.Marshal(&_BindingValue)
   333      b.SetBytes(int64(len(TwitterJson)))
   334      b.ResetTimer()
   335      for i := 0; i < b.N; i++ {
   336          _, _ = json.Marshal(&_BindingValue)
   337      }
   338  }
   339  
   340  func BenchmarkEncoder_Parallel_Generic_Sonic(b *testing.B) {
   341      _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   342      b.SetBytes(int64(len(TwitterJson)))
   343      b.ResetTimer()
   344      b.RunParallel(func(pb *testing.PB) {
   345          for pb.Next() {
   346              _, _ = Encode(_GenericValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   347          }
   348      })
   349  }
   350  
   351  func BenchmarkEncoder_Parallel_Generic_Sonic_Fast(b *testing.B) {
   352      _, _ = Encode(_GenericValue, NoQuoteTextMarshaler)
   353      b.SetBytes(int64(len(TwitterJson)))
   354      b.ResetTimer()
   355      b.RunParallel(func(pb *testing.PB) {
   356          for pb.Next() {
   357              _, _ = Encode(_GenericValue, NoQuoteTextMarshaler)
   358          }
   359      })
   360  }
   361  
   362  func BenchmarkEncoder_Parallel_Generic_StdLib(b *testing.B) {
   363      _, _ = json.Marshal(_GenericValue)
   364      b.SetBytes(int64(len(TwitterJson)))
   365      b.ResetTimer()
   366      b.RunParallel(func(pb *testing.PB) {
   367          for pb.Next() {
   368              _, _ = json.Marshal(_GenericValue)
   369          }
   370      })
   371  }
   372  
   373  func BenchmarkEncoder_Parallel_Binding_Sonic(b *testing.B) {
   374      _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   375      b.SetBytes(int64(len(TwitterJson)))
   376      b.ResetTimer()
   377      b.RunParallel(func(pb *testing.PB) {
   378          for pb.Next() {
   379              _, _ = Encode(&_BindingValue, SortMapKeys | EscapeHTML | CompactMarshaler)
   380          }
   381      })
   382  }
   383  
   384  func BenchmarkEncoder_Parallel_Binding_Sonic_Fast(b *testing.B) {
   385      _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
   386      b.SetBytes(int64(len(TwitterJson)))
   387      b.ResetTimer()
   388      b.RunParallel(func(pb *testing.PB) {
   389          for pb.Next() {
   390              _, _ = Encode(&_BindingValue, NoQuoteTextMarshaler)
   391          }
   392      })
   393  }
   394  
   395  func BenchmarkEncoder_Parallel_Binding_StdLib(b *testing.B) {
   396      _, _ = json.Marshal(&_BindingValue)
   397      b.SetBytes(int64(len(TwitterJson)))
   398      b.ResetTimer()
   399      b.RunParallel(func(pb *testing.PB) {
   400          for pb.Next() {
   401              _, _ = json.Marshal(&_BindingValue)
   402          }
   403      })
   404  }
   405  
   406  func BenchmarkHTMLEscape_Sonic(b *testing.B) {
   407      jsonByte := []byte(TwitterJson)
   408      b.SetBytes(int64(len(TwitterJson)))
   409      b.ResetTimer()
   410      var buf []byte
   411      for i := 0; i < b.N; i++ {
   412          buf = HTMLEscape(nil, jsonByte)
   413      }
   414      _ = buf
   415  }
   416  
   417  func BenchmarkHTMLEscape_StdLib(b *testing.B) {
   418      jsonByte := []byte(TwitterJson)
   419      b.SetBytes(int64(len(TwitterJson)))
   420      b.ResetTimer()
   421      var buf []byte
   422      for i := 0; i < b.N; i++ {
   423          out := bytes.NewBuffer(make([]byte, 0, len(TwitterJson) * 6 / 5))
   424          json.HTMLEscape(out, jsonByte)
   425          buf = out.Bytes()
   426      }
   427      _ = buf
   428  }
   429  
   430  
   431  func BenchmarkValidate_Sonic(b *testing.B) {
   432      var data = rt.Str2Mem(TwitterJson)
   433      ok, s := Valid(data)
   434      if !ok {
   435          b.Fatal(s)
   436      }
   437      b.SetBytes(int64(len(TwitterJson)))
   438      b.ResetTimer()
   439      for i:=0; i<b.N; i++ {
   440          _, _ = Valid(data)
   441      }
   442  }
   443  
   444  func BenchmarkValidate_Std(b *testing.B) {
   445      var data = rt.Str2Mem(TwitterJson)
   446      if !json.Valid(data) {
   447          b.Fatal()
   448      }
   449      b.SetBytes(int64(len(TwitterJson)))
   450      b.ResetTimer()
   451      for i:=0; i<b.N; i++ {
   452          _ = json.Valid(data)
   453      }
   454  }
   455  
   456  func BenchmarkCompact_Std(b *testing.B) {
   457      var data = rt.Str2Mem(TwitterJson)
   458      var dst = bytes.NewBuffer(nil)
   459      if err := json.Compact(dst, data); err != nil {
   460          b.Fatal(err)
   461      }
   462      b.SetBytes(int64(len(TwitterJson)))
   463      b.ResetTimer()
   464      for i:=0; i<b.N; i++ {
   465          dst.Reset()
   466          _ = json.Compact(dst, data)
   467      }
   468  }
   469  
   470  type f64Bench struct {
   471      name    string
   472      float   float64
   473  }
   474  func BenchmarkEncode_Float64(b *testing.B) {
   475      var bench = []f64Bench{
   476          {"Zero", 0},
   477          {"ShortDecimal", 1000},
   478          {"Decimal", 33909},
   479          {"Float", 339.7784},
   480          {"Exp", -5.09e75},
   481          {"NegExp", -5.11e-95},
   482          {"LongExp", 1.234567890123456e-78},
   483          {"Big", 123456789123456789123456789},
   484      
   485      }
   486      maxUint := "18446744073709551615"
   487      for i := 1; i <= len(maxUint); i++ {
   488          name := strconv.FormatInt(int64(i), 10) + "-Digs"
   489          num, _ := strconv.ParseUint(string(maxUint[:i]), 10, 64)
   490          bench = append(bench, f64Bench{name, float64(num)})
   491      }
   492      for _, c := range bench {
   493          libs := []struct {
   494              name string
   495              test func(*testing.B)
   496          }{{
   497              name: "StdLib",
   498              test: func(b *testing.B) {  _, _ = json.Marshal(c.float); for i := 0; i < b.N; i++ { _, _ = json.Marshal(c.float) }},
   499          }, {
   500              name: "Sonic",
   501              test: func(b *testing.B) { _, _ = Encode(c.float, 0); for i := 0; i < b.N; i++ { _, _ = Encode(c.float, 0) }},
   502          }}
   503          for _, lib := range libs {
   504              name := lib.name + "_" + c.name
   505              b.Run(name, lib.test)
   506          }  
   507      }
   508  }
   509  
   510  type f32Bench struct {
   511      name    string
   512      float   float32
   513  }
   514  func BenchmarkEncode_Float32(b *testing.B) {
   515      var bench = []f32Bench{
   516          {"Zero", 0},
   517          {"ShortDecimal", 1000},
   518          {"Decimal", 33909},
   519          {"ExactFraction", 3.375},
   520          {"Point", 339.7784},
   521          {"Exp", -5.09e25},
   522          {"NegExp", -5.11e-25},
   523          {"Shortest", 1.234567e-8},
   524      }
   525  
   526      maxUint := "18446744073709551615"
   527      for i := 1; i <= len(maxUint); i++ {
   528          name := strconv.FormatInt(int64(i), 10) + "-Digs"
   529          num, _ := strconv.ParseUint(string(maxUint[:i]), 10, 64)
   530          bench = append(bench, f32Bench{name, float32(num)})
   531      }
   532      for _, c := range bench {
   533          libs := []struct {
   534              name string
   535              test func(*testing.B)
   536          }{{
   537              name: "StdLib",
   538              test: func(b *testing.B) {  _, _ = json.Marshal(c.float); for i := 0; i < b.N; i++ { _, _ = json.Marshal(c.float) }},
   539          }, {
   540              name: "Sonic",
   541              test: func(b *testing.B) { _, _ = Encode(c.float, 0); for i := 0; i < b.N; i++ { _, _ = Encode(c.float, 0) }},
   542          }}
   543          for _, lib := range libs {
   544              name := lib.name + "_" + c.name
   545              b.Run(name, lib.test)
   546          }  
   547      }
   548  }