github.com/dgraph-io/simdjson-go@v0.3.0/parsed_serialize_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package simdjson
    18  
    19  import (
    20  	"bytes"
    21  	"sync"
    22  	"testing"
    23  )
    24  
    25  func BenchmarkSerialize(b *testing.B) {
    26  	if !SupportedCPU() {
    27  		b.SkipNow()
    28  	}
    29  
    30  	bench := func(b *testing.B, s *Serializer) {
    31  		for _, tt := range testCases {
    32  			s := NewSerializer()
    33  			var once sync.Once
    34  			b.Run(tt.name, func(b *testing.B) {
    35  				org := loadCompressed(b, tt.name)
    36  				pj, err := Parse(org, nil)
    37  				if err != nil {
    38  					b.Fatal(err)
    39  				}
    40  				output := s.Serialize(nil, *pj)
    41  				once.Do(func() {
    42  					b.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%")
    43  				})
    44  				//_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm)
    45  				b.SetBytes(int64(len(org)))
    46  				b.ReportAllocs()
    47  				b.ResetTimer()
    48  				for i := 0; i < b.N; i++ {
    49  					output = s.Serialize(output[:0], *pj)
    50  				}
    51  			})
    52  		}
    53  	}
    54  	b.Run("default", func(b *testing.B) {
    55  		s := NewSerializer()
    56  		bench(b, s)
    57  	})
    58  	b.Run("none", func(b *testing.B) {
    59  		s := NewSerializer()
    60  		s.CompressMode(CompressNone)
    61  		bench(b, s)
    62  	})
    63  	b.Run("fast", func(b *testing.B) {
    64  		s := NewSerializer()
    65  		s.CompressMode(CompressFast)
    66  		bench(b, s)
    67  	})
    68  	b.Run("best", func(b *testing.B) {
    69  		s := NewSerializer()
    70  		s.CompressMode(CompressBest)
    71  		bench(b, s)
    72  	})
    73  }
    74  
    75  func BenchmarkDeSerialize(b *testing.B) {
    76  	if !SupportedCPU() {
    77  		b.SkipNow()
    78  	}
    79  
    80  	bench := func(b *testing.B, s *Serializer) {
    81  		for _, tt := range testCases {
    82  			b.Run(tt.name, func(b *testing.B) {
    83  				org := loadCompressed(b, tt.name)
    84  				pj, err := Parse(org, nil)
    85  				if err != nil {
    86  					b.Fatal(err)
    87  				}
    88  
    89  				output := s.Serialize(nil, *pj)
    90  				if false {
    91  					b.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%")
    92  				}
    93  				//_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm)
    94  				pj2, err := s.Deserialize(output, nil)
    95  				if err != nil {
    96  					b.Fatal(err)
    97  				}
    98  
    99  				b.SetBytes(int64(len(org)))
   100  				b.ReportAllocs()
   101  				b.ResetTimer()
   102  				for i := 0; i < b.N; i++ {
   103  					pj2, err = s.Deserialize(output, pj2)
   104  					if err != nil {
   105  						b.Fatal(err)
   106  					}
   107  				}
   108  			})
   109  		}
   110  	}
   111  
   112  	b.Run("default", func(b *testing.B) {
   113  		s := NewSerializer()
   114  		bench(b, s)
   115  	})
   116  	b.Run("none", func(b *testing.B) {
   117  		s := NewSerializer()
   118  		s.CompressMode(CompressNone)
   119  		bench(b, s)
   120  	})
   121  	b.Run("fast", func(b *testing.B) {
   122  		s := NewSerializer()
   123  		s.CompressMode(CompressFast)
   124  		bench(b, s)
   125  	})
   126  	b.Run("best", func(b *testing.B) {
   127  		s := NewSerializer()
   128  		s.CompressMode(CompressBest)
   129  		bench(b, s)
   130  	})
   131  }
   132  
   133  func BenchmarkSerializeNDJSON(b *testing.B) {
   134  	if !SupportedCPU() {
   135  		b.SkipNow()
   136  	}
   137  
   138  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
   139  
   140  	pj, err := ParseND(ndjson, nil)
   141  	if err != nil {
   142  		b.Fatal(err)
   143  	}
   144  	bench := func(b *testing.B, s *Serializer) {
   145  		output := s.Serialize(nil, *pj)
   146  		if true {
   147  			b.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%")
   148  		}
   149  		//_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm)
   150  		b.SetBytes(int64(len(ndjson)))
   151  		b.ReportAllocs()
   152  		b.ResetTimer()
   153  		for i := 0; i < b.N; i++ {
   154  			output = s.Serialize(output[:0], *pj)
   155  		}
   156  	}
   157  	b.Run("default", func(b *testing.B) {
   158  		s := NewSerializer()
   159  		bench(b, s)
   160  	})
   161  	b.Run("none", func(b *testing.B) {
   162  		s := NewSerializer()
   163  		s.CompressMode(CompressNone)
   164  		bench(b, s)
   165  	})
   166  	b.Run("fast", func(b *testing.B) {
   167  		s := NewSerializer()
   168  		s.CompressMode(CompressFast)
   169  		bench(b, s)
   170  	})
   171  	b.Run("best", func(b *testing.B) {
   172  		s := NewSerializer()
   173  		s.CompressMode(CompressBest)
   174  		bench(b, s)
   175  	})
   176  }
   177  
   178  func BenchmarkDeSerializeNDJSON(b *testing.B) {
   179  	if !SupportedCPU() {
   180  		b.SkipNow()
   181  	}
   182  
   183  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
   184  
   185  	pj, err := ParseND(ndjson, nil)
   186  	if err != nil {
   187  		b.Fatal(err)
   188  	}
   189  	bench := func(b *testing.B, s *Serializer) {
   190  		output := s.Serialize(nil, *pj)
   191  		if true {
   192  			b.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%")
   193  		}
   194  		pj2, err := s.Deserialize(output, nil)
   195  		if err != nil {
   196  			b.Fatal(err)
   197  		}
   198  		// _ = ioutil.WriteFile(filepath.Join("testdata", filepath.Base(b.Name())+".compressed"), output, os.ModePerm)
   199  		b.SetBytes(int64(len(ndjson)))
   200  		b.ReportAllocs()
   201  		b.ResetTimer()
   202  		for i := 0; i < b.N; i++ {
   203  			pj2, err = s.Deserialize(output, pj2)
   204  			if err != nil {
   205  				b.Fatal(err)
   206  			}
   207  		}
   208  	}
   209  	b.Run("default", func(b *testing.B) {
   210  		s := NewSerializer()
   211  		bench(b, s)
   212  	})
   213  	b.Run("none", func(b *testing.B) {
   214  		s := NewSerializer()
   215  		s.CompressMode(CompressNone)
   216  		bench(b, s)
   217  	})
   218  	b.Run("fast", func(b *testing.B) {
   219  		s := NewSerializer()
   220  		s.CompressMode(CompressFast)
   221  		bench(b, s)
   222  	})
   223  	b.Run("best", func(b *testing.B) {
   224  		s := NewSerializer()
   225  		s.CompressMode(CompressBest)
   226  		bench(b, s)
   227  	})
   228  }
   229  
   230  func TestDeSerializeNDJSON(t *testing.T) {
   231  	if !SupportedCPU() {
   232  		t.SkipNow()
   233  	}
   234  	if testing.Short() {
   235  		t.Skip("skipping... too long")
   236  	}
   237  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
   238  
   239  	pj, err := ParseND(ndjson, nil)
   240  	if err != nil {
   241  		t.Fatal(err)
   242  	}
   243  	test := func(t *testing.T, s *Serializer) {
   244  		i := pj.Iter()
   245  		want, err := i.MarshalJSON()
   246  		if err != nil {
   247  			t.Fatal(err)
   248  		}
   249  		output := s.Serialize(nil, *pj)
   250  		if testing.Verbose() {
   251  			t.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%")
   252  		}
   253  		pj2, err := s.Deserialize(output, nil)
   254  		if err != nil {
   255  			t.Fatal(err)
   256  		}
   257  		i = pj2.Iter()
   258  		got, err := i.MarshalJSON()
   259  		if err != nil {
   260  			t.Fatal(err)
   261  		}
   262  		if !bytes.Equal(want, got) {
   263  			t.Fatal("output mismatch")
   264  		}
   265  	}
   266  	t.Run("default", func(b *testing.T) {
   267  		s := NewSerializer()
   268  		test(b, s)
   269  	})
   270  	t.Run("none", func(b *testing.T) {
   271  		s := NewSerializer()
   272  		s.CompressMode(CompressNone)
   273  		test(b, s)
   274  	})
   275  	t.Run("fast", func(b *testing.T) {
   276  		s := NewSerializer()
   277  		s.CompressMode(CompressFast)
   278  		test(b, s)
   279  	})
   280  	t.Run("best", func(b *testing.T) {
   281  		s := NewSerializer()
   282  		s.CompressMode(CompressBest)
   283  		test(b, s)
   284  	})
   285  }
   286  
   287  func TestDeSerializeJSON(t *testing.T) {
   288  	if !SupportedCPU() {
   289  		t.SkipNow()
   290  	}
   291  	test := func(t *testing.T, s *Serializer) {
   292  		for _, tt := range testCases {
   293  			org := loadCompressed(t, tt.name)
   294  			pj, err := Parse(org, nil)
   295  			if err != nil {
   296  				t.Fatal(err)
   297  			}
   298  			var once sync.Once
   299  			t.Run(tt.name, func(t *testing.T) {
   300  				i := pj.Iter()
   301  				want, err := i.MarshalJSON()
   302  				if err != nil {
   303  					t.Fatal(err)
   304  				}
   305  				output := s.Serialize(nil, *pj)
   306  				if testing.Verbose() {
   307  					once.Do(func() {
   308  						t.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%")
   309  					})
   310  				}
   311  				pj2, err := s.Deserialize(output, nil)
   312  				if err != nil {
   313  					t.Fatal(err)
   314  				}
   315  				i = pj2.Iter()
   316  				got, err := i.MarshalJSON()
   317  				if err != nil {
   318  					t.Fatal(err)
   319  				}
   320  				if !bytes.Equal(want, got) {
   321  					t.Fatal("output mismatch")
   322  				}
   323  			})
   324  		}
   325  	}
   326  	t.Run("default", func(b *testing.T) {
   327  		s := NewSerializer()
   328  		test(b, s)
   329  	})
   330  	t.Run("none", func(b *testing.T) {
   331  		s := NewSerializer()
   332  		s.CompressMode(CompressNone)
   333  		test(b, s)
   334  	})
   335  	t.Run("fast", func(b *testing.T) {
   336  		s := NewSerializer()
   337  		s.CompressMode(CompressFast)
   338  		test(b, s)
   339  	})
   340  	t.Run("best", func(b *testing.T) {
   341  		s := NewSerializer()
   342  		s.CompressMode(CompressBest)
   343  		test(b, s)
   344  	})
   345  }