github.com/minio/simdjson-go@v0.4.6-0.20231116094823-04d21cddf993/parsed_serialize_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package simdjson 18 19 import ( 20 "bytes" 21 "sync" 22 "testing" 23 ) 24 25 func BenchmarkSerialize(b *testing.B) { 26 if !SupportedCPU() { 27 b.SkipNow() 28 } 29 30 bench := func(b *testing.B, s *Serializer) { 31 for _, tt := range testCases { 32 b.Run(tt.name, func(b *testing.B) { 33 org := loadCompressed(b, tt.name) 34 pj, err := Parse(org, nil) 35 if err != nil { 36 b.Fatal(err) 37 } 38 output := s.Serialize(nil, *pj) 39 //_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm) 40 b.SetBytes(int64(len(org))) 41 b.ReportAllocs() 42 b.ResetTimer() 43 for i := 0; i < b.N; i++ { 44 output = s.Serialize(output[:0], *pj) 45 } 46 b.ReportMetric(100*float64(len(output))/float64(len(org)), "pct") 47 }) 48 } 49 } 50 b.Run("default", func(b *testing.B) { 51 s := NewSerializer() 52 bench(b, s) 53 }) 54 b.Run("none", func(b *testing.B) { 55 s := NewSerializer() 56 s.CompressMode(CompressNone) 57 bench(b, s) 58 }) 59 b.Run("fast", func(b *testing.B) { 60 s := NewSerializer() 61 s.CompressMode(CompressFast) 62 bench(b, s) 63 }) 64 b.Run("best", func(b *testing.B) { 65 s := NewSerializer() 66 s.CompressMode(CompressBest) 67 bench(b, s) 68 }) 69 } 70 71 func BenchmarkDeSerialize(b *testing.B) { 72 if !SupportedCPU() { 73 b.SkipNow() 74 } 75 76 bench := func(b *testing.B, s *Serializer) { 77 for _, tt := range testCases { 78 b.Run(tt.name, func(b *testing.B) { 79 org := loadCompressed(b, tt.name) 80 pj, err := Parse(org, nil) 81 if err != nil { 82 b.Fatal(err) 83 } 84 85 output := s.Serialize(nil, *pj) 86 //_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm) 87 pj2, err := s.Deserialize(output, nil) 88 if err != nil { 89 b.Fatal(err) 90 } 91 92 b.SetBytes(int64(len(org))) 93 b.ReportAllocs() 94 b.ResetTimer() 95 for i := 0; i < b.N; i++ { 96 pj2, err = s.Deserialize(output, pj2) 97 if err != nil { 98 b.Fatal(err) 99 } 100 } 101 b.ReportMetric(100*float64(len(output))/float64(len(org)), "pct") 102 }) 103 } 104 } 105 106 b.Run("default", func(b *testing.B) { 107 s := NewSerializer() 108 bench(b, s) 109 }) 110 b.Run("none", func(b *testing.B) { 111 s := NewSerializer() 112 s.CompressMode(CompressNone) 113 bench(b, s) 114 }) 115 b.Run("fast", func(b *testing.B) { 116 s := NewSerializer() 117 s.CompressMode(CompressFast) 118 bench(b, s) 119 }) 120 b.Run("best", func(b *testing.B) { 121 s := NewSerializer() 122 s.CompressMode(CompressBest) 123 bench(b, s) 124 }) 125 } 126 127 func BenchmarkSerializeNDJSON(b *testing.B) { 128 if !SupportedCPU() { 129 b.SkipNow() 130 } 131 132 ndjson := loadFile("testdata/parking-citations-1M.json.zst") 133 134 pj, err := ParseND(ndjson, nil) 135 if err != nil { 136 b.Fatal(err) 137 } 138 bench := func(b *testing.B, s *Serializer) { 139 output := s.Serialize(nil, *pj) 140 //_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm) 141 b.SetBytes(int64(len(ndjson))) 142 b.ReportAllocs() 143 b.ResetTimer() 144 for i := 0; i < b.N; i++ { 145 output = s.Serialize(output[:0], *pj) 146 } 147 b.ReportMetric(100*float64(len(output))/float64(len(ndjson)), "pct") 148 } 149 b.Run("default", func(b *testing.B) { 150 s := NewSerializer() 151 bench(b, s) 152 }) 153 b.Run("none", func(b *testing.B) { 154 s := NewSerializer() 155 s.CompressMode(CompressNone) 156 bench(b, s) 157 }) 158 b.Run("fast", func(b *testing.B) { 159 s := NewSerializer() 160 s.CompressMode(CompressFast) 161 bench(b, s) 162 }) 163 b.Run("best", func(b *testing.B) { 164 s := NewSerializer() 165 s.CompressMode(CompressBest) 166 bench(b, s) 167 }) 168 } 169 170 func BenchmarkDeSerializeNDJSON(b *testing.B) { 171 if !SupportedCPU() { 172 b.SkipNow() 173 } 174 175 ndjson := loadFile("testdata/parking-citations-1M.json.zst") 176 177 pj, err := ParseND(ndjson, nil) 178 if err != nil { 179 b.Fatal(err) 180 } 181 bench := func(b *testing.B, s *Serializer) { 182 output := s.Serialize(nil, *pj) 183 pj2, err := s.Deserialize(output, nil) 184 if err != nil { 185 b.Fatal(err) 186 } 187 // _ = ioutil.WriteFile(filepath.Join("testdata", filepath.Base(b.Name())+".compressed"), output, os.ModePerm) 188 b.SetBytes(int64(len(ndjson))) 189 b.ReportAllocs() 190 b.ResetTimer() 191 for i := 0; i < b.N; i++ { 192 pj2, err = s.Deserialize(output, pj2) 193 if err != nil { 194 b.Fatal(err) 195 } 196 } 197 b.ReportMetric(100*float64(len(output))/float64(len(ndjson)), "pct") 198 } 199 b.Run("default", func(b *testing.B) { 200 s := NewSerializer() 201 bench(b, s) 202 }) 203 b.Run("none", func(b *testing.B) { 204 s := NewSerializer() 205 s.CompressMode(CompressNone) 206 bench(b, s) 207 }) 208 b.Run("fast", func(b *testing.B) { 209 s := NewSerializer() 210 s.CompressMode(CompressFast) 211 bench(b, s) 212 }) 213 b.Run("best", func(b *testing.B) { 214 s := NewSerializer() 215 s.CompressMode(CompressBest) 216 bench(b, s) 217 }) 218 } 219 220 func TestDeSerializeNDJSON(t *testing.T) { 221 if !SupportedCPU() { 222 t.SkipNow() 223 } 224 if testing.Short() { 225 t.Skip("skipping... too long") 226 } 227 ndjson := loadFile("testdata/parking-citations.json.zst") 228 229 pj, err := ParseND(ndjson, nil) 230 if err != nil { 231 t.Fatal(err) 232 } 233 test := func(t *testing.T, s *Serializer) { 234 i := pj.Iter() 235 want, err := i.MarshalJSON() 236 if err != nil { 237 t.Fatal(err) 238 } 239 output := s.Serialize(nil, *pj) 240 if testing.Verbose() { 241 t.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%") 242 } 243 pj2, err := s.Deserialize(output, nil) 244 if err != nil { 245 t.Fatal(err) 246 } 247 i = pj2.Iter() 248 got, err := i.MarshalJSON() 249 if err != nil { 250 t.Fatal(err) 251 } 252 if !bytes.Equal(want, got) { 253 t.Fatal("output mismatch") 254 } 255 } 256 t.Run("default", func(b *testing.T) { 257 s := NewSerializer() 258 test(b, s) 259 }) 260 t.Run("none", func(b *testing.T) { 261 s := NewSerializer() 262 s.CompressMode(CompressNone) 263 test(b, s) 264 }) 265 t.Run("fast", func(b *testing.T) { 266 s := NewSerializer() 267 s.CompressMode(CompressFast) 268 test(b, s) 269 }) 270 t.Run("best", func(b *testing.T) { 271 s := NewSerializer() 272 s.CompressMode(CompressBest) 273 test(b, s) 274 }) 275 } 276 277 func TestDeSerializeJSON(t *testing.T) { 278 if !SupportedCPU() { 279 t.SkipNow() 280 } 281 test := func(t *testing.T, s *Serializer) { 282 for _, tt := range testCases { 283 org := loadCompressed(t, tt.name) 284 pj, err := Parse(org, nil) 285 if err != nil { 286 t.Fatal(err) 287 } 288 var once sync.Once 289 t.Run(tt.name, func(t *testing.T) { 290 i := pj.Iter() 291 want, err := i.MarshalJSON() 292 if err != nil { 293 t.Fatal(err) 294 } 295 output := s.Serialize(nil, *pj) 296 if testing.Verbose() { 297 once.Do(func() { 298 t.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%") 299 }) 300 } 301 pj2, err := s.Deserialize(output, nil) 302 if err != nil { 303 t.Fatal(err) 304 } 305 i = pj2.Iter() 306 got, err := i.MarshalJSON() 307 if err != nil { 308 t.Fatal(err) 309 } 310 if !bytes.Equal(want, got) { 311 t.Fatal("output mismatch") 312 } 313 }) 314 } 315 } 316 t.Run("default", func(b *testing.T) { 317 s := NewSerializer() 318 test(b, s) 319 }) 320 t.Run("none", func(b *testing.T) { 321 s := NewSerializer() 322 s.CompressMode(CompressNone) 323 test(b, s) 324 }) 325 t.Run("fast", func(b *testing.T) { 326 s := NewSerializer() 327 s.CompressMode(CompressFast) 328 test(b, s) 329 }) 330 t.Run("best", func(b *testing.T) { 331 s := NewSerializer() 332 s.CompressMode(CompressBest) 333 test(b, s) 334 }) 335 }