github.com/dgraph-io/simdjson-go@v0.3.0/parsed_serialize_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package simdjson 18 19 import ( 20 "bytes" 21 "sync" 22 "testing" 23 ) 24 25 func BenchmarkSerialize(b *testing.B) { 26 if !SupportedCPU() { 27 b.SkipNow() 28 } 29 30 bench := func(b *testing.B, s *Serializer) { 31 for _, tt := range testCases { 32 s := NewSerializer() 33 var once sync.Once 34 b.Run(tt.name, func(b *testing.B) { 35 org := loadCompressed(b, tt.name) 36 pj, err := Parse(org, nil) 37 if err != nil { 38 b.Fatal(err) 39 } 40 output := s.Serialize(nil, *pj) 41 once.Do(func() { 42 b.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%") 43 }) 44 //_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm) 45 b.SetBytes(int64(len(org))) 46 b.ReportAllocs() 47 b.ResetTimer() 48 for i := 0; i < b.N; i++ { 49 output = s.Serialize(output[:0], *pj) 50 } 51 }) 52 } 53 } 54 b.Run("default", func(b *testing.B) { 55 s := NewSerializer() 56 bench(b, s) 57 }) 58 b.Run("none", func(b *testing.B) { 59 s := NewSerializer() 60 s.CompressMode(CompressNone) 61 bench(b, s) 62 }) 63 b.Run("fast", func(b *testing.B) { 64 s := NewSerializer() 65 s.CompressMode(CompressFast) 66 bench(b, s) 67 }) 68 b.Run("best", func(b *testing.B) { 69 s := NewSerializer() 70 s.CompressMode(CompressBest) 71 bench(b, s) 72 }) 73 } 74 75 func BenchmarkDeSerialize(b *testing.B) { 76 if !SupportedCPU() { 77 b.SkipNow() 78 } 79 80 bench := func(b *testing.B, s *Serializer) { 81 for _, tt := range testCases { 82 b.Run(tt.name, func(b *testing.B) { 83 org := loadCompressed(b, tt.name) 84 pj, err := Parse(org, nil) 85 if err != nil { 86 b.Fatal(err) 87 } 88 89 output := s.Serialize(nil, *pj) 90 if false { 91 b.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%") 92 } 93 //_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm) 94 pj2, err := s.Deserialize(output, nil) 95 if err != nil { 96 b.Fatal(err) 97 } 98 99 b.SetBytes(int64(len(org))) 100 b.ReportAllocs() 101 b.ResetTimer() 102 for i := 0; i < b.N; i++ { 103 pj2, err = s.Deserialize(output, pj2) 104 if err != nil { 105 b.Fatal(err) 106 } 107 } 108 }) 109 } 110 } 111 112 b.Run("default", func(b *testing.B) { 113 s := NewSerializer() 114 bench(b, s) 115 }) 116 b.Run("none", func(b *testing.B) { 117 s := NewSerializer() 118 s.CompressMode(CompressNone) 119 bench(b, s) 120 }) 121 b.Run("fast", func(b *testing.B) { 122 s := NewSerializer() 123 s.CompressMode(CompressFast) 124 bench(b, s) 125 }) 126 b.Run("best", func(b *testing.B) { 127 s := NewSerializer() 128 s.CompressMode(CompressBest) 129 bench(b, s) 130 }) 131 } 132 133 func BenchmarkSerializeNDJSON(b *testing.B) { 134 if !SupportedCPU() { 135 b.SkipNow() 136 } 137 138 ndjson := loadFile("testdata/parking-citations-1M.json.zst") 139 140 pj, err := ParseND(ndjson, nil) 141 if err != nil { 142 b.Fatal(err) 143 } 144 bench := func(b *testing.B, s *Serializer) { 145 output := s.Serialize(nil, *pj) 146 if true { 147 b.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%") 148 } 149 //_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".compressed"), output, os.ModePerm) 150 b.SetBytes(int64(len(ndjson))) 151 b.ReportAllocs() 152 b.ResetTimer() 153 for i := 0; i < b.N; i++ { 154 output = s.Serialize(output[:0], *pj) 155 } 156 } 157 b.Run("default", func(b *testing.B) { 158 s := NewSerializer() 159 bench(b, s) 160 }) 161 b.Run("none", func(b *testing.B) { 162 s := NewSerializer() 163 s.CompressMode(CompressNone) 164 bench(b, s) 165 }) 166 b.Run("fast", func(b *testing.B) { 167 s := NewSerializer() 168 s.CompressMode(CompressFast) 169 bench(b, s) 170 }) 171 b.Run("best", func(b *testing.B) { 172 s := NewSerializer() 173 s.CompressMode(CompressBest) 174 bench(b, s) 175 }) 176 } 177 178 func BenchmarkDeSerializeNDJSON(b *testing.B) { 179 if !SupportedCPU() { 180 b.SkipNow() 181 } 182 183 ndjson := loadFile("testdata/parking-citations-1M.json.zst") 184 185 pj, err := ParseND(ndjson, nil) 186 if err != nil { 187 b.Fatal(err) 188 } 189 bench := func(b *testing.B, s *Serializer) { 190 output := s.Serialize(nil, *pj) 191 if true { 192 b.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%") 193 } 194 pj2, err := s.Deserialize(output, nil) 195 if err != nil { 196 b.Fatal(err) 197 } 198 // _ = ioutil.WriteFile(filepath.Join("testdata", filepath.Base(b.Name())+".compressed"), output, os.ModePerm) 199 b.SetBytes(int64(len(ndjson))) 200 b.ReportAllocs() 201 b.ResetTimer() 202 for i := 0; i < b.N; i++ { 203 pj2, err = s.Deserialize(output, pj2) 204 if err != nil { 205 b.Fatal(err) 206 } 207 } 208 } 209 b.Run("default", func(b *testing.B) { 210 s := NewSerializer() 211 bench(b, s) 212 }) 213 b.Run("none", func(b *testing.B) { 214 s := NewSerializer() 215 s.CompressMode(CompressNone) 216 bench(b, s) 217 }) 218 b.Run("fast", func(b *testing.B) { 219 s := NewSerializer() 220 s.CompressMode(CompressFast) 221 bench(b, s) 222 }) 223 b.Run("best", func(b *testing.B) { 224 s := NewSerializer() 225 s.CompressMode(CompressBest) 226 bench(b, s) 227 }) 228 } 229 230 func TestDeSerializeNDJSON(t *testing.T) { 231 if !SupportedCPU() { 232 t.SkipNow() 233 } 234 if testing.Short() { 235 t.Skip("skipping... too long") 236 } 237 ndjson := loadFile("testdata/parking-citations-1M.json.zst") 238 239 pj, err := ParseND(ndjson, nil) 240 if err != nil { 241 t.Fatal(err) 242 } 243 test := func(t *testing.T, s *Serializer) { 244 i := pj.Iter() 245 want, err := i.MarshalJSON() 246 if err != nil { 247 t.Fatal(err) 248 } 249 output := s.Serialize(nil, *pj) 250 if testing.Verbose() { 251 t.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%") 252 } 253 pj2, err := s.Deserialize(output, nil) 254 if err != nil { 255 t.Fatal(err) 256 } 257 i = pj2.Iter() 258 got, err := i.MarshalJSON() 259 if err != nil { 260 t.Fatal(err) 261 } 262 if !bytes.Equal(want, got) { 263 t.Fatal("output mismatch") 264 } 265 } 266 t.Run("default", func(b *testing.T) { 267 s := NewSerializer() 268 test(b, s) 269 }) 270 t.Run("none", func(b *testing.T) { 271 s := NewSerializer() 272 s.CompressMode(CompressNone) 273 test(b, s) 274 }) 275 t.Run("fast", func(b *testing.T) { 276 s := NewSerializer() 277 s.CompressMode(CompressFast) 278 test(b, s) 279 }) 280 t.Run("best", func(b *testing.T) { 281 s := NewSerializer() 282 s.CompressMode(CompressBest) 283 test(b, s) 284 }) 285 } 286 287 func TestDeSerializeJSON(t *testing.T) { 288 if !SupportedCPU() { 289 t.SkipNow() 290 } 291 test := func(t *testing.T, s *Serializer) { 292 for _, tt := range testCases { 293 org := loadCompressed(t, tt.name) 294 pj, err := Parse(org, nil) 295 if err != nil { 296 t.Fatal(err) 297 } 298 var once sync.Once 299 t.Run(tt.name, func(t *testing.T) { 300 i := pj.Iter() 301 want, err := i.MarshalJSON() 302 if err != nil { 303 t.Fatal(err) 304 } 305 output := s.Serialize(nil, *pj) 306 if testing.Verbose() { 307 once.Do(func() { 308 t.Log(len(org), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(org)), "%") 309 }) 310 } 311 pj2, err := s.Deserialize(output, nil) 312 if err != nil { 313 t.Fatal(err) 314 } 315 i = pj2.Iter() 316 got, err := i.MarshalJSON() 317 if err != nil { 318 t.Fatal(err) 319 } 320 if !bytes.Equal(want, got) { 321 t.Fatal("output mismatch") 322 } 323 }) 324 } 325 } 326 t.Run("default", func(b *testing.T) { 327 s := NewSerializer() 328 test(b, s) 329 }) 330 t.Run("none", func(b *testing.T) { 331 s := NewSerializer() 332 s.CompressMode(CompressNone) 333 test(b, s) 334 }) 335 t.Run("fast", func(b *testing.T) { 336 s := NewSerializer() 337 s.CompressMode(CompressFast) 338 test(b, s) 339 }) 340 t.Run("best", func(b *testing.T) { 341 s := NewSerializer() 342 s.CompressMode(CompressBest) 343 test(b, s) 344 }) 345 }