github.com/bytedance/sonic@v1.11.7-0.20240517092252-d2edb31b167b/internal/decoder/stream_test.go (about) 1 /* 2 * Copyright 2021 ByteDance Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package decoder 18 19 import ( 20 `bytes` 21 `encoding/json` 22 `io` 23 `io/ioutil` 24 `strings` 25 `testing` 26 27 `github.com/bytedance/sonic/option` 28 `github.com/stretchr/testify/assert` 29 `github.com/stretchr/testify/require` 30 ) 31 32 var ( 33 DefaultBufferSize = option.DefaultDecoderBufferSize 34 _Single_JSON = `{`+`"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}` + strings.Repeat(" ", int(DefaultBufferSize)) + `{` 35 _Double_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}` + strings.Repeat(" ", int(DefaultBufferSize)) + `{"11111":"` + strings.Repeat("2", int(DefaultBufferSize)) + `"}` 36 _Triple_JSON = `{"aaaaa":"` + strings.Repeat("b", int(DefaultBufferSize)) + `"}{ } {"11111":"` + 37 strings.Repeat("2", int(DefaultBufferSize))+`"} b {}` 38 _SMALL_JSON = `{"a":"b"} [1] {"c":"d"} [2]` 39 ) 40 41 func TestStreamError(t *testing.T) { 42 var qs = []string{ 43 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"`, 44 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"}`, 45 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`""}`, 46 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":}`, 47 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":]`, 48 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x`, 49 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x}`, 50 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":1x]`, 51 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":t`, 52 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":t}`, 53 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":true]`, 54 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":f`, 55 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":f}`, 56 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":false]`, 57 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":n`, 58 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":n}`, 59 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":null]`, 60 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"`, 61 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a`, 62 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a}`, 63 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a"`, 64 `{`+strings.Repeat(" ", int(DefaultBufferSize))+`"":"a"]`, 65 } 66 67 for i, q := range qs { 68 var qq = []byte(q[:int(DefaultBufferSize)]+strings.Repeat(" ", i*100)+q[int(DefaultBufferSize):]) 69 var obj interface{} 70 require.NotNil(t, NewStreamDecoder(bytes.NewReader(qq)).Decode(&obj)) 71 } 72 } 73 74 func TestDecodeEmpty(t *testing.T) { 75 var str = `` 76 var r1 = strings.NewReader(str) 77 var v1 interface{} 78 var d1 = json.NewDecoder(r1) 79 var r2 = strings.NewReader(str) 80 var v2 interface{} 81 var d2 = NewStreamDecoder(r2) 82 es1 := d1.Decode(&v1) 83 ee1 := d2.Decode(&v2) 84 assert.Equal(t, es1, ee1) 85 assert.Equal(t, v1, v2) 86 } 87 88 func TestDecodeRecurse(t *testing.T) { 89 var str = _Single_JSON 90 var r1 = bytes.NewBufferString(str) 91 var v1 interface{} 92 var d1 = json.NewDecoder(r1) 93 var r2 = bytes.NewBufferString(str) 94 var v2 interface{} 95 var d2 = NewStreamDecoder(r2) 96 97 require.Equal(t, d1.More(), d2.More()) 98 es1 := d1.Decode(&v1) 99 ee1 := d2.Decode(&v2) 100 assert.Equal(t, es1, ee1) 101 assert.Equal(t, v1, v2) 102 103 require.Equal(t, d1.More(), d2.More()) 104 r1.WriteString(str[1:]) 105 r2.WriteString(str[1:]) 106 107 require.Equal(t, d1.More(), d2.More()) 108 es1 = d1.Decode(&v1) 109 ee1 = d2.Decode(&v2) 110 assert.Equal(t, es1, ee1) 111 println(es1) 112 assert.Equal(t, v1, v2) 113 require.Equal(t, d1.More(), d2.More()) 114 } 115 116 type HaltReader struct { 117 halts map[int]bool 118 buf string 119 p int 120 } 121 122 func NewHaltReader(buf string, halts map[int]bool) *HaltReader { 123 return &HaltReader{ 124 halts: halts, 125 buf: buf, 126 p: 0, 127 } 128 } 129 130 func (self *HaltReader) Read(p []byte) (int, error) { 131 t := 0 132 for ; t < len(p); { 133 if self.p >= len(self.buf) { 134 return t, io.EOF 135 } 136 if b, ok := self.halts[self.p]; b { 137 self.halts[self.p] = false 138 return t, nil 139 } else if ok { 140 delete(self.halts, self.p) 141 return 0, nil 142 } 143 p[t] = self.buf[self.p] 144 self.p++ 145 t++ 146 } 147 return t, nil 148 } 149 150 func (self *HaltReader) Reset(buf string) { 151 self.p = 0 152 self.buf = buf 153 } 154 155 var testHalts = func () map[int]bool { 156 return map[int]bool{ 157 1: true, 158 10:true, 159 20: true} 160 } 161 162 func TestBuffered(t *testing.T) { 163 var str = _Triple_JSON 164 var r1 = NewHaltReader(str, testHalts()) 165 var v1 map[string]interface{} 166 var d1 = json.NewDecoder(r1) 167 168 var r2 = NewHaltReader(str, testHalts()) 169 var v2 map[string]interface{} 170 var d2 = NewStreamDecoder(r2) 171 172 require.Equal(t, d1.More(), d2.More()) 173 require.Nil(t, d1.Decode(&v1)) 174 require.Nil(t, d2.Decode(&v2)) 175 left1, err1 := ioutil.ReadAll(d1.Buffered()) 176 require.Nil(t, err1) 177 left2, err2 := ioutil.ReadAll(d2.Buffered()) 178 require.Nil(t, err2) 179 require.Equal(t, d1.InputOffset(), d2.InputOffset()) 180 min := len(left1) 181 if min > len(left2) { 182 min = len(left2) 183 } 184 require.Equal(t, left1[:min], left2[:min]) 185 186 require.Equal(t, d1.More(), d2.More()) 187 es4 := d1.Decode(&v1) 188 ee4 := d2.Decode(&v2) 189 assert.Equal(t, es4, ee4) 190 println(str[d1.InputOffset()-5:d1.InputOffset()+5]) 191 assert.Equal(t, d1.InputOffset(), d2.InputOffset()-1) 192 193 require.Equal(t, d1.More(), d2.More()) 194 es2 := d1.Decode(&v1) 195 ee2 := d2.Decode(&v2) 196 assert.Equal(t, es2, ee2) 197 println(str[d1.InputOffset()-5:d1.InputOffset()+5]) 198 assert.Equal(t, d1.InputOffset(), d2.InputOffset()-1) 199 } 200 201 func BenchmarkDecodeStream_Std(b *testing.B) { 202 b.Run("single", func (b *testing.B) { 203 var str = _Single_JSON 204 var r1 = bytes.NewBufferString(str) 205 dc := json.NewDecoder(r1) 206 for i:=0; i<b.N; i++ { 207 var v1 map[string]interface{} 208 e := dc.Decode(&v1) 209 if e != nil { 210 b.Fatal(e) 211 } 212 r1.WriteString(str[1:]) 213 } 214 }) 215 216 b.Run("double", func (b *testing.B) { 217 var str = _Double_JSON 218 for i:=0; i<b.N; i++ { 219 var r1 = strings.NewReader(str) 220 var v1 map[string]interface{} 221 dc := json.NewDecoder(r1) 222 _ = dc.Decode(&v1) 223 if dc.More() { 224 _ = dc.Decode(&v1) 225 } 226 } 227 }) 228 229 b.Run("4x", func (b *testing.B) { 230 var str = _Double_JSON + strings.Repeat(" ", int(DefaultBufferSize-10)) + _Double_JSON 231 b.ResetTimer() 232 for i:=0; i<b.N; i++ { 233 var r1 = strings.NewReader(str) 234 var v1 map[string]interface{} 235 dc := json.NewDecoder(r1) 236 for dc.More() { 237 e := dc.Decode(&v1) 238 if e != nil { 239 b.Fatal(e) 240 } 241 } 242 } 243 }) 244 245 b.Run("halt", func (b *testing.B) { 246 var str = _Double_JSON 247 for i:=0; i<b.N; i++ { 248 var r1 = NewHaltReader(str, testHalts()) 249 var v1 map[string]interface{} 250 dc := json.NewDecoder(r1) 251 _ = dc.Decode(&v1) 252 } 253 }) 254 255 b.Run("small", func (b *testing.B) { 256 var str = _SMALL_JSON 257 for i:=0; i<b.N; i++ { 258 var r1 = strings.NewReader(str) 259 var v1 interface{} 260 dc := json.NewDecoder(r1) 261 for dc.More() { 262 e := dc.Decode(&v1) 263 if e != nil { 264 b.Fatal(e) 265 } 266 } 267 } 268 }) 269 } 270 271 // func BenchmarkDecodeError_Sonic(b *testing.B) { 272 // var str = `\b测试1234` 273 // for i:=0; i<b.N; i++ { 274 // var v1 map[string]interface{} 275 // _ = NewDecoder(str).Decode(&v1) 276 // } 277 // } 278 279 func BenchmarkDecodeStream_Sonic(b *testing.B) { 280 b.Run("single", func (b *testing.B) { 281 var str = _Single_JSON 282 var r1 = bytes.NewBufferString(str) 283 dc := NewStreamDecoder(r1) 284 for i:=0; i<b.N; i++ { 285 var v1 map[string]interface{} 286 e := dc.Decode(&v1) 287 if e != nil { 288 b.Fatal(e) 289 } 290 r1.WriteString(str[1:]) 291 } 292 }) 293 294 b.Run("double", func (b *testing.B) { 295 var str = _Double_JSON 296 for i:=0; i<b.N; i++ { 297 var r1 = strings.NewReader(str) 298 var v1 map[string]interface{} 299 dc := NewStreamDecoder(r1) 300 _ = dc.Decode(&v1) 301 if dc.More() { 302 _ = dc.Decode(&v1) 303 } 304 } 305 }) 306 307 b.Run("4x", func (b *testing.B) { 308 var str = _Double_JSON + strings.Repeat(" ", int(DefaultBufferSize-10)) + _Double_JSON 309 b.ResetTimer() 310 for i:=0; i<b.N; i++ { 311 // println("loop") 312 var r1 = strings.NewReader(str) 313 var v1 map[string]interface{} 314 dc := NewStreamDecoder(r1) 315 for dc.More() { 316 e := dc.Decode(&v1) 317 if e != nil { 318 b.Fatal(e) 319 } 320 } 321 } 322 }) 323 324 b.Run("halt", func (b *testing.B) { 325 var str = _Double_JSON 326 for i:=0; i<b.N; i++ { 327 var r1 = NewHaltReader(str, testHalts()) 328 var v1 map[string]interface{} 329 dc := NewStreamDecoder(r1) 330 _ = dc.Decode(&v1) 331 } 332 }) 333 334 b.Run("small", func (b *testing.B) { 335 var str = _SMALL_JSON 336 for i:=0; i<b.N; i++ { 337 // println("one loop") 338 var r1 = strings.NewReader(str) 339 var v1 interface{} 340 dc := NewStreamDecoder(r1) 341 for dc.More() { 342 e := dc.Decode(&v1) 343 if e != nil { 344 b.Fatal(e) 345 } 346 } 347 } 348 }) 349 }