git.lukeshu.com/go/lowmemjson@v0.3.9-0.20230723050957-72f6d13f6fb2/decode_scan_test.go (about) 1 // Copyright (C) 2022-2023 Luke Shumaker <lukeshu@lukeshu.com> 2 // 3 // SPDX-License-Identifier: GPL-2.0-or-later 4 5 package lowmemjson 6 7 import ( 8 "fmt" 9 "io" 10 "strings" 11 "testing" 12 13 "github.com/stretchr/testify/assert" 14 15 "git.lukeshu.com/go/lowmemjson/internal/jsonparse" 16 ) 17 18 type ReadRuneTypeResult struct { 19 r rune 20 s int 21 t jsonparse.RuneType 22 e error 23 } 24 25 const ( 26 unreadRune = -1 27 pushReadBarrier = -2 28 popReadBarrier = -3 29 reset = -4 30 ) 31 32 func (r ReadRuneTypeResult) String() string { 33 switch r.s { 34 case unreadRune: 35 return fmt.Sprintf("{%q, unreadRune, %#v, %v}", r.r, r.t, r.e) 36 case pushReadBarrier: 37 return fmt.Sprintf("{%q, pushReadBarrier, %#v, %v}", r.r, r.t, r.e) 38 case popReadBarrier: 39 return fmt.Sprintf("{%q, popReadBarrier, %#v, %v}", r.r, r.t, r.e) 40 case reset: 41 return fmt.Sprintf("{%q, reset, %#v, %v}", r.r, r.t, r.e) 42 default: 43 return fmt.Sprintf("{%q, %d, %#v, %v}", r.r, r.s, r.t, r.e) 44 } 45 } 46 47 type runeTypeScannerTestcase struct { 48 Input string 49 ExpRemainder string 50 Exp []ReadRuneTypeResult 51 } 52 53 func TestRuneTypeScanner(t *testing.T) { 54 t.Parallel() 55 testcases := map[string]runeTypeScannerTestcase{ 56 "basic": {`{"foo": 12.0}`, ``, []ReadRuneTypeResult{ 57 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 58 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 59 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 60 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 61 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 62 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 63 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 64 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 65 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 66 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 67 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 68 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 69 {0, 0, jsonparse.RuneTypeEOF, nil}, 70 {0, 0, jsonparse.RuneTypeEOF, nil}, 71 }}, 72 "unread": {`{"foo": 12.0}`, ``, []ReadRuneTypeResult{ 73 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 74 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 75 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 76 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 77 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 78 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 79 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 80 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 81 {0, unreadRune, 0, nil}, 82 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 83 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 84 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 85 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 86 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 87 {0, 0, jsonparse.RuneTypeEOF, nil}, 88 {0, 0, jsonparse.RuneTypeEOF, nil}, 89 }}, 90 "unread2": {`{"foo": 12.0}`, ``, []ReadRuneTypeResult{ 91 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 92 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 93 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 94 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 95 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 96 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 97 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 98 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 99 {0, unreadRune, 0, nil}, 100 {0, unreadRune, 0, ErrInvalidUnreadRune}, 101 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 102 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 103 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 104 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 105 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 106 {0, 0, jsonparse.RuneTypeEOF, nil}, 107 {0, 0, jsonparse.RuneTypeEOF, nil}, 108 }}, 109 "unread-eof": {`[1,2]`, ``, []ReadRuneTypeResult{ 110 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 111 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 112 {',', 1, jsonparse.RuneTypeArrayComma, nil}, 113 {0, pushReadBarrier, 0, nil}, 114 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 115 {0, 0, jsonparse.RuneTypeEOF, nil}, 116 {0, unreadRune, 0, ErrInvalidUnreadRune}, 117 {0, popReadBarrier, 0, nil}, 118 {']', 1, jsonparse.RuneTypeArrayEnd, nil}, 119 {0, 0, jsonparse.RuneTypeEOF, nil}, 120 {0, unreadRune, 0, ErrInvalidUnreadRune}, 121 {0, 0, jsonparse.RuneTypeEOF, nil}, 122 {0, 0, jsonparse.RuneTypeEOF, nil}, 123 }}, 124 "tail-ws": {`{"foo": 12.0} `, ``, []ReadRuneTypeResult{ 125 // Disable auto-child. 126 {0, pushReadBarrier, 0, nil}, 127 {0, popReadBarrier, 0, nil}, 128 // Test main. 129 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 130 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 131 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 132 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 133 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 134 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 135 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 136 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 137 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 138 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 139 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 140 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 141 {0, 0, jsonparse.RuneTypeEOF, nil}, 142 {0, 0, jsonparse.RuneTypeEOF, nil}, 143 }}, 144 "child-tail-ws": {`[1,` + `{"foo": 12.0} `, ` `, []ReadRuneTypeResult{ 145 // Child prefix. 146 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 147 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 148 {',', 1, jsonparse.RuneTypeArrayComma, nil}, 149 {0, pushReadBarrier, 0, nil}, 150 // Test main. 151 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 152 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 153 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 154 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 155 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 156 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 157 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 158 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 159 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 160 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 161 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 162 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 163 {0, 0, jsonparse.RuneTypeEOF, nil}, 164 {0, 0, jsonparse.RuneTypeEOF, nil}, 165 }}, 166 "syntax-error": {`[[0,]`, ``, []ReadRuneTypeResult{ 167 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 168 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 169 {'0', 1, jsonparse.RuneTypeNumberIntZero, nil}, 170 {',', 1, jsonparse.RuneTypeArrayComma, nil}, 171 {']', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}}, 172 {']', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}}, 173 {']', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q looking for beginning of value", ']')}}, 174 }}, 175 "multi-value1": {`1{}`, `{}`, []ReadRuneTypeResult{ 176 {0, pushReadBarrier, 0, nil}, 177 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 178 {0, 0, jsonparse.RuneTypeEOF, nil}, 179 {0, 0, jsonparse.RuneTypeEOF, nil}, 180 {0, 0, jsonparse.RuneTypeEOF, nil}, 181 {0, popReadBarrier, 0, nil}, 182 }}, 183 "multi-value2": {`1{}`, ``, []ReadRuneTypeResult{ 184 {0, pushReadBarrier, 0, nil}, 185 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 186 {0, 0, jsonparse.RuneTypeEOF, nil}, 187 {0, 0, jsonparse.RuneTypeEOF, nil}, 188 {0, 0, jsonparse.RuneTypeEOF, nil}, 189 {0, popReadBarrier, 0, nil}, 190 {0, reset, 0, nil}, 191 {0, pushReadBarrier, 0, nil}, 192 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 193 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 194 {0, 0, jsonparse.RuneTypeEOF, nil}, 195 {0, 0, jsonparse.RuneTypeEOF, nil}, 196 {0, 0, jsonparse.RuneTypeEOF, nil}, 197 {0, popReadBarrier, 0, nil}, 198 {0, 0, jsonparse.RuneTypeEOF, nil}, 199 {0, 0, jsonparse.RuneTypeEOF, nil}, 200 {0, 0, jsonparse.RuneTypeEOF, nil}, 201 }}, 202 "early-eof": {` {`, ``, []ReadRuneTypeResult{ 203 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 204 {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 2, Err: io.ErrUnexpectedEOF}}, 205 {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 2, Err: io.ErrUnexpectedEOF}}, 206 {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 2, Err: io.ErrUnexpectedEOF}}, 207 }}, 208 "empty": {``, ``, []ReadRuneTypeResult{ 209 {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 0, Err: io.EOF}}, 210 {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 0, Err: io.EOF}}, 211 {0, 0, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 0, Err: io.EOF}}, 212 }}, 213 "basic2": {`1`, ``, []ReadRuneTypeResult{ 214 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 215 {0, 0, jsonparse.RuneTypeEOF, nil}, 216 {0, 0, jsonparse.RuneTypeEOF, nil}, 217 {0, 0, jsonparse.RuneTypeEOF, nil}, 218 }}, 219 "fragment": {`1,`, ``, []ReadRuneTypeResult{ 220 // Disable auto-child. 221 {0, pushReadBarrier, 0, nil}, 222 {0, popReadBarrier, 0, nil}, 223 // Test main. 224 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 225 {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, 226 {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, 227 {',', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 1, Err: fmt.Errorf("invalid character %q after top-level value", ',')}}, 228 }}, 229 "child-fragment": {`[1,` + `1,`, `,`, []ReadRuneTypeResult{ 230 // Child prefix. 231 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 232 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 233 {',', 1, jsonparse.RuneTypeArrayComma, nil}, 234 {0, pushReadBarrier, 0, nil}, 235 // Test main. 236 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 237 {0, 0, jsonparse.RuneTypeEOF, nil}, 238 {0, 0, jsonparse.RuneTypeEOF, nil}, 239 {0, 0, jsonparse.RuneTypeEOF, nil}, 240 }}, 241 "elem": {` { "foo" : 12.0 } `, ``, []ReadRuneTypeResult{ 242 // Disable auto-child. 243 {0, pushReadBarrier, 0, nil}, 244 {0, popReadBarrier, 0, nil}, 245 // Test main. 246 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 247 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 248 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 249 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 250 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 251 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 252 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 253 {0, pushReadBarrier, 0, nil}, 254 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 255 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 256 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 257 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 258 {0, 0, jsonparse.RuneTypeEOF, nil}, 259 {0, 0, jsonparse.RuneTypeEOF, nil}, 260 {0, popReadBarrier, 0, nil}, 261 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 262 {0, 0, jsonparse.RuneTypeEOF, nil}, 263 {0, 0, jsonparse.RuneTypeEOF, nil}, 264 }}, 265 "child-elem": {`[1,` + ` { "foo" : 12.0 } `, ` `, []ReadRuneTypeResult{ 266 // Child prefix. 267 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 268 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 269 {',', 1, jsonparse.RuneTypeArrayComma, nil}, 270 {0, pushReadBarrier, 0, nil}, 271 // Test main. 272 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 273 {'"', 1, jsonparse.RuneTypeStringBeg, nil}, 274 {'f', 1, jsonparse.RuneTypeStringChar, nil}, 275 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 276 {'o', 1, jsonparse.RuneTypeStringChar, nil}, 277 {'"', 1, jsonparse.RuneTypeStringEnd, nil}, 278 {':', 1, jsonparse.RuneTypeObjectColon, nil}, 279 {0, pushReadBarrier, 0, nil}, 280 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 281 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 282 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 283 {'0', 1, jsonparse.RuneTypeNumberFracDig, nil}, 284 {0, 0, jsonparse.RuneTypeEOF, nil}, 285 {0, 0, jsonparse.RuneTypeEOF, nil}, 286 {0, popReadBarrier, 0, nil}, 287 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 288 {0, 0, jsonparse.RuneTypeEOF, nil}, 289 {0, 0, jsonparse.RuneTypeEOF, nil}, 290 }}, 291 "invalid-number": {`1.2.3`, ``, []ReadRuneTypeResult{ 292 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 293 {'.', 1, jsonparse.RuneTypeNumberFracDot, nil}, 294 {'2', 1, jsonparse.RuneTypeNumberFracDig, nil}, 295 {'.', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 3, Err: fmt.Errorf("invalid character %q after top-level value", '.')}}, 296 {0, reset, 0, nil}, 297 {'3', 1, jsonparse.RuneTypeNumberIntDig, nil}, 298 {0, 0, jsonparse.RuneTypeEOF, nil}, 299 }}, 300 "trailing-garbage": {" 42 x", ``, []ReadRuneTypeResult{ 301 {0, pushReadBarrier, 0, nil}, 302 {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, 303 {0, unreadRune, 0, nil}, 304 {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, 305 {0, unreadRune, 0, nil}, 306 {0, pushReadBarrier, 0, nil}, 307 {'4', 1, jsonparse.RuneTypeNumberIntDig, nil}, 308 {'2', 1, jsonparse.RuneTypeNumberIntDig, nil}, 309 {0, 0, jsonparse.RuneTypeEOF, nil}, 310 {0, popReadBarrier, 0, nil}, 311 {0, popReadBarrier, 0, nil}, 312 {'x', 1, jsonparse.RuneTypeError, &DecodeSyntaxError{Offset: 4, Err: fmt.Errorf("invalid character %q after top-level value", 'x')}}, 313 }}, 314 "unread-reset": {`{}`, ``, []ReadRuneTypeResult{ 315 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 316 {0, unreadRune, 0, nil}, 317 {0, reset, 0, nil}, 318 {'{', 1, jsonparse.RuneTypeObjectBeg, nil}, 319 {'}', 1, jsonparse.RuneTypeObjectEnd, nil}, 320 {0, 0, jsonparse.RuneTypeEOF, nil}, 321 {0, 0, jsonparse.RuneTypeEOF, nil}, 322 {0, 0, jsonparse.RuneTypeEOF, nil}, 323 }}, 324 } 325 func() { 326 childTestcases := make(map[string]runeTypeScannerTestcase) 327 for tcName, tc := range testcases { 328 canChild := true 329 for _, res := range tc.Exp { 330 if res.s == pushReadBarrier || res.s == reset { 331 canChild = false 332 break 333 } 334 } 335 if !canChild { 336 continue 337 } 338 tc.Input = `[1,` + tc.Input 339 tc.Exp = append([]ReadRuneTypeResult{ 340 {'[', 1, jsonparse.RuneTypeArrayBeg, nil}, 341 {'1', 1, jsonparse.RuneTypeNumberIntDig, nil}, 342 {',', 1, jsonparse.RuneTypeArrayComma, nil}, 343 {0, pushReadBarrier, 0, nil}, 344 }, tc.Exp...) 345 for i := 2; i < len(tc.Exp); i++ { 346 if se, ok := tc.Exp[i].e.(*DecodeSyntaxError); ok { 347 seCopy := *se 348 seCopy.Offset += 3 349 tc.Exp[i].e = &seCopy 350 } 351 } 352 childTestcases["child-"+tcName] = tc 353 } 354 for tcName, tc := range childTestcases { 355 testcases[tcName] = tc 356 } 357 }() 358 for tcName, tc := range testcases { 359 tc := tc 360 t.Run(tcName, func(t *testing.T) { 361 t.Parallel() 362 t.Logf("input=%q", tc.Input) 363 reader := strings.NewReader(tc.Input) 364 sc := &runeTypeScanner{inner: reader} 365 var exp, act []string 366 for _, iExp := range tc.Exp { 367 var iAct ReadRuneTypeResult 368 switch iExp.s { 369 case unreadRune: 370 iAct.s = iExp.s 371 iAct.e = sc.UnreadRune() 372 case pushReadBarrier: 373 sc.PushReadBarrier() 374 iAct.s = iExp.s 375 case popReadBarrier: 376 sc.PopReadBarrier() 377 iAct.s = iExp.s 378 case reset: 379 sc.Reset() 380 iAct.s = iExp.s 381 default: 382 iAct.r, iAct.s, iAct.t, iAct.e = sc.ReadRuneType() 383 } 384 exp = append(exp, iExp.String()) 385 act = append(act, iAct.String()) 386 } 387 assert.Equal(t, exp, act) 388 assert.Equal(t, tc.ExpRemainder, tc.Input[len(tc.Input)-reader.Len():]) 389 }) 390 } 391 }