github.com/3JoB/go-json@v0.10.4/stream_test.go (about) 1 // Copyright 2010 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package json_test 6 7 import ( 8 "bytes" 9 "compress/gzip" 10 "fmt" 11 "io" 12 "log" 13 "net" 14 "net/http" 15 "net/http/httptest" 16 "strconv" 17 "strings" 18 "testing" 19 20 "github.com/3JoB/go-reflect" 21 22 "github.com/3JoB/go-json" 23 ) 24 25 // Test values for the stream test. 26 // One of each JSON kind. 27 var streamTest = []any{ 28 0.1, 29 "hello", 30 nil, 31 true, 32 false, 33 []any{"a", "b", "c"}, 34 map[string]any{"K": "Kelvin", "ß": "long s"}, 35 3.14, // another value to make sure something can follow map 36 } 37 38 var streamEncoded = `0.1 39 "hello" 40 null 41 true 42 false 43 ["a","b","c"] 44 {"ß":"long s","K":"Kelvin"} 45 3.14 46 ` 47 48 func TestStreamEncoder(t *testing.T) { 49 for i := 0; i <= len(streamTest); i++ { 50 var buf bytes.Buffer 51 enc := json.NewEncoder(&buf) 52 // Check that enc.SetIndent("", "") turns off indentation. 53 enc.SetIndent(">", ".") 54 enc.SetIndent("", "") 55 for j, v := range streamTest[0:i] { 56 if err := enc.Encode(v); err != nil { 57 t.Fatalf("encode #%d: %v", j, err) 58 } 59 } 60 if have, want := buf.String(), nlines(streamEncoded, i); have != want { 61 t.Errorf("encoding %d items: mismatch", i) 62 diff(t, []byte(have), []byte(want)) 63 break 64 } 65 } 66 } 67 68 var streamEncodedIndent = `0.1 69 "hello" 70 null 71 true 72 false 73 [ 74 >."a", 75 >."b", 76 >."c" 77 >] 78 { 79 >."ß": "long s", 80 >."K": "Kelvin" 81 >} 82 3.14 83 ` 84 85 func TestEncoderIndent(t *testing.T) { 86 var buf bytes.Buffer 87 enc := json.NewEncoder(&buf) 88 enc.SetIndent(">", ".") 89 for _, v := range streamTest { 90 enc.Encode(v) 91 } 92 if have, want := buf.String(), streamEncodedIndent; have != want { 93 t.Error("indented encoding mismatch") 94 diff(t, []byte(have), []byte(want)) 95 } 96 } 97 98 type strMarshaler string 99 100 func (s strMarshaler) MarshalJSON() ([]byte, error) { 101 return []byte(s), nil 102 } 103 104 type strPtrMarshaler string 105 106 func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) { 107 return []byte(*s), nil 108 } 109 110 func TestEncoderSetEscapeHTML(t *testing.T) { 111 var c C 112 var ct CText 113 var tagStruct struct { 114 Valid int `json:"<>&#! "` 115 Invalid int `json:"\\"` 116 } 117 118 // This case is particularly interesting, as we force the encoder to 119 // take the address of the Ptr field to use its MarshalJSON method. This 120 // is why the '&' is important. 121 marshalerStruct := &struct { 122 NonPtr strMarshaler 123 Ptr strPtrMarshaler 124 }{NonPtr: `"<str>"`, Ptr: `"<str>"`} 125 126 // https://golang.org/issue/34154 127 stringOption := struct { 128 Bar string `json:"bar,string"` 129 }{Bar: `<html>foobar</html>`} 130 131 for _, tt := range []struct { 132 name string 133 v any 134 wantEscape string 135 want string 136 }{ 137 {name: "c", v: c, wantEscape: `"\u003c\u0026\u003e"`, want: `"<&>"`}, 138 {name: "ct", v: ct, wantEscape: `"\"\u003c\u0026\u003e\""`, want: `"\"<&>\""`}, 139 {name: `"<&>"`, v: "<&>", wantEscape: `"\u003c\u0026\u003e"`, want: `"<&>"`}, 140 { 141 name: "tagStruct", v: tagStruct, 142 wantEscape: `{"\u003c\u003e\u0026#! ":0,"Invalid":0}`, 143 want: `{"<>&#! ":0,"Invalid":0}`, 144 }, 145 { 146 name: `"<str>"`, v: marshalerStruct, 147 wantEscape: `{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`, 148 want: `{"NonPtr":"<str>","Ptr":"<str>"}`, 149 }, 150 { 151 name: "stringOption", v: stringOption, 152 wantEscape: `{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`, 153 want: `{"bar":"\"<html>foobar</html>\""}`, 154 }, 155 } { 156 var buf bytes.Buffer 157 enc := json.NewEncoder(&buf) 158 if err := enc.Encode(tt.v); err != nil { 159 t.Errorf("Encode(%s): %s", tt.name, err) 160 continue 161 } 162 if got := strings.TrimSpace(buf.String()); got != tt.wantEscape { 163 t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape) 164 } 165 buf.Reset() 166 enc.SetEscapeHTML(false) 167 if err := enc.Encode(tt.v); err != nil { 168 t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) 169 continue 170 } 171 if got := strings.TrimSpace(buf.String()); got != tt.want { 172 t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q", 173 tt.name, got, tt.want) 174 } 175 } 176 } 177 178 func nlines(s string, n int) string { 179 if n <= 0 { 180 return "" 181 } 182 for i, c := range s { 183 if c == '\n' { 184 if n--; n == 0 { 185 return s[0 : i+1] 186 } 187 } 188 } 189 return s 190 } 191 192 func TestDecoder(t *testing.T) { 193 for i := 0; i <= len(streamTest); i++ { 194 // Use stream without newlines as input, 195 // just to stress the decoder even more. 196 // Our test input does not include back-to-back numbers. 197 // Otherwise stripping the newlines would 198 // merge two adjacent JSON values. 199 var buf bytes.Buffer 200 for _, c := range nlines(streamEncoded, i) { 201 if c != '\n' { 202 buf.WriteRune(c) 203 } 204 } 205 out := make([]any, i) 206 dec := json.NewDecoder(&buf) 207 for j := range out { 208 if err := dec.Decode(&out[j]); err != nil { 209 t.Fatalf("decode #%d/%d: %v", j, i, err) 210 } 211 } 212 if !reflect.DeepEqual(out, streamTest[0:i]) { 213 t.Errorf("decoding %d items: mismatch", i) 214 for j := range out { 215 if !reflect.DeepEqual(out[j], streamTest[j]) { 216 t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j]) 217 } 218 } 219 break 220 } 221 } 222 } 223 224 func TestDecoderBuffered(t *testing.T) { 225 r := strings.NewReader(`{"Name": "Gopher"} extra `) 226 var m struct { 227 Name string 228 } 229 d := json.NewDecoder(r) 230 err := d.Decode(&m) 231 if err != nil { 232 t.Fatal(err) 233 } 234 if m.Name != "Gopher" { 235 t.Errorf("Name = %q; want Gopher", m.Name) 236 } 237 rest, err := io.ReadAll(d.Buffered()) 238 if err != nil { 239 t.Fatal(err) 240 } 241 if g, w := string(rest), " extra "; g != w { 242 t.Errorf("Remaining = %q; want %q", g, w) 243 } 244 } 245 246 func TestRawMessage(t *testing.T) { 247 var data struct { 248 X float64 249 Id json.RawMessage 250 Y float32 251 } 252 const raw = `["\u0056",null]` 253 const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}` 254 err := json.Unmarshal([]byte(msg), &data) 255 if err != nil { 256 t.Fatalf("Unmarshal: %v", err) 257 } 258 if string([]byte(data.Id)) != raw { 259 t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw) 260 } 261 b, err := json.Marshal(&data) 262 if err != nil { 263 t.Fatalf("Marshal: %v", err) 264 } 265 if string(b) != msg { 266 t.Fatalf("Marshal: have %#q want %#q", b, msg) 267 } 268 } 269 270 func TestNullRawMessage(t *testing.T) { 271 var data struct { 272 X float64 273 Id json.RawMessage 274 IdPtr *json.RawMessage 275 Y float32 276 } 277 const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}` 278 err := json.Unmarshal([]byte(msg), &data) 279 if err != nil { 280 t.Fatalf("Unmarshal: %v", err) 281 } 282 if want, got := "null", string(data.Id); want != got { 283 t.Fatalf("Raw mismatch: have %q, want %q", got, want) 284 } 285 if data.IdPtr != nil { 286 t.Fatalf("Raw pointer mismatch: have non-nil, want nil") 287 } 288 b, err := json.Marshal(&data) 289 if err != nil { 290 t.Fatalf("Marshal: %v", err) 291 } 292 if string(b) != msg { 293 t.Fatalf("Marshal: have %#q want %#q", b, msg) 294 } 295 } 296 297 var blockingTests = []string{ 298 `{"x": 1}`, 299 `[1, 2, 3]`, 300 } 301 302 func TestBlocking(t *testing.T) { 303 for _, enc := range blockingTests { 304 r, w := net.Pipe() 305 go w.Write([]byte(enc)) 306 var val any 307 308 // If Decode reads beyond what w.Write writes above, 309 // it will block, and the test will deadlock. 310 if err := json.NewDecoder(r).Decode(&val); err != nil { 311 t.Errorf("decoding %s: %v", enc, err) 312 } 313 r.Close() 314 w.Close() 315 } 316 } 317 318 type tokenStreamCase struct { 319 json string 320 expTokens []any 321 } 322 323 type decodeThis struct { 324 v any 325 } 326 327 var tokenStreamCases = []tokenStreamCase{ 328 // streaming token cases 329 {json: `10`, expTokens: []any{float64(10)}}, 330 {json: ` [10] `, expTokens: []any{ 331 json.Delim('['), float64(10), json.Delim(']')}}, 332 {json: ` [false,10,"b"] `, expTokens: []any{ 333 json.Delim('['), false, float64(10), "b", json.Delim(']')}}, 334 {json: `{ "a": 1 }`, expTokens: []any{ 335 json.Delim('{'), "a", float64(1), json.Delim('}')}}, 336 {json: `{"a": 1, "b":"3"}`, expTokens: []any{ 337 json.Delim('{'), "a", float64(1), "b", "3", json.Delim('}')}}, 338 {json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{ 339 json.Delim('['), 340 json.Delim('{'), "a", float64(1), json.Delim('}'), 341 json.Delim('{'), "a", float64(2), json.Delim('}'), 342 json.Delim(']')}}, 343 {json: `{"obj": {"a": 1}}`, expTokens: []any{ 344 json.Delim('{'), "obj", json.Delim('{'), "a", float64(1), json.Delim('}'), 345 json.Delim('}'), 346 }}, 347 {json: `{"obj": [{"a": 1}]}`, expTokens: []any{ 348 json.Delim('{'), "obj", json.Delim('['), 349 json.Delim('{'), "a", float64(1), json.Delim('}'), 350 json.Delim(']'), json.Delim('}')}}, 351 352 // streaming tokens with intermittent Decode() 353 {json: `{ "a": 1 }`, expTokens: []any{ 354 json.Delim('{'), "a", 355 decodeThis{v: float64(1)}, 356 json.Delim('}')}}, 357 {json: ` [ { "a" : 1 } ] `, expTokens: []any{ 358 json.Delim('['), 359 decodeThis{v: map[string]any{"a": float64(1)}}, 360 json.Delim(']')}}, 361 {json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{ 362 json.Delim('['), 363 decodeThis{v: map[string]any{"a": float64(1)}}, 364 decodeThis{v: map[string]any{"a": float64(2)}}, 365 json.Delim(']')}}, 366 {json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []any{ 367 json.Delim('{'), "obj", json.Delim('['), 368 decodeThis{v: map[string]any{"a": float64(1)}}, 369 json.Delim(']'), json.Delim('}')}}, 370 371 {json: `{"obj": {"a": 1}}`, expTokens: []any{ 372 json.Delim('{'), "obj", 373 decodeThis{v: map[string]any{"a": float64(1)}}, 374 json.Delim('}')}}, 375 {json: `{"obj": [{"a": 1}]}`, expTokens: []any{ 376 json.Delim('{'), "obj", 377 decodeThis{v: []any{ 378 map[string]any{"a": float64(1)}, 379 }}, 380 json.Delim('}')}}, 381 /* 382 {json: ` [{"a": 1} {"a": 2}] `, expTokens: []interface{}{ 383 json.Delim('['), 384 decodeThis{map[string]interface{}{"a": float64(1)}}, 385 decodeThis{json.NewSyntaxError("expected comma after array element", 11)}, 386 }}, 387 {json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []interface{}{ 388 json.Delim('{'), strings.Repeat("a", 513), 389 decodeThis{json.NewSyntaxError("expected colon after object key", 518)}, 390 }}, 391 {json: `{ "\a" }`, expTokens: []interface{}{ 392 json.Delim('{'), 393 json.NewSyntaxError("invalid character 'a' in string escape code", 3), 394 }}, 395 {json: ` \a`, expTokens: []interface{}{ 396 json.NewSyntaxError("invalid character '\\\\' looking for beginning of value", 1), 397 }}, 398 */ 399 } 400 401 func TestDecodeInStream(t *testing.T) { 402 for ci, tcase := range tokenStreamCases { 403 dec := json.NewDecoder(strings.NewReader(tcase.json)) 404 for i, etk := range tcase.expTokens { 405 var tk any 406 var err error 407 408 if dt, ok := etk.(decodeThis); ok { 409 etk = dt.v 410 err = dec.Decode(&tk) 411 } else { 412 tk, err = dec.Token() 413 } 414 if experr, ok := etk.(error); ok { 415 if err == nil || !reflect.DeepEqual(err, experr) { 416 t.Errorf("case %v: Expected error %#v in %q, but was %#v", ci, experr, tcase.json, err) 417 } 418 break 419 } else if err == io.EOF { 420 t.Errorf("case %v: Unexpected EOF in %q", ci, tcase.json) 421 break 422 } else if err != nil { 423 t.Errorf("case %v: Unexpected error '%#v' in %q", ci, err, tcase.json) 424 break 425 } 426 if !reflect.DeepEqual(tk, etk) { 427 t.Errorf(`case %v: %q @ %v expected %T(%v) was %T(%v)`, ci, tcase.json, i, etk, etk, tk, tk) 428 break 429 } 430 } 431 } 432 } 433 434 func TestDecodeStreamUseNumber(t *testing.T) { 435 dec := json.NewDecoder(strings.NewReader(`3.14`)) 436 dec.UseNumber() 437 v, err := dec.Token() 438 if err != nil { 439 t.Errorf("unexpected error: %#v", err) 440 } 441 assertEq(t, "json.Number", "json.Number", fmt.Sprintf("%T", v)) 442 } 443 444 // Test from golang.org/issue/11893 445 func TestHTTPDecoding(t *testing.T) { 446 const raw = `{ "foo": "bar" }` 447 448 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 449 w.Write([]byte(raw)) 450 })) 451 defer ts.Close() 452 res, err := http.Get(ts.URL) 453 if err != nil { 454 log.Fatalf("GET failed: %v", err) 455 } 456 defer res.Body.Close() 457 458 foo := struct { 459 Foo string 460 }{} 461 462 d := json.NewDecoder(res.Body) 463 err = d.Decode(&foo) 464 if err != nil { 465 t.Fatalf("Decode: %v", err) 466 } 467 if foo.Foo != "bar" { 468 t.Errorf("decoded %q; want \"bar\"", foo.Foo) 469 } 470 471 // make sure we get the EOF the second time 472 err = d.Decode(&foo) 473 if err != io.EOF { 474 t.Errorf("err = %v; want io.EOF", err) 475 } 476 } 477 478 func TestGzipStreaming(t *testing.T) { 479 type someStruct struct { 480 ID int `json:"id"` 481 Text []string `json:"text"` 482 } 483 484 manyItems := strings.Repeat(`"Quis autem vel eum iure reprehenderit, qui in ea voluptate velit esse, quam`+ 485 ` nihil molestiae consequatur, vel illum, qui dolorem eum fugiat, quo voluptas nulla pariatur?",`, 400) 486 487 longJSON := `{"id":123,"text":[` + manyItems[0:len(manyItems)-1] + `]}` 488 489 compressed := bytes.NewBuffer(nil) 490 gw := gzip.NewWriter(compressed) 491 492 _, err := io.Copy(gw, bytes.NewReader([]byte(longJSON))) 493 if err != nil { 494 t.Fatalf("Unexpected error: %v", err) 495 } 496 497 err = gw.Close() 498 if err != nil { 499 t.Fatalf("Unexpected error: %v", err) 500 } 501 502 gr, err := gzip.NewReader(bytes.NewReader(compressed.Bytes())) 503 if err != nil { 504 t.Fatalf("Unexpected error: %v", err) 505 } 506 507 var v someStruct 508 dec := json.NewDecoder(gr) 509 510 err = dec.Decode(&v) 511 if err != nil { 512 t.Fatalf("Unexpected error: %v", err) 513 } 514 } 515 516 func TestLongUTF8(t *testing.T) { 517 want := strings.Repeat("あ", 342) 518 r := strings.NewReader(strconv.Quote(want)) 519 520 var got string 521 if err := json.NewDecoder(r).Decode(&got); err != nil { 522 t.Fatalf("Unexpected error: %v", err) 523 } 524 if got != want { 525 t.Errorf("string %q; want = %q", got, want) 526 } 527 } 528 529 func TestIssue278(t *testing.T) { 530 a := `{"嗷嗷":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7"}` 531 r := strings.NewReader(a) 532 var m map[string]string 533 if err := json.NewDecoder(r).Decode(&m); err != nil { 534 t.Fatalf("Unexpected error: %v", err) 535 } 536 want := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7" 537 if got := m["嗷嗷"]; got != want { 538 t.Errorf("string %q; want = %q", got, want) 539 } 540 }