github.com/goccy/go-json@v0.10.3-0.20240509105655-5e2ae3f23c1d/stream_test.go (about)

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package json_test
     6  
     7  import (
     8  	"bytes"
     9  	"compress/gzip"
    10  	"fmt"
    11  	"io"
    12  	"log"
    13  	"net"
    14  	"net/http"
    15  	"net/http/httptest"
    16  	"reflect"
    17  	"strconv"
    18  	"strings"
    19  	"testing"
    20  
    21  	"github.com/goccy/go-json"
    22  )
    23  
    24  // Test values for the stream test.
    25  // One of each JSON kind.
    26  var streamTest = []interface{}{
    27  	0.1,
    28  	"hello",
    29  	nil,
    30  	true,
    31  	false,
    32  	[]interface{}{"a", "b", "c"},
    33  	map[string]interface{}{"K": "Kelvin", "ß": "long s"},
    34  	3.14, // another value to make sure something can follow map
    35  }
    36  
    37  var streamEncoded = `0.1
    38  "hello"
    39  null
    40  true
    41  false
    42  ["a","b","c"]
    43  {"ß":"long s","K":"Kelvin"}
    44  3.14
    45  `
    46  
    47  func TestStreamEncoder(t *testing.T) {
    48  	for i := 0; i <= len(streamTest); i++ {
    49  		var buf bytes.Buffer
    50  		enc := json.NewEncoder(&buf)
    51  		// Check that enc.SetIndent("", "") turns off indentation.
    52  		enc.SetIndent(">", ".")
    53  		enc.SetIndent("", "")
    54  		for j, v := range streamTest[0:i] {
    55  			if err := enc.Encode(v); err != nil {
    56  				t.Fatalf("encode #%d: %v", j, err)
    57  			}
    58  		}
    59  		if have, want := buf.String(), nlines(streamEncoded, i); have != want {
    60  			t.Errorf("encoding %d items: mismatch", i)
    61  			diff(t, []byte(have), []byte(want))
    62  			break
    63  		}
    64  	}
    65  }
    66  
    67  var streamEncodedIndent = `0.1
    68  "hello"
    69  null
    70  true
    71  false
    72  [
    73  >."a",
    74  >."b",
    75  >."c"
    76  >]
    77  {
    78  >."ß": "long s",
    79  >."K": "Kelvin"
    80  >}
    81  3.14
    82  `
    83  
    84  func TestEncoderIndent(t *testing.T) {
    85  	var buf bytes.Buffer
    86  	enc := json.NewEncoder(&buf)
    87  	enc.SetIndent(">", ".")
    88  	for _, v := range streamTest {
    89  		enc.Encode(v)
    90  	}
    91  	if have, want := buf.String(), streamEncodedIndent; have != want {
    92  		t.Error("indented encoding mismatch")
    93  		diff(t, []byte(have), []byte(want))
    94  	}
    95  }
    96  
    97  type strMarshaler string
    98  
    99  func (s strMarshaler) MarshalJSON() ([]byte, error) {
   100  	return []byte(s), nil
   101  }
   102  
   103  type strPtrMarshaler string
   104  
   105  func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) {
   106  	return []byte(*s), nil
   107  }
   108  
   109  func TestEncoderSetEscapeHTML(t *testing.T) {
   110  	var c C
   111  	var ct CText
   112  	var tagStruct struct {
   113  		Valid   int `json:"<>&#! "`
   114  		Invalid int `json:"\\"`
   115  	}
   116  
   117  	// This case is particularly interesting, as we force the encoder to
   118  	// take the address of the Ptr field to use its MarshalJSON method. This
   119  	// is why the '&' is important.
   120  	marshalerStruct := &struct {
   121  		NonPtr strMarshaler
   122  		Ptr    strPtrMarshaler
   123  	}{`"<str>"`, `"<str>"`}
   124  
   125  	// https://golang.org/issue/34154
   126  	stringOption := struct {
   127  		Bar string `json:"bar,string"`
   128  	}{`<html>foobar</html>`}
   129  
   130  	for _, tt := range []struct {
   131  		name       string
   132  		v          interface{}
   133  		wantEscape string
   134  		want       string
   135  	}{
   136  		{"c", c, `"\u003c\u0026\u003e"`, `"<&>"`},
   137  		{"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`},
   138  		{`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`},
   139  		{
   140  			"tagStruct", tagStruct,
   141  			`{"\u003c\u003e\u0026#! ":0,"Invalid":0}`,
   142  			`{"<>&#! ":0,"Invalid":0}`,
   143  		},
   144  		{
   145  			`"<str>"`, marshalerStruct,
   146  			`{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`,
   147  			`{"NonPtr":"<str>","Ptr":"<str>"}`,
   148  		},
   149  		{
   150  			"stringOption", stringOption,
   151  			`{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
   152  			`{"bar":"\"<html>foobar</html>\""}`,
   153  		},
   154  	} {
   155  		var buf bytes.Buffer
   156  		enc := json.NewEncoder(&buf)
   157  		if err := enc.Encode(tt.v); err != nil {
   158  			t.Errorf("Encode(%s): %s", tt.name, err)
   159  			continue
   160  		}
   161  		if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
   162  			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape)
   163  		}
   164  		buf.Reset()
   165  		enc.SetEscapeHTML(false)
   166  		if err := enc.Encode(tt.v); err != nil {
   167  			t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
   168  			continue
   169  		}
   170  		if got := strings.TrimSpace(buf.String()); got != tt.want {
   171  			t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q",
   172  				tt.name, got, tt.want)
   173  		}
   174  	}
   175  }
   176  
   177  func nlines(s string, n int) string {
   178  	if n <= 0 {
   179  		return ""
   180  	}
   181  	for i, c := range s {
   182  		if c == '\n' {
   183  			if n--; n == 0 {
   184  				return s[0 : i+1]
   185  			}
   186  		}
   187  	}
   188  	return s
   189  }
   190  
   191  func TestDecoder(t *testing.T) {
   192  	for i := 0; i <= len(streamTest); i++ {
   193  		// Use stream without newlines as input,
   194  		// just to stress the decoder even more.
   195  		// Our test input does not include back-to-back numbers.
   196  		// Otherwise stripping the newlines would
   197  		// merge two adjacent JSON values.
   198  		var buf bytes.Buffer
   199  		for _, c := range nlines(streamEncoded, i) {
   200  			if c != '\n' {
   201  				buf.WriteRune(c)
   202  			}
   203  		}
   204  		out := make([]interface{}, i)
   205  		dec := json.NewDecoder(&buf)
   206  		for j := range out {
   207  			if err := dec.Decode(&out[j]); err != nil {
   208  				t.Fatalf("decode #%d/%d: %v", j, i, err)
   209  			}
   210  		}
   211  		if !reflect.DeepEqual(out, streamTest[0:i]) {
   212  			t.Errorf("decoding %d items: mismatch", i)
   213  			for j := range out {
   214  				if !reflect.DeepEqual(out[j], streamTest[j]) {
   215  					t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j])
   216  				}
   217  			}
   218  			break
   219  		}
   220  	}
   221  }
   222  
   223  func TestDecoderBuffered(t *testing.T) {
   224  	r := strings.NewReader(`{"Name": "Gopher"} extra `)
   225  	var m struct {
   226  		Name string
   227  	}
   228  	d := json.NewDecoder(r)
   229  	err := d.Decode(&m)
   230  	if err != nil {
   231  		t.Fatal(err)
   232  	}
   233  	if m.Name != "Gopher" {
   234  		t.Errorf("Name = %q; want Gopher", m.Name)
   235  	}
   236  	rest, err := io.ReadAll(d.Buffered())
   237  	if err != nil {
   238  		t.Fatal(err)
   239  	}
   240  	if g, w := string(rest), " extra "; g != w {
   241  		t.Errorf("Remaining = %q; want %q", g, w)
   242  	}
   243  }
   244  
   245  func TestRawMessage(t *testing.T) {
   246  	var data struct {
   247  		X  float64
   248  		Id json.RawMessage
   249  		Y  float32
   250  	}
   251  	const raw = `["\u0056",null]`
   252  	const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
   253  	err := json.Unmarshal([]byte(msg), &data)
   254  	if err != nil {
   255  		t.Fatalf("Unmarshal: %v", err)
   256  	}
   257  	if string([]byte(data.Id)) != raw {
   258  		t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw)
   259  	}
   260  	b, err := json.Marshal(&data)
   261  	if err != nil {
   262  		t.Fatalf("Marshal: %v", err)
   263  	}
   264  	if string(b) != msg {
   265  		t.Fatalf("Marshal: have %#q want %#q", b, msg)
   266  	}
   267  }
   268  
   269  func TestNullRawMessage(t *testing.T) {
   270  	var data struct {
   271  		X     float64
   272  		Id    json.RawMessage
   273  		IdPtr *json.RawMessage
   274  		Y     float32
   275  	}
   276  	const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}`
   277  	err := json.Unmarshal([]byte(msg), &data)
   278  	if err != nil {
   279  		t.Fatalf("Unmarshal: %v", err)
   280  	}
   281  	if want, got := "null", string(data.Id); want != got {
   282  		t.Fatalf("Raw mismatch: have %q, want %q", got, want)
   283  	}
   284  	if data.IdPtr != nil {
   285  		t.Fatalf("Raw pointer mismatch: have non-nil, want nil")
   286  	}
   287  	b, err := json.Marshal(&data)
   288  	if err != nil {
   289  		t.Fatalf("Marshal: %v", err)
   290  	}
   291  	if string(b) != msg {
   292  		t.Fatalf("Marshal: have %#q want %#q", b, msg)
   293  	}
   294  }
   295  
   296  var blockingTests = []string{
   297  	`{"x": 1}`,
   298  	`[1, 2, 3]`,
   299  }
   300  
   301  func TestBlocking(t *testing.T) {
   302  	for _, enc := range blockingTests {
   303  		r, w := net.Pipe()
   304  		go w.Write([]byte(enc))
   305  		var val interface{}
   306  
   307  		// If Decode reads beyond what w.Write writes above,
   308  		// it will block, and the test will deadlock.
   309  		if err := json.NewDecoder(r).Decode(&val); err != nil {
   310  			t.Errorf("decoding %s: %v", enc, err)
   311  		}
   312  		r.Close()
   313  		w.Close()
   314  	}
   315  }
   316  
   317  type tokenStreamCase struct {
   318  	json      string
   319  	expTokens []interface{}
   320  }
   321  
   322  type decodeThis struct {
   323  	v interface{}
   324  }
   325  
   326  var tokenStreamCases = []tokenStreamCase{
   327  	// streaming token cases
   328  	{json: `10`, expTokens: []interface{}{float64(10)}},
   329  	{json: ` [10] `, expTokens: []interface{}{
   330  		json.Delim('['), float64(10), json.Delim(']')}},
   331  	{json: ` [false,10,"b"] `, expTokens: []interface{}{
   332  		json.Delim('['), false, float64(10), "b", json.Delim(']')}},
   333  	{json: `{ "a": 1 }`, expTokens: []interface{}{
   334  		json.Delim('{'), "a", float64(1), json.Delim('}')}},
   335  	{json: `{"a": 1, "b":"3"}`, expTokens: []interface{}{
   336  		json.Delim('{'), "a", float64(1), "b", "3", json.Delim('}')}},
   337  	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{
   338  		json.Delim('['),
   339  		json.Delim('{'), "a", float64(1), json.Delim('}'),
   340  		json.Delim('{'), "a", float64(2), json.Delim('}'),
   341  		json.Delim(']')}},
   342  	{json: `{"obj": {"a": 1}}`, expTokens: []interface{}{
   343  		json.Delim('{'), "obj", json.Delim('{'), "a", float64(1), json.Delim('}'),
   344  		json.Delim('}')}},
   345  	{json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{
   346  		json.Delim('{'), "obj", json.Delim('['),
   347  		json.Delim('{'), "a", float64(1), json.Delim('}'),
   348  		json.Delim(']'), json.Delim('}')}},
   349  
   350  	// streaming tokens with intermittent Decode()
   351  	{json: `{ "a": 1 }`, expTokens: []interface{}{
   352  		json.Delim('{'), "a",
   353  		decodeThis{float64(1)},
   354  		json.Delim('}')}},
   355  	{json: ` [ { "a" : 1 } ] `, expTokens: []interface{}{
   356  		json.Delim('['),
   357  		decodeThis{map[string]interface{}{"a": float64(1)}},
   358  		json.Delim(']')}},
   359  	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{
   360  		json.Delim('['),
   361  		decodeThis{map[string]interface{}{"a": float64(1)}},
   362  		decodeThis{map[string]interface{}{"a": float64(2)}},
   363  		json.Delim(']')}},
   364  	{json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []interface{}{
   365  		json.Delim('{'), "obj", json.Delim('['),
   366  		decodeThis{map[string]interface{}{"a": float64(1)}},
   367  		json.Delim(']'), json.Delim('}')}},
   368  
   369  	{json: `{"obj": {"a": 1}}`, expTokens: []interface{}{
   370  		json.Delim('{'), "obj",
   371  		decodeThis{map[string]interface{}{"a": float64(1)}},
   372  		json.Delim('}')}},
   373  	{json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{
   374  		json.Delim('{'), "obj",
   375  		decodeThis{[]interface{}{
   376  			map[string]interface{}{"a": float64(1)},
   377  		}},
   378  		json.Delim('}')}},
   379  	/*
   380  		{json: ` [{"a": 1} {"a": 2}] `, expTokens: []interface{}{
   381  			json.Delim('['),
   382  			decodeThis{map[string]interface{}{"a": float64(1)}},
   383  			decodeThis{json.NewSyntaxError("expected comma after array element", 11)},
   384  		}},
   385  		{json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []interface{}{
   386  			json.Delim('{'), strings.Repeat("a", 513),
   387  			decodeThis{json.NewSyntaxError("expected colon after object key", 518)},
   388  		}},
   389  		{json: `{ "\a" }`, expTokens: []interface{}{
   390  			json.Delim('{'),
   391  			json.NewSyntaxError("invalid character 'a' in string escape code", 3),
   392  		}},
   393  		{json: ` \a`, expTokens: []interface{}{
   394  			json.NewSyntaxError("invalid character '\\\\' looking for beginning of value", 1),
   395  		}},
   396  	*/
   397  }
   398  
   399  func TestDecodeInStream(t *testing.T) {
   400  	for ci, tcase := range tokenStreamCases {
   401  
   402  		dec := json.NewDecoder(strings.NewReader(tcase.json))
   403  		for i, etk := range tcase.expTokens {
   404  
   405  			var tk interface{}
   406  			var err error
   407  
   408  			if dt, ok := etk.(decodeThis); ok {
   409  				etk = dt.v
   410  				err = dec.Decode(&tk)
   411  			} else {
   412  				tk, err = dec.Token()
   413  			}
   414  			if experr, ok := etk.(error); ok {
   415  				if err == nil || !reflect.DeepEqual(err, experr) {
   416  					t.Errorf("case %v: Expected error %#v in %q, but was %#v", ci, experr, tcase.json, err)
   417  				}
   418  				break
   419  			} else if err == io.EOF {
   420  				t.Errorf("case %v: Unexpected EOF in %q", ci, tcase.json)
   421  				break
   422  			} else if err != nil {
   423  				t.Errorf("case %v: Unexpected error '%#v' in %q", ci, err, tcase.json)
   424  				break
   425  			}
   426  			if !reflect.DeepEqual(tk, etk) {
   427  				t.Errorf(`case %v: %q @ %v expected %T(%v) was %T(%v)`, ci, tcase.json, i, etk, etk, tk, tk)
   428  				break
   429  			}
   430  		}
   431  	}
   432  }
   433  
   434  func TestDecodeStreamUseNumber(t *testing.T) {
   435  	dec := json.NewDecoder(strings.NewReader(`3.14`))
   436  	dec.UseNumber()
   437  	v, err := dec.Token()
   438  	if err != nil {
   439  		t.Errorf("unexpected error: %#v", err)
   440  	}
   441  	assertEq(t, "json.Number", "json.Number", fmt.Sprintf("%T", v))
   442  }
   443  
   444  // Test from golang.org/issue/11893
   445  func TestHTTPDecoding(t *testing.T) {
   446  	const raw = `{ "foo": "bar" }`
   447  
   448  	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   449  		w.Write([]byte(raw))
   450  	}))
   451  	defer ts.Close()
   452  	res, err := http.Get(ts.URL)
   453  	if err != nil {
   454  		log.Fatalf("GET failed: %v", err)
   455  	}
   456  	defer res.Body.Close()
   457  
   458  	foo := struct {
   459  		Foo string
   460  	}{}
   461  
   462  	d := json.NewDecoder(res.Body)
   463  	err = d.Decode(&foo)
   464  	if err != nil {
   465  		t.Fatalf("Decode: %v", err)
   466  	}
   467  	if foo.Foo != "bar" {
   468  		t.Errorf("decoded %q; want \"bar\"", foo.Foo)
   469  	}
   470  
   471  	// make sure we get the EOF the second time
   472  	err = d.Decode(&foo)
   473  	if err != io.EOF {
   474  		t.Errorf("err = %v; want io.EOF", err)
   475  	}
   476  }
   477  
   478  func TestGzipStreaming(t *testing.T) {
   479  	type someStruct struct {
   480  		ID   int      `json:"id"`
   481  		Text []string `json:"text"`
   482  	}
   483  
   484  	manyItems := strings.Repeat(`"Quis autem vel eum iure reprehenderit, qui in ea voluptate velit esse, quam`+
   485  		` nihil molestiae consequatur, vel illum, qui dolorem eum fugiat, quo voluptas nulla pariatur?",`, 400)
   486  
   487  	longJSON := `{"id":123,"text":[` + manyItems[0:len(manyItems)-1] + `]}`
   488  
   489  	compressed := bytes.NewBuffer(nil)
   490  	gw := gzip.NewWriter(compressed)
   491  
   492  	_, err := io.Copy(gw, bytes.NewReader([]byte(longJSON)))
   493  	if err != nil {
   494  		t.Fatalf("Unexpected error: %v", err)
   495  	}
   496  
   497  	err = gw.Close()
   498  	if err != nil {
   499  		t.Fatalf("Unexpected error: %v", err)
   500  	}
   501  
   502  	gr, err := gzip.NewReader(bytes.NewReader(compressed.Bytes()))
   503  	if err != nil {
   504  		t.Fatalf("Unexpected error: %v", err)
   505  	}
   506  
   507  	var v someStruct
   508  	dec := json.NewDecoder(gr)
   509  
   510  	err = dec.Decode(&v)
   511  	if err != nil {
   512  		t.Fatalf("Unexpected error: %v", err)
   513  	}
   514  }
   515  
   516  func TestLongUTF8(t *testing.T) {
   517  	want := strings.Repeat("あ", 342)
   518  	r := strings.NewReader(strconv.Quote(want))
   519  
   520  	var got string
   521  	if err := json.NewDecoder(r).Decode(&got); err != nil {
   522  		t.Fatalf("Unexpected error: %v", err)
   523  	}
   524  	if got != want {
   525  		t.Errorf("string %q; want = %q", got, want)
   526  	}
   527  }
   528  
   529  func TestIssue278(t *testing.T) {
   530  	a := `{"嗷嗷":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7"}`
   531  	r := strings.NewReader(a)
   532  	var m map[string]string
   533  	if err := json.NewDecoder(r).Decode(&m); err != nil {
   534  		t.Fatalf("Unexpected error: %v", err)
   535  	}
   536  	want := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7"
   537  	if got := m["嗷嗷"]; got != want {
   538  		t.Errorf("string %q; want = %q", got, want)
   539  	}
   540  }