github.com/grbit/go-json@v0.11.0/stream_test.go (about)

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package json_test
     6  
     7  import (
     8  	"bytes"
     9  	"compress/gzip"
    10  	"fmt"
    11  	"io"
    12  	"io/ioutil"
    13  	"log"
    14  	"net"
    15  	"net/http"
    16  	"net/http/httptest"
    17  	"reflect"
    18  	"strconv"
    19  	"strings"
    20  	"testing"
    21  
    22  	"github.com/grbit/go-json"
    23  )
    24  
    25  // Test values for the stream test.
    26  // One of each JSON kind.
    27  var streamTest = []interface{}{
    28  	0.1,
    29  	"hello",
    30  	nil,
    31  	true,
    32  	false,
    33  	[]interface{}{"a", "b", "c"},
    34  	map[string]interface{}{"K": "Kelvin", "ß": "long s"},
    35  	3.14, // another value to make sure something can follow map
    36  }
    37  
    38  var streamEncoded = `0.1
    39  "hello"
    40  null
    41  true
    42  false
    43  ["a","b","c"]
    44  {"ß":"long s","K":"Kelvin"}
    45  3.14
    46  `
    47  
    48  func TestStreamEncoder(t *testing.T) {
    49  	for i := 0; i <= len(streamTest); i++ {
    50  		var buf bytes.Buffer
    51  		enc := json.NewEncoder(&buf)
    52  		// Check that enc.SetIndent("", "") turns off indentation.
    53  		enc.SetIndent(">", ".")
    54  		enc.SetIndent("", "")
    55  		for j, v := range streamTest[0:i] {
    56  			if err := enc.Encode(v); err != nil {
    57  				t.Fatalf("encode #%d: %v", j, err)
    58  			}
    59  		}
    60  		if have, want := buf.String(), nlines(streamEncoded, i); have != want {
    61  			t.Errorf("encoding %d items: mismatch", i)
    62  			diff(t, []byte(have), []byte(want))
    63  			break
    64  		}
    65  	}
    66  }
    67  
    68  var streamEncodedIndent = `0.1
    69  "hello"
    70  null
    71  true
    72  false
    73  [
    74  >."a",
    75  >."b",
    76  >."c"
    77  >]
    78  {
    79  >."ß": "long s",
    80  >."K": "Kelvin"
    81  >}
    82  3.14
    83  `
    84  
    85  func TestEncoderIndent(t *testing.T) {
    86  	var buf bytes.Buffer
    87  	enc := json.NewEncoder(&buf)
    88  	enc.SetIndent(">", ".")
    89  	for _, v := range streamTest {
    90  		enc.Encode(v)
    91  	}
    92  	if have, want := buf.String(), streamEncodedIndent; have != want {
    93  		t.Error("indented encoding mismatch")
    94  		diff(t, []byte(have), []byte(want))
    95  	}
    96  }
    97  
    98  type strMarshaler string
    99  
   100  func (s strMarshaler) MarshalJSON() ([]byte, error) {
   101  	return []byte(s), nil
   102  }
   103  
   104  type strPtrMarshaler string
   105  
   106  func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) {
   107  	return []byte(*s), nil
   108  }
   109  
   110  func TestEncoderSetEscapeHTML(t *testing.T) {
   111  	var c C
   112  	var ct CText
   113  	var tagStruct struct {
   114  		Valid   int `json:"<>&#! "`
   115  		Invalid int `json:"\\"`
   116  	}
   117  
   118  	// This case is particularly interesting, as we force the encoder to
   119  	// take the address of the Ptr field to use its MarshalJSON method. This
   120  	// is why the '&' is important.
   121  	marshalerStruct := &struct {
   122  		NonPtr strMarshaler
   123  		Ptr    strPtrMarshaler
   124  	}{`"<str>"`, `"<str>"`}
   125  
   126  	// https://golang.org/issue/34154
   127  	stringOption := struct {
   128  		Bar string `json:"bar,string"`
   129  	}{`<html>foobar</html>`}
   130  
   131  	for _, tt := range []struct {
   132  		name       string
   133  		v          interface{}
   134  		wantEscape string
   135  		want       string
   136  	}{
   137  		{"c", c, `"\u003c\u0026\u003e"`, `"<&>"`},
   138  		{"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`},
   139  		{`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`},
   140  		{
   141  			"tagStruct", tagStruct,
   142  			`{"\u003c\u003e\u0026#! ":0,"Invalid":0}`,
   143  			`{"<>&#! ":0,"Invalid":0}`,
   144  		},
   145  		{
   146  			`"<str>"`, marshalerStruct,
   147  			`{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`,
   148  			`{"NonPtr":"<str>","Ptr":"<str>"}`,
   149  		},
   150  		{
   151  			"stringOption", stringOption,
   152  			`{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
   153  			`{"bar":"\"<html>foobar</html>\""}`,
   154  		},
   155  	} {
   156  		var buf bytes.Buffer
   157  		enc := json.NewEncoder(&buf)
   158  		if err := enc.Encode(tt.v); err != nil {
   159  			t.Errorf("Encode(%s): %s", tt.name, err)
   160  			continue
   161  		}
   162  		if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
   163  			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape)
   164  		}
   165  		buf.Reset()
   166  		enc.SetEscapeHTML(false)
   167  		if err := enc.Encode(tt.v); err != nil {
   168  			t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
   169  			continue
   170  		}
   171  		if got := strings.TrimSpace(buf.String()); got != tt.want {
   172  			t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q",
   173  				tt.name, got, tt.want)
   174  		}
   175  	}
   176  }
   177  
   178  func nlines(s string, n int) string {
   179  	if n <= 0 {
   180  		return ""
   181  	}
   182  	for i, c := range s {
   183  		if c == '\n' {
   184  			if n--; n == 0 {
   185  				return s[0 : i+1]
   186  			}
   187  		}
   188  	}
   189  	return s
   190  }
   191  
   192  func TestDecoder(t *testing.T) {
   193  	for i := 0; i <= len(streamTest); i++ {
   194  		// Use stream without newlines as input,
   195  		// just to stress the decoder even more.
   196  		// Our test input does not include back-to-back numbers.
   197  		// Otherwise stripping the newlines would
   198  		// merge two adjacent JSON values.
   199  		var buf bytes.Buffer
   200  		for _, c := range nlines(streamEncoded, i) {
   201  			if c != '\n' {
   202  				buf.WriteRune(c)
   203  			}
   204  		}
   205  		out := make([]interface{}, i)
   206  		dec := json.NewDecoder(&buf)
   207  		for j := range out {
   208  			if err := dec.Decode(&out[j]); err != nil {
   209  				t.Fatalf("decode #%d/%d: %v", j, i, err)
   210  			}
   211  		}
   212  		if !reflect.DeepEqual(out, streamTest[0:i]) {
   213  			t.Errorf("decoding %d items: mismatch", i)
   214  			for j := range out {
   215  				if !reflect.DeepEqual(out[j], streamTest[j]) {
   216  					t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j])
   217  				}
   218  			}
   219  			break
   220  		}
   221  	}
   222  }
   223  
   224  func TestDecoderBuffered(t *testing.T) {
   225  	r := strings.NewReader(`{"Name": "Gopher"} extra `)
   226  	var m struct {
   227  		Name string
   228  	}
   229  	d := json.NewDecoder(r)
   230  	err := d.Decode(&m)
   231  	if err != nil {
   232  		t.Fatal(err)
   233  	}
   234  	if m.Name != "Gopher" {
   235  		t.Errorf("Name = %q; want Gopher", m.Name)
   236  	}
   237  	rest, err := ioutil.ReadAll(d.Buffered())
   238  	if err != nil {
   239  		t.Fatal(err)
   240  	}
   241  	if g, w := string(rest), " extra "; g != w {
   242  		t.Errorf("Remaining = %q; want %q", g, w)
   243  	}
   244  }
   245  
   246  func TestRawMessage(t *testing.T) {
   247  	var data struct {
   248  		X  float64
   249  		Id json.RawMessage
   250  		Y  float32
   251  	}
   252  	const raw = `["\u0056",null]`
   253  	const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
   254  	err := json.Unmarshal([]byte(msg), &data)
   255  	if err != nil {
   256  		t.Fatalf("Unmarshal: %v", err)
   257  	}
   258  	if string([]byte(data.Id)) != raw {
   259  		t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw)
   260  	}
   261  	b, err := json.Marshal(&data)
   262  	if err != nil {
   263  		t.Fatalf("Marshal: %v", err)
   264  	}
   265  	if string(b) != msg {
   266  		t.Fatalf("Marshal: have %#q want %#q", b, msg)
   267  	}
   268  }
   269  
   270  func TestNullRawMessage(t *testing.T) {
   271  	var data struct {
   272  		X     float64
   273  		Id    json.RawMessage
   274  		IdPtr *json.RawMessage
   275  		Y     float32
   276  	}
   277  	const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}`
   278  	err := json.Unmarshal([]byte(msg), &data)
   279  	if err != nil {
   280  		t.Fatalf("Unmarshal: %v", err)
   281  	}
   282  	if want, got := "null", string(data.Id); want != got {
   283  		t.Fatalf("Raw mismatch: have %q, want %q", got, want)
   284  	}
   285  	if data.IdPtr != nil {
   286  		t.Fatalf("Raw pointer mismatch: have non-nil, want nil")
   287  	}
   288  	b, err := json.Marshal(&data)
   289  	if err != nil {
   290  		t.Fatalf("Marshal: %v", err)
   291  	}
   292  	if string(b) != msg {
   293  		t.Fatalf("Marshal: have %#q want %#q", b, msg)
   294  	}
   295  }
   296  
   297  var blockingTests = []string{
   298  	`{"x": 1}`,
   299  	`[1, 2, 3]`,
   300  }
   301  
   302  func TestBlocking(t *testing.T) {
   303  	for _, enc := range blockingTests {
   304  		r, w := net.Pipe()
   305  		go w.Write([]byte(enc))
   306  		var val interface{}
   307  
   308  		// If Decode reads beyond what w.Write writes above,
   309  		// it will block, and the test will deadlock.
   310  		if err := json.NewDecoder(r).Decode(&val); err != nil {
   311  			t.Errorf("decoding %s: %v", enc, err)
   312  		}
   313  		r.Close()
   314  		w.Close()
   315  	}
   316  }
   317  
   318  type tokenStreamCase struct {
   319  	json      string
   320  	expTokens []interface{}
   321  }
   322  
   323  type decodeThis struct {
   324  	v interface{}
   325  }
   326  
   327  var tokenStreamCases = []tokenStreamCase{
   328  	// streaming token cases
   329  	{json: `10`, expTokens: []interface{}{float64(10)}},
   330  	{json: ` [10] `, expTokens: []interface{}{
   331  		json.Delim('['), float64(10), json.Delim(']')}},
   332  	{json: ` [false,10,"b"] `, expTokens: []interface{}{
   333  		json.Delim('['), false, float64(10), "b", json.Delim(']')}},
   334  	{json: `{ "a": 1 }`, expTokens: []interface{}{
   335  		json.Delim('{'), "a", float64(1), json.Delim('}')}},
   336  	{json: `{"a": 1, "b":"3"}`, expTokens: []interface{}{
   337  		json.Delim('{'), "a", float64(1), "b", "3", json.Delim('}')}},
   338  	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{
   339  		json.Delim('['),
   340  		json.Delim('{'), "a", float64(1), json.Delim('}'),
   341  		json.Delim('{'), "a", float64(2), json.Delim('}'),
   342  		json.Delim(']')}},
   343  	{json: `{"obj": {"a": 1}}`, expTokens: []interface{}{
   344  		json.Delim('{'), "obj", json.Delim('{'), "a", float64(1), json.Delim('}'),
   345  		json.Delim('}')}},
   346  	{json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{
   347  		json.Delim('{'), "obj", json.Delim('['),
   348  		json.Delim('{'), "a", float64(1), json.Delim('}'),
   349  		json.Delim(']'), json.Delim('}')}},
   350  
   351  	// streaming tokens with intermittent Decode()
   352  	{json: `{ "a": 1 }`, expTokens: []interface{}{
   353  		json.Delim('{'), "a",
   354  		decodeThis{float64(1)},
   355  		json.Delim('}')}},
   356  	{json: ` [ { "a" : 1 } ] `, expTokens: []interface{}{
   357  		json.Delim('['),
   358  		decodeThis{map[string]interface{}{"a": float64(1)}},
   359  		json.Delim(']')}},
   360  	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{
   361  		json.Delim('['),
   362  		decodeThis{map[string]interface{}{"a": float64(1)}},
   363  		decodeThis{map[string]interface{}{"a": float64(2)}},
   364  		json.Delim(']')}},
   365  	{json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []interface{}{
   366  		json.Delim('{'), "obj", json.Delim('['),
   367  		decodeThis{map[string]interface{}{"a": float64(1)}},
   368  		json.Delim(']'), json.Delim('}')}},
   369  
   370  	{json: `{"obj": {"a": 1}}`, expTokens: []interface{}{
   371  		json.Delim('{'), "obj",
   372  		decodeThis{map[string]interface{}{"a": float64(1)}},
   373  		json.Delim('}')}},
   374  	{json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{
   375  		json.Delim('{'), "obj",
   376  		decodeThis{[]interface{}{
   377  			map[string]interface{}{"a": float64(1)},
   378  		}},
   379  		json.Delim('}')}},
   380  	/*
   381  		{json: ` [{"a": 1} {"a": 2}] `, expTokens: []interface{}{
   382  			json.Delim('['),
   383  			decodeThis{map[string]interface{}{"a": float64(1)}},
   384  			decodeThis{json.NewSyntaxError("expected comma after array element", 11)},
   385  		}},
   386  		{json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []interface{}{
   387  			json.Delim('{'), strings.Repeat("a", 513),
   388  			decodeThis{json.NewSyntaxError("expected colon after object key", 518)},
   389  		}},
   390  		{json: `{ "\a" }`, expTokens: []interface{}{
   391  			json.Delim('{'),
   392  			json.NewSyntaxError("invalid character 'a' in string escape code", 3),
   393  		}},
   394  		{json: ` \a`, expTokens: []interface{}{
   395  			json.NewSyntaxError("invalid character '\\\\' looking for beginning of value", 1),
   396  		}},
   397  	*/
   398  }
   399  
   400  func TestDecodeInStream(t *testing.T) {
   401  	for ci, tcase := range tokenStreamCases {
   402  
   403  		dec := json.NewDecoder(strings.NewReader(tcase.json))
   404  		for i, etk := range tcase.expTokens {
   405  
   406  			var tk interface{}
   407  			var err error
   408  
   409  			if dt, ok := etk.(decodeThis); ok {
   410  				etk = dt.v
   411  				err = dec.Decode(&tk)
   412  			} else {
   413  				tk, err = dec.Token()
   414  			}
   415  			if experr, ok := etk.(error); ok {
   416  				if err == nil || !reflect.DeepEqual(err, experr) {
   417  					t.Errorf("case %v: Expected error %#v in %q, but was %#v", ci, experr, tcase.json, err)
   418  				}
   419  				break
   420  			} else if err == io.EOF {
   421  				t.Errorf("case %v: Unexpected EOF in %q", ci, tcase.json)
   422  				break
   423  			} else if err != nil {
   424  				t.Errorf("case %v: Unexpected error '%#v' in %q", ci, err, tcase.json)
   425  				break
   426  			}
   427  			if !reflect.DeepEqual(tk, etk) {
   428  				t.Errorf(`case %v: %q @ %v expected %T(%v) was %T(%v)`, ci, tcase.json, i, etk, etk, tk, tk)
   429  				break
   430  			}
   431  		}
   432  	}
   433  }
   434  
   435  func TestDecodeStreamUseNumber(t *testing.T) {
   436  	dec := json.NewDecoder(strings.NewReader(`3.14`))
   437  	dec.UseNumber()
   438  	v, err := dec.Token()
   439  	if err != nil {
   440  		t.Errorf("unexpected error: %#v", err)
   441  	}
   442  	assertEq(t, "json.Number", "json.Number", fmt.Sprintf("%T", v))
   443  }
   444  
   445  // Test from golang.org/issue/11893
   446  func TestHTTPDecoding(t *testing.T) {
   447  	const raw = `{ "foo": "bar" }`
   448  
   449  	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   450  		w.Write([]byte(raw))
   451  	}))
   452  	defer ts.Close()
   453  	res, err := http.Get(ts.URL)
   454  	if err != nil {
   455  		log.Fatalf("GET failed: %v", err)
   456  	}
   457  	defer res.Body.Close()
   458  
   459  	foo := struct {
   460  		Foo string
   461  	}{}
   462  
   463  	d := json.NewDecoder(res.Body)
   464  	err = d.Decode(&foo)
   465  	if err != nil {
   466  		t.Fatalf("Decode: %v", err)
   467  	}
   468  	if foo.Foo != "bar" {
   469  		t.Errorf("decoded %q; want \"bar\"", foo.Foo)
   470  	}
   471  
   472  	// make sure we get the EOF the second time
   473  	err = d.Decode(&foo)
   474  	if err != io.EOF {
   475  		t.Errorf("err = %v; want io.EOF", err)
   476  	}
   477  }
   478  
   479  func TestGzipStreaming(t *testing.T) {
   480  	type someStruct struct {
   481  		ID   int      `json:"id"`
   482  		Text []string `json:"text"`
   483  	}
   484  
   485  	manyItems := strings.Repeat(`"Quis autem vel eum iure reprehenderit, qui in ea voluptate velit esse, quam`+
   486  		` nihil molestiae consequatur, vel illum, qui dolorem eum fugiat, quo voluptas nulla pariatur?",`, 400)
   487  
   488  	longJSON := `{"id":123,"text":[` + manyItems[0:len(manyItems)-1] + `]}`
   489  
   490  	compressed := bytes.NewBuffer(nil)
   491  	gw := gzip.NewWriter(compressed)
   492  
   493  	_, err := io.Copy(gw, bytes.NewReader([]byte(longJSON)))
   494  	if err != nil {
   495  		t.Fatalf("Unexpected error: %v", err)
   496  	}
   497  
   498  	err = gw.Close()
   499  	if err != nil {
   500  		t.Fatalf("Unexpected error: %v", err)
   501  	}
   502  
   503  	gr, err := gzip.NewReader(bytes.NewReader(compressed.Bytes()))
   504  	if err != nil {
   505  		t.Fatalf("Unexpected error: %v", err)
   506  	}
   507  
   508  	var v someStruct
   509  	dec := json.NewDecoder(gr)
   510  
   511  	err = dec.Decode(&v)
   512  	if err != nil {
   513  		t.Fatalf("Unexpected error: %v", err)
   514  	}
   515  }
   516  
   517  func TestLongUTF8(t *testing.T) {
   518  	want := strings.Repeat("あ", 342)
   519  	r := strings.NewReader(strconv.Quote(want))
   520  
   521  	var got string
   522  	if err := json.NewDecoder(r).Decode(&got); err != nil {
   523  		t.Fatalf("Unexpected error: %v", err)
   524  	}
   525  	if got != want {
   526  		t.Errorf("string %q; want = %q", got, want)
   527  	}
   528  }
   529  
   530  func TestIssue278(t *testing.T) {
   531  	a := `{"嗷嗷":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7"}`
   532  	r := strings.NewReader(a)
   533  	var m map[string]string
   534  	if err := json.NewDecoder(r).Decode(&m); err != nil {
   535  		t.Fatalf("Unexpected error: %v", err)
   536  	}
   537  	want := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u55f7"
   538  	if got := m["嗷嗷"]; got != want {
   539  		t.Errorf("string %q; want = %q", got, want)
   540  	}
   541  }