github.com/minio/simdjson-go@v0.4.6-0.20231116094823-04d21cddf993/parse_json_amd64_test.go (about)

     1  //go:build !noasm && !appengine && gc
     2  // +build !noasm,!appengine,gc
     3  
     4  /*
     5   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     6   *
     7   * Licensed under the Apache License, Version 2.0 (the "License");
     8   * you may not use this file except in compliance with the License.
     9   * You may obtain a copy of the License at
    10   *
    11   *     http://www.apache.org/licenses/LICENSE-2.0
    12   *
    13   * Unless required by applicable law or agreed to in writing, software
    14   * distributed under the License is distributed on an "AS IS" BASIS,
    15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16   * See the License for the specific language governing permissions and
    17   * limitations under the License.
    18   */
    19  
    20  package simdjson
    21  
    22  import (
    23  	"bytes"
    24  	"errors"
    25  	"fmt"
    26  	"io"
    27  	"math"
    28  	"runtime"
    29  	"strconv"
    30  	"strings"
    31  	"testing"
    32  )
    33  
    34  func TestDemoNdjson(t *testing.T) {
    35  	if !SupportedCPU() {
    36  		t.SkipNow()
    37  	}
    38  	pj := internalParsedJson{}
    39  
    40  	if err := pj.parseMessage([]byte(demo_ndjson), true); err != nil {
    41  		t.Errorf("TestDemoNdjson: got: %v want: nil", err)
    42  	}
    43  
    44  	verifyDemoNdjson(pj, t, 0)
    45  }
    46  
    47  func TestNdjsonEmptyLines(t *testing.T) {
    48  	if !SupportedCPU() {
    49  		t.SkipNow()
    50  	}
    51  
    52  	ndjson_emptylines := []string{`{"zero":"emptylines"}
    53  {"c":"d"}`,
    54  		`{"single":"emptyline"}
    55  
    56  {"c":"d"}`,
    57  		`{"dual":"emptylines"}
    58  
    59  
    60  {"c":"d"}`,
    61  		`{"triple":"emptylines"}
    62  
    63  
    64  
    65  {"c":"d"}`}
    66  
    67  	pj := internalParsedJson{}
    68  
    69  	for _, json := range ndjson_emptylines {
    70  		if err := pj.parseMessage([]byte(json), true); err != nil {
    71  			t.Errorf("TestNdjsonEmptyLine: got: %v want: nil", err)
    72  		}
    73  	}
    74  }
    75  
    76  func BenchmarkNdjsonStage2(b *testing.B) {
    77  	if !SupportedCPU() {
    78  		b.SkipNow()
    79  	}
    80  
    81  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
    82  	pj := internalParsedJson{}
    83  
    84  	b.SetBytes(int64(len(ndjson)))
    85  	b.ReportAllocs()
    86  	b.ResetTimer()
    87  	for i := 0; i < b.N; i++ {
    88  		err := pj.parseMessage(ndjson, true)
    89  		if err != nil {
    90  			panic(err)
    91  		}
    92  	}
    93  }
    94  
    95  func BenchmarkNdjsonStage1(b *testing.B) {
    96  	if !SupportedCPU() {
    97  		b.SkipNow()
    98  	}
    99  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
   100  
   101  	pj := internalParsedJson{}
   102  
   103  	b.SetBytes(int64(len(ndjson)))
   104  	b.ReportAllocs()
   105  	b.ResetTimer()
   106  
   107  	pj.Message = ndjson
   108  	for i := 0; i < b.N; i++ {
   109  		// Create new channel (large enough so we won't block)
   110  		pj.indexChans = make(chan indexChan, 128*10240)
   111  		pj.findStructuralIndices()
   112  	}
   113  }
   114  
   115  func BenchmarkNdjsonColdCountStar(b *testing.B) {
   116  	if !SupportedCPU() {
   117  		b.SkipNow()
   118  	}
   119  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
   120  
   121  	b.SetBytes(int64(len(ndjson)))
   122  	b.ReportAllocs()
   123  	// Allocate stuff
   124  	pj := internalParsedJson{}
   125  
   126  	b.ResetTimer()
   127  
   128  	for i := 0; i < b.N; i++ {
   129  		pj.parseMessage(ndjson, true)
   130  		count_raw_tape(pj.Tape)
   131  	}
   132  }
   133  
   134  func BenchmarkNdjsonColdCountStarWithWhere(b *testing.B) {
   135  	if !SupportedCPU() {
   136  		b.SkipNow()
   137  	}
   138  	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
   139  	const want = 110349
   140  	runtime.GC()
   141  	pj := internalParsedJson{}
   142  
   143  	b.Run("iter", func(b *testing.B) {
   144  		b.SetBytes(int64(len(ndjson)))
   145  		b.ReportAllocs()
   146  
   147  		for i := 0; i < b.N; i++ {
   148  			err := pj.parseMessage(ndjson, true)
   149  			if err != nil {
   150  				b.Fatal(err)
   151  			}
   152  			got := countWhere("Make", "HOND", pj.ParsedJson)
   153  			if got != want {
   154  				b.Fatal(got, "!=", want)
   155  			}
   156  		}
   157  	})
   158  	b.Run("chan", func(b *testing.B) {
   159  		b.SetBytes(int64(len(ndjson)))
   160  		b.ReportAllocs()
   161  
   162  		for i := 0; i < b.N; i++ {
   163  			// Temp values.
   164  			obj := &Object{}
   165  			elem := &Element{}
   166  			var tmp Iter
   167  			var nFound int
   168  			reuse := make(chan *ParsedJson, 1000)
   169  			res := make(chan Stream, 10)
   170  
   171  			ParseNDStream(bytes.NewBuffer(ndjson), res, reuse)
   172  			for got := range res {
   173  				if got.Error != nil {
   174  					if got.Error == io.EOF {
   175  						break
   176  					}
   177  					b.Fatal(got.Error)
   178  				}
   179  
   180  				all := got.Value.Iter()
   181  				// NDJSON is a separated by root objects.
   182  				for all.Advance() == TypeRoot {
   183  					// Read inside root.
   184  					t, i, err := all.Root(&tmp)
   185  					if t != TypeObject {
   186  						b.Log("got type", t.String())
   187  						continue
   188  					}
   189  
   190  					// Prepare object.
   191  					obj, err = i.Object(obj)
   192  					if err != nil {
   193  						b.Log("got err", err)
   194  						continue
   195  					}
   196  
   197  					// Find Make key.
   198  					elem = obj.FindKey("Make", elem)
   199  					if elem.Type != TypeString {
   200  						b.Log("got type", err)
   201  						continue
   202  					}
   203  					asB, err := elem.Iter.StringBytes()
   204  					if err != nil {
   205  						b.Log("got err", err)
   206  						continue
   207  					}
   208  					if bytes.Equal(asB, []byte("HOND")) {
   209  						nFound++
   210  					}
   211  				}
   212  				reuse <- got.Value
   213  			}
   214  			if nFound != want {
   215  				b.Fatal(nFound, "!=", want)
   216  			}
   217  		}
   218  	})
   219  
   220  }
   221  
   222  func TestParseNumber(t *testing.T) {
   223  	testCases := []struct {
   224  		input     string
   225  		wantTag   Tag
   226  		expectedD float64
   227  		expectedI int64
   228  		expectedU uint64
   229  		flags     FloatFlags
   230  	}{
   231  		{input: "1", wantTag: TagInteger, expectedI: 1},
   232  		{input: "-1", wantTag: TagInteger, expectedI: -1},
   233  		{input: "10000000000000000000", wantTag: TagUint, expectedU: 10000000000000000000},
   234  		{input: "10000000000000000001", wantTag: TagUint, expectedU: 10000000000000000001},
   235  		// math.MinInt64 - 1
   236  		{input: "-9223372036854775809", wantTag: TagFloat, expectedD: -9.223372036854776e+18, flags: FloatOverflowedInteger.Flags()},
   237  		{input: "-10000000000000000000", wantTag: TagFloat, expectedD: -10000000000000000000, flags: FloatOverflowedInteger.Flags()},
   238  		{input: "100000000000000000000", wantTag: TagFloat, expectedD: 100000000000000000000, flags: FloatOverflowedInteger.Flags()},
   239  		// math.MaxUint64 +1
   240  		{input: "18446744073709551616", wantTag: TagFloat, expectedD: 1.8446744073709552e+19, flags: FloatOverflowedInteger.Flags()},
   241  		{input: "1.0", wantTag: TagFloat, expectedD: 1.0},
   242  		{input: "1234567890", wantTag: TagInteger, expectedI: 1234567890},
   243  		{input: "9876.543210", wantTag: TagFloat, expectedD: 9876.543210},
   244  		{input: "0.123456789e-12", wantTag: TagFloat, expectedD: 1.23456789e-13},
   245  		{input: "1.234567890E+34", wantTag: TagFloat, expectedD: 1.234567890e+34},
   246  		{input: "23456789012E66", wantTag: TagFloat, expectedD: 23456789012e66},
   247  		{input: "-9876.543210", wantTag: TagFloat, expectedD: -9876.543210},
   248  		{input: "-65.619720000000029", wantTag: TagFloat, expectedD: -65.61972000000003},
   249  	}
   250  
   251  	for _, tc := range testCases {
   252  		id, val := parseNumber([]byte(fmt.Sprintf(`%s:`, tc.input)))
   253  		tag := Tag(id >> JSONTAGOFFSET)
   254  		flags := id & JSONVALUEMASK
   255  		if tag != tc.wantTag {
   256  			t.Errorf("TestParseNumber: got: %v want: %v", tag, tc.wantTag)
   257  		}
   258  		switch tag {
   259  		case TagFloat:
   260  			got := math.Float64frombits(val)
   261  			if !closeEnough(got, tc.expectedD) {
   262  				t.Errorf("TestParseNumber: got: %g want: %g", got, tc.expectedD)
   263  			}
   264  		case TagInteger:
   265  			if tc.expectedI != int64(val) {
   266  				t.Errorf("TestParseNumber: got: %d want: %d", int64(val), tc.expectedI)
   267  			}
   268  		case TagUint:
   269  			if tc.expectedU != val {
   270  				t.Errorf("TestParseNumber: got: %d want: %d", val, tc.expectedU)
   271  			}
   272  		}
   273  		if flags != uint64(tc.flags) {
   274  			t.Errorf("TestParseNumber flags; got: %d want: %d", flags, tc.flags)
   275  		}
   276  	}
   277  }
   278  
   279  // The following code is borrowed from Golang (https://golang.org/src/strconv/atoi_test.go)
   280  
   281  type parseInt64Test struct {
   282  	in  string
   283  	out int64
   284  	tag Tag
   285  }
   286  
   287  var parseInt64Tests = []parseInt64Test{
   288  	{"", 0, TagEnd},
   289  	{"0", 0, TagInteger},
   290  	{"-0", 0, TagInteger},
   291  	{"1", 1, TagInteger},
   292  	{"-1", -1, TagInteger},
   293  	{"12345", 12345, TagInteger},
   294  	{"-12345", -12345, TagInteger},
   295  	{"012345", 0, TagEnd},
   296  	{"-012345", 0, TagEnd},
   297  	{"98765432100", 98765432100, TagInteger},
   298  	{"-98765432100", -98765432100, TagInteger},
   299  	{"9223372036854775807", 1<<63 - 1, TagInteger},
   300  	{"-9223372036854775807", -(1<<63 - 1), TagInteger},
   301  	{"9223372036854775808", 1<<63 - 1, TagUint},
   302  	{"-9223372036854775808", -1 << 63, TagInteger},
   303  	{"9223372036854775809", 1<<63 - 1, TagUint},
   304  	{"-9223372036854775809", -1 << 63, TagFloat},
   305  	{"-1_2_3_4_5", 0, TagEnd}, // base=10 so no underscores allowed
   306  	{"-_12345", 0, TagEnd},
   307  	{"_12345", 0, TagEnd},
   308  	{"1__2345", 0, TagEnd},
   309  	{"12345_", 0, TagEnd},
   310  
   311  	// zero (originate from atof tests below, but returned as int for simdjson)
   312  	{"0e0", 0, TagFloat},
   313  	{"-0e0", 0, TagFloat},
   314  	{"0e-0", 0, TagFloat},
   315  	{"-0e-0", 0, TagFloat},
   316  	{"0e+0", 0, TagFloat},
   317  	{"-0e+0", 0, TagFloat},
   318  }
   319  
   320  func TestParseInt64(t *testing.T) {
   321  	for i := range parseInt64Tests {
   322  		test := &parseInt64Tests[i]
   323  		t.Run(test.in, func(t *testing.T) {
   324  
   325  			id, val := parseNumber([]byte(fmt.Sprintf(`%s:`, test.in)))
   326  			tag := Tag(id >> JSONTAGOFFSET)
   327  			if tag != test.tag {
   328  				// Ignore intentionally bad syntactical errors
   329  				t.Errorf("TestParseInt64: got: %v want: %v", tag, test.tag)
   330  				return // skip testing the rest for this test case
   331  			}
   332  			if tag == TagInteger && int64(val) != test.out {
   333  				// Ignore intentionally wrong conversions
   334  				t.Errorf("TestParseInt64: got value: %v want: %v", int64(val), test.out)
   335  			}
   336  		})
   337  
   338  	}
   339  }
   340  
   341  // The following code is borrowed from Golang (https://golang.org/src/strconv/atof_test.go)
   342  
   343  type atofTest struct {
   344  	in  string
   345  	out string
   346  	err error
   347  }
   348  
   349  var atoftests = []atofTest{
   350  	{"", "0", strconv.ErrSyntax}, /* fails for simdjson */
   351  	{"1", "1", nil},              /* parsed as int for simdjson */
   352  	{"+1", "1", nil},             /* parsed as int for simdjson */
   353  
   354  	{"1x", "0", strconv.ErrSyntax},
   355  	{"1.1.", "0", strconv.ErrSyntax},
   356  	{"1e23", "1e+23", nil},
   357  	{"1E23", "1e+23", nil},
   358  	{"100000000000000000000000", "1e+23", nil}, /* parsed as int for simdjson */
   359  	{"1e-100", "1e-100", nil},
   360  	{"123456700", "123456700", nil},                             /* parsed as int for simdjson */
   361  	{"99999999999999974834176", "9.999999999999997e+22", nil},   /* parsed as int for simdjson */
   362  	{"100000000000000000000001", "1.0000000000000001e+23", nil}, /* parsed as int for simdjson */
   363  	{"100000000000000008388608", "1.0000000000000001e+23", nil}, /* parsed as int for simdjson */
   364  	{"100000000000000016777215", "1.0000000000000001e+23", nil}, /* parsed as int for simdjson */
   365  	{"100000000000000016777216", "1.0000000000000003e+23", nil}, /* parsed as int for simdjson */
   366  	{"-1", "-1", nil},                                           /* parsed as int for simdjson */
   367  	{"-0.1", "-0.1", nil},
   368  	{"-0", "0", nil}, /* parsed as int for simdjson */
   369  	{"1e-20", "1e-20", nil},
   370  	{"625e-3", "0.625", nil},
   371  
   372  	// zeros (several test cases for zero have been moved up because they are detected as ints)
   373  	{"+0e0", "0", nil},
   374  	{"+0e-0", "0", nil},
   375  	{"+0e+0", "0", nil},
   376  	{"0e+01234567890123456789", "0", nil},
   377  	{"0.00e-01234567890123456789", "0", nil},
   378  	{"-0e+01234567890123456789", "-0", nil},
   379  	{"-0.00e-01234567890123456789", "-0", nil},
   380  
   381  	{"0e291", "0", nil},   // issue 15364
   382  	{"0e292", "0", nil},   // issue 15364
   383  	{"0e347", "0", nil},   // issue 15364
   384  	{"0e348", "0", nil},   // issue 15364
   385  	{"-0e291", "-0", nil}, /* returns "0" */
   386  	{"-0e292", "-0", nil}, /* returns "0" */
   387  	{"-0e347", "-0", nil}, /* returns "0" */
   388  	{"-0e348", "-0", nil}, /* returns "0" */
   389  
   390  	// NaNs
   391  	{"nan", "NaN", errors.New("invalid json")},
   392  	{"NaN", "NaN", errors.New("invalid json")},
   393  	{"NAN", "NaN", errors.New("invalid json")},
   394  
   395  	// Infs
   396  	{"inf", "+Inf", errors.New("invalid json")},
   397  	{"-Inf", "-Inf", errors.New("invalid json")},
   398  	{"+INF", "+Inf", errors.New("invalid json")},
   399  	{"-Infinity", "-Inf", errors.New("invalid json")},
   400  	{"+INFINITY", "+Inf", errors.New("invalid json")},
   401  	{"Infinity", "+Inf", errors.New("invalid json")},
   402  
   403  	// largest float64
   404  	{"1.7976931348623157e308", "1.7976931348623157e+308", nil},
   405  	{"-1.7976931348623157e308", "-1.7976931348623157e+308", nil},
   406  
   407  	// next float64 - too large
   408  	{"1.7976931348623159e308", "+Inf", strconv.ErrRange},
   409  	{"-1.7976931348623159e308", "-Inf", strconv.ErrRange},
   410  
   411  	{"1.7976931348623158e308", "1.7976931348623157e+308", nil},
   412  	{"-1.7976931348623158e308", "-1.7976931348623157e+308", nil},
   413  
   414  	// borderline - too large
   415  	{"1.797693134862315808e308", "+Inf", strconv.ErrRange},
   416  	{"-1.797693134862315808e308", "-Inf", strconv.ErrRange},
   417  
   418  	// a little too large
   419  	{"1e308", "1e+308", nil},
   420  	{"2e308", "+Inf", strconv.ErrRange},
   421  	{"1e309", "+Inf", strconv.ErrRange},
   422  
   423  	// way too large
   424  	{"1e310", "+Inf", strconv.ErrRange},
   425  	{"-1e310", "-Inf", strconv.ErrRange},
   426  	{"1e400", "+Inf", strconv.ErrRange},
   427  	{"-1e400", "-Inf", strconv.ErrRange},
   428  	{"1e400000", "+Inf", strconv.ErrRange},
   429  	{"-1e400000", "-Inf", strconv.ErrRange},
   430  
   431  	// denormalized
   432  	{"1e-305", "1e-305", nil},
   433  	{"1e-306", "1e-306", nil},
   434  	{"1e-307", "1e-307", nil},
   435  	{"1e-308", "1e-308", nil},
   436  	{"1e-309", "1e-309", nil},
   437  	{"1e-310", "1e-310", nil},
   438  	{"1e-322", "1e-322", nil},
   439  	// smallest denormal
   440  	{"5e-324", "5e-324", nil},
   441  	{"4e-324", "5e-324", nil},
   442  	{"3e-324", "5e-324", nil},
   443  	// too small
   444  	{"2e-324", "0", nil},
   445  	// way too small
   446  	{"1e-350", "0", nil},
   447  	{"1e-400000", "0", nil},
   448  
   449  	// try to overflow exponent
   450  	{"1e-4294967296", "0", nil},
   451  	{"1e+4294967296", "+Inf", strconv.ErrRange},
   452  	{"1e-18446744073709551616", "0", nil},
   453  	{"1e+18446744073709551616", "+Inf", strconv.ErrRange},
   454  
   455  	// Parse errors
   456  	{"1e", "0", strconv.ErrSyntax},
   457  	{"1e-", "0", strconv.ErrSyntax},
   458  	{".e-1", "0", strconv.ErrSyntax},
   459  
   460  	// https://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
   461  	{"2.2250738585072012e-308", "2.2250738585072014e-308", nil},
   462  	// https://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/
   463  	{"2.2250738585072011e-308", "2.225073858507201e-308", nil},
   464  
   465  	// A very large number (initially wrongly parsed by the fast algorithm).
   466  	{"4.630813248087435e+307", "4.630813248087435e+307", nil},
   467  
   468  	// A different kind of very large number.
   469  	{"22.222222222222222", "22.22222222222222", nil},
   470  	{"2." + strings.Repeat("2", 4000) + "e+1", "22.22222222222222", nil},
   471  
   472  	// Exactly halfway between 1 and math.Nextafter(1, 2).
   473  	// Round to even (down).
   474  	{"1.00000000000000011102230246251565404236316680908203125", "1", nil},
   475  	// Slightly lower; still round down.
   476  	{"1.00000000000000011102230246251565404236316680908203124", "1", nil},
   477  	// Slightly higher; round up.
   478  	{"1.00000000000000011102230246251565404236316680908203126", "1.0000000000000002", nil},
   479  	// Slightly higher, but you have to read all the way to the end.
   480  	{"1.00000000000000011102230246251565404236316680908203125" + strings.Repeat("0", 10000) + "1", "1.0000000000000002", nil},
   481  
   482  	// Halfway between x := math.Nextafter(1, 2) and math.Nextafter(x, 2)
   483  	// Round to even (up).
   484  	{"1.00000000000000033306690738754696212708950042724609375", "1.0000000000000004", nil},
   485  
   486  	// Underscores.
   487  	{"1_23.50_0_0e+1_2", "1.235e+14", strconv.ErrSyntax},
   488  	{"-_123.5e+12", "0", strconv.ErrSyntax},
   489  	{"+_123.5e+12", "0", strconv.ErrSyntax},
   490  	{"_123.5e+12", "0", strconv.ErrSyntax},
   491  	{"1__23.5e+12", "0", strconv.ErrSyntax},
   492  	{"123_.5e+12", "0", strconv.ErrSyntax},
   493  	{"123._5e+12", "0", strconv.ErrSyntax},
   494  	{"123.5_e+12", "0", strconv.ErrSyntax},
   495  	{"123.5__0e+12", "0", strconv.ErrSyntax},
   496  	{"123.5e_+12", "0", strconv.ErrSyntax},
   497  	{"123.5e+_12", "0", strconv.ErrSyntax},
   498  	{"123.5e_-12", "0", strconv.ErrSyntax},
   499  	{"123.5e-_12", "0", strconv.ErrSyntax},
   500  	{"123.5e+1__2", "0", strconv.ErrSyntax},
   501  	{"123.5e+12_", "0", strconv.ErrSyntax},
   502  }
   503  
   504  func TestParseFloat64(t *testing.T) {
   505  
   506  	for i := 0; i < len(atoftests); i++ {
   507  		test := &atoftests[i]
   508  		t.Run(test.in, func(t *testing.T) {
   509  			id, val := parseNumber([]byte(fmt.Sprintf(`%s:`, test.in)))
   510  			tag := Tag(id >> JSONTAGOFFSET)
   511  			switch tag {
   512  			case TagEnd:
   513  				if test.err == nil {
   514  					t.Errorf("TestParseFloat64: got error, none")
   515  				}
   516  			case TagFloat:
   517  				got := math.Float64frombits(val)
   518  				outs := strconv.FormatFloat(got, 'g', -1, 64)
   519  				if outs != test.out {
   520  					t.Errorf("TestParseFloat64: got: %v want: %v", outs, test.out)
   521  				}
   522  			case TagInteger:
   523  				got := int64(val)
   524  				outs := fmt.Sprint(got)
   525  				if outs != test.out {
   526  					t.Errorf("TestParseFloat64: got: %v want: %v", outs, test.out)
   527  				}
   528  			case TagUint:
   529  				got := val
   530  				outs := fmt.Sprint(got)
   531  				if outs != test.out {
   532  					t.Errorf("TestParseFloat64: got: %v want: %v", outs, test.out)
   533  				}
   534  			default:
   535  			}
   536  		})
   537  	}
   538  }
   539  
   540  func TestParseString(t *testing.T) {
   541  	if !SupportedCPU() {
   542  		t.SkipNow()
   543  	}
   544  	for _, tt := range tests {
   545  		t.Run(tt.name, func(t *testing.T) {
   546  			// enclose test string in quotes (as validated by stage 1)
   547  			buf := []byte(fmt.Sprintf(`"%s"`, tt.str))
   548  			dest := make([]byte, 0, len(buf)+32 /* safety margin as parseString writes full AVX2 words */)
   549  
   550  			success := parseStringSimd(buf, &dest)
   551  
   552  			if success != tt.success {
   553  				t.Errorf("TestParseString() got = %v, want %v", success, tt.success)
   554  			}
   555  			if success {
   556  				size := len(dest)
   557  				if size != len(tt.want) {
   558  					t.Errorf("TestParseString() got = %d, want %d", size, len(tt.want))
   559  				}
   560  				if !bytes.Equal(dest[:size], tt.want) {
   561  					t.Errorf("TestParseString() got = %v, want %v", dest[:size], tt.want)
   562  				}
   563  			}
   564  		})
   565  	}
   566  }
   567  
   568  func TestParseStringValidateOnly(t *testing.T) {
   569  	if !SupportedCPU() {
   570  		t.SkipNow()
   571  	}
   572  
   573  	for _, tt := range tests {
   574  		t.Run(tt.name, func(t *testing.T) {
   575  			// enclose test string in quotes (as validated by stage 1)
   576  			buf := []byte(fmt.Sprintf(`"%s"`, tt.str))
   577  
   578  			dst_length := uint64(0)
   579  			need_copy := false
   580  			l := uint64(len(buf))
   581  			success := parseStringSimdValidateOnly(buf, &l, &dst_length, &need_copy)
   582  
   583  			if success != tt.success {
   584  				t.Errorf("TestParseString() got = %v, want %v", success, tt.success)
   585  			}
   586  			if success && !need_copy {
   587  				if dst_length != uint64(len(tt.want)) {
   588  					t.Errorf("TestParseString() got = %d, want %d", dst_length, len(tt.want))
   589  				}
   590  			}
   591  		})
   592  	}
   593  }
   594  
   595  func TestParseStringValidateOnlyBeyondBuffer(t *testing.T) {
   596  	if !SupportedCPU() {
   597  		t.SkipNow()
   598  	}
   599  
   600  	t.Skip()
   601  
   602  	buf := []byte(fmt.Sprintf(`"%s`, "   "))
   603  
   604  	dst_length := uint64(0)
   605  	need_copy := false
   606  	l := uint64(len(buf)) + 32
   607  	success := parseStringSimdValidateOnly(buf, &l, &dst_length, &need_copy)
   608  	if !success {
   609  		t.Errorf("TestParseStringValidateOnlyBeyondBuffer() got = %v, want %v", success, false)
   610  	}
   611  }
   612  
   613  // Benchmarking code for integers
   614  
   615  func BenchmarkParseNumber(b *testing.B) {
   616  	b.Run("Pos", func(b *testing.B) {
   617  		benchmarkParseNumber(b, 1)
   618  	})
   619  	b.Run("Neg", func(b *testing.B) {
   620  		benchmarkParseNumber(b, -1)
   621  	})
   622  }
   623  
   624  func benchmarkParseNumber(b *testing.B, neg int) {
   625  	cases := []benchCase{
   626  		{"63bit", 1<<63 - 1},
   627  	}
   628  	for _, cs := range cases {
   629  		b.Run(cs.name, func(b *testing.B) {
   630  			s := fmt.Sprintf("%d", cs.num*int64(neg))
   631  			s = fmt.Sprintf(`%s:`, s) // append delimiter
   632  			for i := 0; i < b.N; i++ {
   633  				parseNumber([]byte(s))
   634  			}
   635  		})
   636  	}
   637  }
   638  
   639  func BenchmarkParseNumberFloat(b *testing.B) {
   640  	b.SetBytes(1)
   641  	for i := 0; i < b.N; i++ {
   642  		parseNumber([]byte("339.7784:"))
   643  	}
   644  }
   645  
   646  func BenchmarkParseAtof64FloatGolang(b *testing.B) {
   647  	b.SetBytes(1)
   648  	for i := 0; i < b.N; i++ {
   649  		strconv.ParseFloat("339.7784", 64)
   650  	}
   651  }
   652  
   653  func BenchmarkParseNumberFloatExp(b *testing.B) {
   654  	b.SetBytes(1)
   655  	for i := 0; i < b.N; i++ {
   656  		parseNumber([]byte("-5.09e75:"))
   657  	}
   658  }
   659  
   660  func BenchmarkParseNumberBig(b *testing.B) {
   661  	b.SetBytes(1)
   662  	x := []byte("123456789123456789123456789:")
   663  	for i := 0; i < b.N; i++ {
   664  		parseNumber(x)
   665  	}
   666  }
   667  
   668  func BenchmarkParseNumberRandomBits(b *testing.B) {
   669  	initAtof()
   670  	for i := 0; i < b.N; i++ {
   671  		parseNumber([]byte(benchmarksRandomBitsSimd[i%1024]))
   672  	}
   673  }
   674  
   675  func BenchmarkParseNumberRandomFloats(b *testing.B) {
   676  	initAtof()
   677  	for i := 0; i < b.N; i++ {
   678  		parseNumber([]byte(benchmarksRandomNormalSimd[i%1024]))
   679  	}
   680  }
   681  
   682  func TestVerifyTape(t *testing.T) {
   683  	if !SupportedCPU() {
   684  		t.SkipNow()
   685  	}
   686  
   687  	for _, tt := range testCases {
   688  		t.Run(tt.name, func(t *testing.T) {
   689  			ref := loadCompressed(t, tt.name)
   690  
   691  			pj := internalParsedJson{}
   692  			if err := pj.parseMessage(ref, false); err != nil {
   693  				t.Errorf("parseMessage failed: %v\n", err)
   694  				return
   695  			}
   696  		})
   697  	}
   698  }