github.com/goshafaq/sonic@v0.0.0-20231026082336-871835fb94c6/search_test.go

//go:build amd64 && go1.16 && !go1.22
// +build amd64,go1.16,!go1.22

/*
 * Copyright 2021 ByteDance Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sonic

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"math/rand"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/goshafaq/sonic/ast"
	"github.com/stretchr/testify/assert"
)

func Parse(src string) (*ast.Node, error) {
	node, err := ast.NewParser(src).Parse()
	if err != 0 {
		return &node, fmt.Errorf("parsing error: %v", err)
	}
	return &node, nil
}

func assertCond(cond bool) {
	if !cond {
		panic("assertCond failed")
	}
}

func TestExampleSearch(t *testing.T) {
	data := []byte(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `)

	node, e := Get(data, "test", 0)
	x, _ := node.Bool()
	if e != nil || x != true {
		t.Fatalf("node: %v, err: %v", node, e)
	}

	node, e = Get(data, "test", 1)
	a, _ := node.Float64()
	if e != nil || a != 0.1 {
		t.Fatalf("node: %v, err: %v", node, e)
	}

	node, e = Get(data, "test", 2)
	b, _ := node.String()
	if e != nil || b != "abc" {
		t.Fatalf("node: %v, err: %v", node, e)
	}

	node, e = Get(data, "test", 3)
	arr, _ := node.Array()
	if e != nil || arr[0] != "h" {
		t.Fatalf("node: %v, err: %v", node, e)
	}

	node, e = Get(data, "test", 4, "a")
	c, _ := node.String()
	if e != nil || c != "bc" {
		t.Fatalf("node: %v, err: %v", node, e)
	}
}

func TestExampleSearchEscapedKey(t *testing.T) {
	data := []byte(`
{
    "xx" : [] ,
    "yy" :{ }, 
    "test\"" : [
        true ,
        0.1 ,
        "abc",
        [
            "h"
        ], 
        {
            "a\u0008": {},
            "b\\\\": null,
            "\u2028\u2028": "\u2028\u2029",
            "\u0026":2,
            "0":1
        } 
    ],
    ",,,,,,,,,,(,,15": ",",
    ",,,,,,,,,,(,,,16": "a,]}",
    ",,,,,,,,,,(,,,,17": 1,
    ",,,,,,,,,,(,,,,,,,,,,,,,,,,(,,,,34": "c"
} `)

	type getTest struct {
		path   []interface{}
		expect interface{}
	}

	tests := []getTest{
		{[]interface{}{"test\"", 0}, true},
		{[]interface{}{"test\"", 1}, 0.1},
		{[]interface{}{"test\"", 2}, "abc"},
		{[]interface{}{"test\"", 3}, []interface{}{"h"}},
		{[]interface{}{"test\"", 4, "a\u0008"}, map[string]interface{}{}},
		{[]interface{}{"test\"", 4, "b\\\\"}, nil},
		{[]interface{}{"test\"", 4, "\u2028\u2028"}, "\u2028\u2029"},
		{[]interface{}{"test\"", 4, "0"}, float64(1)},
		{[]interface{}{",,,,,,,,,,(,,15"}, ","},
		{[]interface{}{",,,,,,,,,,(,,,16"}, "a,]}"},
		{[]interface{}{",,,,,,,,,,(,,,,17"}, float64(1)},
		{[]interface{}{",,,,,,,,,,(,,,,,,,,,,,,,,,,(,,,,34"}, "c"},
	}

	for _, test := range tests {
		node, err := Get(data, test.path...)
		assert.NoErrorf(t, err, "get return errors")
		got, err := node.Interface()
		assert.NoErrorf(t, err, "get convert errors")
		assert.Equalf(t, test.expect, got, "get result is wrong from path %#v", test.path)
	}
}

func TestExampleSearchErr(t *testing.T) {
	data := []byte(` { "xx" : [] ,"yy" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `)
	node, e := Get(data, "zz")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)

	node, e = Get(data, "xx", 4)
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)

	node, e = Get(data, "yy", "a")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)

	node, e = Get(data, "test", 4, "x")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)
}

func TestExampleSearchEscapedKeyError(t *testing.T) {
	data := []byte(` { "xx" : [] ,"yy" :{ }, "x\u0008" : [] ,"y\\\"y" :{ }, "test" : [ true , 0.1 , "abc", ["h"], {"a":"bc"} ] } `)
	node, e := Get(data, "zz")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)

	node, e = Get(data, "x\u0008", 4)
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)

	node, e = Get(data, "yy", "a")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)

	node, e = Get(data, "test", 4, "x")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}

	node, e = Get(data, "y\\\"y", 4, "x")
	if e == nil {
		t.Fatalf("node: %v, err: %v", node, e)
	}
	fmt.Println(e)
}

func TestRandomData(t *testing.T) {
	var lstr string
	defer func() {
		if v := recover(); v != nil {
			println("'" + hex.Dump([]byte(lstr)) + "'")
			println(lstr)
			panic(v)
		}
	}()
	data := []byte(`"�-mp�`)
	_, err := ast.NewParser(string(data)).Parse()
	if err != 0 {
		fmt.Println(hex.Dump(data))
		fmt.Println(string(data))
	}
	rand.Seed(time.Now().UnixNano())
	b := make([]byte, 200)
	for i := 0; i < 1000000; i++ {
		n, ee := rand.Read(b[:rand.Int()%len(b)])
		if ee != nil {
			t.Fatalf("get random bytes failed: %v,", ee)
			return
		}
		lstr = string(b[:n])
		_, _ = ast.NewParser(lstr).Parse()
	}
}

func TestRandomValidStrings(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	b := make([]byte, 200)
	for i := 0; i < 1000; i++ {
		n, err := rand.Read(b[:rand.Int()%len(b)])
		if err != nil {
			t.Fatal("get random data failed:", err)
		}
		sm, err := json.Marshal(string(b[:n]))
		if err != nil {
			t.Fatal("marshal data failed:", err)
		}
		var su string
		if err := json.Unmarshal(sm, &su); err != nil {
			t.Fatal("unmarshal data failed:", err)
		}
		token, err := GetFromString(`{"str":`+string(sm)+`}`, "str")
		if err != nil {
			spew.Dump(string(sm))
			t.Fatal("search data failed:", err)
		}
		x, _ := token.Interface()
		st, ok := x.(string)
		if !ok {
			t.Fatalf("type mismatch, exp: %v, got: %v", su, x)
		}
		if st != su {
			t.Fatalf("string mismatch, exp: %v, got: %v", su, x)
		}
	}
}

func TestEmoji(t *testing.T) {
	var input = []byte(`{"utf8":"Example emoji, KO: \ud83d\udd13, \ud83c\udfc3 ` +
		`OK: \u2764\ufe0f "}`)
	value, err := Get(input, "utf8")
	if err != nil {
		t.Fatal(err)
	}
	var v map[string]interface{}
	if err := json.Unmarshal(input, &v); err != nil {
		t.Fatal(err)
	}
	s, _ := v["utf8"].(string)
	x, _ := value.String()
	if x != s {
		t.Fatalf("expected '%v', got '%v'", s, x)
	}
}

func testEscapePath(t *testing.T, json, expect string, path ...interface{}) {
	n, e := Get([]byte(json), path...)
	if e != nil {
		t.Fatal(e)
	}
	x, _ := n.String()
	if x != expect {
		x, _ := n.Interface()
		t.Fatalf("expected '%v', got '%v'", expect, x)
	}
}

func TestEscapePath(t *testing.T) {
	data := `{
        "test":{
            "*":"valZ",
            "*v":"val0",
            "keyv*":"val1",
            "key*v":"val2",
            "keyv?":"val3",
            "key?v":"val4",
            "keyv.":"val5",
            "key.v":"val6",
            "keyk*":{"key?":"val7"}
        }
    }`

	testEscapePath(t, data, "valZ", "test", "*")
	testEscapePath(t, data, "val0", "test", "*v")
	testEscapePath(t, data, "val1", "test", "keyv*")
	testEscapePath(t, data, "val2", "test", "key*v")
	testEscapePath(t, data, "val3", "test", "keyv?")
	testEscapePath(t, data, "val4", "test", "key?v")
	testEscapePath(t, data, "val5", "test", "keyv.")
	testEscapePath(t, data, "val6", "test", "key.v")
	testEscapePath(t, data, "val7", "test", "keyk*", "key?")
}

func TestParseAny(t *testing.T) {
	n, e := Parse("100")
	assertCond(e == nil)
	if n == nil {
		panic("n is nil")
	}
	x, _ := n.Float64()
	assertCond(x == 100)
	n, e = Parse("true")
	assertCond(e == nil)
	if n == nil {
		panic("n is nil")
	}

	a, _ := n.Bool()
	assertCond(a)
	n, e = Parse("false")
	assertCond(e == nil)
	if n == nil {
		panic("n is nil")
	}
	b, _ := n.Bool()
	assertCond(b == false)
	n, e = Parse("yikes")
	assertCond(e != nil)
}

func TestTime(t *testing.T) {
	data := []byte(`{
      "code": 0,
      "msg": "",
      "data": {
        "sz002024": {
          "qfqday": [
            [
              "2014-01-02",
              "8.93",
              "9.03",
              "9.17",
              "8.88",
              "621143.00"
            ],
            [
              "2014-01-03",
              "9.03",
              "9.30",
              "9.47",
              "8.98",
              "1624438.00"
            ]
          ]
        }
      }
    }`)

	var num []string
	n, e := Get(data, "data", "sz002024", "qfqday", 0)
	if e != nil {
		t.Fatal(e)
	}

	arr, _ := n.Array()
	for _, v := range arr {
		s := v.(string)
		num = append(num, s)
	}
	if fmt.Sprintf("%v", num) != "[2014-01-02 8.93 9.03 9.17 8.88 621143.00]" {
		t.Fatalf("invalid result")
	}
}

var exampleJSON = `{
    "widget": {
        "debug": "on",
        "window": {
            "title": "Sample Konfabulator Widget",
            "name": "main_window",
            "width": 500,
            "height": 500
        },
        "image": {
            "src": "Images/Sun.png",
            "hOffset": 250,
            "vOffset": 250,
            "alignment": "center"
        },
        "text": {
            "data": "Click Here",
            "size": 36,
            "style": "bold",
            "vOffset": 100,
            "alignment": "center",
            "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
        }
    }
}`

func TestUnmarshalMap(t *testing.T) {
	n, err := Parse(exampleJSON)
	if err != nil || n == nil {
		t.Fatal(err)
	}
	m1, _ := n.Map()
	var m2 map[string]interface{}
	if err := json.Unmarshal([]byte(exampleJSON), &m2); err != nil {
		t.Fatal(err)
	}
	b1, err := json.Marshal(m1)
	if err != nil {
		t.Fatal(err)
	}
	b2, err := json.Marshal(m2)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(b1, b2) {
		t.Fatalf("b1 != b2\n b1: %v\nb2: %v", string(b1), string(b2))
	}
}

func GetMany(src2 string, path ...string) (ret []string) {
	src := []byte(src2)
	for _, p := range path {
		pathes := strings.Split(p, ".")
		if len(pathes) == 0 {
			panic(fmt.Sprintf("invalid path: %v", p))
		}
		ps := make([]interface{}, 0, len(pathes))
		for _, p := range pathes {
			ps = append(ps, p)
		}
		n, e := Get(src, ps...)
		if e != nil {
			ret = append(ret, "")
			continue
		}
		x, _ := n.Interface()
		ret = append(ret, fmt.Sprintf("%v", x))
	}
	return
}

func get(str string, path string) *ast.Node {
	src := []byte(str)
	pathes := strings.Split(path, ".")
	if len(pathes) == 0 {
		panic(fmt.Sprintf("invalid path: %v", path))
	}
	ps := make([]interface{}, 0, len(pathes))
	for _, p := range pathes {
		ps = append(ps, p)
	}
	n, e := Get(src, ps...)
	if e != nil {
		return nil
	}
	return &n
}

func TestSingleArrayValue(t *testing.T) {
	var data = []byte(`{"key": "value","key2":[1,2,3,4,"A"]}`)
	array, _ := get(string(data), "key2").Array()
	if len(array) != 5 {
		t.Fatalf("got '%v', expected '%v'", len(array), 5)
	}

	_, e := Get(data, "key3")
	if e == nil {
		t.Fatalf("got '%v', expected '%v'", e, nil)
	}
}

var manyJSON = `  {
    "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
        "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
        "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
        "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
        "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
        "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{
        "a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"a":{"hello":"world"
        }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},
        "position":{"type":"Point","coordinates":[-115.24,33.09]},
        "loves":["world peace"],
        "name":{"last":"Anderson","first":"Nancy"},
        "age":31,
        "x":{"a":"emptya","b":"emptyb"},
        "name_last":"Yellow",
        "name_first":"Cat",
}`

func TestManyBasic(t *testing.T) {
	testMany := func(shouldFallback bool, expect string, paths ...string) {
		println()
		rs := GetMany(
			manyJSON,
			paths...,
		)
		// if len(rs) != len(paths) {
		//     t.Fatalf("expected %v, got %v", len(paths), len(rs))
		// }
		var results = "[" + strings.Join(rs, " ") + "]"
		if results != expect {
			fmt.Printf("%v\n", paths)
			t.Fatalf("expected %v, got %v", expect, results)
		}
	}
	testMany(false, "[Point]", "position.type")
	testMany(false, `[emptya [world peace] 31]`, "x.a", "loves", "age")
	testMany(false, `[[world peace]]`, "loves")
	testMany(false, `[map[first:Nancy last:Anderson] Nancy]`, "name",
		"name.first")
	testMany(true, `[]`, strings.Repeat("a.", 40)+"hello")
	res := get(manyJSON, strings.Repeat("a.", 48)+"a")
	assertCond(res != nil)
	x, _ := res.Interface()
	testMany(true, "["+fmt.Sprint(x)+"]", strings.Repeat("a.", 48)+"a")
	// these should fallback
	testMany(true, `[Cat Nancy]`, "name_first", "name.first")
	testMany(true, `[world]`, strings.Repeat("a.", 70)+"hello")
}

func testMany(t *testing.T, json string, paths, expected []string) {
	testManyAny(t, json, paths, expected)
	testManyAny(t, json, paths, expected)
}

func testManyAny(t *testing.T, json string, paths, expected []string) {
	var result []string
	for i := 0; i < 2; i++ {
		var which string
		if i == 0 {
			which = "Get"
			result = nil
			for j := 0; j < len(expected); j++ {
				x, _ := get(json, paths[j]).Interface()
				result = append(result, fmt.Sprintf("%v", x))
			}
		} else if i == 1 {
			which = "GetMany"
			result = GetMany(json, paths...)
		}
		if result == nil {
			panic("result is nil")
		}
		for j := 0; j < len(expected); j++ {
			if result[j] != expected[j] {
				t.Fatalf("Using key '%s' for '%s'\nexpected '%v', got '%v'",
					paths[j], which, expected[j], result[j])
			}
		}
	}
}

func TestNested(t *testing.T) {
	data := `{ "name": "FirstName", "name1": "FirstName1", ` +
		`"address": "address1", "addressDetails": "address2", }`
	paths := []string{"name", "name1", "address", "addressDetails"}
	expected := []string{"FirstName", "FirstName1", "address1", "address2"}
	t.Run("SingleMany", func(t *testing.T) {
		testMany(t, data, paths,
			expected)
	})
}

func TestMultiLevelFields(t *testing.T) {
	data := `{ "Level1Field1":3, 
               "Level1Field4":4, 
               "Level1Field2":{ "Level2Field1":[ "value1", "value2" ], 
               "Level2Field2":{ "Level3Field1":[ { "key1":"value1" } ] } } }`
	paths := []string{"Level1Field1", "Level1Field2.Level2Field1",
		"Level1Field2.Level2Field2.Level3Field1", "Level1Field4"}
	expected := []string{"3", `[value1 value2]`,
		`[map[key1:value1]]`, "4"}
	t.Run("SingleMany", func(t *testing.T) {
		testMany(t, data, paths,
			expected)
	})
}

func TestRandomMany(t *testing.T) {
	var lstr string
	defer func() {
		if v := recover(); v != nil {
			println("'" + hex.EncodeToString([]byte(lstr)) + "'")
			println("'" + lstr + "'")
			panic(v)
		}
	}()
	rand.Seed(time.Now().UnixNano())
	b := make([]byte, 512)
	for i := 0; i < 5000; i++ {
		n, err := rand.Read(b[:rand.Int()%len(b)])
		if err != nil {
			t.Fatal(err)
		}
		lstr = string(b[:n])
		paths := make([]string, rand.Int()%64)
		for i := range paths {
			var b []byte
			n := rand.Int() % 5
			for j := 0; j < n; j++ {
				if j > 0 {
					b = append(b, '.')
				}
				nn := rand.Int() % 10
				for k := 0; k < nn; k++ {
					b = append(b, 'a'+byte(rand.Int()%26))
				}
			}
			paths[i] = string(b)
		}
		GetMany(lstr, paths...)
	}
}

func TestGetMany(t *testing.T) {
	data := `{"bar": {"id": 99, "mybar": "my mybar" }, "foo": ` +
		`{"myfoo": [605]}}`
	paths := []string{"foo.myfoo", "bar.id", "bar.mybar", "bar.mybarx"}
	expected := []string{"[605]", "99", "my mybar", ""}
	results := GetMany(data, paths...)
	if len(expected) != len(results) {
		t.Fatalf("expected %v, got %v", len(expected), len(results))
	}
	for i, path := range paths {
		if results[i] != expected[i] {
			t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i],
				results[i], path)
		}
	}
}

func TestGetMany2(t *testing.T) {
	data := `{"bar": {"id": 99, "xyz": "my xyz"}, "foo": {"myfoo": [605]}}`
	paths := []string{"foo.myfoo", "bar.id", "bar.xyz", "bar.abc"}
	expected := []string{"[605]", "99", "my xyz", ""}
	results := GetMany(data, paths...)
	if len(expected) != len(results) {
		t.Fatalf("expected %v, got %v", len(expected), len(results))
	}
	for i, path := range paths {
		if results[i] != expected[i] {
			t.Fatalf("expected '%v', got '%v' for path '%v'", expected[i],
				results[i], path)
		}
	}
}

func TestNullArray(t *testing.T) {
	n, _ := get(`{"data":null}`, "data").Interface()
	if n != nil {
		t.Fatalf("expected '%v', got '%v'", nil, n)
	}
	n = get(`{}`, "data")
	if reflect.DeepEqual(n, nil) {
		t.Fatalf("expected '%v', got '%v'", nil, n)
	}
	n = get(`{"data":[]}`, "data")
	if reflect.DeepEqual(n, &ast.Node{}) {
		t.Fatalf("expected '%v', got '%v'", nil, n)
	}
	arr, _ := get(`{"data":[null]}`, "data").Array()
	n = len(arr)
	if n != 1 {
		t.Fatalf("expected '%v', got '%v'", 1, n)
	}
}

func TestGetMany3(t *testing.T) {
	var r []string
	data := `{"MarketName":null,"Nounce":6115}`
	r = GetMany(data, "Nounce", "Buys", "Sells", "Fills")
	if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" {
		t.Fatalf("expected '%v', got '%v'", "[6115]",
			strings.Replace(fmt.Sprintf("%v", r), " ", "", -1))
	}
	r = GetMany(data, "Nounce", "Buys", "Sells")
	if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" {
		t.Fatalf("expected '%v', got '%v'", "[6115]",
			strings.Replace(fmt.Sprintf("%v", r), " ", "", -1))
	}
	r = GetMany(data, "Nounce")
	if strings.Replace(fmt.Sprintf("%v", r), " ", "", -1) != "[6115]" {
		t.Fatalf("expected '%v', got '%v'", "[6115]",
			strings.Replace(fmt.Sprintf("%v", r), " ", "", -1))
	}
}

func TestGetMany4(t *testing.T) {
	data := `{"one": {"two": 2, "three": 3}, "four": 4, "five": 5}`
	results := GetMany(data, "four", "five", "one.two", "one.six")
	expected := []string{"4", "5", "2", ""}
	for i, r := range results {
		if r != expected[i] {
			t.Fatalf("expected %v, got %v", expected[i], r)
		}
	}
}

func TestGetNotExist(t *testing.T) {
	var dataStr = `{"m1":{"m2":3}}`
	ret, err := GetFromString(dataStr, "not_exist", "m3")
	if err == nil || ret.Exists() {
		t.Fatal("Get exist!")
	}
	if ret.Type() != ast.V_NONE {
		t.Fatal(ret.Type())
	}
	ret, err = GetFromString(dataStr)
	if !ret.IsRaw() || ret.Type() != ast.V_OBJECT {
		t.Fatal(ret.Type())
	}
	v11 := ret.Get("not_exist")
	if v11.Exists() {
		t.Fatal()
	}
	v2 := ret.GetByPath("m1", "m2")
	if !v2.Exists() || !v2.IsRaw() {
		t.Fatal(ret.Type())
	}
	x, _ := v2.Int64()
	if x != 3 {
		t.Fatal(x)
	}
}