storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/pkg/s3select/simdj/reader_amd64_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package simdj
    18  
    19  import (
    20  	"bytes"
    21  	"io"
    22  	"io/ioutil"
    23  	"path/filepath"
    24  	"testing"
    25  
    26  	"github.com/klauspost/compress/zstd"
    27  	"github.com/minio/simdjson-go"
    28  
    29  	"storj.io/minio/pkg/s3select/json"
    30  	"storj.io/minio/pkg/s3select/sql"
    31  )
    32  
    33  type tester interface {
    34  	Fatal(args ...interface{})
    35  }
    36  
    37  func loadCompressed(t tester, file string) (js []byte) {
    38  	dec, err := zstd.NewReader(nil)
    39  	if err != nil {
    40  		t.Fatal(err)
    41  	}
    42  	defer dec.Close()
    43  	js, err = ioutil.ReadFile(filepath.Join("testdata", file+".json.zst"))
    44  	if err != nil {
    45  		t.Fatal(err)
    46  	}
    47  	js, err = dec.DecodeAll(js, nil)
    48  	if err != nil {
    49  		t.Fatal(err)
    50  	}
    51  
    52  	return js
    53  }
    54  
    55  var testCases = []struct {
    56  	name  string
    57  	array bool
    58  }{
    59  	{
    60  		name: "parking-citations-10",
    61  	},
    62  }
    63  
    64  func TestNDJSON(t *testing.T) {
    65  	if !simdjson.SupportedCPU() {
    66  		t.Skip("Unsupported cpu")
    67  	}
    68  
    69  	for _, tt := range testCases {
    70  
    71  		t.Run(tt.name, func(t *testing.T) {
    72  			ref := loadCompressed(t, tt.name)
    73  
    74  			var err error
    75  			dst := make(chan simdjson.Object, 100)
    76  			dec := NewElementReader(dst, &err, &json.ReaderArgs{ContentType: "json"})
    77  			pj, err := simdjson.ParseND(ref, nil)
    78  			if err != nil {
    79  				t.Fatal(err)
    80  			}
    81  			i := pj.Iter()
    82  			cpy := i
    83  			b, err := cpy.MarshalJSON()
    84  			if err != nil {
    85  				t.Fatal(err)
    86  			}
    87  			if false {
    88  				t.Log(string(b))
    89  			}
    90  			//_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".json"), b, os.ModePerm)
    91  
    92  		parser:
    93  			for {
    94  				var next simdjson.Iter
    95  				typ, err := i.AdvanceIter(&next)
    96  				if err != nil {
    97  					t.Fatal(err)
    98  				}
    99  				switch typ {
   100  				case simdjson.TypeNone:
   101  					close(dst)
   102  					break parser
   103  				case simdjson.TypeRoot:
   104  					typ, obj, err := next.Root(nil)
   105  					if err != nil {
   106  						t.Fatal(err)
   107  					}
   108  					if typ != simdjson.TypeObject {
   109  						if typ == simdjson.TypeNone {
   110  							close(dst)
   111  							break parser
   112  						}
   113  						t.Fatal("Unexpected type:", typ.String())
   114  					}
   115  
   116  					o, err := obj.Object(nil)
   117  					if err != nil {
   118  						t.Fatal(err)
   119  					}
   120  					dst <- *o
   121  				default:
   122  					t.Fatal("unexpected type:", typ.String())
   123  				}
   124  			}
   125  			refDec := json.NewReader(ioutil.NopCloser(bytes.NewBuffer(ref)), &json.ReaderArgs{ContentType: "json"})
   126  
   127  			for {
   128  				rec, err := dec.Read(nil)
   129  				if err == io.EOF {
   130  					break
   131  				}
   132  				if err != nil {
   133  					t.Error(err)
   134  				}
   135  				want, err := refDec.Read(nil)
   136  				if err != nil {
   137  					t.Error(err)
   138  				}
   139  				var gotB, wantB bytes.Buffer
   140  				opts := sql.WriteCSVOpts{
   141  					FieldDelimiter: ',',
   142  					Quote:          '"',
   143  					QuoteEscape:    '"',
   144  					AlwaysQuote:    false,
   145  				}
   146  				err = rec.WriteCSV(&gotB, opts)
   147  				if err != nil {
   148  					t.Error(err)
   149  				}
   150  				err = want.WriteCSV(&wantB, opts)
   151  				if err != nil {
   152  					t.Error(err)
   153  				}
   154  
   155  				if !bytes.Equal(gotB.Bytes(), wantB.Bytes()) {
   156  					t.Errorf("CSV output mismatch.\nwant: %s(%x)\ngot:  %s(%x)", wantB.String(), wantB.Bytes(), gotB.String(), gotB.Bytes())
   157  				}
   158  				gotB.Reset()
   159  				wantB.Reset()
   160  
   161  				err = rec.WriteJSON(&gotB)
   162  				if err != nil {
   163  					t.Error(err)
   164  				}
   165  				err = want.WriteJSON(&wantB)
   166  				if err != nil {
   167  					t.Error(err)
   168  				}
   169  				// truncate newline from 'want'
   170  				wantB.Truncate(wantB.Len() - 1)
   171  				if !bytes.Equal(gotB.Bytes(), wantB.Bytes()) {
   172  					t.Errorf("JSON output mismatch.\nwant: %s\ngot:  %s", wantB.String(), gotB.String())
   173  				}
   174  			}
   175  		})
   176  	}
   177  }