storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/pkg/s3select/internal/parquet-go/data/column-primitivelist_test.go (about)

     1  /*
     2   * Minio Cloud Storage, (C) 2019 Minio, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package data
    18  
    19  import (
    20  	"reflect"
    21  	"testing"
    22  
    23  	"storj.io/minio/pkg/s3select/internal/parquet-go/gen-go/parquet"
    24  	"storj.io/minio/pkg/s3select/internal/parquet-go/schema"
    25  )
    26  
    27  func TestPopulatePrimitiveList(t *testing.T) {
    28  	requiredList1 := schema.NewTree()
    29  	{
    30  		requiredCol, err := schema.NewElement("col", parquet.FieldRepetitionType_REQUIRED,
    31  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST),
    32  			nil, nil, nil)
    33  		if err != nil {
    34  			t.Fatal(err)
    35  		}
    36  
    37  		list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED,
    38  			nil, nil,
    39  			nil, nil, nil)
    40  		if err != nil {
    41  			t.Fatal(err)
    42  		}
    43  
    44  		requiredElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED,
    45  			parquet.TypePtr(parquet.Type_INT32), nil,
    46  			nil, nil, nil)
    47  		if err != nil {
    48  			t.Fatal(err)
    49  		}
    50  
    51  		if err = requiredList1.Set("col", requiredCol); err != nil {
    52  			t.Fatal(err)
    53  		}
    54  		if err = requiredList1.Set("col.list", list); err != nil {
    55  			t.Fatal(err)
    56  		}
    57  		if err = requiredList1.Set("col.list.element", requiredElement); err != nil {
    58  			t.Fatal(err)
    59  		}
    60  
    61  		if _, _, err = requiredList1.ToParquetSchema(); err != nil {
    62  			t.Fatal(err)
    63  		}
    64  	}
    65  
    66  	requiredList2 := schema.NewTree()
    67  	{
    68  		requiredCol, err := schema.NewElement("col", parquet.FieldRepetitionType_REQUIRED,
    69  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST),
    70  			nil, nil, nil)
    71  		if err != nil {
    72  			t.Fatal(err)
    73  		}
    74  
    75  		list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED,
    76  			nil, nil,
    77  			nil, nil, nil)
    78  		if err != nil {
    79  			t.Fatal(err)
    80  		}
    81  
    82  		optionalElement, err := schema.NewElement("element", parquet.FieldRepetitionType_OPTIONAL,
    83  			parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_UINT_32),
    84  			nil, nil, nil)
    85  		if err != nil {
    86  			t.Fatal(err)
    87  		}
    88  
    89  		if err = requiredList2.Set("col", requiredCol); err != nil {
    90  			t.Fatal(err)
    91  		}
    92  		if err = requiredList2.Set("col.list", list); err != nil {
    93  			t.Fatal(err)
    94  		}
    95  		if err = requiredList2.Set("col.list.element", optionalElement); err != nil {
    96  			t.Fatal(err)
    97  		}
    98  
    99  		if _, _, err = requiredList2.ToParquetSchema(); err != nil {
   100  			t.Fatal(err)
   101  		}
   102  	}
   103  
   104  	optionalList1 := schema.NewTree()
   105  	{
   106  		optionalCol, err := schema.NewElement("col", parquet.FieldRepetitionType_OPTIONAL,
   107  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST),
   108  			nil, nil, nil)
   109  		if err != nil {
   110  			t.Fatal(err)
   111  		}
   112  
   113  		list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED,
   114  			nil, nil,
   115  			nil, nil, nil)
   116  		if err != nil {
   117  			t.Fatal(err)
   118  		}
   119  
   120  		requiredElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED,
   121  			parquet.TypePtr(parquet.Type_INT32), nil,
   122  			nil, nil, nil)
   123  		if err != nil {
   124  			t.Fatal(err)
   125  		}
   126  
   127  		if err = optionalList1.Set("col", optionalCol); err != nil {
   128  			t.Fatal(err)
   129  		}
   130  		if err = optionalList1.Set("col.list", list); err != nil {
   131  			t.Fatal(err)
   132  		}
   133  		if err = optionalList1.Set("col.list.element", requiredElement); err != nil {
   134  			t.Fatal(err)
   135  		}
   136  
   137  		if _, _, err = optionalList1.ToParquetSchema(); err != nil {
   138  			t.Fatal(err)
   139  		}
   140  	}
   141  
   142  	optionalList2 := schema.NewTree()
   143  	{
   144  		optionalCol, err := schema.NewElement("col", parquet.FieldRepetitionType_OPTIONAL,
   145  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST),
   146  			nil, nil, nil)
   147  		if err != nil {
   148  			t.Fatal(err)
   149  		}
   150  
   151  		list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED,
   152  			nil, nil,
   153  			nil, nil, nil)
   154  		if err != nil {
   155  			t.Fatal(err)
   156  		}
   157  
   158  		optionalElement, err := schema.NewElement("element", parquet.FieldRepetitionType_OPTIONAL,
   159  			parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_UINT_32),
   160  			nil, nil, nil)
   161  		if err != nil {
   162  			t.Fatal(err)
   163  		}
   164  
   165  		if err = optionalList2.Set("col", optionalCol); err != nil {
   166  			t.Fatal(err)
   167  		}
   168  		if err = optionalList2.Set("col.list", list); err != nil {
   169  			t.Fatal(err)
   170  		}
   171  		if err = optionalList2.Set("col.list.element", optionalElement); err != nil {
   172  			t.Fatal(err)
   173  		}
   174  
   175  		if _, _, err = optionalList2.ToParquetSchema(); err != nil {
   176  			t.Fatal(err)
   177  		}
   178  	}
   179  
   180  	result1 := map[string]*Column{
   181  		"col.list.element": {
   182  			parquetType:      parquet.Type_INT32,
   183  			values:           []interface{}{v10},
   184  			definitionLevels: []int64{1},
   185  			repetitionLevels: []int64{0},
   186  			rowCount:         1,
   187  			maxBitWidth:      4,
   188  			minValue:         v10,
   189  			maxValue:         v10,
   190  		},
   191  	}
   192  
   193  	result2 := map[string]*Column{
   194  		"col.list.element": {
   195  			parquetType:      parquet.Type_INT32,
   196  			values:           []interface{}{v10, v20, v30},
   197  			definitionLevels: []int64{1, 1, 1},
   198  			repetitionLevels: []int64{0, 1, 1},
   199  			rowCount:         1,
   200  			maxBitWidth:      5,
   201  			minValue:         v10,
   202  			maxValue:         v30,
   203  		},
   204  	}
   205  
   206  	result3 := map[string]*Column{
   207  		"col.list.element": {
   208  			parquetType:      parquet.Type_INT32,
   209  			values:           []interface{}{nil},
   210  			definitionLevels: []int64{1},
   211  			repetitionLevels: []int64{0},
   212  			rowCount:         1,
   213  		},
   214  	}
   215  
   216  	result4 := map[string]*Column{
   217  		"col.list.element": {
   218  			parquetType:      parquet.Type_INT32,
   219  			values:           []interface{}{v10},
   220  			definitionLevels: []int64{2},
   221  			repetitionLevels: []int64{0},
   222  			rowCount:         1,
   223  			maxBitWidth:      4,
   224  			minValue:         v10,
   225  			maxValue:         v10,
   226  		},
   227  	}
   228  
   229  	result5 := map[string]*Column{
   230  		"col.list.element": {
   231  			parquetType:      parquet.Type_INT32,
   232  			values:           []interface{}{v10, v20, v30},
   233  			definitionLevels: []int64{2, 2, 2},
   234  			repetitionLevels: []int64{0, 1, 1},
   235  			rowCount:         1,
   236  			maxBitWidth:      5,
   237  			minValue:         v10,
   238  			maxValue:         v30,
   239  		},
   240  	}
   241  
   242  	result6 := map[string]*Column{
   243  		"col.list.element": {
   244  			parquetType:      parquet.Type_INT32,
   245  			values:           []interface{}{nil},
   246  			definitionLevels: []int64{0},
   247  			repetitionLevels: []int64{0},
   248  			rowCount:         1,
   249  		},
   250  	}
   251  
   252  	result7 := map[string]*Column{
   253  		"col.list.element": {
   254  			parquetType:      parquet.Type_INT32,
   255  			values:           []interface{}{nil},
   256  			definitionLevels: []int64{2},
   257  			repetitionLevels: []int64{0},
   258  			rowCount:         1,
   259  		},
   260  	}
   261  
   262  	result8 := map[string]*Column{
   263  		"col.list.element": {
   264  			parquetType:      parquet.Type_INT32,
   265  			values:           []interface{}{v10},
   266  			definitionLevels: []int64{3},
   267  			repetitionLevels: []int64{0},
   268  			rowCount:         1,
   269  			maxBitWidth:      4,
   270  			minValue:         v10,
   271  			maxValue:         v10,
   272  		},
   273  	}
   274  
   275  	result9 := map[string]*Column{
   276  		"col.list.element": {
   277  			parquetType:      parquet.Type_INT32,
   278  			values:           []interface{}{v10, v20, v30},
   279  			definitionLevels: []int64{3, 3, 3},
   280  			repetitionLevels: []int64{0, 1, 1},
   281  			rowCount:         1,
   282  			maxBitWidth:      5,
   283  			minValue:         v10,
   284  			maxValue:         v30,
   285  		},
   286  	}
   287  
   288  	testCases := []struct {
   289  		schemaTree     *schema.Tree
   290  		data           string
   291  		expectedResult map[string]*Column
   292  		expectErr      bool
   293  	}{
   294  		{requiredList1, `{}`, nil, true},              // err: col: nil value for required field
   295  		{requiredList1, `{"col": null}`, nil, true},   // err: col: nil value for required field
   296  		{requiredList1, `{"col": [null]}`, nil, true}, // err: col.list.element: nil value for required field
   297  		{requiredList1, `{"col": [10]}`, result1, false},
   298  		{requiredList1, `{"col": [10, 20, 30]}`, result2, false},
   299  		{requiredList2, `{}`, nil, true},            // err: col: nil value for required field
   300  		{requiredList2, `{"col": null}`, nil, true}, // err: col: nil value for required field
   301  		{requiredList2, `{"col": [null]}`, result3, false},
   302  		{requiredList2, `{"col": [10]}`, result4, false},
   303  		{requiredList2, `{"col": [10, 20, 30]}`, result5, false},
   304  		{optionalList1, `{}`, result6, false},
   305  		{optionalList1, `{"col": null}`, result6, false},
   306  		{optionalList1, `{"col": [null]}`, nil, true}, // err: col.list.element: nil value for required field
   307  		{optionalList1, `{"col": [10]}`, result4, false},
   308  		{optionalList1, `{"col": [10, 20, 30]}`, result5, false},
   309  		{optionalList2, `{}`, result6, false},
   310  		{optionalList2, `{"col": null}`, result6, false},
   311  		{optionalList2, `{"col": [null]}`, result7, false},
   312  		{optionalList2, `{"col": [10]}`, result8, false},
   313  		{optionalList2, `{"col": [10, 20, 30]}`, result9, false},
   314  	}
   315  
   316  	for i, testCase := range testCases {
   317  		result, err := UnmarshalJSON([]byte(testCase.data), testCase.schemaTree)
   318  		expectErr := (err != nil)
   319  
   320  		if testCase.expectErr != expectErr {
   321  			t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
   322  		}
   323  
   324  		if !testCase.expectErr {
   325  			if !reflect.DeepEqual(result, testCase.expectedResult) {
   326  				t.Fatalf("case %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
   327  			}
   328  		}
   329  	}
   330  }