storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/pkg/s3select/internal/parquet-go/data/column-map_test.go (about)

     1  /*
     2   * Minio Cloud Storage, (C) 2019 Minio, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package data
    18  
    19  import (
    20  	"reflect"
    21  	"testing"
    22  
    23  	"storj.io/minio/pkg/s3select/internal/parquet-go/gen-go/parquet"
    24  	"storj.io/minio/pkg/s3select/internal/parquet-go/schema"
    25  )
    26  
    27  func TestPopulateMap(t *testing.T) {
    28  	t.Skip("Broken")
    29  	requiredMap1 := schema.NewTree()
    30  	{
    31  		mapElement, err := schema.NewElement("map", parquet.FieldRepetitionType_REQUIRED,
    32  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_MAP),
    33  			nil, nil, nil)
    34  		if err != nil {
    35  			t.Fatal(err)
    36  		}
    37  
    38  		keyValue, err := schema.NewElement("key_value", parquet.FieldRepetitionType_REPEATED,
    39  			nil, nil,
    40  			nil, nil, nil)
    41  		if err != nil {
    42  			t.Fatal(err)
    43  		}
    44  
    45  		requiredKey, err := schema.NewElement("key", parquet.FieldRepetitionType_REQUIRED,
    46  			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
    47  			nil, nil, nil)
    48  		if err != nil {
    49  			t.Fatal(err)
    50  		}
    51  
    52  		requiredValue, err := schema.NewElement("value", parquet.FieldRepetitionType_REQUIRED,
    53  			parquet.TypePtr(parquet.Type_INT32), nil,
    54  			nil, nil, nil)
    55  		if err != nil {
    56  			t.Fatal(err)
    57  		}
    58  
    59  		if err = requiredMap1.Set("map", mapElement); err != nil {
    60  			t.Fatal(err)
    61  		}
    62  
    63  		if err = requiredMap1.Set("map.key_value", keyValue); err != nil {
    64  			t.Fatal(err)
    65  		}
    66  
    67  		if err = requiredMap1.Set("map.key_value.key", requiredKey); err != nil {
    68  			t.Fatal(err)
    69  		}
    70  
    71  		if err = requiredMap1.Set("map.key_value.value", requiredValue); err != nil {
    72  			t.Fatal(err)
    73  		}
    74  
    75  		if _, _, err = requiredMap1.ToParquetSchema(); err != nil {
    76  			t.Fatal(err)
    77  		}
    78  	}
    79  
    80  	requiredMap2 := schema.NewTree()
    81  	{
    82  		mapElement, err := schema.NewElement("map", parquet.FieldRepetitionType_REQUIRED,
    83  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_MAP),
    84  			nil, nil, nil)
    85  		if err != nil {
    86  			t.Fatal(err)
    87  		}
    88  
    89  		keyValue, err := schema.NewElement("key_value", parquet.FieldRepetitionType_REPEATED,
    90  			nil, nil,
    91  			nil, nil, nil)
    92  		if err != nil {
    93  			t.Fatal(err)
    94  		}
    95  
    96  		requiredKey, err := schema.NewElement("key", parquet.FieldRepetitionType_REQUIRED,
    97  			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
    98  			nil, nil, nil)
    99  		if err != nil {
   100  			t.Fatal(err)
   101  		}
   102  
   103  		optionalValue, err := schema.NewElement("value", parquet.FieldRepetitionType_OPTIONAL,
   104  			parquet.TypePtr(parquet.Type_INT32), nil,
   105  			nil, nil, nil)
   106  		if err != nil {
   107  			t.Fatal(err)
   108  		}
   109  
   110  		if err = requiredMap2.Set("map", mapElement); err != nil {
   111  			t.Fatal(err)
   112  		}
   113  
   114  		if err = requiredMap2.Set("map.key_value", keyValue); err != nil {
   115  			t.Fatal(err)
   116  		}
   117  
   118  		if err = requiredMap2.Set("map.key_value.key", requiredKey); err != nil {
   119  			t.Fatal(err)
   120  		}
   121  
   122  		if err = requiredMap2.Set("map.key_value.value", optionalValue); err != nil {
   123  			t.Fatal(err)
   124  		}
   125  
   126  		if _, _, err = requiredMap2.ToParquetSchema(); err != nil {
   127  			t.Fatal(err)
   128  		}
   129  	}
   130  
   131  	optionalMap1 := schema.NewTree()
   132  	{
   133  		mapElement, err := schema.NewElement("map", parquet.FieldRepetitionType_OPTIONAL,
   134  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_MAP),
   135  			nil, nil, nil)
   136  		if err != nil {
   137  			t.Fatal(err)
   138  		}
   139  
   140  		keyValue, err := schema.NewElement("key_value", parquet.FieldRepetitionType_REPEATED,
   141  			nil, nil,
   142  			nil, nil, nil)
   143  		if err != nil {
   144  			t.Fatal(err)
   145  		}
   146  
   147  		requiredKey, err := schema.NewElement("key", parquet.FieldRepetitionType_REQUIRED,
   148  			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
   149  			nil, nil, nil)
   150  		if err != nil {
   151  			t.Fatal(err)
   152  		}
   153  
   154  		requiredValue, err := schema.NewElement("value", parquet.FieldRepetitionType_REQUIRED,
   155  			parquet.TypePtr(parquet.Type_INT32), nil,
   156  			nil, nil, nil)
   157  		if err != nil {
   158  			t.Fatal(err)
   159  		}
   160  
   161  		if err = optionalMap1.Set("map", mapElement); err != nil {
   162  			t.Fatal(err)
   163  		}
   164  
   165  		if err = optionalMap1.Set("map.key_value", keyValue); err != nil {
   166  			t.Fatal(err)
   167  		}
   168  
   169  		if err = optionalMap1.Set("map.key_value.key", requiredKey); err != nil {
   170  			t.Fatal(err)
   171  		}
   172  
   173  		if err = optionalMap1.Set("map.key_value.value", requiredValue); err != nil {
   174  			t.Fatal(err)
   175  		}
   176  
   177  		if _, _, err = optionalMap1.ToParquetSchema(); err != nil {
   178  			t.Fatal(err)
   179  		}
   180  	}
   181  
   182  	optionalMap2 := schema.NewTree()
   183  	{
   184  		mapElement, err := schema.NewElement("map", parquet.FieldRepetitionType_OPTIONAL,
   185  			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_MAP),
   186  			nil, nil, nil)
   187  		if err != nil {
   188  			t.Fatal(err)
   189  		}
   190  
   191  		keyValue, err := schema.NewElement("key_value", parquet.FieldRepetitionType_REPEATED,
   192  			nil, nil,
   193  			nil, nil, nil)
   194  		if err != nil {
   195  			t.Fatal(err)
   196  		}
   197  
   198  		requiredKey, err := schema.NewElement("key", parquet.FieldRepetitionType_REQUIRED,
   199  			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
   200  			nil, nil, nil)
   201  		if err != nil {
   202  			t.Fatal(err)
   203  		}
   204  
   205  		optionalValue, err := schema.NewElement("value", parquet.FieldRepetitionType_OPTIONAL,
   206  			parquet.TypePtr(parquet.Type_INT32), nil,
   207  			nil, nil, nil)
   208  		if err != nil {
   209  			t.Fatal(err)
   210  		}
   211  
   212  		if err = optionalMap2.Set("map", mapElement); err != nil {
   213  			t.Fatal(err)
   214  		}
   215  
   216  		if err = optionalMap2.Set("map.key_value", keyValue); err != nil {
   217  			t.Fatal(err)
   218  		}
   219  
   220  		if err = optionalMap2.Set("map.key_value.key", requiredKey); err != nil {
   221  			t.Fatal(err)
   222  		}
   223  
   224  		if err = optionalMap2.Set("map.key_value.value", optionalValue); err != nil {
   225  			t.Fatal(err)
   226  		}
   227  
   228  		if _, _, err = optionalMap2.ToParquetSchema(); err != nil {
   229  			t.Fatal(err)
   230  		}
   231  	}
   232  
   233  	result1 := map[string]*Column{
   234  		"map.key_value.key": {
   235  			parquetType:      parquet.Type_BYTE_ARRAY,
   236  			values:           []interface{}{ten},
   237  			definitionLevels: []int64{1},
   238  			repetitionLevels: []int64{0},
   239  		},
   240  		"map.key_value.value": {
   241  			parquetType:      parquet.Type_INT32,
   242  			values:           []interface{}{v10},
   243  			definitionLevels: []int64{1},
   244  			repetitionLevels: []int64{1},
   245  		},
   246  	}
   247  
   248  	result2 := map[string]*Column{
   249  		"map.key_value.key": {
   250  			parquetType:      parquet.Type_BYTE_ARRAY,
   251  			values:           []interface{}{ten},
   252  			definitionLevels: []int64{1},
   253  			repetitionLevels: []int64{0},
   254  		},
   255  		"map.key_value.value": {
   256  			parquetType:      parquet.Type_INT32,
   257  			values:           []interface{}{nil},
   258  			definitionLevels: []int64{1},
   259  			repetitionLevels: []int64{1},
   260  		},
   261  	}
   262  
   263  	result3 := map[string]*Column{
   264  		"map.key_value.key": {
   265  			parquetType:      parquet.Type_BYTE_ARRAY,
   266  			values:           []interface{}{ten},
   267  			definitionLevels: []int64{1},
   268  			repetitionLevels: []int64{0},
   269  		},
   270  		"map.key_value.value": {
   271  			parquetType:      parquet.Type_INT32,
   272  			values:           []interface{}{v10},
   273  			definitionLevels: []int64{2},
   274  			repetitionLevels: []int64{1},
   275  		},
   276  	}
   277  
   278  	result4 := map[string]*Column{
   279  		"map.key_value.key": {
   280  			parquetType:      parquet.Type_BYTE_ARRAY,
   281  			values:           []interface{}{nil},
   282  			definitionLevels: []int64{0},
   283  			repetitionLevels: []int64{0},
   284  		},
   285  	}
   286  
   287  	result5 := map[string]*Column{
   288  		"map.key_value.key": {
   289  			parquetType:      parquet.Type_BYTE_ARRAY,
   290  			values:           []interface{}{ten},
   291  			definitionLevels: []int64{2},
   292  			repetitionLevels: []int64{0},
   293  		},
   294  		"map.key_value.value": {
   295  			parquetType:      parquet.Type_INT32,
   296  			values:           []interface{}{v10},
   297  			definitionLevels: []int64{2},
   298  			repetitionLevels: []int64{1},
   299  		},
   300  	}
   301  
   302  	result6 := map[string]*Column{
   303  		"map.key_value.key": {
   304  			parquetType:      parquet.Type_BYTE_ARRAY,
   305  			values:           []interface{}{ten},
   306  			definitionLevels: []int64{2},
   307  			repetitionLevels: []int64{0},
   308  		},
   309  		"map.key_value.value": {
   310  			parquetType:      parquet.Type_INT32,
   311  			values:           []interface{}{nil},
   312  			definitionLevels: []int64{2},
   313  			repetitionLevels: []int64{1},
   314  		},
   315  	}
   316  
   317  	result7 := map[string]*Column{
   318  		"map.key_value.key": {
   319  			parquetType:      parquet.Type_BYTE_ARRAY,
   320  			values:           []interface{}{ten},
   321  			definitionLevels: []int64{2},
   322  			repetitionLevels: []int64{0},
   323  		},
   324  		"map.key_value.value": {
   325  			parquetType:      parquet.Type_INT32,
   326  			values:           []interface{}{v10},
   327  			definitionLevels: []int64{3},
   328  			repetitionLevels: []int64{1},
   329  		},
   330  	}
   331  
   332  	testCases := []struct {
   333  		schemaTree     *schema.Tree
   334  		data           string
   335  		expectedResult map[string]*Column
   336  		expectErr      bool
   337  	}{
   338  		{requiredMap1, `{}`, nil, true},                     // err: map: nil value for required field
   339  		{requiredMap1, `{"map": null}`, nil, true},          // err: map: nil value for required field
   340  		{requiredMap1, `{"map": {"ten": null}}`, nil, true}, // err: map.key_value.value: nil value for required field
   341  		{requiredMap1, `{"map": {"ten": 10}}`, result1, false},
   342  		{requiredMap2, `{}`, nil, true},            // err: map: nil value for required field
   343  		{requiredMap2, `{"map": null}`, nil, true}, // err: map: nil value for required field
   344  		{requiredMap2, `{"map": {"ten": null}}`, result2, false},
   345  		{requiredMap2, `{"map": {"ten": 10}}`, result3, false},
   346  		{optionalMap1, `{}`, result4, false},
   347  		{optionalMap1, `{"map": null}`, result4, false},
   348  		{optionalMap1, `{"map": {"ten": null}}`, nil, true}, // err: map.key_value.value: nil value for required field
   349  		{optionalMap1, `{"map": {"ten": 10}}`, result5, false},
   350  		{optionalMap2, `{}`, result4, false},
   351  		{optionalMap2, `{"map": null}`, result4, false},
   352  		{optionalMap2, `{"map": {"ten": null}}`, result6, false},
   353  		{optionalMap2, `{"map": {"ten": 10}}`, result7, false},
   354  	}
   355  
   356  	for i, testCase := range testCases {
   357  		result, err := UnmarshalJSON([]byte(testCase.data), testCase.schemaTree)
   358  		expectErr := (err != nil)
   359  
   360  		if testCase.expectErr != expectErr {
   361  			t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
   362  		}
   363  
   364  		if !testCase.expectErr {
   365  			if !reflect.DeepEqual(result, testCase.expectedResult) {
   366  				t.Errorf("case %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
   367  			}
   368  		}
   369  	}
   370  }