storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/pkg/s3select/internal/parquet-go/data/column-grouplist_test.go (about) 1 /* 2 * Minio Cloud Storage, (C) 2019 Minio, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package data 18 19 import ( 20 "reflect" 21 "testing" 22 23 "storj.io/minio/pkg/s3select/internal/parquet-go/gen-go/parquet" 24 "storj.io/minio/pkg/s3select/internal/parquet-go/schema" 25 ) 26 27 func TestPopulateGroupList(t *testing.T) { 28 requiredList1 := schema.NewTree() 29 { 30 requiredGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_REQUIRED, 31 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 32 nil, nil, nil) 33 if err != nil { 34 t.Fatal(err) 35 } 36 37 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 38 nil, nil, 39 nil, nil, nil) 40 if err != nil { 41 t.Fatal(err) 42 } 43 44 requiredElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED, 45 nil, nil, 46 nil, nil, nil) 47 if err != nil { 48 t.Fatal(err) 49 } 50 51 requiredCol, err := schema.NewElement("col", parquet.FieldRepetitionType_REQUIRED, 52 parquet.TypePtr(parquet.Type_INT32), nil, 53 nil, nil, nil) 54 if err != nil { 55 t.Fatal(err) 56 } 57 58 if err = requiredList1.Set("group", requiredGroup); err != nil { 59 t.Fatal(err) 60 } 61 if err = requiredList1.Set("group.list", list); err != nil { 62 t.Fatal(err) 63 } 64 if err = requiredList1.Set("group.list.element", requiredElement); err != nil { 65 t.Fatal(err) 66 } 67 if err = requiredList1.Set("group.list.element.col", requiredCol); err != nil { 68 t.Fatal(err) 69 } 70 71 if _, _, err := requiredList1.ToParquetSchema(); err != nil { 72 t.Fatal(err) 73 } 74 } 75 76 requiredList2 := schema.NewTree() 77 { 78 requiredGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_REQUIRED, 79 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 80 nil, nil, nil) 81 if err != nil { 82 t.Fatal(err) 83 } 84 85 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 86 nil, nil, 87 nil, nil, nil) 88 if err != nil { 89 t.Fatal(err) 90 } 91 92 requiredElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED, 93 nil, nil, 94 nil, nil, nil) 95 if err != nil { 96 t.Fatal(err) 97 } 98 99 optionalCol, err := schema.NewElement("col", parquet.FieldRepetitionType_OPTIONAL, 100 parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_UINT_32), 101 nil, nil, nil) 102 if err != nil { 103 t.Fatal(err) 104 } 105 106 if err = requiredList2.Set("group", requiredGroup); err != nil { 107 t.Fatal(err) 108 } 109 if err = requiredList2.Set("group.list", list); err != nil { 110 t.Fatal(err) 111 } 112 if err = requiredList2.Set("group.list.element", requiredElement); err != nil { 113 t.Fatal(err) 114 } 115 if err = requiredList2.Set("group.list.element.col", optionalCol); err != nil { 116 t.Fatal(err) 117 } 118 119 if _, _, err := requiredList2.ToParquetSchema(); err != nil { 120 t.Fatal(err) 121 } 122 } 123 124 requiredList3 := schema.NewTree() 125 { 126 requiredGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_REQUIRED, 127 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 128 nil, nil, nil) 129 if err != nil { 130 t.Fatal(err) 131 } 132 133 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 134 nil, nil, 135 nil, nil, nil) 136 if err != nil { 137 t.Fatal(err) 138 } 139 140 optionalElement, err := schema.NewElement("element", parquet.FieldRepetitionType_OPTIONAL, 141 nil, nil, 142 nil, nil, nil) 143 if err != nil { 144 t.Fatal(err) 145 } 146 147 requiredCol, err := schema.NewElement("col", parquet.FieldRepetitionType_REQUIRED, 148 parquet.TypePtr(parquet.Type_INT32), nil, 149 nil, nil, nil) 150 if err != nil { 151 t.Fatal(err) 152 } 153 154 if err = requiredList3.Set("group", requiredGroup); err != nil { 155 t.Fatal(err) 156 } 157 if err = requiredList3.Set("group.list", list); err != nil { 158 t.Fatal(err) 159 } 160 if err = requiredList3.Set("group.list.element", optionalElement); err != nil { 161 t.Fatal(err) 162 } 163 if err = requiredList3.Set("group.list.element.col", requiredCol); err != nil { 164 t.Fatal(err) 165 } 166 167 if _, _, err := requiredList3.ToParquetSchema(); err != nil { 168 t.Fatal(err) 169 } 170 } 171 172 requiredList4 := schema.NewTree() 173 { 174 requiredGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_REQUIRED, 175 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 176 nil, nil, nil) 177 if err != nil { 178 t.Fatal(err) 179 } 180 181 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 182 nil, nil, 183 nil, nil, nil) 184 if err != nil { 185 t.Fatal(err) 186 } 187 188 optionalElement, err := schema.NewElement("element", parquet.FieldRepetitionType_OPTIONAL, 189 nil, nil, 190 nil, nil, nil) 191 if err != nil { 192 t.Fatal(err) 193 } 194 195 optionalCol, err := schema.NewElement("col", parquet.FieldRepetitionType_OPTIONAL, 196 parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_UINT_32), 197 nil, nil, nil) 198 if err != nil { 199 t.Fatal(err) 200 } 201 202 if err = requiredList4.Set("group", requiredGroup); err != nil { 203 t.Fatal(err) 204 } 205 if err = requiredList4.Set("group.list", list); err != nil { 206 t.Fatal(err) 207 } 208 if err = requiredList4.Set("group.list.element", optionalElement); err != nil { 209 t.Fatal(err) 210 } 211 if err = requiredList4.Set("group.list.element.col", optionalCol); err != nil { 212 t.Fatal(err) 213 } 214 215 if _, _, err := requiredList4.ToParquetSchema(); err != nil { 216 t.Fatal(err) 217 } 218 } 219 220 optionalList1 := schema.NewTree() 221 { 222 optionalGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_OPTIONAL, 223 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 224 nil, nil, nil) 225 if err != nil { 226 t.Fatal(err) 227 } 228 229 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 230 nil, nil, 231 nil, nil, nil) 232 if err != nil { 233 t.Fatal(err) 234 } 235 236 requiredElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED, 237 nil, nil, 238 nil, nil, nil) 239 if err != nil { 240 t.Fatal(err) 241 } 242 243 requiredCol, err := schema.NewElement("col", parquet.FieldRepetitionType_REQUIRED, 244 parquet.TypePtr(parquet.Type_INT32), nil, 245 nil, nil, nil) 246 if err != nil { 247 t.Fatal(err) 248 } 249 250 if err = optionalList1.Set("group", optionalGroup); err != nil { 251 t.Fatal(err) 252 } 253 if err = optionalList1.Set("group.list", list); err != nil { 254 t.Fatal(err) 255 } 256 if err = optionalList1.Set("group.list.element", requiredElement); err != nil { 257 t.Fatal(err) 258 } 259 if err = optionalList1.Set("group.list.element.col", requiredCol); err != nil { 260 t.Fatal(err) 261 } 262 263 if _, _, err := optionalList1.ToParquetSchema(); err != nil { 264 t.Fatal(err) 265 } 266 } 267 268 optionalList2 := schema.NewTree() 269 { 270 optionalGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_OPTIONAL, 271 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 272 nil, nil, nil) 273 if err != nil { 274 t.Fatal(err) 275 } 276 277 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 278 nil, nil, 279 nil, nil, nil) 280 if err != nil { 281 t.Fatal(err) 282 } 283 284 requiredElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED, 285 nil, nil, 286 nil, nil, nil) 287 if err != nil { 288 t.Fatal(err) 289 } 290 291 optionalCol, err := schema.NewElement("col", parquet.FieldRepetitionType_OPTIONAL, 292 parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_UINT_32), 293 nil, nil, nil) 294 if err != nil { 295 t.Fatal(err) 296 } 297 298 if err = optionalList2.Set("group", optionalGroup); err != nil { 299 t.Fatal(err) 300 } 301 if err = optionalList2.Set("group.list", list); err != nil { 302 t.Fatal(err) 303 } 304 if err = optionalList2.Set("group.list.element", requiredElement); err != nil { 305 t.Fatal(err) 306 } 307 if err = optionalList2.Set("group.list.element.col", optionalCol); err != nil { 308 t.Fatal(err) 309 } 310 311 if _, _, err := optionalList2.ToParquetSchema(); err != nil { 312 t.Fatal(err) 313 } 314 } 315 316 optionalList3 := schema.NewTree() 317 { 318 optionalGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_OPTIONAL, 319 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 320 nil, nil, nil) 321 if err != nil { 322 t.Fatal(err) 323 } 324 325 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 326 nil, nil, 327 nil, nil, nil) 328 if err != nil { 329 t.Fatal(err) 330 } 331 332 optionalElement, err := schema.NewElement("element", parquet.FieldRepetitionType_OPTIONAL, 333 nil, nil, 334 nil, nil, nil) 335 if err != nil { 336 t.Fatal(err) 337 } 338 339 requiredCol, err := schema.NewElement("col", parquet.FieldRepetitionType_REQUIRED, 340 parquet.TypePtr(parquet.Type_INT32), nil, 341 nil, nil, nil) 342 if err != nil { 343 t.Fatal(err) 344 } 345 346 if err = optionalList3.Set("group", optionalGroup); err != nil { 347 t.Fatal(err) 348 } 349 if err = optionalList3.Set("group.list", list); err != nil { 350 t.Fatal(err) 351 } 352 if err = optionalList3.Set("group.list.element", optionalElement); err != nil { 353 t.Fatal(err) 354 } 355 if err = optionalList3.Set("group.list.element.col", requiredCol); err != nil { 356 t.Fatal(err) 357 } 358 359 if _, _, err := optionalList3.ToParquetSchema(); err != nil { 360 t.Fatal(err) 361 } 362 } 363 364 optionalList4 := schema.NewTree() 365 { 366 optionalGroup, err := schema.NewElement("group", parquet.FieldRepetitionType_OPTIONAL, 367 nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST), 368 nil, nil, nil) 369 if err != nil { 370 t.Fatal(err) 371 } 372 373 list, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED, 374 nil, nil, 375 nil, nil, nil) 376 if err != nil { 377 t.Fatal(err) 378 } 379 380 optionalElement, err := schema.NewElement("element", parquet.FieldRepetitionType_OPTIONAL, 381 nil, nil, 382 nil, nil, nil) 383 if err != nil { 384 t.Fatal(err) 385 } 386 387 optionalCol, err := schema.NewElement("col", parquet.FieldRepetitionType_OPTIONAL, 388 parquet.TypePtr(parquet.Type_INT32), parquet.ConvertedTypePtr(parquet.ConvertedType_UINT_32), 389 nil, nil, nil) 390 if err != nil { 391 t.Fatal(err) 392 } 393 394 if err = optionalList4.Set("group", optionalGroup); err != nil { 395 t.Fatal(err) 396 } 397 if err = optionalList4.Set("group.list", list); err != nil { 398 t.Fatal(err) 399 } 400 if err = optionalList4.Set("group.list.element", optionalElement); err != nil { 401 t.Fatal(err) 402 } 403 if err = optionalList4.Set("group.list.element.col", optionalCol); err != nil { 404 t.Fatal(err) 405 } 406 407 if _, _, err := optionalList4.ToParquetSchema(); err != nil { 408 t.Fatal(err) 409 } 410 } 411 412 result1 := map[string]*Column{ 413 "group.list.element.col": { 414 parquetType: parquet.Type_INT32, 415 values: []interface{}{v10}, 416 definitionLevels: []int64{1}, 417 repetitionLevels: []int64{0}, 418 rowCount: 1, 419 maxBitWidth: 4, 420 minValue: v10, 421 maxValue: v10, 422 }, 423 } 424 425 result2 := map[string]*Column{ 426 "group.list.element.col": { 427 parquetType: parquet.Type_INT32, 428 values: []interface{}{v10, v20}, 429 definitionLevels: []int64{1, 1}, 430 repetitionLevels: []int64{0, 1}, 431 rowCount: 1, 432 maxBitWidth: 5, 433 minValue: v10, 434 maxValue: v20, 435 }, 436 } 437 438 result3 := map[string]*Column{ 439 "group.list.element.col": { 440 parquetType: parquet.Type_INT32, 441 values: []interface{}{nil}, 442 definitionLevels: []int64{1}, 443 repetitionLevels: []int64{0}, 444 rowCount: 1, 445 }, 446 } 447 448 result4 := map[string]*Column{ 449 "group.list.element.col": { 450 parquetType: parquet.Type_INT32, 451 values: []interface{}{v10}, 452 definitionLevels: []int64{2}, 453 repetitionLevels: []int64{0}, 454 rowCount: 1, 455 maxBitWidth: 4, 456 minValue: v10, 457 maxValue: v10, 458 }, 459 } 460 461 result5 := map[string]*Column{ 462 "group.list.element.col": { 463 parquetType: parquet.Type_INT32, 464 values: []interface{}{v10, v20}, 465 definitionLevels: []int64{2, 2}, 466 repetitionLevels: []int64{0, 1}, 467 rowCount: 1, 468 maxBitWidth: 5, 469 minValue: v10, 470 maxValue: v20, 471 }, 472 } 473 474 result6 := map[string]*Column{ 475 "group.list.element.col": { 476 parquetType: parquet.Type_INT32, 477 values: []interface{}{nil}, 478 definitionLevels: []int64{2}, 479 repetitionLevels: []int64{0}, 480 rowCount: 1, 481 }, 482 } 483 484 result7 := map[string]*Column{ 485 "group.list.element.col": { 486 parquetType: parquet.Type_INT32, 487 values: []interface{}{v10}, 488 definitionLevels: []int64{3}, 489 repetitionLevels: []int64{0}, 490 rowCount: 1, 491 maxBitWidth: 4, 492 minValue: v10, 493 maxValue: v10, 494 }, 495 } 496 497 result8 := map[string]*Column{ 498 "group.list.element.col": { 499 parquetType: parquet.Type_INT32, 500 values: []interface{}{v10, v20}, 501 definitionLevels: []int64{3, 3}, 502 repetitionLevels: []int64{0, 1}, 503 rowCount: 1, 504 maxBitWidth: 5, 505 minValue: v10, 506 maxValue: v20, 507 }, 508 } 509 510 result9 := map[string]*Column{ 511 "group.list.element.col": { 512 parquetType: parquet.Type_INT32, 513 values: []interface{}{nil}, 514 definitionLevels: []int64{0}, 515 repetitionLevels: []int64{0}, 516 rowCount: 1, 517 }, 518 } 519 520 result10 := map[string]*Column{ 521 "group.list.element.col": { 522 parquetType: parquet.Type_INT32, 523 values: []interface{}{nil}, 524 definitionLevels: []int64{3}, 525 repetitionLevels: []int64{0}, 526 rowCount: 1, 527 }, 528 } 529 530 result11 := map[string]*Column{ 531 "group.list.element.col": { 532 parquetType: parquet.Type_INT32, 533 values: []interface{}{v10}, 534 definitionLevels: []int64{4}, 535 repetitionLevels: []int64{0}, 536 rowCount: 1, 537 maxBitWidth: 4, 538 minValue: v10, 539 maxValue: v10, 540 }, 541 } 542 543 result12 := map[string]*Column{ 544 "group.list.element.col": { 545 parquetType: parquet.Type_INT32, 546 values: []interface{}{v10, v20}, 547 definitionLevels: []int64{4, 4}, 548 repetitionLevels: []int64{0, 1}, 549 rowCount: 1, 550 maxBitWidth: 5, 551 minValue: v10, 552 maxValue: v20, 553 }, 554 } 555 556 testCases := []struct { 557 schemaTree *schema.Tree 558 data string 559 expectedResult map[string]*Column 560 expectErr bool 561 }{ 562 {requiredList1, `{}`, nil, true}, // err: group: nil value for required field 563 {requiredList1, `{"group": null}`, nil, true}, // err: group: nil value for required field 564 {requiredList1, `{"group": [{"col": null}]}`, nil, true}, // err: group.list.element.col: nil value for required field 565 {requiredList1, `{"group": [{"col": 10}]}`, result1, false}, 566 {requiredList1, `{"group": [{"col": 10}, {"col": 20}]}`, result2, false}, 567 {requiredList2, `{}`, nil, true}, // err: group: nil value for required field 568 {requiredList2, `{"group": null}`, nil, true}, // err: group: nil value for required field 569 {requiredList2, `{"group": [{"col": null}]}`, result3, false}, 570 {requiredList2, `{"group": [{"col": 10}]}`, result4, false}, 571 {requiredList2, `{"group": [{"col": 10}, {"col": 20}]}`, result5, false}, 572 {requiredList3, `{}`, nil, true}, // err: group: nil value for required field 573 {requiredList3, `{"group": null}`, nil, true}, // err: group: nil value for required field 574 {requiredList3, `{"group": [{"col": null}]}`, nil, true}, // err: group.list.element.col: nil value for required field 575 {requiredList3, `{"group": [{"col": 10}]}`, result4, false}, 576 {requiredList3, `{"group": [{"col": 10}, {"col": 20}]}`, result5, false}, 577 {requiredList4, `{}`, nil, true}, // err: group: nil value for required field 578 {requiredList4, `{"group": null}`, nil, true}, // err: group: nil value for required field 579 {requiredList4, `{"group": [{"col": null}]}`, result6, false}, 580 {requiredList4, `{"group": [{"col": 10}]}`, result7, false}, 581 {requiredList4, `{"group": [{"col": 10}, {"col": 20}]}`, result8, false}, 582 {optionalList1, `{}`, result9, false}, 583 {optionalList1, `{"group": null}`, result9, false}, 584 {optionalList1, `{"group": [{"col": null}]}`, nil, true}, // err: group.list.element.col: nil value for required field 585 {optionalList1, `{"group": [{"col": 10}]}`, result4, false}, 586 {optionalList1, `{"group": [{"col": 10}, {"col": 20}]}`, result5, false}, 587 {optionalList2, `{}`, result9, false}, 588 {optionalList2, `{"group": null}`, result9, false}, 589 {optionalList2, `{"group": [{"col": null}]}`, result6, false}, 590 {optionalList2, `{"group": [{"col": 10}]}`, result7, false}, 591 {optionalList2, `{"group": [{"col": 10}, {"col": 20}]}`, result8, false}, 592 {optionalList3, `{}`, result9, false}, 593 {optionalList3, `{"group": null}`, result9, false}, 594 {optionalList3, `{"group": [{"col": null}]}`, nil, true}, // err: group.list.element.col: nil value for required field 595 {optionalList3, `{"group": [{"col": 10}]}`, result7, false}, 596 {optionalList3, `{"group": [{"col": 10}, {"col": 20}]}`, result8, false}, 597 {optionalList4, `{}`, result9, false}, 598 {optionalList4, `{"group": null}`, result9, false}, 599 {optionalList4, `{"group": [{"col": null}]}`, result10, false}, 600 {optionalList4, `{"group": [{"col": 10}]}`, result11, false}, 601 {optionalList4, `{"group": [{"col": 10}, {"col": 20}]}`, result12, false}, 602 } 603 604 for i, testCase := range testCases { 605 result, err := UnmarshalJSON([]byte(testCase.data), testCase.schemaTree) 606 expectErr := (err != nil) 607 608 if testCase.expectErr != expectErr { 609 t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) 610 } 611 612 if !testCase.expectErr { 613 if !reflect.DeepEqual(result, testCase.expectedResult) { 614 t.Fatalf("case %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) 615 } 616 } 617 } 618 }