github.com/fraugster/parquet-go@v0.12.0/parquet_test.go (about)

     1  package goparquet
     2  
     3  import (
     4  	"os"
     5  	"path/filepath"
     6  	"testing"
     7  
     8  	"github.com/stretchr/testify/require"
     9  )
    10  
    11  func TestParquetTesting(t *testing.T) {
    12  	testingRoot := os.Getenv("PARQUET_TESTING_ROOT") // path where https://github.com/apache/parquet-testing has been cloned to.
    13  	if testingRoot == "" {
    14  		t.Skip("PARQUET_TESTING_ROOT not set, skipping test")
    15  	}
    16  
    17  	testFiles := []string{
    18  		"data/alltypes_dictionary.parquet",
    19  		"data/alltypes_plain.parquet",
    20  		"data/alltypes_plain.snappy.parquet",
    21  		"data/binary.parquet",
    22  		"data/byte_array_decimal.parquet",
    23  		"data/datapage_v2.snappy.parquet",
    24  		"data/delta_binary_packed.parquet",
    25  		//"data/delta_byte_array.parquet",
    26  		"data/delta_encoding_optional_column.parquet",
    27  		"data/delta_encoding_required_column.parquet",
    28  		// "data/dict-page-offset-zero.parquet",
    29  		"data/fixed_length_decimal.parquet",
    30  		"data/fixed_length_decimal_legacy.parquet",
    31  		// "data/hadoop_lz4_compressed.parquet", // LZ4 is currently unsupported out of the box.
    32  		// "data/hadoop_lz4_compressed_larger.parquet",
    33  		"data/int32_decimal.parquet",
    34  		"data/int64_decimal.parquet",
    35  		"data/list_columns.parquet",
    36  		"data/nested_lists.snappy.parquet",
    37  		"data/nested_maps.snappy.parquet",
    38  		// "data/nested_structs.rust.parquet", // uses ZSTD which is currently unsupported out of the box.
    39  		"data/nonnullable.impala.parquet",
    40  		"data/nullable.impala.parquet",
    41  		"data/nulls.snappy.parquet",
    42  		"data/repeated_no_annotation.parquet",
    43  	}
    44  
    45  	for _, file := range testFiles {
    46  		t.Run(file, func(t *testing.T) {
    47  			fullPath := filepath.Join(testingRoot, file)
    48  
    49  			f, err := os.Open(fullPath)
    50  			require.NoError(t, err)
    51  			defer f.Close()
    52  
    53  			r, err := NewFileReader(f)
    54  			require.NoError(t, err)
    55  
    56  			numRows := r.NumRows()
    57  
    58  			t.Logf("%s: got %d rows", file, numRows)
    59  			t.Logf("%s: schema = %s", file, r.GetSchemaDefinition().String())
    60  
    61  			for i := int64(0); i < numRows; i++ {
    62  				_, err := r.NextRow()
    63  				require.NoError(t, err)
    64  			}
    65  		})
    66  	}
    67  }