github.com/snowflakedb/gosnowflake@v1.9.0/file_compression_type.go (about)

     1  // Copyright (c) 2021-2022 Snowflake Computing Inc. All rights reserved.
     2  
     3  package gosnowflake
     4  
     5  import (
     6  	"bytes"
     7  	"strings"
     8  
     9  	"github.com/gabriel-vasile/mimetype"
    10  )
    11  
    12  type compressionType struct {
    13  	name          string
    14  	fileExtension string
    15  	mimeSubtypes  []string
    16  	isSupported   bool
    17  }
    18  
    19  var compressionTypes = map[string]*compressionType{
    20  	"GZIP": {
    21  		"GZIP",
    22  		".gz",
    23  		[]string{"gzip", "x-gzip"},
    24  		true,
    25  	},
    26  	"DEFLATE": {
    27  		"DEFLATE",
    28  		".deflate",
    29  		[]string{"zlib", "deflate"},
    30  		true,
    31  	},
    32  	"RAW_DEFLATE": {
    33  		"RAW_DEFLATE",
    34  		".raw_deflate",
    35  		[]string{"raw_deflate"},
    36  		true,
    37  	},
    38  	"BZIP2": {
    39  		"BZIP2",
    40  		".bz2",
    41  		[]string{"bzip2", "x-bzip2", "x-bz2", "x-bzip", "bz2"},
    42  		true,
    43  	},
    44  	"LZIP": {
    45  		"LZIP",
    46  		".lz",
    47  		[]string{"lzip", "x-lzip"},
    48  		false,
    49  	},
    50  	"LZMA": {
    51  		"LZMA",
    52  		".lzma",
    53  		[]string{"lzma", "x-lzma"},
    54  		false,
    55  	},
    56  	"LZO": {
    57  		"LZO",
    58  		".lzo",
    59  		[]string{"lzo", "x-lzo"},
    60  		false,
    61  	},
    62  	"XZ": {
    63  		"XZ",
    64  		".xz",
    65  		[]string{"xz", "x-xz"},
    66  		false,
    67  	},
    68  	"COMPRESS": {
    69  		"COMPRESS",
    70  		".Z",
    71  		[]string{"compress", "x-compress"},
    72  		false,
    73  	},
    74  	"PARQUET": {
    75  		"PARQUET",
    76  		".parquet",
    77  		[]string{"parquet"},
    78  		true,
    79  	},
    80  	"ZSTD": {
    81  		"ZSTD",
    82  		".zst",
    83  		[]string{"zstd", "x-zstd"},
    84  		true,
    85  	},
    86  	"BROTLI": {
    87  		"BROTLI",
    88  		".br",
    89  		[]string{"br", "x-br"},
    90  		true,
    91  	},
    92  	"ORC": {
    93  		"ORC",
    94  		".orc",
    95  		[]string{"orc"},
    96  		true,
    97  	},
    98  }
    99  
   100  var mimeSubTypeToCompression map[string]*compressionType
   101  var extensionToCompression map[string]*compressionType
   102  
   103  func init() {
   104  	mimeSubTypeToCompression = make(map[string]*compressionType)
   105  	extensionToCompression = make(map[string]*compressionType)
   106  	for _, meta := range compressionTypes {
   107  		extensionToCompression[meta.fileExtension] = meta
   108  		for _, subtype := range meta.mimeSubtypes {
   109  			mimeSubTypeToCompression[subtype] = meta
   110  		}
   111  	}
   112  	mimetype.Extend(func(raw []byte, limit uint32) bool {
   113  		return bytes.HasPrefix(raw, []byte("PAR1"))
   114  	}, "snowflake/parquet", ".parquet")
   115  	mimetype.Extend(func(raw []byte, limit uint32) bool {
   116  		return bytes.HasPrefix(raw, []byte("ORC"))
   117  	}, "snowflake/orc", ".orc")
   118  }
   119  
   120  func lookupByMimeSubType(mimeSubType string) *compressionType {
   121  	if val, ok := mimeSubTypeToCompression[strings.ToLower(mimeSubType)]; ok {
   122  		return val
   123  	}
   124  	return nil
   125  }
   126  
   127  func lookupByExtension(extension string) *compressionType {
   128  	if val, ok := extensionToCompression[strings.ToLower(extension)]; ok {
   129  		return val
   130  	}
   131  	return nil
   132  }