github.com/bir3/gocompiler@v0.9.2202/extra/compress/zstd/decoder_options.go (about) 1 // Copyright 2019+ Klaus Post. All rights reserved. 2 // License information can be found in the LICENSE file. 3 // Based on work by Yann Collet, released under BSD License. 4 5 package zstd 6 7 import ( 8 "errors" 9 "fmt" 10 "math/bits" 11 "runtime" 12 ) 13 14 // DOption is an option for creating a decoder. 15 type DOption func(*decoderOptions) error 16 17 // options retains accumulated state of multiple options. 18 type decoderOptions struct { 19 lowMem bool 20 concurrent int 21 maxDecodedSize uint64 22 maxWindowSize uint64 23 dicts []*dict 24 ignoreChecksum bool 25 limitToCap bool 26 decodeBufsBelow int 27 } 28 29 func (o *decoderOptions) setDefault() { 30 *o = decoderOptions{ 31 // use less ram: true for now, but may change. 32 lowMem: true, 33 concurrent: runtime.GOMAXPROCS(0), 34 maxWindowSize: MaxWindowSize, 35 decodeBufsBelow: 128 << 10, 36 } 37 if o.concurrent > 4 { 38 o.concurrent = 4 39 } 40 o.maxDecodedSize = 64 << 30 41 } 42 43 // WithDecoderLowmem will set whether to use a lower amount of memory, 44 // but possibly have to allocate more while running. 45 func WithDecoderLowmem(b bool) DOption { 46 return func(o *decoderOptions) error { o.lowMem = b; return nil } 47 } 48 49 // WithDecoderConcurrency sets the number of created decoders. 50 // When decoding block with DecodeAll, this will limit the number 51 // of possible concurrently running decodes. 52 // When decoding streams, this will limit the number of 53 // inflight blocks. 54 // When decoding streams and setting maximum to 1, 55 // no async decoding will be done. 56 // When a value of 0 is provided GOMAXPROCS will be used. 57 // By default this will be set to 4 or GOMAXPROCS, whatever is lower. 58 func WithDecoderConcurrency(n int) DOption { 59 return func(o *decoderOptions) error { 60 if n < 0 { 61 return errors.New("concurrency must be at least 1") 62 } 63 if n == 0 { 64 o.concurrent = runtime.GOMAXPROCS(0) 65 } else { 66 o.concurrent = n 67 } 68 return nil 69 } 70 } 71 72 // WithDecoderMaxMemory allows to set a maximum decoded size for in-memory 73 // non-streaming operations or maximum window size for streaming operations. 74 // This can be used to control memory usage of potentially hostile content. 75 // Maximum is 1 << 63 bytes. Default is 64GiB. 76 func WithDecoderMaxMemory(n uint64) DOption { 77 return func(o *decoderOptions) error { 78 if n == 0 { 79 return errors.New("WithDecoderMaxMemory must be at least 1") 80 } 81 if n > 1<<63 { 82 return errors.New("WithDecoderMaxmemory must be less than 1 << 63") 83 } 84 o.maxDecodedSize = n 85 return nil 86 } 87 } 88 89 // WithDecoderDicts allows to register one or more dictionaries for the decoder. 90 // 91 // Each slice in dict must be in the [dictionary format] produced by 92 // "zstd --train" from the Zstandard reference implementation. 93 // 94 // If several dictionaries with the same ID are provided, the last one will be used. 95 // 96 // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format 97 func WithDecoderDicts(dicts ...[]byte) DOption { 98 return func(o *decoderOptions) error { 99 for _, b := range dicts { 100 d, err := loadDict(b) 101 if err != nil { 102 return err 103 } 104 o.dicts = append(o.dicts, d) 105 } 106 return nil 107 } 108 } 109 110 // WithEncoderDictRaw registers a dictionary that may be used by the decoder. 111 // The slice content can be arbitrary data. 112 func WithDecoderDictRaw(id uint32, content []byte) DOption { 113 return func(o *decoderOptions) error { 114 if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { 115 return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) 116 } 117 o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) 118 return nil 119 } 120 } 121 122 // WithDecoderMaxWindow allows to set a maximum window size for decodes. 123 // This allows rejecting packets that will cause big memory usage. 124 // The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. 125 // If WithDecoderMaxMemory is set to a lower value, that will be used. 126 // Default is 512MB, Maximum is ~3.75 TB as per zstandard spec. 127 func WithDecoderMaxWindow(size uint64) DOption { 128 return func(o *decoderOptions) error { 129 if size < MinWindowSize { 130 return errors.New("WithMaxWindowSize must be at least 1KB, 1024 bytes") 131 } 132 if size > (1<<41)+7*(1<<38) { 133 return errors.New("WithMaxWindowSize must be less than (1<<41) + 7*(1<<38) ~ 3.75TB") 134 } 135 o.maxWindowSize = size 136 return nil 137 } 138 } 139 140 // WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes, 141 // or any size set in WithDecoderMaxMemory. 142 // This can be used to limit decoding to a specific maximum output size. 143 // Disabled by default. 144 func WithDecodeAllCapLimit(b bool) DOption { 145 return func(o *decoderOptions) error { 146 o.limitToCap = b 147 return nil 148 } 149 } 150 151 // WithDecodeBuffersBelow will fully decode readers that have a 152 // `Bytes() []byte` and `Len() int` interface similar to bytes.Buffer. 153 // This typically uses less allocations but will have the full decompressed object in memory. 154 // Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less. 155 // Default is 128KiB. 156 func WithDecodeBuffersBelow(size int) DOption { 157 return func(o *decoderOptions) error { 158 o.decodeBufsBelow = size 159 return nil 160 } 161 } 162 163 // IgnoreChecksum allows to forcibly ignore checksum checking. 164 func IgnoreChecksum(b bool) DOption { 165 return func(o *decoderOptions) error { 166 o.ignoreChecksum = b 167 return nil 168 } 169 }