github.com/grafana/pyroscope@v1.18.0/pkg/block/block.go (about)

     1  package block
     2  
     3  import (
     4  	"github.com/grafana/pyroscope/pkg/tenant"
     5  )
     6  
     7  const (
     8  	DirNameSegment    = "segments"
     9  	DirNameBlock      = "blocks"
    10  	DirNameDLQ        = "dlq"
    11  	DirNameAnonTenant = tenant.DefaultTenantID
    12  
    13  	FileNameProfilesParquet = "profiles.parquet"
    14  	FileNameDataObject      = "block.bin"
    15  	FileNameMetadataObject  = "meta.pb"
    16  )
    17  
    18  const (
    19  	defaultObjectSizeLoadInMemory        = 1 << 20
    20  	defaultTenantDatasetSizeLoadInMemory = 1 << 20
    21  
    22  	maxRowsPerRowGroup  = 10 << 10
    23  	symbolsPrefetchSize = 32 << 10
    24  )
    25  
    26  func estimateReadBufferSize(s int64) int {
    27  	const minSize = 64 << 10
    28  	const maxSize = 1 << 20
    29  	// Parquet has global buffer map, where buffer size is key,
    30  	// so we want a low cardinality here.
    31  	e := nextPowerOfTwo(uint32(s / 10))
    32  	if e < minSize {
    33  		return minSize
    34  	}
    35  	return int(min(e, maxSize))
    36  }
    37  
    38  // This is a verbatim copy of estimateReadBufferSize.
    39  // It's kept for the sake of clarity and to avoid confusion.
    40  func estimatePageBufferSize(s int64) int {
    41  	const minSize = 64 << 10
    42  	const maxSize = 1 << 20
    43  	e := nextPowerOfTwo(uint32(s / 10))
    44  	if e < minSize {
    45  		return minSize
    46  	}
    47  	return int(min(e, maxSize))
    48  }
    49  
    50  func estimateFooterSize(size int64) int64 {
    51  	var s int64
    52  	// as long as we don't keep the exact footer sizes in the meta estimate it
    53  	if size > 0 {
    54  		s = size / 10000
    55  	}
    56  	// set a minimum footer size of 32KiB
    57  	if s < 32<<10 {
    58  		s = 32 << 10
    59  	}
    60  	// set a maximum footer size of 512KiB
    61  	if s > 512<<10 {
    62  		s = 512 << 10
    63  	}
    64  	// now check clamp it to the actual size of the whole object
    65  	if s > size {
    66  		s = size
    67  	}
    68  	return s
    69  }
    70  
    71  func nextPowerOfTwo(n uint32) uint32 {
    72  	if n == 0 {
    73  		return 1
    74  	}
    75  	n--
    76  	n |= n >> 1
    77  	n |= n >> 2
    78  	n |= n >> 4
    79  	n |= n >> 8
    80  	n |= n >> 16
    81  	n++
    82  	return n
    83  }