github.com/thanos-io/thanos@v0.32.5/pkg/store/postings.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package store
     5  
     6  import (
     7  	"bufio"
     8  	"context"
     9  	"encoding/binary"
    10  	"fmt"
    11  	"io"
    12  
    13  	"github.com/pkg/errors"
    14  )
    15  
    16  type postingsReaderBuilder struct {
    17  	e       error
    18  	readBuf []byte
    19  
    20  	r        *bufio.Reader
    21  	postings []postingPtr
    22  
    23  	lastOffset int64
    24  	pi         int
    25  
    26  	start, length         int64
    27  	cur                   []byte
    28  	keyID                 int
    29  	repeatFor             int
    30  	numberOfPostingsInCur uint64
    31  	uvarintEncodeBuf      []byte
    32  	ctx                   context.Context
    33  }
    34  
    35  // newPostingsReaderBuilder is a builder that reads directly from the index
    36  // and builds a diff varint encoded []byte that could be later used directly.
    37  func newPostingsReaderBuilder(ctx context.Context, r *bufio.Reader, postings []postingPtr, start, length int64) *postingsReaderBuilder {
    38  	prb := &postingsReaderBuilder{
    39  		r:                r,
    40  		readBuf:          make([]byte, 4),
    41  		start:            start,
    42  		length:           length,
    43  		postings:         postings,
    44  		uvarintEncodeBuf: make([]byte, binary.MaxVarintLen64),
    45  		ctx:              ctx,
    46  	}
    47  
    48  	return prb
    49  }
    50  
    51  func getInt32(r io.Reader, buf []byte) (uint32, error) {
    52  	read, err := io.ReadFull(r, buf)
    53  	if err != nil {
    54  		return 0, errors.Wrap(err, "reading")
    55  	}
    56  	if read != 4 {
    57  		return 0, fmt.Errorf("read got %d bytes instead of 4", read)
    58  	}
    59  	return binary.BigEndian.Uint32(buf), nil
    60  }
    61  
    62  func (r *postingsReaderBuilder) Next() bool {
    63  	if r.ctx.Err() != nil {
    64  		r.e = r.ctx.Err()
    65  		return false
    66  	}
    67  	if r.repeatFor > 0 {
    68  		r.keyID = r.postings[r.pi-r.repeatFor].keyID
    69  		r.repeatFor--
    70  		return true
    71  	}
    72  	if r.pi >= len(r.postings) {
    73  		return false
    74  	}
    75  	if r.Error() != nil {
    76  		return false
    77  	}
    78  	from := r.postings[r.pi].ptr.Start - r.start
    79  
    80  	if from-r.lastOffset < 0 {
    81  		panic("would have skipped negative bytes")
    82  	}
    83  
    84  	_, err := r.r.Discard(int(from - r.lastOffset))
    85  	if err != nil {
    86  		r.e = err
    87  		return false
    88  	}
    89  	r.lastOffset += from - r.lastOffset
    90  
    91  	postingsCount, err := getInt32(r.r, r.readBuf[:])
    92  	if err != nil {
    93  		r.e = err
    94  		return false
    95  	}
    96  	r.lastOffset += 4
    97  
    98  	// Assume 1.25 bytes per compressed posting.
    99  	r.cur = make([]byte, 0, int(float64(postingsCount)*1.25))
   100  
   101  	prev := uint32(0)
   102  
   103  	for i := 0; i < int(postingsCount); i++ {
   104  		posting, err := getInt32(r.r, r.readBuf[:])
   105  		if err != nil {
   106  			r.e = err
   107  			return false
   108  		}
   109  		r.lastOffset += 4
   110  
   111  		uvarintSize := binary.PutUvarint(r.uvarintEncodeBuf, uint64(posting-prev))
   112  		r.cur = append(r.cur, r.uvarintEncodeBuf[:uvarintSize]...)
   113  		prev = posting
   114  	}
   115  	r.numberOfPostingsInCur = uint64(postingsCount)
   116  
   117  	r.keyID = r.postings[r.pi].keyID
   118  	r.pi++
   119  	for {
   120  		if r.pi >= len(r.postings) {
   121  			break
   122  		}
   123  
   124  		if r.postings[r.pi].ptr.Start == r.postings[r.pi-1].ptr.Start &&
   125  			r.postings[r.pi].ptr.End == r.postings[r.pi-1].ptr.End {
   126  			r.repeatFor++
   127  			r.pi++
   128  			continue
   129  		}
   130  
   131  		break
   132  	}
   133  
   134  	return true
   135  }
   136  
   137  func (r *postingsReaderBuilder) Error() error {
   138  	return r.e
   139  }
   140  
   141  func (r *postingsReaderBuilder) AtDiffVarint() ([]byte, uint64, int) {
   142  	return r.cur, r.numberOfPostingsInCur, r.keyID
   143  }