github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/bigchunk.go (about) 1 package chunk 2 3 import ( 4 "bytes" 5 "encoding/binary" 6 "errors" 7 "io" 8 9 "github.com/prometheus/common/model" 10 "github.com/prometheus/prometheus/tsdb/chunkenc" 11 12 "github.com/grafana/loki/pkg/util/filter" 13 ) 14 15 const samplesPerChunk = 120 16 17 var errOutOfBounds = errors.New("out of bounds") 18 19 type smallChunk struct { 20 chunkenc.XORChunk 21 start int64 22 } 23 24 // bigchunk is a set of prometheus/tsdb chunks. It grows over time and has no 25 // upperbound on number of samples it can contain. 26 type bigchunk struct { 27 chunks []smallChunk 28 29 appender chunkenc.Appender 30 remainingSamples int 31 } 32 33 func newBigchunk() *bigchunk { 34 return &bigchunk{} 35 } 36 37 // TODO(owen-d): remove bigchunk from our code, we don't use it. 38 // Hack an Entries() impl 39 func (b *bigchunk) Entries() int { return 0 } 40 41 func (b *bigchunk) Add(sample model.SamplePair) (Data, error) { 42 if b.remainingSamples == 0 { 43 if bigchunkSizeCapBytes > 0 && b.Size() > bigchunkSizeCapBytes { 44 return addToOverflowChunk(sample) 45 } 46 if err := b.addNextChunk(sample.Timestamp); err != nil { 47 return nil, err 48 } 49 } 50 51 b.appender.Append(int64(sample.Timestamp), float64(sample.Value)) 52 b.remainingSamples-- 53 return nil, nil 54 } 55 56 // addNextChunk adds a new XOR "subchunk" to the internal list of chunks. 57 func (b *bigchunk) addNextChunk(start model.Time) error { 58 // To save memory, we "compact" the previous chunk - the array backing the slice 59 // will be upto 2x too big, and we can save this space. 60 const chunkCapacityExcess = 32 // don't bother copying if it's within this range 61 if l := len(b.chunks); l > 0 { 62 oldBuf := b.chunks[l-1].XORChunk.Bytes() 63 if cap(oldBuf) > len(oldBuf)+chunkCapacityExcess { 64 buf := make([]byte, len(oldBuf)) 65 copy(buf, oldBuf) 66 compacted, err := chunkenc.FromData(chunkenc.EncXOR, buf) 67 if err != nil { 68 return err 69 } 70 b.chunks[l-1].XORChunk = *compacted.(*chunkenc.XORChunk) 71 } 72 } 73 74 // Explicitly reallocate slice to avoid up to 2x overhead if we let append() do it 75 if len(b.chunks)+1 > cap(b.chunks) { 76 newChunks := make([]smallChunk, len(b.chunks), len(b.chunks)+1) 77 copy(newChunks, b.chunks) 78 b.chunks = newChunks 79 } 80 b.chunks = append(b.chunks, smallChunk{ 81 XORChunk: *chunkenc.NewXORChunk(), 82 start: int64(start), 83 }) 84 85 appender, err := b.chunks[len(b.chunks)-1].Appender() 86 if err != nil { 87 return err 88 } 89 b.appender = appender 90 b.remainingSamples = samplesPerChunk 91 return nil 92 } 93 94 func (b *bigchunk) Rebound(start, end model.Time, filter filter.Func) (Data, error) { 95 return nil, errors.New("not implemented") 96 } 97 98 func (b *bigchunk) Marshal(wio io.Writer) error { 99 w := writer{wio} 100 if err := w.WriteVarInt16(uint16(len(b.chunks))); err != nil { 101 return err 102 } 103 for _, chunk := range b.chunks { 104 buf := chunk.Bytes() 105 if err := w.WriteVarInt16(uint16(len(buf))); err != nil { 106 return err 107 } 108 if _, err := w.Write(buf); err != nil { 109 return err 110 } 111 } 112 return nil 113 } 114 115 func (b *bigchunk) MarshalToBuf(buf []byte) error { 116 writer := bytes.NewBuffer(buf) 117 return b.Marshal(writer) 118 } 119 120 func (b *bigchunk) UnmarshalFromBuf(buf []byte) error { 121 r := reader{buf: buf} 122 numChunks, err := r.ReadUint16() 123 if err != nil { 124 return err 125 } 126 127 b.chunks = make([]smallChunk, 0, numChunks+1) // allow one extra space in case we want to add new data 128 var reuseIter chunkenc.Iterator 129 for i := uint16(0); i < numChunks; i++ { 130 chunkLen, err := r.ReadUint16() 131 if err != nil { 132 return err 133 } 134 135 chunkBuf, err := r.ReadBytes(int(chunkLen)) 136 if err != nil { 137 return err 138 } 139 140 chunk, err := chunkenc.FromData(chunkenc.EncXOR, chunkBuf) 141 if err != nil { 142 return err 143 } 144 145 var start int64 146 start, reuseIter, err = firstTime(chunk, reuseIter) 147 if err != nil { 148 return err 149 } 150 151 b.chunks = append(b.chunks, smallChunk{ 152 XORChunk: *chunk.(*chunkenc.XORChunk), 153 start: start, 154 }) 155 } 156 return nil 157 } 158 159 func (b *bigchunk) Encoding() Encoding { 160 return Bigchunk 161 } 162 163 func (b *bigchunk) Utilization() float64 { 164 return 1.0 165 } 166 167 func (b *bigchunk) Len() int { 168 sum := 0 169 for _, c := range b.chunks { 170 sum += c.NumSamples() 171 } 172 return sum 173 } 174 175 // Unused, but for compatibility 176 func (b *bigchunk) UncompressedSize() int { return b.Size() } 177 178 func (b *bigchunk) Size() int { 179 sum := 2 // For the number of sub chunks. 180 for _, c := range b.chunks { 181 sum += 2 // For the length of the sub chunk. 182 sum += len(c.Bytes()) 183 } 184 return sum 185 } 186 187 func (b *bigchunk) Slice(start, end model.Time) Data { 188 i, j := 0, len(b.chunks) 189 for k := 0; k < len(b.chunks); k++ { 190 if b.chunks[k].start <= int64(start) { 191 i = k 192 } 193 if b.chunks[k].start > int64(end) { 194 j = k 195 break 196 } 197 } 198 return &bigchunk{ 199 chunks: b.chunks[i:j], 200 } 201 } 202 203 type writer struct { 204 io.Writer 205 } 206 207 func (w writer) WriteVarInt16(i uint16) error { 208 var b [2]byte 209 binary.LittleEndian.PutUint16(b[:], i) 210 _, err := w.Write(b[:]) 211 return err 212 } 213 214 type reader struct { 215 i int 216 buf []byte 217 } 218 219 func (r *reader) ReadUint16() (uint16, error) { 220 if r.i+2 > len(r.buf) { 221 return 0, errOutOfBounds 222 } 223 result := binary.LittleEndian.Uint16(r.buf[r.i:]) 224 r.i += 2 225 return result, nil 226 } 227 228 func (r *reader) ReadBytes(count int) ([]byte, error) { 229 if r.i+count > len(r.buf) { 230 return nil, errOutOfBounds 231 } 232 result := r.buf[r.i : r.i+count] 233 r.i += count 234 return result, nil 235 } 236 237 // addToOverflowChunk is a utility function that creates a new chunk as overflow 238 // chunk, adds the provided sample to it, and returns a chunk slice containing 239 // the provided old chunk followed by the new overflow chunk. 240 func addToOverflowChunk(s model.SamplePair) (Data, error) { 241 overflowChunk := New() 242 _, err := overflowChunk.(*bigchunk).Add(s) 243 if err != nil { 244 return nil, err 245 } 246 return overflowChunk, nil 247 } 248 249 func firstTime(c chunkenc.Chunk, iter chunkenc.Iterator) (int64, chunkenc.Iterator, error) { 250 var first int64 251 iter = c.Iterator(iter) 252 if iter.Next() { 253 first, _ = iter.At() 254 } 255 return first, iter, iter.Err() 256 }