github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/chunk.go (about) 1 package chunk 2 3 import ( 4 "bytes" 5 "encoding/binary" 6 "hash/crc32" 7 "reflect" 8 "strconv" 9 "strings" 10 "sync" 11 "unsafe" 12 13 "github.com/golang/snappy" 14 jsoniter "github.com/json-iterator/go" 15 "github.com/pkg/errors" 16 "github.com/prometheus/common/model" 17 "github.com/prometheus/prometheus/model/labels" 18 errs "github.com/weaveworks/common/errors" 19 20 "github.com/grafana/loki/pkg/logproto" 21 ) 22 23 const ( 24 ErrInvalidChecksum = errs.Error("invalid chunk checksum") 25 ErrWrongMetadata = errs.Error("wrong chunk metadata") 26 ErrMetadataLength = errs.Error("chunk metadata wrong length") 27 ErrDataLength = errs.Error("chunk data wrong length") 28 ErrSliceOutOfRange = errs.Error("chunk can't be sliced out of its data range") 29 ) 30 31 var castagnoliTable = crc32.MakeTable(crc32.Castagnoli) 32 33 func errInvalidChunkID(s string) error { 34 return errors.Errorf("invalid chunk ID %q", s) 35 } 36 37 // Chunk contains encoded timeseries data 38 type Chunk struct { 39 logproto.ChunkRef 40 41 Metric labels.Labels `json:"metric"` 42 43 // We never use Delta encoding (the zero value), so if this entry is 44 // missing, we default to DoubleDelta. 45 Encoding Encoding `json:"encoding"` 46 Data Data `json:"-"` 47 48 // The encoded version of the chunk, held so we don't need to re-encode it 49 encoded []byte 50 } 51 52 // NewChunk creates a new chunk 53 func NewChunk(userID string, fp model.Fingerprint, metric labels.Labels, c Data, from, through model.Time) Chunk { 54 return Chunk{ 55 ChunkRef: logproto.ChunkRef{ 56 Fingerprint: uint64(fp), 57 UserID: userID, 58 From: from, 59 Through: through, 60 }, 61 Metric: metric, 62 Encoding: c.Encoding(), 63 Data: c, 64 } 65 } 66 67 // ParseExternalKey is used to construct a partially-populated chunk from the 68 // key in DynamoDB. This chunk can then be used to calculate the key needed 69 // to fetch the Chunk data from Memcache/S3, and then fully populate the chunk 70 // with decode(). 71 // 72 // Pre-checksums, the keys written to DynamoDB looked like 73 // `<fingerprint>:<start time>:<end time>` (aka the ID), and the key for 74 // memcache and S3 was `<user id>/<fingerprint>:<start time>:<end time>. 75 // Finger prints and times were written in base-10. 76 // 77 // Post-checksums, externals keys become the same across DynamoDB, Memcache 78 // and S3. Numbers become hex encoded. Keys look like: 79 // `<user id>/<fingerprint>:<start time>:<end time>:<checksum>`. 80 // 81 // v12+, fingerprint is now a prefix to support better read and write request parallelization: 82 // `<user>/<fprint>/<start>:<end>:<checksum>` 83 func ParseExternalKey(userID, externalKey string) (Chunk, error) { 84 if strings.Count(externalKey, "/") == 2 { // v12+ 85 return parseNewerExternalKey(userID, externalKey) 86 } 87 return parseNewExternalKey(userID, externalKey) 88 } 89 90 // post-checksum 91 func parseNewExternalKey(userID, key string) (Chunk, error) { 92 userIdx := strings.Index(key, "/") 93 if userIdx == -1 || userIdx+1 >= len(key) { 94 return Chunk{}, errInvalidChunkID(key) 95 } 96 if userID != key[:userIdx] { 97 return Chunk{}, errors.WithStack(ErrWrongMetadata) 98 } 99 hexParts := key[userIdx+1:] 100 partsBytes := unsafeGetBytes(hexParts) 101 h0, i := readOneHexPart(partsBytes) 102 if i == 0 || i+1 >= len(partsBytes) { 103 return Chunk{}, errInvalidChunkID(key) 104 } 105 fingerprint, err := strconv.ParseUint(unsafeGetString(h0), 16, 64) 106 if err != nil { 107 return Chunk{}, err 108 } 109 partsBytes = partsBytes[i+1:] 110 h1, i := readOneHexPart(partsBytes) 111 if i == 0 || i+1 >= len(partsBytes) { 112 return Chunk{}, errInvalidChunkID(key) 113 } 114 from, err := strconv.ParseInt(unsafeGetString(h1), 16, 64) 115 if err != nil { 116 return Chunk{}, err 117 } 118 partsBytes = partsBytes[i+1:] 119 h2, i := readOneHexPart(partsBytes) 120 if i == 0 || i+1 >= len(partsBytes) { 121 return Chunk{}, errInvalidChunkID(key) 122 } 123 through, err := strconv.ParseInt(unsafeGetString(h2), 16, 64) 124 if err != nil { 125 return Chunk{}, err 126 } 127 checksum, err := strconv.ParseUint(unsafeGetString(partsBytes[i+1:]), 16, 32) 128 if err != nil { 129 return Chunk{}, err 130 } 131 return Chunk{ 132 ChunkRef: logproto.ChunkRef{ 133 UserID: userID, 134 Fingerprint: fingerprint, 135 From: model.Time(from), 136 Through: model.Time(through), 137 Checksum: uint32(checksum), 138 }, 139 }, nil 140 } 141 142 // v12+ 143 func parseNewerExternalKey(userID, key string) (Chunk, error) { 144 // Parse user 145 userIdx := strings.Index(key, "/") 146 if userIdx == -1 || userIdx+1 >= len(key) { 147 return Chunk{}, errInvalidChunkID(key) 148 } 149 if userID != key[:userIdx] { 150 return Chunk{}, errors.WithStack(ErrWrongMetadata) 151 } 152 hexParts := key[userIdx+1:] 153 partsBytes := unsafeGetBytes(hexParts) 154 // Parse fingerprint 155 h, i := readOneHexPart(partsBytes) 156 if i == 0 || i+1 >= len(partsBytes) { 157 return Chunk{}, errors.Wrap(errInvalidChunkID(key), "decoding fingerprint") 158 } 159 fingerprint, err := strconv.ParseUint(unsafeGetString(h), 16, 64) 160 if err != nil { 161 return Chunk{}, errors.Wrap(err, "parsing fingerprint") 162 } 163 partsBytes = partsBytes[i+1:] 164 // Parse start 165 h, i = readOneHexPart(partsBytes) 166 if i == 0 || i+1 >= len(partsBytes) { 167 return Chunk{}, errors.Wrap(errInvalidChunkID(key), "decoding start") 168 } 169 from, err := strconv.ParseInt(unsafeGetString(h), 16, 64) 170 if err != nil { 171 return Chunk{}, errors.Wrap(err, "parsing start") 172 } 173 partsBytes = partsBytes[i+1:] 174 // Parse through 175 h, i = readOneHexPart(partsBytes) 176 if i == 0 || i+1 >= len(partsBytes) { 177 return Chunk{}, errors.Wrap(errInvalidChunkID(key), "decoding through") 178 } 179 through, err := strconv.ParseInt(unsafeGetString(h), 16, 64) 180 if err != nil { 181 return Chunk{}, errors.Wrap(err, "parsing through") 182 } 183 partsBytes = partsBytes[i+1:] 184 // Parse checksum 185 checksum, err := strconv.ParseUint(unsafeGetString(partsBytes), 16, 64) 186 if err != nil { 187 return Chunk{}, errors.Wrap(err, "parsing checksum") 188 } 189 return Chunk{ 190 ChunkRef: logproto.ChunkRef{ 191 UserID: userID, 192 Fingerprint: fingerprint, 193 From: model.Time(from), 194 Through: model.Time(through), 195 Checksum: uint32(checksum), 196 }, 197 }, nil 198 } 199 200 func readOneHexPart(hex []byte) (part []byte, i int) { 201 for i < len(hex) { 202 if hex[i] != ':' && hex[i] != '/' { 203 i++ 204 continue 205 } 206 return hex[:i], i 207 } 208 return nil, 0 209 } 210 211 func unsafeGetBytes(s string) []byte { 212 var buf []byte 213 p := unsafe.Pointer(&buf) 214 *(*string)(p) = s 215 (*reflect.SliceHeader)(p).Cap = len(s) 216 return buf 217 } 218 219 func unsafeGetString(buf []byte) string { 220 return *((*string)(unsafe.Pointer(&buf))) 221 } 222 223 var writerPool = sync.Pool{ 224 New: func() interface{} { return snappy.NewBufferedWriter(nil) }, 225 } 226 227 // Encode writes the chunk into a buffer, and calculates the checksum. 228 func (c *Chunk) Encode() error { 229 return c.EncodeTo(nil) 230 } 231 232 // EncodeTo is like Encode but you can provide your own buffer to use. 233 func (c *Chunk) EncodeTo(buf *bytes.Buffer) error { 234 if buf == nil { 235 buf = bytes.NewBuffer(nil) 236 } 237 // Write 4 empty bytes first - we will come back and put the len in here. 238 metadataLenBytes := [4]byte{} 239 if _, err := buf.Write(metadataLenBytes[:]); err != nil { 240 return err 241 } 242 243 // Encode chunk metadata into snappy-compressed buffer 244 writer := writerPool.Get().(*snappy.Writer) 245 defer writerPool.Put(writer) 246 writer.Reset(buf) 247 json := jsoniter.ConfigFastest 248 if err := json.NewEncoder(writer).Encode(c); err != nil { 249 return err 250 } 251 writer.Close() 252 253 // Write the metadata length back at the start of the buffer. 254 // (note this length includes the 4 bytes for the length itself) 255 metadataLen := buf.Len() 256 binary.BigEndian.PutUint32(metadataLenBytes[:], uint32(metadataLen)) 257 copy(buf.Bytes(), metadataLenBytes[:]) 258 259 // Write another 4 empty bytes - we will come back and put the len in here. 260 dataLenBytes := [4]byte{} 261 if _, err := buf.Write(dataLenBytes[:]); err != nil { 262 return err 263 } 264 265 // And now the chunk data 266 if err := c.Data.Marshal(buf); err != nil { 267 return err 268 } 269 270 // Now write the data len back into the buf. 271 binary.BigEndian.PutUint32(dataLenBytes[:], uint32(buf.Len()-metadataLen-4)) 272 copy(buf.Bytes()[metadataLen:], dataLenBytes[:]) 273 274 // Now work out the checksum 275 c.encoded = buf.Bytes() 276 c.Checksum = crc32.Checksum(c.encoded, castagnoliTable) 277 return nil 278 } 279 280 // Encoded returns the buffer created by Encoded() 281 func (c *Chunk) Encoded() ([]byte, error) { 282 if c.encoded == nil { 283 if err := c.Encode(); err != nil { 284 return nil, err 285 } 286 } 287 return c.encoded, nil 288 } 289 290 // DecodeContext holds data that can be re-used between decodes of different chunks 291 type DecodeContext struct { 292 reader *snappy.Reader 293 } 294 295 // NewDecodeContext creates a new, blank, DecodeContext 296 func NewDecodeContext() *DecodeContext { 297 return &DecodeContext{ 298 reader: snappy.NewReader(nil), 299 } 300 } 301 302 // Decode the chunk from the given buffer, and confirm the chunk is the one we 303 // expected. 304 func (c *Chunk) Decode(decodeContext *DecodeContext, input []byte) error { 305 // First, calculate the checksum of the chunk and confirm it matches 306 // what we expected. 307 if c.Checksum != crc32.Checksum(input, castagnoliTable) { 308 return errors.WithStack(ErrInvalidChecksum) 309 } 310 311 // Now unmarshal the chunk metadata. 312 r := bytes.NewReader(input) 313 var metadataLen uint32 314 if err := binary.Read(r, binary.BigEndian, &metadataLen); err != nil { 315 return errors.Wrap(err, "when reading metadata length from chunk") 316 } 317 var tempMetadata Chunk 318 decodeContext.reader.Reset(r) 319 json := jsoniter.ConfigFastest 320 err := json.NewDecoder(decodeContext.reader).Decode(&tempMetadata) 321 if err != nil { 322 return errors.Wrap(err, "when decoding chunk metadata") 323 } 324 metadataRead := len(input) - r.Len() 325 // Older versions of Cortex included the initial length word; newer versions do not. 326 if !(metadataRead == int(metadataLen) || metadataRead == int(metadataLen)+4) { 327 return errors.Wrapf(ErrMetadataLength, "expected %d, got %d", metadataLen, metadataRead) 328 } 329 330 // Next, confirm the chunks matches what we expected. Easiest way to do this 331 // is to compare what the decoded data thinks its external ID would be, but 332 // we don't write the checksum to s3, so we have to copy the checksum in. 333 334 tempMetadata.Checksum = c.Checksum 335 if !equalByKey(*c, tempMetadata) { 336 return errors.WithStack(ErrWrongMetadata) 337 } 338 339 *c = tempMetadata 340 341 // Finally, unmarshal the actual chunk data. 342 c.Data, err = NewForEncoding(c.Encoding) 343 if err != nil { 344 return errors.Wrap(err, "when creating new chunk") 345 } 346 347 var dataLen uint32 348 if err := binary.Read(r, binary.BigEndian, &dataLen); err != nil { 349 return errors.Wrap(err, "when reading data length from chunk") 350 } 351 352 c.encoded = input 353 remainingData := input[len(input)-r.Len():] 354 if int(dataLen) != len(remainingData) { 355 return ErrDataLength 356 } 357 358 return c.Data.UnmarshalFromBuf(remainingData[:int(dataLen)]) 359 } 360 361 func equalByKey(a, b Chunk) bool { 362 return a.UserID == b.UserID && a.Fingerprint == b.Fingerprint && 363 a.From == b.From && a.Through == b.Through && a.Checksum == b.Checksum 364 }