github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/chunk/chunk_store_utils.go (about) 1 package chunk 2 3 import ( 4 "context" 5 "sync" 6 7 "github.com/go-kit/log/level" 8 "github.com/prometheus/common/model" 9 "github.com/prometheus/prometheus/pkg/labels" 10 "github.com/prometheus/prometheus/promql" 11 12 "github.com/cortexproject/cortex/pkg/chunk/cache" 13 util_log "github.com/cortexproject/cortex/pkg/util/log" 14 "github.com/cortexproject/cortex/pkg/util/spanlogger" 15 ) 16 17 const chunkDecodeParallelism = 16 18 19 func filterChunksByTime(from, through model.Time, chunks []Chunk) []Chunk { 20 filtered := make([]Chunk, 0, len(chunks)) 21 for _, chunk := range chunks { 22 if chunk.Through < from || through < chunk.From { 23 continue 24 } 25 filtered = append(filtered, chunk) 26 } 27 return filtered 28 } 29 30 func keysFromChunks(chunks []Chunk) []string { 31 keys := make([]string, 0, len(chunks)) 32 for _, chk := range chunks { 33 keys = append(keys, chk.ExternalKey()) 34 } 35 36 return keys 37 } 38 39 func labelNamesFromChunks(chunks []Chunk) []string { 40 var result UniqueStrings 41 for _, c := range chunks { 42 for _, l := range c.Metric { 43 result.Add(l.Name) 44 } 45 } 46 return result.Strings() 47 } 48 49 func filterChunksByUniqueFingerprint(chunks []Chunk) ([]Chunk, []string) { 50 filtered := make([]Chunk, 0, len(chunks)) 51 keys := make([]string, 0, len(chunks)) 52 uniqueFp := map[model.Fingerprint]struct{}{} 53 54 for _, chunk := range chunks { 55 if _, ok := uniqueFp[chunk.Fingerprint]; ok { 56 continue 57 } 58 filtered = append(filtered, chunk) 59 keys = append(keys, chunk.ExternalKey()) 60 uniqueFp[chunk.Fingerprint] = struct{}{} 61 } 62 return filtered, keys 63 } 64 65 func filterChunksByMatchers(chunks []Chunk, filters []*labels.Matcher) []Chunk { 66 filteredChunks := make([]Chunk, 0, len(chunks)) 67 outer: 68 for _, chunk := range chunks { 69 for _, filter := range filters { 70 if !filter.Matches(chunk.Metric.Get(filter.Name)) { 71 continue outer 72 } 73 } 74 filteredChunks = append(filteredChunks, chunk) 75 } 76 return filteredChunks 77 } 78 79 // Fetcher deals with fetching chunk contents from the cache/store, 80 // and writing back any misses to the cache. Also responsible for decoding 81 // chunks from the cache, in parallel. 82 type Fetcher struct { 83 storage Client 84 cache cache.Cache 85 cacheStubs bool 86 87 wait sync.WaitGroup 88 decodeRequests chan decodeRequest 89 quit chan struct{} 90 } 91 92 type decodeRequest struct { 93 chunk Chunk 94 buf []byte 95 responses chan decodeResponse 96 } 97 type decodeResponse struct { 98 chunk Chunk 99 err error 100 } 101 102 // NewChunkFetcher makes a new ChunkFetcher. 103 func NewChunkFetcher(cacher cache.Cache, cacheStubs bool, storage Client) (*Fetcher, error) { 104 c := &Fetcher{ 105 storage: storage, 106 cache: cacher, 107 cacheStubs: cacheStubs, 108 decodeRequests: make(chan decodeRequest), 109 quit: make(chan struct{}), 110 } 111 112 c.wait.Add(chunkDecodeParallelism) 113 for i := 0; i < chunkDecodeParallelism; i++ { 114 go c.worker() 115 } 116 117 return c, nil 118 } 119 120 // Stop the ChunkFetcher. 121 func (c *Fetcher) Stop() { 122 select { 123 case <-c.quit: 124 default: 125 close(c.quit) 126 } 127 128 c.wait.Wait() 129 c.cache.Stop() 130 } 131 132 func (c *Fetcher) worker() { 133 defer c.wait.Done() 134 decodeContext := NewDecodeContext() 135 for { 136 select { 137 case <-c.quit: 138 return 139 case req := <-c.decodeRequests: 140 err := req.chunk.Decode(decodeContext, req.buf) 141 if err != nil { 142 cacheCorrupt.Inc() 143 } 144 req.responses <- decodeResponse{ 145 chunk: req.chunk, 146 err: err, 147 } 148 } 149 } 150 } 151 152 // FetchChunks fetches a set of chunks from cache and store. Note that the keys passed in must be 153 // lexicographically sorted, while the returned chunks are not in the same order as the passed in chunks. 154 func (c *Fetcher) FetchChunks(ctx context.Context, chunks []Chunk, keys []string) ([]Chunk, error) { 155 log, ctx := spanlogger.New(ctx, "ChunkStore.FetchChunks") 156 defer log.Span.Finish() 157 158 // Now fetch the actual chunk data from Memcache / S3 159 cacheHits, cacheBufs, _ := c.cache.Fetch(ctx, keys) 160 161 fromCache, missing, err := c.processCacheResponse(ctx, chunks, cacheHits, cacheBufs) 162 if err != nil { 163 level.Warn(log).Log("msg", "error fetching from cache", "err", err) 164 } 165 166 var fromStorage []Chunk 167 if len(missing) > 0 { 168 fromStorage, err = c.storage.GetChunks(ctx, missing) 169 } 170 171 // Always cache any chunks we did get 172 if cacheErr := c.writeBackCache(ctx, fromStorage); cacheErr != nil { 173 level.Warn(log).Log("msg", "could not store chunks in chunk cache", "err", cacheErr) 174 } 175 176 if err != nil { 177 // Don't rely on Cortex error translation here. 178 return nil, promql.ErrStorage{Err: err} 179 } 180 181 allChunks := append(fromCache, fromStorage...) 182 return allChunks, nil 183 } 184 185 func (c *Fetcher) writeBackCache(ctx context.Context, chunks []Chunk) error { 186 keys := make([]string, 0, len(chunks)) 187 bufs := make([][]byte, 0, len(chunks)) 188 for i := range chunks { 189 var encoded []byte 190 var err error 191 if !c.cacheStubs { 192 encoded, err = chunks[i].Encoded() 193 // TODO don't fail, just log and continue? 194 if err != nil { 195 return err 196 } 197 } 198 199 keys = append(keys, chunks[i].ExternalKey()) 200 bufs = append(bufs, encoded) 201 } 202 203 c.cache.Store(ctx, keys, bufs) 204 return nil 205 } 206 207 // ProcessCacheResponse decodes the chunks coming back from the cache, separating 208 // hits and misses. 209 func (c *Fetcher) processCacheResponse(ctx context.Context, chunks []Chunk, keys []string, bufs [][]byte) ([]Chunk, []Chunk, error) { 210 var ( 211 requests = make([]decodeRequest, 0, len(keys)) 212 responses = make(chan decodeResponse) 213 missing []Chunk 214 ) 215 log, _ := spanlogger.New(ctx, "Fetcher.processCacheResponse") 216 defer log.Span.Finish() 217 218 i, j := 0, 0 219 for i < len(chunks) && j < len(keys) { 220 chunkKey := chunks[i].ExternalKey() 221 222 if chunkKey < keys[j] { 223 missing = append(missing, chunks[i]) 224 i++ 225 } else if chunkKey > keys[j] { 226 level.Warn(util_log.Logger).Log("msg", "got chunk from cache we didn't ask for") 227 j++ 228 } else { 229 requests = append(requests, decodeRequest{ 230 chunk: chunks[i], 231 buf: bufs[j], 232 responses: responses, 233 }) 234 i++ 235 j++ 236 } 237 } 238 for ; i < len(chunks); i++ { 239 missing = append(missing, chunks[i]) 240 } 241 level.Debug(log).Log("chunks", len(chunks), "decodeRequests", len(requests), "missing", len(missing)) 242 243 go func() { 244 for _, request := range requests { 245 select { 246 case <-c.quit: 247 return 248 case c.decodeRequests <- request: 249 } 250 } 251 }() 252 253 var ( 254 err error 255 found []Chunk 256 ) 257 258 loopResponses: 259 for i := 0; i < len(requests); i++ { 260 select { 261 case <-c.quit: 262 break loopResponses 263 case response := <-responses: 264 // Don't exit early, as we don't want to block the workers. 265 if response.err != nil { 266 err = response.err 267 } else { 268 found = append(found, response.chunk) 269 } 270 } 271 } 272 return found, missing, err 273 }