github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/store.go (about) 1 package storage 2 3 import ( 4 "context" 5 "math" 6 "time" 7 8 "github.com/go-kit/log" 9 "github.com/pkg/errors" 10 "github.com/prometheus/client_golang/prometheus" 11 "github.com/prometheus/common/model" 12 "github.com/prometheus/prometheus/model/labels" 13 14 "github.com/grafana/dskit/tenant" 15 16 "github.com/grafana/loki/pkg/iter" 17 "github.com/grafana/loki/pkg/logproto" 18 "github.com/grafana/loki/pkg/logql" 19 "github.com/grafana/loki/pkg/logqlmodel/stats" 20 "github.com/grafana/loki/pkg/querier/astmapper" 21 "github.com/grafana/loki/pkg/storage/chunk" 22 "github.com/grafana/loki/pkg/storage/chunk/cache" 23 "github.com/grafana/loki/pkg/storage/chunk/client" 24 "github.com/grafana/loki/pkg/storage/chunk/fetcher" 25 "github.com/grafana/loki/pkg/storage/config" 26 "github.com/grafana/loki/pkg/storage/stores" 27 "github.com/grafana/loki/pkg/storage/stores/indexshipper" 28 "github.com/grafana/loki/pkg/storage/stores/indexshipper/gatewayclient" 29 "github.com/grafana/loki/pkg/storage/stores/series" 30 "github.com/grafana/loki/pkg/storage/stores/series/index" 31 "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" 32 "github.com/grafana/loki/pkg/storage/stores/tsdb" 33 "github.com/grafana/loki/pkg/usagestats" 34 "github.com/grafana/loki/pkg/util" 35 "github.com/grafana/loki/pkg/util/deletion" 36 ) 37 38 var ( 39 indexTypeStats = usagestats.NewString("store_index_type") 40 objectTypeStats = usagestats.NewString("store_object_type") 41 schemaStats = usagestats.NewString("store_schema") 42 43 errWritingChunkUnsupported = errors.New("writing chunks is not supported while running store in read-only mode") 44 ) 45 46 // Store is the Loki chunk store to retrieve and save chunks. 47 type Store interface { 48 stores.Store 49 SelectSamples(ctx context.Context, req logql.SelectSampleParams) (iter.SampleIterator, error) 50 SelectLogs(ctx context.Context, req logql.SelectLogParams) (iter.EntryIterator, error) 51 Series(ctx context.Context, req logql.SelectLogParams) ([]logproto.SeriesIdentifier, error) 52 GetSchemaConfigs() []config.PeriodConfig 53 SetChunkFilterer(chunkFilter chunk.RequestChunkFilterer) 54 } 55 56 type store struct { 57 stores.Store 58 composite *stores.CompositeStore 59 60 cfg Config 61 storeCfg config.ChunkStoreConfig 62 schemaCfg config.SchemaConfig 63 64 chunkMetrics *ChunkMetrics 65 chunkClientMetrics client.ChunkClientMetrics 66 clientMetrics ClientMetrics 67 registerer prometheus.Registerer 68 69 indexReadCache cache.Cache 70 chunksCache cache.Cache 71 writeDedupeCache cache.Cache 72 73 limits StoreLimits 74 logger log.Logger 75 76 chunkFilterer chunk.RequestChunkFilterer 77 } 78 79 // NewStore creates a new Loki Store using configuration supplied. 80 func NewStore(cfg Config, storeCfg config.ChunkStoreConfig, schemaCfg config.SchemaConfig, 81 limits StoreLimits, clientMetrics ClientMetrics, registerer prometheus.Registerer, logger log.Logger, 82 ) (Store, error) { 83 if len(schemaCfg.Configs) != 0 { 84 if index := config.ActivePeriodConfig(schemaCfg.Configs); index != -1 && index < len(schemaCfg.Configs) { 85 indexTypeStats.Set(schemaCfg.Configs[index].IndexType) 86 objectTypeStats.Set(schemaCfg.Configs[index].ObjectType) 87 schemaStats.Set(schemaCfg.Configs[index].Schema) 88 } 89 } 90 91 indexReadCache, err := cache.New(cfg.IndexQueriesCacheConfig, registerer, logger, stats.IndexCache) 92 if err != nil { 93 return nil, err 94 } 95 96 writeDedupeCache, err := cache.New(storeCfg.WriteDedupeCacheConfig, registerer, logger, stats.WriteDedupeCache) 97 if err != nil { 98 return nil, err 99 } 100 101 chunkCacheCfg := storeCfg.ChunkCacheConfig 102 chunkCacheCfg.Prefix = "chunks" 103 chunksCache, err := cache.New(chunkCacheCfg, registerer, logger, stats.ChunkCache) 104 if err != nil { 105 return nil, err 106 } 107 108 // Cache is shared by multiple stores, which means they will try and Stop 109 // it more than once. Wrap in a StopOnce to prevent this. 110 indexReadCache = cache.StopOnce(indexReadCache) 111 chunksCache = cache.StopOnce(chunksCache) 112 writeDedupeCache = cache.StopOnce(writeDedupeCache) 113 114 // Lets wrap all caches except chunksCache with CacheGenMiddleware to facilitate cache invalidation using cache generation numbers. 115 // chunksCache is not wrapped because chunks content can't be anyways modified without changing its ID so there is no use of 116 // invalidating chunks cache. Also chunks can be fetched only by their ID found in index and we are anyways removing the index and invalidating index cache here. 117 indexReadCache = cache.NewCacheGenNumMiddleware(indexReadCache) 118 writeDedupeCache = cache.NewCacheGenNumMiddleware(writeDedupeCache) 119 120 err = schemaCfg.Load() 121 if err != nil { 122 return nil, errors.Wrap(err, "error loading schema config") 123 } 124 stores := stores.NewCompositeStore(limits) 125 126 s := &store{ 127 Store: stores, 128 composite: stores, 129 cfg: cfg, 130 storeCfg: storeCfg, 131 schemaCfg: schemaCfg, 132 133 chunkClientMetrics: client.NewChunkClientMetrics(registerer), 134 clientMetrics: clientMetrics, 135 chunkMetrics: NewChunkMetrics(registerer, cfg.MaxChunkBatchSize), 136 registerer: registerer, 137 138 indexReadCache: indexReadCache, 139 chunksCache: chunksCache, 140 writeDedupeCache: writeDedupeCache, 141 142 logger: logger, 143 limits: limits, 144 } 145 if err := s.init(); err != nil { 146 return nil, err 147 } 148 return s, nil 149 } 150 151 func (s *store) init() error { 152 for _, p := range s.schemaCfg.Configs { 153 chunkClient, err := s.chunkClientForPeriod(p) 154 if err != nil { 155 return err 156 } 157 f, err := fetcher.New(s.chunksCache, s.storeCfg.ChunkCacheStubs(), s.schemaCfg, chunkClient, s.storeCfg.ChunkCacheConfig.AsyncCacheWriteBackConcurrency, s.storeCfg.ChunkCacheConfig.AsyncCacheWriteBackBufferSize) 158 if err != nil { 159 return err 160 } 161 162 w, idx, stop, err := s.storeForPeriod(p, chunkClient, f) 163 if err != nil { 164 return err 165 } 166 s.composite.AddStore(p.From.Time, f, idx, w, stop) 167 } 168 169 if s.cfg.EnableAsyncStore { 170 s.Store = NewAsyncStore(s.cfg.AsyncStoreConfig, s.Store, s.schemaCfg) 171 } 172 return nil 173 } 174 175 func (s *store) chunkClientForPeriod(p config.PeriodConfig) (client.Client, error) { 176 objectStoreType := p.ObjectType 177 if objectStoreType == "" { 178 objectStoreType = p.IndexType 179 } 180 chunkClientReg := prometheus.WrapRegistererWith( 181 prometheus.Labels{"component": "chunk-store-" + p.From.String()}, s.registerer) 182 183 chunks, err := NewChunkClient(objectStoreType, s.cfg, s.schemaCfg, s.clientMetrics, chunkClientReg) 184 if err != nil { 185 return nil, errors.Wrap(err, "error creating object client") 186 } 187 188 chunks = client.NewMetricsChunkClient(chunks, s.chunkClientMetrics) 189 return chunks, nil 190 } 191 192 func shouldUseIndexGatewayClient(cfg indexshipper.Config) bool { 193 if cfg.Mode != indexshipper.ModeReadOnly || cfg.IndexGatewayClientConfig.Disabled { 194 return false 195 } 196 197 gatewayCfg := cfg.IndexGatewayClientConfig 198 if gatewayCfg.Mode == indexgateway.SimpleMode && gatewayCfg.Address == "" { 199 return false 200 } 201 202 return true 203 } 204 205 func (s *store) storeForPeriod(p config.PeriodConfig, chunkClient client.Client, f *fetcher.Fetcher) (stores.ChunkWriter, series.IndexStore, func(), error) { 206 indexClientReg := prometheus.WrapRegistererWith( 207 prometheus.Labels{"component": "index-store-" + p.From.String()}, s.registerer) 208 209 if p.IndexType == config.TSDBType { 210 if shouldUseIndexGatewayClient(s.cfg.TSDBShipperConfig) { 211 // inject the index-gateway client into the index store 212 gw, err := gatewayclient.NewGatewayClient(s.cfg.TSDBShipperConfig.IndexGatewayClientConfig, indexClientReg, s.logger) 213 if err != nil { 214 return nil, nil, nil, err 215 } 216 idx := series.NewIndexGatewayClientStore(gw, nil) 217 218 return failingChunkWriter{}, idx, func() { 219 f.Stop() 220 gw.Stop() 221 }, nil 222 } 223 224 objectClient, err := NewObjectClient(s.cfg.TSDBShipperConfig.SharedStoreType, s.cfg, s.clientMetrics) 225 if err != nil { 226 return nil, nil, nil, err 227 } 228 229 writer, idx, stopTSDBStoreFunc, err := tsdb.NewStore(s.cfg.TSDBShipperConfig, p, f, objectClient, s.limits, 230 getIndexStoreTableRanges(config.TSDBType, s.schemaCfg.Configs), indexClientReg) 231 if err != nil { 232 return nil, nil, nil, err 233 } 234 235 return writer, idx, 236 func() { 237 f.Stop() 238 chunkClient.Stop() 239 stopTSDBStoreFunc() 240 objectClient.Stop() 241 }, nil 242 } 243 244 idx, err := NewIndexClient(p.IndexType, s.cfg, s.schemaCfg, s.limits, s.clientMetrics, nil, indexClientReg) 245 if err != nil { 246 return nil, nil, nil, errors.Wrap(err, "error creating index client") 247 } 248 idx = index.NewCachingIndexClient(idx, s.indexReadCache, s.cfg.IndexCacheValidity, s.limits, s.logger, s.cfg.DisableBroadIndexQueries) 249 schema, err := index.CreateSchema(p) 250 if err != nil { 251 return nil, nil, nil, err 252 } 253 if s.storeCfg.CacheLookupsOlderThan != 0 { 254 schema = index.NewSchemaCaching(schema, time.Duration(s.storeCfg.CacheLookupsOlderThan)) 255 } 256 257 var ( 258 writer stores.ChunkWriter = series.NewWriter(f, s.schemaCfg, idx, schema, s.writeDedupeCache, s.storeCfg.DisableIndexDeduplication) 259 indexStore = series.NewIndexStore(s.schemaCfg, schema, idx, f, s.cfg.MaxChunkBatchSize) 260 ) 261 262 // (Sandeep): Disable IndexGatewayClientStore for stores other than tsdb until we are ready to enable it again 263 /*if s.cfg.BoltDBShipperConfig != nil && shouldUseIndexGatewayClient(s.cfg.BoltDBShipperConfig) { 264 // inject the index-gateway client into the index store 265 gw, err := shipper.NewGatewayClient(s.cfg.BoltDBShipperConfig.IndexGatewayClientConfig, indexClientReg, s.logger) 266 if err != nil { 267 return nil, nil, nil, err 268 } 269 indexStore = series.NewIndexGatewayClientStore(gw, indexStore) 270 }*/ 271 272 return writer, 273 indexStore, 274 func() { 275 chunkClient.Stop() 276 f.Stop() 277 idx.Stop() 278 }, 279 nil 280 } 281 282 // decodeReq sanitizes an incoming request, rounds bounds, appends the __name__ matcher, 283 // and adds the "__cortex_shard__" label if this is a sharded query. 284 // todo(cyriltovena) refactor this. 285 func decodeReq(req logql.QueryParams) ([]*labels.Matcher, model.Time, model.Time, error) { 286 expr, err := req.LogSelector() 287 if err != nil { 288 return nil, 0, 0, err 289 } 290 291 matchers := expr.Matchers() 292 nameLabelMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, "logs") 293 if err != nil { 294 return nil, 0, 0, err 295 } 296 matchers = append(matchers, nameLabelMatcher) 297 if err != nil { 298 return nil, 0, 0, err 299 } 300 matchers, err = injectShardLabel(req.GetShards(), matchers) 301 if err != nil { 302 return nil, 0, 0, err 303 } 304 from, through := util.RoundToMilliseconds(req.GetStart(), req.GetEnd()) 305 return matchers, from, through, nil 306 } 307 308 func injectShardLabel(shards []string, matchers []*labels.Matcher) ([]*labels.Matcher, error) { 309 if shards != nil { 310 parsed, err := logql.ParseShards(shards) 311 if err != nil { 312 return nil, err 313 } 314 for _, s := range parsed { 315 shardMatcher, err := labels.NewMatcher( 316 labels.MatchEqual, 317 astmapper.ShardLabel, 318 s.String(), 319 ) 320 if err != nil { 321 return nil, err 322 } 323 matchers = append(matchers, shardMatcher) 324 break // nolint:staticcheck 325 } 326 } 327 return matchers, nil 328 } 329 330 func (s *store) SetChunkFilterer(chunkFilterer chunk.RequestChunkFilterer) { 331 s.chunkFilterer = chunkFilterer 332 s.Store.SetChunkFilterer(chunkFilterer) 333 } 334 335 // lazyChunks is an internal function used to resolve a set of lazy chunks from the store without actually loading them. It's used internally by `LazyQuery` and `GetSeries` 336 func (s *store) lazyChunks(ctx context.Context, matchers []*labels.Matcher, from, through model.Time) ([]*LazyChunk, error) { 337 userID, err := tenant.TenantID(ctx) 338 if err != nil { 339 return nil, err 340 } 341 342 stats := stats.FromContext(ctx) 343 344 chks, fetchers, err := s.GetChunkRefs(ctx, userID, from, through, matchers...) 345 if err != nil { 346 return nil, err 347 } 348 349 var prefiltered int 350 var filtered int 351 for i := range chks { 352 prefiltered += len(chks[i]) 353 stats.AddChunksRef(int64(len(chks[i]))) 354 chks[i] = filterChunksByTime(from, through, chks[i]) 355 filtered += len(chks[i]) 356 } 357 358 s.chunkMetrics.refs.WithLabelValues(statusDiscarded).Add(float64(prefiltered - filtered)) 359 s.chunkMetrics.refs.WithLabelValues(statusMatched).Add(float64(filtered)) 360 361 // creates lazychunks with chunks ref. 362 lazyChunks := make([]*LazyChunk, 0, filtered) 363 for i := range chks { 364 for _, c := range chks[i] { 365 lazyChunks = append(lazyChunks, &LazyChunk{Chunk: c, Fetcher: fetchers[i]}) 366 } 367 } 368 return lazyChunks, nil 369 } 370 371 func (s *store) Series(ctx context.Context, req logql.SelectLogParams) ([]logproto.SeriesIdentifier, error) { 372 userID, err := tenant.TenantID(ctx) 373 if err != nil { 374 return nil, err 375 } 376 var from, through model.Time 377 var matchers []*labels.Matcher 378 379 // The Loki parser doesn't allow for an empty label matcher but for the Series API 380 // we allow this to select all series in the time range. 381 if req.Selector == "" { 382 from, through = util.RoundToMilliseconds(req.Start, req.End) 383 nameLabelMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, "logs") 384 if err != nil { 385 return nil, err 386 } 387 matchers = []*labels.Matcher{nameLabelMatcher} 388 matchers, err = injectShardLabel(req.GetShards(), matchers) 389 if err != nil { 390 return nil, err 391 } 392 } else { 393 var err error 394 matchers, from, through, err = decodeReq(req) 395 if err != nil { 396 return nil, err 397 } 398 } 399 series, err := s.Store.GetSeries(ctx, userID, from, through, matchers...) 400 if err != nil { 401 return nil, err 402 } 403 result := make([]logproto.SeriesIdentifier, len(series)) 404 for i, s := range series { 405 result[i] = logproto.SeriesIdentifier{ 406 Labels: s.Map(), 407 } 408 } 409 return result, nil 410 } 411 412 // SelectLogs returns an iterator that will query the store for more chunks while iterating instead of fetching all chunks upfront 413 // for that request. 414 func (s *store) SelectLogs(ctx context.Context, req logql.SelectLogParams) (iter.EntryIterator, error) { 415 matchers, from, through, err := decodeReq(req) 416 if err != nil { 417 return nil, err 418 } 419 420 lazyChunks, err := s.lazyChunks(ctx, matchers, from, through) 421 if err != nil { 422 return nil, err 423 } 424 425 if len(lazyChunks) == 0 { 426 return iter.NoopIterator, nil 427 } 428 429 expr, err := req.LogSelector() 430 if err != nil { 431 return nil, err 432 } 433 434 pipeline, err := expr.Pipeline() 435 if err != nil { 436 return nil, err 437 } 438 439 pipeline, err = deletion.SetupPipeline(req, pipeline) 440 if err != nil { 441 return nil, err 442 } 443 444 var chunkFilterer chunk.Filterer 445 if s.chunkFilterer != nil { 446 chunkFilterer = s.chunkFilterer.ForRequest(ctx) 447 } 448 449 return newLogBatchIterator(ctx, s.schemaCfg, s.chunkMetrics, lazyChunks, s.cfg.MaxChunkBatchSize, matchers, pipeline, req.Direction, req.Start, req.End, chunkFilterer) 450 } 451 452 func (s *store) SelectSamples(ctx context.Context, req logql.SelectSampleParams) (iter.SampleIterator, error) { 453 matchers, from, through, err := decodeReq(req) 454 if err != nil { 455 return nil, err 456 } 457 458 lazyChunks, err := s.lazyChunks(ctx, matchers, from, through) 459 if err != nil { 460 return nil, err 461 } 462 463 if len(lazyChunks) == 0 { 464 return iter.NoopIterator, nil 465 } 466 467 expr, err := req.Expr() 468 if err != nil { 469 return nil, err 470 } 471 472 extractor, err := expr.Extractor() 473 if err != nil { 474 return nil, err 475 } 476 477 extractor, err = deletion.SetupExtractor(req, extractor) 478 if err != nil { 479 return nil, err 480 } 481 482 var chunkFilterer chunk.Filterer 483 if s.chunkFilterer != nil { 484 chunkFilterer = s.chunkFilterer.ForRequest(ctx) 485 } 486 487 return newSampleBatchIterator(ctx, s.schemaCfg, s.chunkMetrics, lazyChunks, s.cfg.MaxChunkBatchSize, matchers, extractor, req.Start, req.End, chunkFilterer) 488 } 489 490 func (s *store) GetSchemaConfigs() []config.PeriodConfig { 491 return s.schemaCfg.Configs 492 } 493 494 func filterChunksByTime(from, through model.Time, chunks []chunk.Chunk) []chunk.Chunk { 495 filtered := make([]chunk.Chunk, 0, len(chunks)) 496 for _, chunk := range chunks { 497 if chunk.Through < from || through < chunk.From { 498 continue 499 } 500 filtered = append(filtered, chunk) 501 } 502 return filtered 503 } 504 505 type failingChunkWriter struct{} 506 507 func (f failingChunkWriter) Put(_ context.Context, _ []chunk.Chunk) error { 508 return errWritingChunkUnsupported 509 } 510 511 func (f failingChunkWriter) PutOne(_ context.Context, _, _ model.Time, _ chunk.Chunk) error { 512 return errWritingChunkUnsupported 513 } 514 515 func getIndexStoreTableRanges(indexType string, periodicConfigs []config.PeriodConfig) config.TableRanges { 516 var ranges config.TableRanges 517 for i := range periodicConfigs { 518 if periodicConfigs[i].IndexType != indexType { 519 continue 520 } 521 522 periodEndTime := config.DayTime{Time: math.MaxInt64} 523 if i < len(periodicConfigs)-1 { 524 periodEndTime = config.DayTime{Time: periodicConfigs[i+1].From.Time.Add(-time.Millisecond)} 525 } 526 527 ranges = append(ranges, periodicConfigs[i].GetIndexTableNumberRange(periodEndTime)) 528 } 529 530 return ranges 531 }