github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/factory.go (about) 1 package storage 2 3 import ( 4 "context" 5 "flag" 6 "fmt" 7 "strings" 8 "time" 9 10 "github.com/go-kit/log/level" 11 "github.com/pkg/errors" 12 "github.com/prometheus/client_golang/prometheus" 13 14 "github.com/grafana/loki/pkg/storage/chunk/cache" 15 "github.com/grafana/loki/pkg/storage/chunk/client" 16 "github.com/grafana/loki/pkg/storage/chunk/client/aws" 17 "github.com/grafana/loki/pkg/storage/chunk/client/azure" 18 "github.com/grafana/loki/pkg/storage/chunk/client/baidubce" 19 "github.com/grafana/loki/pkg/storage/chunk/client/cassandra" 20 "github.com/grafana/loki/pkg/storage/chunk/client/gcp" 21 "github.com/grafana/loki/pkg/storage/chunk/client/grpc" 22 "github.com/grafana/loki/pkg/storage/chunk/client/hedging" 23 "github.com/grafana/loki/pkg/storage/chunk/client/local" 24 "github.com/grafana/loki/pkg/storage/chunk/client/openstack" 25 "github.com/grafana/loki/pkg/storage/chunk/client/testutils" 26 "github.com/grafana/loki/pkg/storage/config" 27 "github.com/grafana/loki/pkg/storage/stores/indexshipper" 28 "github.com/grafana/loki/pkg/storage/stores/indexshipper/downloads" 29 "github.com/grafana/loki/pkg/storage/stores/indexshipper/gatewayclient" 30 "github.com/grafana/loki/pkg/storage/stores/series/index" 31 "github.com/grafana/loki/pkg/storage/stores/shipper" 32 util_log "github.com/grafana/loki/pkg/util/log" 33 ) 34 35 // BoltDB Shipper is supposed to be run as a singleton. 36 // This could also be done in NewBoltDBIndexClientWithShipper factory method but we are doing it here because that method is used 37 // in tests for creating multiple instances of it at a time. 38 var boltDBIndexClientWithShipper index.Client 39 40 // ResetBoltDBIndexClientWithShipper allows to reset the singleton. 41 // MUST ONLY BE USED IN TESTS 42 func ResetBoltDBIndexClientWithShipper() { 43 boltDBIndexClientWithShipper.Stop() 44 boltDBIndexClientWithShipper = nil 45 } 46 47 // StoreLimits helps get Limits specific to Queries for Stores 48 type StoreLimits interface { 49 downloads.Limits 50 CardinalityLimit(userID string) int 51 MaxChunksPerQueryFromStore(userID string) int 52 MaxQueryLength(userID string) time.Duration 53 } 54 55 // Config chooses which storage client to use. 56 type Config struct { 57 AWSStorageConfig aws.StorageConfig `yaml:"aws"` 58 AzureStorageConfig azure.BlobStorageConfig `yaml:"azure"` 59 BOSStorageConfig baidubce.BOSStorageConfig `yaml:"bos"` 60 GCPStorageConfig gcp.Config `yaml:"bigtable"` 61 GCSConfig gcp.GCSConfig `yaml:"gcs"` 62 CassandraStorageConfig cassandra.Config `yaml:"cassandra"` 63 BoltDBConfig local.BoltDBConfig `yaml:"boltdb"` 64 FSConfig local.FSConfig `yaml:"filesystem"` 65 Swift openstack.SwiftConfig `yaml:"swift"` 66 GrpcConfig grpc.Config `yaml:"grpc_store"` 67 Hedging hedging.Config `yaml:"hedging"` 68 69 IndexCacheValidity time.Duration `yaml:"index_cache_validity"` 70 71 IndexQueriesCacheConfig cache.Config `yaml:"index_queries_cache_config"` 72 DisableBroadIndexQueries bool `yaml:"disable_broad_index_queries"` 73 MaxParallelGetChunk int `yaml:"max_parallel_get_chunk"` 74 75 MaxChunkBatchSize int `yaml:"max_chunk_batch_size"` 76 BoltDBShipperConfig shipper.Config `yaml:"boltdb_shipper"` 77 TSDBShipperConfig indexshipper.Config `yaml:"tsdb_shipper"` 78 79 // Config for using AsyncStore when using async index stores like `boltdb-shipper`. 80 // It is required for getting chunk ids of recently flushed chunks from the ingesters. 81 EnableAsyncStore bool `yaml:"-"` 82 AsyncStoreConfig AsyncStoreCfg `yaml:"-"` 83 } 84 85 // RegisterFlags adds the flags required to configure this flag set. 86 func (cfg *Config) RegisterFlags(f *flag.FlagSet) { 87 cfg.AWSStorageConfig.RegisterFlags(f) 88 cfg.AzureStorageConfig.RegisterFlags(f) 89 cfg.BOSStorageConfig.RegisterFlags(f) 90 cfg.GCPStorageConfig.RegisterFlags(f) 91 cfg.GCSConfig.RegisterFlags(f) 92 cfg.CassandraStorageConfig.RegisterFlags(f) 93 cfg.BoltDBConfig.RegisterFlags(f) 94 cfg.FSConfig.RegisterFlags(f) 95 cfg.Swift.RegisterFlags(f) 96 cfg.GrpcConfig.RegisterFlags(f) 97 cfg.Hedging.RegisterFlagsWithPrefix("store.", f) 98 99 cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "Cache config for index entry reading.", f) 100 f.DurationVar(&cfg.IndexCacheValidity, "store.index-cache-validity", 5*time.Minute, "Cache validity for active index entries. Should be no higher than -ingester.max-chunk-idle.") 101 f.BoolVar(&cfg.DisableBroadIndexQueries, "store.disable-broad-index-queries", false, "Disable broad index queries which results in reduced cache usage and faster query performance at the expense of somewhat higher QPS on the index store.") 102 f.IntVar(&cfg.MaxParallelGetChunk, "store.max-parallel-get-chunk", 150, "Maximum number of parallel chunk reads.") 103 cfg.BoltDBShipperConfig.RegisterFlags(f) 104 f.IntVar(&cfg.MaxChunkBatchSize, "store.max-chunk-batch-size", 50, "The maximum number of chunks to fetch per batch.") 105 cfg.TSDBShipperConfig.RegisterFlagsWithPrefix("tsdb.", f) 106 } 107 108 // Validate config and returns error on failure 109 func (cfg *Config) Validate() error { 110 if err := cfg.CassandraStorageConfig.Validate(); err != nil { 111 return errors.Wrap(err, "invalid Cassandra Storage config") 112 } 113 if err := cfg.GCPStorageConfig.Validate(util_log.Logger); err != nil { 114 return errors.Wrap(err, "invalid GCP Storage Storage config") 115 } 116 if err := cfg.Swift.Validate(); err != nil { 117 return errors.Wrap(err, "invalid Swift Storage config") 118 } 119 if err := cfg.IndexQueriesCacheConfig.Validate(); err != nil { 120 return errors.Wrap(err, "invalid Index Queries Cache config") 121 } 122 if err := cfg.AzureStorageConfig.Validate(); err != nil { 123 return errors.Wrap(err, "invalid Azure Storage config") 124 } 125 if err := cfg.AWSStorageConfig.Validate(); err != nil { 126 return errors.Wrap(err, "invalid AWS Storage config") 127 } 128 if err := cfg.BoltDBShipperConfig.Validate(); err != nil { 129 return errors.Wrap(err, "invalid boltdb-shipper config") 130 } 131 if err := cfg.TSDBShipperConfig.Validate(); err != nil { 132 return errors.Wrap(err, "invalid tsdb config") 133 } 134 return nil 135 } 136 137 // NewIndexClient makes a new index client of the desired type. 138 func NewIndexClient(name string, cfg Config, schemaCfg config.SchemaConfig, limits StoreLimits, cm ClientMetrics, ownsTenantFn downloads.IndexGatewayOwnsTenant, registerer prometheus.Registerer) (index.Client, error) { 139 switch name { 140 case config.StorageTypeInMemory: 141 store := testutils.NewMockStorage() 142 return store, nil 143 case config.StorageTypeAWS, config.StorageTypeAWSDynamo: 144 if cfg.AWSStorageConfig.DynamoDB.URL == nil { 145 return nil, fmt.Errorf("Must set -dynamodb.url in aws mode") 146 } 147 path := strings.TrimPrefix(cfg.AWSStorageConfig.DynamoDB.URL.Path, "/") 148 if len(path) > 0 { 149 level.Warn(util_log.Logger).Log("msg", "ignoring DynamoDB URL path", "path", path) 150 } 151 return aws.NewDynamoDBIndexClient(cfg.AWSStorageConfig.DynamoDBConfig, schemaCfg, registerer) 152 case config.StorageTypeGCP: 153 return gcp.NewStorageClientV1(context.Background(), cfg.GCPStorageConfig, schemaCfg) 154 case config.StorageTypeGCPColumnKey, config.StorageTypeBigTable: 155 return gcp.NewStorageClientColumnKey(context.Background(), cfg.GCPStorageConfig, schemaCfg) 156 case config.StorageTypeBigTableHashed: 157 cfg.GCPStorageConfig.DistributeKeys = true 158 return gcp.NewStorageClientColumnKey(context.Background(), cfg.GCPStorageConfig, schemaCfg) 159 case config.StorageTypeCassandra: 160 return cassandra.NewStorageClient(cfg.CassandraStorageConfig, schemaCfg, registerer) 161 case config.StorageTypeBoltDB: 162 return local.NewBoltDBIndexClient(cfg.BoltDBConfig) 163 case config.StorageTypeGrpc: 164 return grpc.NewStorageClient(cfg.GrpcConfig, schemaCfg) 165 case config.BoltDBShipperType: 166 if boltDBIndexClientWithShipper != nil { 167 return boltDBIndexClientWithShipper, nil 168 } 169 170 if shouldUseIndexGatewayClient(cfg.BoltDBShipperConfig.Config) { 171 gateway, err := gatewayclient.NewGatewayClient(cfg.BoltDBShipperConfig.IndexGatewayClientConfig, registerer, util_log.Logger) 172 if err != nil { 173 return nil, err 174 } 175 176 boltDBIndexClientWithShipper = gateway 177 return gateway, nil 178 } 179 180 objectClient, err := NewObjectClient(cfg.BoltDBShipperConfig.SharedStoreType, cfg, cm) 181 if err != nil { 182 return nil, err 183 } 184 185 tableRanges := getIndexStoreTableRanges(config.BoltDBShipperType, schemaCfg.Configs) 186 187 boltDBIndexClientWithShipper, err = shipper.NewShipper(cfg.BoltDBShipperConfig, objectClient, limits, 188 ownsTenantFn, tableRanges, registerer) 189 190 return boltDBIndexClientWithShipper, err 191 default: 192 return nil, fmt.Errorf("Unrecognized storage client %v, choose one of: %v, %v, %v, %v, %v, %v", name, config.StorageTypeAWS, config.StorageTypeCassandra, config.StorageTypeInMemory, config.StorageTypeGCP, config.StorageTypeBigTable, config.StorageTypeBigTableHashed) 193 } 194 } 195 196 // NewChunkClient makes a new chunk.Client of the desired types. 197 func NewChunkClient(name string, cfg Config, schemaCfg config.SchemaConfig, clientMetrics ClientMetrics, registerer prometheus.Registerer) (client.Client, error) { 198 switch name { 199 case config.StorageTypeInMemory: 200 return testutils.NewMockStorage(), nil 201 case config.StorageTypeAWS, config.StorageTypeS3: 202 c, err := aws.NewS3ObjectClient(cfg.AWSStorageConfig.S3Config, cfg.Hedging) 203 if err != nil { 204 return nil, err 205 } 206 return client.NewClientWithMaxParallel(c, nil, cfg.MaxParallelGetChunk, schemaCfg), nil 207 case config.StorageTypeAWSDynamo: 208 if cfg.AWSStorageConfig.DynamoDB.URL == nil { 209 return nil, fmt.Errorf("Must set -dynamodb.url in aws mode") 210 } 211 path := strings.TrimPrefix(cfg.AWSStorageConfig.DynamoDB.URL.Path, "/") 212 if len(path) > 0 { 213 level.Warn(util_log.Logger).Log("msg", "ignoring DynamoDB URL path", "path", path) 214 } 215 return aws.NewDynamoDBChunkClient(cfg.AWSStorageConfig.DynamoDBConfig, schemaCfg, registerer) 216 case config.StorageTypeAzure: 217 c, err := azure.NewBlobStorage(&cfg.AzureStorageConfig, clientMetrics.AzureMetrics, cfg.Hedging) 218 if err != nil { 219 return nil, err 220 } 221 return client.NewClientWithMaxParallel(c, nil, cfg.MaxParallelGetChunk, schemaCfg), nil 222 case config.StorageTypeBOS: 223 c, err := baidubce.NewBOSObjectStorage(&cfg.BOSStorageConfig) 224 if err != nil { 225 return nil, err 226 } 227 return client.NewClientWithMaxParallel(c, nil, cfg.MaxChunkBatchSize, schemaCfg), nil 228 case config.StorageTypeGCP: 229 return gcp.NewBigtableObjectClient(context.Background(), cfg.GCPStorageConfig, schemaCfg) 230 case config.StorageTypeGCPColumnKey, config.StorageTypeBigTable, config.StorageTypeBigTableHashed: 231 return gcp.NewBigtableObjectClient(context.Background(), cfg.GCPStorageConfig, schemaCfg) 232 case config.StorageTypeGCS: 233 c, err := gcp.NewGCSObjectClient(context.Background(), cfg.GCSConfig, cfg.Hedging) 234 if err != nil { 235 return nil, err 236 } 237 return client.NewClientWithMaxParallel(c, nil, cfg.MaxParallelGetChunk, schemaCfg), nil 238 case config.StorageTypeSwift: 239 c, err := openstack.NewSwiftObjectClient(cfg.Swift, cfg.Hedging) 240 if err != nil { 241 return nil, err 242 } 243 return client.NewClientWithMaxParallel(c, nil, cfg.MaxParallelGetChunk, schemaCfg), nil 244 case config.StorageTypeCassandra: 245 return cassandra.NewObjectClient(cfg.CassandraStorageConfig, schemaCfg, registerer, cfg.MaxParallelGetChunk) 246 case config.StorageTypeFileSystem: 247 store, err := local.NewFSObjectClient(cfg.FSConfig) 248 if err != nil { 249 return nil, err 250 } 251 return client.NewClientWithMaxParallel(store, client.FSEncoder, cfg.MaxParallelGetChunk, schemaCfg), nil 252 case config.StorageTypeGrpc: 253 return grpc.NewStorageClient(cfg.GrpcConfig, schemaCfg) 254 default: 255 return nil, fmt.Errorf("Unrecognized storage client %v, choose one of: %v, %v, %v, %v, %v, %v, %v, %v", name, config.StorageTypeAWS, config.StorageTypeAzure, config.StorageTypeCassandra, config.StorageTypeInMemory, config.StorageTypeGCP, config.StorageTypeBigTable, config.StorageTypeBigTableHashed, config.StorageTypeGrpc) 256 } 257 } 258 259 // NewTableClient makes a new table client based on the configuration. 260 func NewTableClient(name string, cfg Config, cm ClientMetrics, registerer prometheus.Registerer) (index.TableClient, error) { 261 switch name { 262 case config.StorageTypeInMemory: 263 return testutils.NewMockStorage(), nil 264 case config.StorageTypeAWS, config.StorageTypeAWSDynamo: 265 if cfg.AWSStorageConfig.DynamoDB.URL == nil { 266 return nil, fmt.Errorf("Must set -dynamodb.url in aws mode") 267 } 268 path := strings.TrimPrefix(cfg.AWSStorageConfig.DynamoDB.URL.Path, "/") 269 if len(path) > 0 { 270 level.Warn(util_log.Logger).Log("msg", "ignoring DynamoDB URL path", "path", path) 271 } 272 return aws.NewDynamoDBTableClient(cfg.AWSStorageConfig.DynamoDBConfig, registerer) 273 case config.StorageTypeGCP, config.StorageTypeGCPColumnKey, config.StorageTypeBigTable, config.StorageTypeBigTableHashed: 274 return gcp.NewTableClient(context.Background(), cfg.GCPStorageConfig) 275 case config.StorageTypeCassandra: 276 return cassandra.NewTableClient(context.Background(), cfg.CassandraStorageConfig, registerer) 277 case config.StorageTypeBoltDB: 278 return local.NewTableClient(cfg.BoltDBConfig.Directory) 279 case config.StorageTypeGrpc: 280 return grpc.NewTableClient(cfg.GrpcConfig) 281 case config.BoltDBShipperType, config.TSDBType: 282 objectClient, err := NewObjectClient(cfg.BoltDBShipperConfig.SharedStoreType, cfg, cm) 283 if err != nil { 284 return nil, err 285 } 286 sharedStoreKeyPrefix := cfg.BoltDBShipperConfig.SharedStoreKeyPrefix 287 if name == config.TSDBType { 288 sharedStoreKeyPrefix = cfg.TSDBShipperConfig.SharedStoreKeyPrefix 289 } 290 return indexshipper.NewTableClient(objectClient, sharedStoreKeyPrefix), nil 291 default: 292 return nil, fmt.Errorf("Unrecognized storage client %v, choose one of: %v, %v, %v, %v, %v, %v, %v", name, config.StorageTypeAWS, config.StorageTypeCassandra, config.StorageTypeInMemory, config.StorageTypeGCP, config.StorageTypeBigTable, config.StorageTypeBigTableHashed, config.StorageTypeGrpc) 293 } 294 } 295 296 // // NewTableClient creates a TableClient for managing tables for index/chunk store. 297 // // ToDo: Add support in Cortex for registering custom table client like index client. 298 // func NewTableClient(name string, cfg Config) (chunk.TableClient, error) { 299 // if name == shipper.BoltDBShipperType { 300 // name = "boltdb" 301 // cfg.FSConfig = chunk_local.FSConfig{Directory: cfg.BoltDBShipperConfig.ActiveIndexDirectory} 302 // } 303 // return storage.NewTableClient(name, cfg.Config, prometheus.DefaultRegisterer) 304 // } 305 306 // NewBucketClient makes a new bucket client based on the configuration. 307 func NewBucketClient(storageConfig Config) (index.BucketClient, error) { 308 if storageConfig.FSConfig.Directory != "" { 309 return local.NewFSObjectClient(storageConfig.FSConfig) 310 } 311 312 return nil, nil 313 } 314 315 type ClientMetrics struct { 316 AzureMetrics azure.BlobStorageMetrics 317 } 318 319 func NewClientMetrics() ClientMetrics { 320 return ClientMetrics{ 321 AzureMetrics: azure.NewBlobStorageMetrics(), 322 } 323 } 324 325 func (c *ClientMetrics) Unregister() { 326 c.AzureMetrics.Unregister() 327 } 328 329 // NewObjectClient makes a new StorageClient of the desired types. 330 func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (client.ObjectClient, error) { 331 switch name { 332 case config.StorageTypeAWS, config.StorageTypeS3: 333 return aws.NewS3ObjectClient(cfg.AWSStorageConfig.S3Config, cfg.Hedging) 334 case config.StorageTypeGCS: 335 return gcp.NewGCSObjectClient(context.Background(), cfg.GCSConfig, cfg.Hedging) 336 case config.StorageTypeAzure: 337 return azure.NewBlobStorage(&cfg.AzureStorageConfig, clientMetrics.AzureMetrics, cfg.Hedging) 338 case config.StorageTypeSwift: 339 return openstack.NewSwiftObjectClient(cfg.Swift, cfg.Hedging) 340 case config.StorageTypeInMemory: 341 return testutils.NewMockStorage(), nil 342 case config.StorageTypeFileSystem: 343 return local.NewFSObjectClient(cfg.FSConfig) 344 case config.StorageTypeBOS: 345 return baidubce.NewBOSObjectStorage(&cfg.BOSStorageConfig) 346 default: 347 return nil, fmt.Errorf("Unrecognized storage client %v, choose one of: %v, %v, %v, %v, %v", name, config.StorageTypeAWS, config.StorageTypeS3, config.StorageTypeGCS, config.StorageTypeAzure, config.StorageTypeFileSystem) 348 } 349 }