github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/stores/series/index/schema_config.go (about)

     1  package index
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/prometheus/common/model"
     9  
    10  	"github.com/grafana/loki/pkg/storage/config"
    11  	"github.com/grafana/loki/pkg/util/math"
    12  )
    13  
    14  const (
    15  	secondsInDay      = int64(24 * time.Hour / time.Second)
    16  	millisecondsInDay = int64(24 * time.Hour / time.Millisecond)
    17  	v12               = "v12"
    18  )
    19  
    20  var (
    21  	errInvalidSchemaVersion = errors.New("invalid schema version")
    22  	errInvalidTablePeriod   = errors.New("the table period must be a multiple of 24h (1h for schema v1)")
    23  )
    24  
    25  // CreateSchema returns the schema defined by the PeriodConfig
    26  func CreateSchema(cfg config.PeriodConfig) (SeriesStoreSchema, error) {
    27  	buckets, bucketsPeriod := dailyBuckets(cfg), 24*time.Hour
    28  
    29  	// Ensure the tables period is a multiple of the bucket period
    30  	if cfg.IndexTables.Period > 0 && cfg.IndexTables.Period%bucketsPeriod != 0 {
    31  		return nil, errInvalidTablePeriod
    32  	}
    33  
    34  	if cfg.ChunkTables.Period > 0 && cfg.ChunkTables.Period%bucketsPeriod != 0 {
    35  		return nil, errInvalidTablePeriod
    36  	}
    37  
    38  	switch cfg.Schema {
    39  	case "v9":
    40  		return newSeriesStoreSchema(buckets, v9Entries{}), nil
    41  	case "v10", "v11", v12:
    42  		if cfg.RowShards == 0 {
    43  			return nil, fmt.Errorf("must have row_shards > 0 (current: %d) for schema (%s)", cfg.RowShards, cfg.Schema)
    44  		}
    45  
    46  		v10 := v10Entries{rowShards: cfg.RowShards}
    47  		if cfg.Schema == "v10" {
    48  			return newSeriesStoreSchema(buckets, v10), nil
    49  		} else if cfg.Schema == "v11" {
    50  			return newSeriesStoreSchema(buckets, v11Entries{v10}), nil
    51  		} else { // v12
    52  			return newSeriesStoreSchema(buckets, v12Entries{v11Entries{v10}}), nil
    53  		}
    54  	default:
    55  		return nil, errInvalidSchemaVersion
    56  	}
    57  }
    58  
    59  // Bucket describes a range of time with a tableName and hashKey
    60  type Bucket struct {
    61  	from       uint32
    62  	through    uint32
    63  	tableName  string
    64  	hashKey    string
    65  	bucketSize uint32 // helps with deletion of series ids in series store. Size in milliseconds.
    66  }
    67  
    68  func dailyBuckets(cfg config.PeriodConfig) schemaBucketsFunc {
    69  	return func(from, through model.Time, userID string) []Bucket {
    70  		var (
    71  			fromDay    = from.Unix() / secondsInDay
    72  			throughDay = through.Unix() / secondsInDay
    73  			result     = []Bucket{}
    74  		)
    75  
    76  		for i := fromDay; i <= throughDay; i++ {
    77  			// The idea here is that the hash key contains the bucket start time (rounded to
    78  			// the nearest day).  The range key can contain the offset from that, to the
    79  			// (start/end) of the chunk. For chunks that span multiple buckets, these
    80  			// offsets will be capped to the bucket boundaries, i.e. start will be
    81  			// positive in the first bucket, then zero in the next etc.
    82  			//
    83  			// The reason for doing all this is to reduce the size of the time stamps we
    84  			// include in the range keys - we use a uint32 - as we then have to base 32
    85  			// encode it.
    86  
    87  			relativeFrom := math.Max64(0, int64(from)-(i*millisecondsInDay))
    88  			relativeThrough := math.Min64(millisecondsInDay, int64(through)-(i*millisecondsInDay))
    89  			result = append(result, Bucket{
    90  				from:       uint32(relativeFrom),
    91  				through:    uint32(relativeThrough),
    92  				tableName:  cfg.IndexTables.TableFor(model.TimeFromUnix(i * secondsInDay)),
    93  				hashKey:    fmt.Sprintf("%s:d%d", userID, i),
    94  				bucketSize: uint32(millisecondsInDay), // helps with deletion of series ids in series store
    95  			})
    96  		}
    97  		return result
    98  	}
    99  }