github.com/thanos-io/thanos@v0.32.5/pkg/compact/retention_test.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package compact_test
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"encoding/json"
    10  	"path/filepath"
    11  	"strings"
    12  	"testing"
    13  	"time"
    14  
    15  	"github.com/go-kit/log"
    16  	"github.com/oklog/ulid"
    17  	"github.com/prometheus/client_golang/prometheus"
    18  	"github.com/prometheus/client_golang/prometheus/promauto"
    19  	promtest "github.com/prometheus/client_golang/prometheus/testutil"
    20  	"github.com/prometheus/prometheus/tsdb"
    21  	"github.com/thanos-io/objstore"
    22  
    23  	"github.com/efficientgo/core/testutil"
    24  	"github.com/thanos-io/thanos/pkg/block"
    25  	"github.com/thanos-io/thanos/pkg/block/metadata"
    26  	"github.com/thanos-io/thanos/pkg/compact"
    27  )
    28  
    29  func TestApplyRetentionPolicyByResolution(t *testing.T) {
    30  	type testBlock struct {
    31  		id         string
    32  		minTime    time.Time
    33  		maxTime    time.Time
    34  		resolution compact.ResolutionLevel
    35  	}
    36  
    37  	logger := log.NewNopLogger()
    38  	ctx := context.TODO()
    39  
    40  	for _, tt := range []struct {
    41  		name                  string
    42  		blocks                []testBlock
    43  		retentionByResolution map[compact.ResolutionLevel]time.Duration
    44  		want                  []string
    45  		wantErr               bool
    46  	}{
    47  		{
    48  			"empty bucket",
    49  			[]testBlock{},
    50  			map[compact.ResolutionLevel]time.Duration{
    51  				compact.ResolutionLevelRaw: 24 * time.Hour,
    52  				compact.ResolutionLevel5m:  7 * 24 * time.Hour,
    53  				compact.ResolutionLevel1h:  14 * 24 * time.Hour,
    54  			},
    55  			[]string{},
    56  			false,
    57  		},
    58  		{
    59  			"only raw retention",
    60  			[]testBlock{
    61  				{
    62  					"01CPHBEX20729MJQZXE3W0BW48",
    63  					time.Now().Add(-3 * 24 * time.Hour),
    64  					time.Now().Add(-2 * 24 * time.Hour),
    65  					compact.ResolutionLevelRaw,
    66  				},
    67  				{
    68  					"01CPHBEX20729MJQZXE3W0BW49",
    69  					time.Now().Add(-2 * 24 * time.Hour),
    70  					time.Now().Add(-24 * time.Hour),
    71  					compact.ResolutionLevel5m,
    72  				},
    73  				{
    74  					"01CPHBEX20729MJQZXE3W0BW50",
    75  					time.Now().Add(-24 * time.Hour),
    76  					time.Now().Add(-23 * time.Hour),
    77  					compact.ResolutionLevel1h,
    78  				},
    79  				{
    80  					"01CPHBEX20729MJQZXE3W0BW51",
    81  					time.Now().Add(-23 * time.Hour),
    82  					time.Now().Add(-6 * time.Hour),
    83  					compact.ResolutionLevelRaw,
    84  				},
    85  			},
    86  			map[compact.ResolutionLevel]time.Duration{
    87  				compact.ResolutionLevelRaw: 24 * time.Hour,
    88  				compact.ResolutionLevel5m:  0,
    89  				compact.ResolutionLevel1h:  0,
    90  			},
    91  			[]string{
    92  				"01CPHBEX20729MJQZXE3W0BW49/",
    93  				"01CPHBEX20729MJQZXE3W0BW50/",
    94  				"01CPHBEX20729MJQZXE3W0BW51/",
    95  			},
    96  			false,
    97  		},
    98  		{
    99  			"no retention",
   100  			[]testBlock{
   101  				{
   102  					"01CPHBEX20729MJQZXE3W0BW48",
   103  					time.Now().Add(-3 * 24 * time.Hour),
   104  					time.Now().Add(-2 * 24 * time.Hour),
   105  					compact.ResolutionLevelRaw,
   106  				},
   107  				{
   108  					"01CPHBEX20729MJQZXE3W0BW49",
   109  					time.Now().Add(-2 * 24 * time.Hour),
   110  					time.Now().Add(-24 * time.Hour),
   111  					compact.ResolutionLevel5m,
   112  				},
   113  				{
   114  					"01CPHBEX20729MJQZXE3W0BW50",
   115  					time.Now().Add(-24 * time.Hour),
   116  					time.Now().Add(-23 * time.Hour),
   117  					compact.ResolutionLevel1h,
   118  				},
   119  				{
   120  					"01CPHBEX20729MJQZXE3W0BW51",
   121  					time.Now().Add(-23 * time.Hour),
   122  					time.Now().Add(-6 * time.Hour),
   123  					compact.ResolutionLevelRaw,
   124  				},
   125  			},
   126  			map[compact.ResolutionLevel]time.Duration{
   127  				compact.ResolutionLevelRaw: 0,
   128  				compact.ResolutionLevel5m:  0,
   129  				compact.ResolutionLevel1h:  0,
   130  			},
   131  			[]string{
   132  				"01CPHBEX20729MJQZXE3W0BW48/",
   133  				"01CPHBEX20729MJQZXE3W0BW49/",
   134  				"01CPHBEX20729MJQZXE3W0BW50/",
   135  				"01CPHBEX20729MJQZXE3W0BW51/",
   136  			},
   137  			false,
   138  		},
   139  		{
   140  			"no retention 1900",
   141  			[]testBlock{
   142  				{
   143  					"01CPHBEX20729MJQZXE3W0BW48",
   144  					time.Date(1900, 1, 1, 1, 0, 0, 0, time.Local),
   145  					time.Date(1900, 1, 1, 2, 0, 0, 0, time.Local),
   146  					compact.ResolutionLevelRaw,
   147  				},
   148  				{
   149  					"01CPHBEX20729MJQZXE3W0BW49",
   150  					time.Date(1900, 1, 1, 1, 0, 0, 0, time.Local),
   151  					time.Date(1900, 1, 1, 2, 0, 0, 0, time.Local),
   152  					compact.ResolutionLevel5m,
   153  				},
   154  				{
   155  					"01CPHBEX20729MJQZXE3W0BW50",
   156  					time.Date(1900, 1, 1, 1, 0, 0, 0, time.Local),
   157  					time.Date(1900, 1, 1, 2, 0, 0, 0, time.Local),
   158  					compact.ResolutionLevel1h,
   159  				},
   160  			},
   161  			map[compact.ResolutionLevel]time.Duration{
   162  				compact.ResolutionLevelRaw: 0,
   163  				compact.ResolutionLevel5m:  0,
   164  				compact.ResolutionLevel1h:  0,
   165  			},
   166  			[]string{
   167  				"01CPHBEX20729MJQZXE3W0BW48/",
   168  				"01CPHBEX20729MJQZXE3W0BW49/",
   169  				"01CPHBEX20729MJQZXE3W0BW50/",
   170  			},
   171  			false,
   172  		},
   173  		{
   174  			"unknown resolution",
   175  			[]testBlock{
   176  				{
   177  					"01CPHBEX20729MJQZXE3W0BW48",
   178  					time.Now().Add(-3 * 24 * time.Hour),
   179  					time.Now().Add(-2 * 24 * time.Hour),
   180  					compact.ResolutionLevel(1),
   181  				},
   182  			},
   183  			map[compact.ResolutionLevel]time.Duration{},
   184  			[]string{
   185  				"01CPHBEX20729MJQZXE3W0BW48/",
   186  			},
   187  			false,
   188  		},
   189  		{
   190  			"every retention deletes",
   191  			[]testBlock{
   192  				{
   193  					"01CPHBEX20729MJQZXE3W0BW40",
   194  					time.Now().Add(-1 * 24 * time.Hour),
   195  					time.Now().Add(-0 * 24 * time.Hour),
   196  					compact.ResolutionLevelRaw,
   197  				},
   198  				{
   199  					"01CPHBEX20729MJQZXE3W0BW41",
   200  					time.Now().Add(-2 * 24 * time.Hour),
   201  					time.Now().Add(-1 * 24 * time.Hour),
   202  					compact.ResolutionLevelRaw,
   203  				},
   204  				{
   205  					"01CPHBEX20729MJQZXE3W0BW42",
   206  					time.Now().Add(-7 * 24 * time.Hour),
   207  					time.Now().Add(-6 * 24 * time.Hour),
   208  					compact.ResolutionLevel5m,
   209  				},
   210  				{
   211  					"01CPHBEX20729MJQZXE3W0BW43",
   212  					time.Now().Add(-8 * 24 * time.Hour),
   213  					time.Now().Add(-7 * 24 * time.Hour),
   214  					compact.ResolutionLevel5m,
   215  				},
   216  				{
   217  					"01CPHBEX20729MJQZXE3W0BW44",
   218  					time.Now().Add(-14 * 24 * time.Hour),
   219  					time.Now().Add(-13 * 24 * time.Hour),
   220  					compact.ResolutionLevel1h,
   221  				},
   222  				{
   223  					"01CPHBEX20729MJQZXE3W0BW45",
   224  					time.Now().Add(-15 * 24 * time.Hour),
   225  					time.Now().Add(-14 * 24 * time.Hour),
   226  					compact.ResolutionLevel1h,
   227  				},
   228  			},
   229  			map[compact.ResolutionLevel]time.Duration{
   230  				compact.ResolutionLevelRaw: 24 * time.Hour,
   231  				compact.ResolutionLevel5m:  7 * 24 * time.Hour,
   232  				compact.ResolutionLevel1h:  14 * 24 * time.Hour,
   233  			},
   234  			[]string{
   235  				"01CPHBEX20729MJQZXE3W0BW40/",
   236  				"01CPHBEX20729MJQZXE3W0BW42/",
   237  				"01CPHBEX20729MJQZXE3W0BW44/",
   238  			},
   239  			false,
   240  		},
   241  	} {
   242  		t.Run(tt.name, func(t *testing.T) {
   243  			bkt := objstore.WithNoopInstr(objstore.NewInMemBucket())
   244  			for _, b := range tt.blocks {
   245  				uploadMockBlock(t, bkt, b.id, b.minTime, b.maxTime, int64(b.resolution))
   246  			}
   247  
   248  			metaFetcher, err := block.NewMetaFetcher(logger, 32, bkt, "", nil, nil)
   249  			testutil.Ok(t, err)
   250  
   251  			blocksMarkedForDeletion := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
   252  
   253  			metas, _, err := metaFetcher.Fetch(ctx)
   254  			testutil.Ok(t, err)
   255  
   256  			if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, metas, tt.retentionByResolution, blocksMarkedForDeletion); (err != nil) != tt.wantErr {
   257  				t.Errorf("ApplyRetentionPolicyByResolution() error = %v, wantErr %v", err, tt.wantErr)
   258  			}
   259  
   260  			got := []string{}
   261  			gotMarkedBlocksCount := 0.0
   262  			testutil.Ok(t, bkt.Iter(context.TODO(), "", func(name string) error {
   263  				exists, err := bkt.Exists(ctx, filepath.Join(name, metadata.DeletionMarkFilename))
   264  				if err != nil {
   265  					return err
   266  				}
   267  				if !exists {
   268  					got = append(got, name)
   269  					return nil
   270  				}
   271  				gotMarkedBlocksCount += 1.0
   272  				return nil
   273  			}))
   274  
   275  			testutil.Equals(t, got, tt.want)
   276  			testutil.Equals(t, gotMarkedBlocksCount, promtest.ToFloat64(blocksMarkedForDeletion))
   277  		})
   278  	}
   279  }
   280  
   281  func uploadMockBlock(t *testing.T, bkt objstore.Bucket, id string, minTime, maxTime time.Time, resolutionLevel int64) {
   282  	t.Helper()
   283  	meta1 := metadata.Meta{
   284  		BlockMeta: tsdb.BlockMeta{
   285  			ULID:    ulid.MustParse(id),
   286  			MinTime: minTime.Unix() * 1000,
   287  			MaxTime: maxTime.Unix() * 1000,
   288  			Version: 1,
   289  		},
   290  		Thanos: metadata.Thanos{
   291  			Downsample: metadata.ThanosDownsample{
   292  				Resolution: resolutionLevel,
   293  			},
   294  		},
   295  	}
   296  
   297  	b, err := json.Marshal(meta1)
   298  	testutil.Ok(t, err)
   299  
   300  	testutil.Ok(t, bkt.Upload(context.Background(), id+"/meta.json", bytes.NewReader(b)))
   301  	testutil.Ok(t, bkt.Upload(context.Background(), id+"/chunks/000001", strings.NewReader("@test-data@")))
   302  	testutil.Ok(t, bkt.Upload(context.Background(), id+"/chunks/000002", strings.NewReader("@test-data@")))
   303  	testutil.Ok(t, bkt.Upload(context.Background(), id+"/chunks/000003", strings.NewReader("@test-data@")))
   304  }