github.com/thanos-io/thanos@v0.32.5/pkg/compact/clean_test.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package compact
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"encoding/json"
    10  	"path"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/go-kit/log"
    15  	"github.com/oklog/ulid"
    16  	"github.com/prometheus/client_golang/prometheus"
    17  	"github.com/prometheus/client_golang/prometheus/promauto"
    18  	promtest "github.com/prometheus/client_golang/prometheus/testutil"
    19  	"github.com/thanos-io/objstore"
    20  
    21  	"github.com/efficientgo/core/testutil"
    22  	"github.com/thanos-io/thanos/pkg/block"
    23  	"github.com/thanos-io/thanos/pkg/block/metadata"
    24  )
    25  
    26  func TestBestEffortCleanAbortedPartialUploads(t *testing.T) {
    27  	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    28  	defer cancel()
    29  
    30  	bkt := objstore.WithNoopInstr(objstore.NewInMemBucket())
    31  	logger := log.NewNopLogger()
    32  
    33  	metaFetcher, err := block.NewMetaFetcher(nil, 32, bkt, "", nil, nil)
    34  	testutil.Ok(t, err)
    35  
    36  	// 1. No meta, old block, should be removed.
    37  	shouldDeleteID, err := ulid.New(uint64(time.Now().Add(-PartialUploadThresholdAge-1*time.Hour).Unix()*1000), nil)
    38  	testutil.Ok(t, err)
    39  
    40  	var fakeChunk bytes.Buffer
    41  	fakeChunk.Write([]byte{0, 1, 2, 3})
    42  	testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldDeleteID.String(), "chunks", "000001"), &fakeChunk))
    43  
    44  	// 2.  Old block with meta, so should be kept.
    45  	shouldIgnoreID1, err := ulid.New(uint64(time.Now().Add(-PartialUploadThresholdAge-2*time.Hour).Unix()*1000), nil)
    46  	testutil.Ok(t, err)
    47  	var meta metadata.Meta
    48  	meta.Version = 1
    49  	meta.ULID = shouldIgnoreID1
    50  
    51  	var buf bytes.Buffer
    52  	testutil.Ok(t, json.NewEncoder(&buf).Encode(&meta))
    53  	testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID1.String(), metadata.MetaFilename), &buf))
    54  	testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID1.String(), "chunks", "000001"), &fakeChunk))
    55  
    56  	// 3. No meta, newer block that should be kept.
    57  	shouldIgnoreID2, err := ulid.New(uint64(time.Now().Add(-2*time.Hour).Unix()*1000), nil)
    58  	testutil.Ok(t, err)
    59  
    60  	testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID2.String(), "chunks", "000001"), &fakeChunk))
    61  
    62  	deleteAttempts := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
    63  	blockCleanups := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
    64  	blockCleanupFailures := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
    65  	_, partial, err := metaFetcher.Fetch(ctx)
    66  	testutil.Ok(t, err)
    67  
    68  	BestEffortCleanAbortedPartialUploads(ctx, logger, partial, bkt, deleteAttempts, blockCleanups, blockCleanupFailures)
    69  	testutil.Equals(t, 1.0, promtest.ToFloat64(deleteAttempts))
    70  	testutil.Equals(t, 1.0, promtest.ToFloat64(blockCleanups))
    71  	testutil.Equals(t, 0.0, promtest.ToFloat64(blockCleanupFailures))
    72  
    73  	exists, err := bkt.Exists(ctx, path.Join(shouldDeleteID.String(), "chunks", "000001"))
    74  	testutil.Ok(t, err)
    75  	testutil.Equals(t, false, exists)
    76  
    77  	exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreID1.String(), "chunks", "000001"))
    78  	testutil.Ok(t, err)
    79  	testutil.Equals(t, true, exists)
    80  
    81  	exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreID2.String(), "chunks", "000001"))
    82  	testutil.Ok(t, err)
    83  	testutil.Equals(t, true, exists)
    84  }