github.com/thanos-io/thanos@v0.32.5/pkg/compact/clean.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package compact
     5  
     6  import (
     7  	"context"
     8  	"time"
     9  
    10  	"github.com/go-kit/log"
    11  	"github.com/go-kit/log/level"
    12  	"github.com/oklog/ulid"
    13  	"github.com/prometheus/client_golang/prometheus"
    14  	"github.com/thanos-io/objstore"
    15  
    16  	"github.com/thanos-io/thanos/pkg/block"
    17  )
    18  
    19  const (
    20  	// PartialUploadThresholdAge is a time after partial block is assumed aborted and ready to be cleaned.
    21  	// Keep it long as it is based on block creation time not upload start time.
    22  	PartialUploadThresholdAge = 2 * 24 * time.Hour
    23  )
    24  
    25  func BestEffortCleanAbortedPartialUploads(
    26  	ctx context.Context,
    27  	logger log.Logger,
    28  	partial map[ulid.ULID]error,
    29  	bkt objstore.Bucket,
    30  	deleteAttempts prometheus.Counter,
    31  	blockCleanups prometheus.Counter,
    32  	blockCleanupFailures prometheus.Counter,
    33  ) {
    34  	level.Info(logger).Log("msg", "started cleaning of aborted partial uploads")
    35  
    36  	// Delete partial blocks that are older than partialUploadThresholdAge.
    37  	// TODO(bwplotka): This is can cause data loss if blocks are:
    38  	// * being uploaded longer than partialUploadThresholdAge
    39  	// * being uploaded and started after their partialUploadThresholdAge
    40  	// can be assumed in this case. Keep partialUploadThresholdAge long for now.
    41  	// Mitigate this by adding ModifiedTime to bkt and check that instead of ULID (block creation time).
    42  	for id := range partial {
    43  		if ulid.Now()-id.Time() <= uint64(PartialUploadThresholdAge/time.Millisecond) {
    44  			// Minimum delay has not expired, ignore for now.
    45  			continue
    46  		}
    47  
    48  		deleteAttempts.Inc()
    49  		level.Info(logger).Log("msg", "found partially uploaded block; marking for deletion", "block", id)
    50  		// We don't gather any information about deletion marks for partial blocks, so let's simply remove it. We waited
    51  		// long PartialUploadThresholdAge already.
    52  		// TODO(bwplotka): Fix some edge cases: https://github.com/thanos-io/thanos/issues/2470 .
    53  		if err := block.Delete(ctx, logger, bkt, id); err != nil {
    54  			blockCleanupFailures.Inc()
    55  			level.Warn(logger).Log("msg", "failed to delete aborted partial upload; will retry in next iteration", "block", id, "thresholdAge", PartialUploadThresholdAge, "err", err)
    56  			continue
    57  		}
    58  		blockCleanups.Inc()
    59  		level.Info(logger).Log("msg", "deleted aborted partial upload", "block", id, "thresholdAge", PartialUploadThresholdAge)
    60  	}
    61  	level.Info(logger).Log("msg", "cleaning of aborted partial uploads done")
    62  }