github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/storage/tsdb/bucketindex/markers.go (about)

     1  package bucketindex
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"path"
     7  	"path/filepath"
     8  	"strings"
     9  
    10  	"github.com/oklog/ulid"
    11  	"github.com/pkg/errors"
    12  	tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
    13  	"github.com/thanos-io/thanos/pkg/block"
    14  	"github.com/thanos-io/thanos/pkg/block/metadata"
    15  	"github.com/thanos-io/thanos/pkg/objstore"
    16  
    17  	"github.com/cortexproject/cortex/pkg/storage/bucket"
    18  )
    19  
    20  const (
    21  	MarkersPathname = "markers"
    22  )
    23  
    24  // BlockDeletionMarkFilepath returns the path, relative to the tenant's bucket location,
    25  // of a block deletion mark in the bucket markers location.
    26  func BlockDeletionMarkFilepath(blockID ulid.ULID) string {
    27  	return fmt.Sprintf("%s/%s-%s", MarkersPathname, blockID.String(), metadata.DeletionMarkFilename)
    28  }
    29  
    30  // IsBlockDeletionMarkFilename returns whether the input filename matches the expected pattern
    31  // of block deletion markers stored in the markers location.
    32  func IsBlockDeletionMarkFilename(name string) (ulid.ULID, bool) {
    33  	parts := strings.SplitN(name, "-", 2)
    34  	if len(parts) != 2 {
    35  		return ulid.ULID{}, false
    36  	}
    37  
    38  	// Ensure the 2nd part matches the block deletion mark filename.
    39  	if parts[1] != metadata.DeletionMarkFilename {
    40  		return ulid.ULID{}, false
    41  	}
    42  
    43  	// Ensure the 1st part is a valid block ID.
    44  	id, err := ulid.Parse(filepath.Base(parts[0]))
    45  	return id, err == nil
    46  }
    47  
    48  // MigrateBlockDeletionMarksToGlobalLocation list all tenant's blocks and, for each of them, look for
    49  // a deletion mark in the block location. Found deletion marks are copied to the global markers location.
    50  // The migration continues on error and returns once all blocks have been checked.
    51  func MigrateBlockDeletionMarksToGlobalLocation(ctx context.Context, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider) error {
    52  	bucket := bucket.NewUserBucketClient(userID, bkt, cfgProvider)
    53  	userBucket := bucket.WithExpectedErrs(bucket.IsObjNotFoundErr)
    54  
    55  	// Find all blocks in the storage.
    56  	var blocks []ulid.ULID
    57  	err := userBucket.Iter(ctx, "", func(name string) error {
    58  		if id, ok := block.IsBlockDir(name); ok {
    59  			blocks = append(blocks, id)
    60  		}
    61  		return nil
    62  	})
    63  	if err != nil {
    64  		return errors.Wrap(err, "list blocks")
    65  	}
    66  
    67  	errs := tsdb_errors.NewMulti()
    68  
    69  	for _, blockID := range blocks {
    70  		// Look up the deletion mark (if any).
    71  		reader, err := userBucket.Get(ctx, path.Join(blockID.String(), metadata.DeletionMarkFilename))
    72  		if userBucket.IsObjNotFoundErr(err) {
    73  			continue
    74  		} else if err != nil {
    75  			errs.Add(err)
    76  			continue
    77  		}
    78  
    79  		// Upload it to the global markers location.
    80  		uploadErr := userBucket.Upload(ctx, BlockDeletionMarkFilepath(blockID), reader)
    81  		if closeErr := reader.Close(); closeErr != nil {
    82  			errs.Add(closeErr)
    83  		}
    84  		if uploadErr != nil {
    85  			errs.Add(uploadErr)
    86  		}
    87  	}
    88  
    89  	return errs.Err()
    90  }