github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/planprocessor/heartbeat.go (about)

     1  package planprocessor
     2  
     3  import (
     4  	"context"
     5  	"strconv"
     6  	"strings"
     7  	"time"
     8  
     9  	"github.com/go-kit/log"
    10  	"github.com/go-kit/log/level"
    11  	"github.com/grafana/dskit/services"
    12  	"github.com/pkg/errors"
    13  	"github.com/thanos-io/thanos/pkg/objstore"
    14  
    15  	"github.com/cortexproject/cortex/tools/blocksconvert"
    16  )
    17  
    18  type heartbeat struct {
    19  	services.Service
    20  
    21  	log              log.Logger
    22  	bucket           objstore.Bucket
    23  	planFileBasename string
    24  
    25  	lastProgressFile string
    26  }
    27  
    28  func newHeartbeat(log log.Logger, bucket objstore.Bucket, interval time.Duration, planFileBasename, lastProgressFile string) *heartbeat {
    29  	hb := &heartbeat{
    30  		log:              log,
    31  		bucket:           bucket,
    32  		planFileBasename: planFileBasename,
    33  		lastProgressFile: lastProgressFile,
    34  	}
    35  
    36  	hb.Service = services.NewTimerService(interval, hb.heartbeat, hb.heartbeat, hb.stopping)
    37  	return hb
    38  }
    39  
    40  func (hb *heartbeat) heartbeat(ctx context.Context) error {
    41  	if hb.lastProgressFile != "" {
    42  		ok, err := hb.bucket.Exists(ctx, hb.lastProgressFile)
    43  		if err != nil {
    44  			level.Warn(hb.log).Log("msg", "failed to check last progress file", "err", err)
    45  			return errors.Wrapf(err, "cannot check if progress file exists: %s", hb.lastProgressFile)
    46  		}
    47  
    48  		if !ok {
    49  			level.Warn(hb.log).Log("msg", "previous progress file doesn't exist")
    50  			return errors.Errorf("previous progress file doesn't exist: %s", hb.lastProgressFile)
    51  		}
    52  	}
    53  
    54  	now := time.Now()
    55  	newProgressFile := blocksconvert.ProgressFilename(hb.planFileBasename, now)
    56  	if newProgressFile == hb.lastProgressFile {
    57  		// when scheduler creates progress file, it can have the same timestamp.
    58  		return nil
    59  	}
    60  
    61  	if err := hb.bucket.Upload(ctx, newProgressFile, strings.NewReader(strconv.FormatInt(now.Unix(), 10))); err != nil {
    62  		return errors.Wrap(err, "failed to upload new progress file")
    63  	}
    64  
    65  	if hb.lastProgressFile != "" {
    66  		if err := hb.bucket.Delete(ctx, hb.lastProgressFile); err != nil {
    67  			return errors.Wrap(err, "failed to delete old progress file")
    68  		}
    69  	}
    70  
    71  	level.Info(hb.log).Log("msg", "updated progress", "file", newProgressFile)
    72  	hb.lastProgressFile = newProgressFile
    73  	return nil
    74  }
    75  
    76  func (hb *heartbeat) stopping(failure error) error {
    77  	// Only delete progress file if there was no failure until now.
    78  	if failure != nil {
    79  		return nil
    80  	}
    81  
    82  	level.Info(hb.log).Log("msg", "deleting last progress file", "file", hb.lastProgressFile)
    83  
    84  	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    85  	defer cancel()
    86  
    87  	if hb.lastProgressFile != "" {
    88  		if err := hb.bucket.Delete(ctx, hb.lastProgressFile); err != nil {
    89  			return errors.Wrap(err, "failed to delete last progress file")
    90  		}
    91  	}
    92  
    93  	return nil
    94  }
    95  
    96  func (hb *heartbeat) String() string {
    97  	return "heartbeat"
    98  }