github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/planprocessor/heartbeat.go (about) 1 package planprocessor 2 3 import ( 4 "context" 5 "strconv" 6 "strings" 7 "time" 8 9 "github.com/go-kit/log" 10 "github.com/go-kit/log/level" 11 "github.com/grafana/dskit/services" 12 "github.com/pkg/errors" 13 "github.com/thanos-io/thanos/pkg/objstore" 14 15 "github.com/cortexproject/cortex/tools/blocksconvert" 16 ) 17 18 type heartbeat struct { 19 services.Service 20 21 log log.Logger 22 bucket objstore.Bucket 23 planFileBasename string 24 25 lastProgressFile string 26 } 27 28 func newHeartbeat(log log.Logger, bucket objstore.Bucket, interval time.Duration, planFileBasename, lastProgressFile string) *heartbeat { 29 hb := &heartbeat{ 30 log: log, 31 bucket: bucket, 32 planFileBasename: planFileBasename, 33 lastProgressFile: lastProgressFile, 34 } 35 36 hb.Service = services.NewTimerService(interval, hb.heartbeat, hb.heartbeat, hb.stopping) 37 return hb 38 } 39 40 func (hb *heartbeat) heartbeat(ctx context.Context) error { 41 if hb.lastProgressFile != "" { 42 ok, err := hb.bucket.Exists(ctx, hb.lastProgressFile) 43 if err != nil { 44 level.Warn(hb.log).Log("msg", "failed to check last progress file", "err", err) 45 return errors.Wrapf(err, "cannot check if progress file exists: %s", hb.lastProgressFile) 46 } 47 48 if !ok { 49 level.Warn(hb.log).Log("msg", "previous progress file doesn't exist") 50 return errors.Errorf("previous progress file doesn't exist: %s", hb.lastProgressFile) 51 } 52 } 53 54 now := time.Now() 55 newProgressFile := blocksconvert.ProgressFilename(hb.planFileBasename, now) 56 if newProgressFile == hb.lastProgressFile { 57 // when scheduler creates progress file, it can have the same timestamp. 58 return nil 59 } 60 61 if err := hb.bucket.Upload(ctx, newProgressFile, strings.NewReader(strconv.FormatInt(now.Unix(), 10))); err != nil { 62 return errors.Wrap(err, "failed to upload new progress file") 63 } 64 65 if hb.lastProgressFile != "" { 66 if err := hb.bucket.Delete(ctx, hb.lastProgressFile); err != nil { 67 return errors.Wrap(err, "failed to delete old progress file") 68 } 69 } 70 71 level.Info(hb.log).Log("msg", "updated progress", "file", newProgressFile) 72 hb.lastProgressFile = newProgressFile 73 return nil 74 } 75 76 func (hb *heartbeat) stopping(failure error) error { 77 // Only delete progress file if there was no failure until now. 78 if failure != nil { 79 return nil 80 } 81 82 level.Info(hb.log).Log("msg", "deleting last progress file", "file", hb.lastProgressFile) 83 84 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 85 defer cancel() 86 87 if hb.lastProgressFile != "" { 88 if err := hb.bucket.Delete(ctx, hb.lastProgressFile); err != nil { 89 return errors.Wrap(err, "failed to delete last progress file") 90 } 91 } 92 93 return nil 94 } 95 96 func (hb *heartbeat) String() string { 97 return "heartbeat" 98 }