github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/storage/track/gc.go (about)

     1  package track
     2  
     3  import (
     4  	"context"
     5  	"time"
     6  
     7  	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
     8  	"github.com/sirupsen/logrus"
     9  )
    10  
    11  // Deleter is used to delete data external to a tracker associated with a tracked object
    12  type Deleter interface {
    13  	Delete(ctx context.Context, id string) error
    14  }
    15  
    16  // DeleterMux returns a Deleter based on the id being deleted
    17  type DeleterMux func(string) Deleter
    18  
    19  // Delete implements Deleter
    20  func (dm DeleterMux) Delete(ctx context.Context, id string) error {
    21  	deleter := dm(id)
    22  	if deleter == nil {
    23  		return errors.Errorf("deleter mux does not have deleter for (%s)", id)
    24  	}
    25  	return deleter.Delete(ctx, id)
    26  }
    27  
    28  // GarbageCollector periodically runs garbage collection on tracker objects
    29  type GarbageCollector struct {
    30  	tracker Tracker
    31  	period  time.Duration
    32  	deleter Deleter
    33  }
    34  
    35  // NewGarbageCollector returns a garbage collector monitoring tracker, and kicking off a cycle every period.
    36  // It will use deleter to deleted associated data before deleting objects from the Tracker
    37  func NewGarbageCollector(tracker Tracker, period time.Duration, deleter Deleter) *GarbageCollector {
    38  	return &GarbageCollector{
    39  		tracker: tracker,
    40  		period:  period,
    41  		deleter: deleter,
    42  	}
    43  }
    44  
    45  // Run runs the gc loop, until the context is cancelled. It returns ErrContextCancell on exit.
    46  func (gc *GarbageCollector) Run(ctx context.Context) error {
    47  	ticker := time.NewTicker(gc.period)
    48  	defer ticker.Stop()
    49  	for {
    50  		if err := func() error {
    51  			ctx, cf := context.WithTimeout(ctx, gc.period/2)
    52  			defer cf()
    53  			return gc.runUntilEmpty(ctx)
    54  		}(); err != nil {
    55  			logrus.Errorf("gc: %v", err)
    56  		}
    57  		select {
    58  		case <-ctx.Done():
    59  			return ctx.Err()
    60  		case <-ticker.C:
    61  		}
    62  	}
    63  }
    64  
    65  func (gc *GarbageCollector) runUntilEmpty(ctx context.Context) error {
    66  	for {
    67  		n, err := gc.runOnce(ctx)
    68  		if err != nil {
    69  			return err
    70  		}
    71  		if n == 0 {
    72  			break
    73  		}
    74  	}
    75  	return nil
    76  }
    77  
    78  func (gc *GarbageCollector) runOnce(ctx context.Context) (int, error) {
    79  	var n int
    80  	err := gc.tracker.IterateDeletable(ctx, func(id string) error {
    81  		if err := gc.deleteObject(ctx, id); err != nil {
    82  			logrus.Errorf("error deleting object (%s): %v", id, err)
    83  		} else {
    84  			n++
    85  		}
    86  		return nil
    87  	})
    88  	return n, err
    89  }
    90  
    91  func (gc *GarbageCollector) deleteObject(ctx context.Context, id string) error {
    92  	if err := gc.tracker.MarkTombstone(ctx, id); err != nil {
    93  		return err
    94  	}
    95  	if err := gc.deleter.Delete(ctx, id); err != nil {
    96  		return err
    97  	}
    98  	return gc.tracker.FinishDelete(ctx, id)
    99  }