github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/gc/container_collector.go (about)

     1  package gc
     2  
     3  import (
     4  	"context"
     5  	"time"
     6  
     7  	"code.cloudfoundry.org/lager"
     8  	"code.cloudfoundry.org/lager/lagerctx"
     9  	"github.com/pf-qiu/concourse/v6/atc/db"
    10  	"github.com/pf-qiu/concourse/v6/atc/metric"
    11  	"github.com/hashicorp/go-multierror"
    12  )
    13  
    14  type containerCollector struct {
    15  	containerRepository         db.ContainerRepository
    16  	missingContainerGracePeriod time.Duration
    17  	hijackContainerGracePeriod  time.Duration
    18  }
    19  
    20  func NewContainerCollector(
    21  	containerRepository db.ContainerRepository,
    22  	missingContainerGracePeriod time.Duration,
    23  	hijackContainerGracePeriod time.Duration,
    24  ) *containerCollector {
    25  	return &containerCollector{
    26  		containerRepository:         containerRepository,
    27  		missingContainerGracePeriod: missingContainerGracePeriod,
    28  		hijackContainerGracePeriod:  hijackContainerGracePeriod,
    29  	}
    30  }
    31  
    32  func (c *containerCollector) Run(ctx context.Context) error {
    33  	logger := lagerctx.FromContext(ctx).Session("container-collector")
    34  
    35  	logger.Debug("start")
    36  	defer logger.Debug("done")
    37  
    38  	start := time.Now()
    39  	defer func() {
    40  		metric.ContainerCollectorDuration{
    41  			Duration: time.Since(start),
    42  		}.Emit(logger)
    43  	}()
    44  
    45  	var errs error
    46  
    47  	err := c.cleanupOrphanedContainers(logger.Session("orphaned-containers"))
    48  	if err != nil {
    49  		errs = multierror.Append(errs, err)
    50  		logger.Error("failed-to-clean-up-orphaned-containers", err)
    51  	}
    52  
    53  	err = c.markFailedContainersAsDestroying(logger.Session("failed-containers"))
    54  	if err != nil {
    55  		errs = multierror.Append(errs, err)
    56  		logger.Error("failed-to-clean-up-failed-containers", err)
    57  	}
    58  
    59  	_, err = c.containerRepository.RemoveMissingContainers(c.missingContainerGracePeriod)
    60  	if err != nil {
    61  		errs = multierror.Append(errs, err)
    62  		logger.Error("failed-to-clean-up-missing-containers", err)
    63  	}
    64  
    65  	return errs
    66  }
    67  
    68  func (c *containerCollector) markFailedContainersAsDestroying(logger lager.Logger) error {
    69  
    70  	numFailedContainers, err := c.containerRepository.DestroyFailedContainers()
    71  	if err != nil {
    72  		logger.Error("failed-to-find-failed-containers-for-deletion", err)
    73  		return err
    74  	}
    75  
    76  	if numFailedContainers > 0 {
    77  		logger.Debug("found-failed-containers-for-deletion", lager.Data{
    78  			"number": numFailedContainers,
    79  		})
    80  	}
    81  
    82  	metric.FailedContainersToBeGarbageCollected{
    83  		Containers: numFailedContainers,
    84  	}.Emit(logger)
    85  
    86  	return nil
    87  }
    88  
    89  func (c *containerCollector) cleanupOrphanedContainers(logger lager.Logger) error {
    90  
    91  	creatingContainers, createdContainers, destroyingContainers, err := c.containerRepository.FindOrphanedContainers()
    92  	if err != nil {
    93  		logger.Error("failed-to-get-orphaned-containers-for-deletion", err)
    94  		return err
    95  	}
    96  
    97  	if len(creatingContainers) > 0 || len(createdContainers) > 0 || len(destroyingContainers) > 0 {
    98  		logger.Debug("found-orphaned-containers-for-deletion", lager.Data{
    99  			"creating-containers-num":   len(creatingContainers),
   100  			"created-containers-num":    len(createdContainers),
   101  			"destroying-containers-num": len(destroyingContainers),
   102  		})
   103  	}
   104  
   105  	metric.CreatingContainersToBeGarbageCollected{
   106  		Containers: len(creatingContainers),
   107  	}.Emit(logger)
   108  
   109  	metric.CreatedContainersToBeGarbageCollected{
   110  		Containers: len(createdContainers),
   111  	}.Emit(logger)
   112  
   113  	metric.DestroyingContainersToBeGarbageCollected{
   114  		Containers: len(destroyingContainers),
   115  	}.Emit(logger)
   116  
   117  	for _, createdContainer := range createdContainers {
   118  
   119  		if time.Since(createdContainer.LastHijack()) > c.hijackContainerGracePeriod {
   120  			_, err := createdContainer.Destroying()
   121  			if err != nil {
   122  				logger.Error("failed-to-transition", err, lager.Data{"container": createdContainer.Handle()})
   123  				continue
   124  			}
   125  		}
   126  	}
   127  
   128  	return nil
   129  }