github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/gc/container_collector.go (about) 1 package gc 2 3 import ( 4 "context" 5 "time" 6 7 "code.cloudfoundry.org/lager" 8 "code.cloudfoundry.org/lager/lagerctx" 9 "github.com/pf-qiu/concourse/v6/atc/db" 10 "github.com/pf-qiu/concourse/v6/atc/metric" 11 "github.com/hashicorp/go-multierror" 12 ) 13 14 type containerCollector struct { 15 containerRepository db.ContainerRepository 16 missingContainerGracePeriod time.Duration 17 hijackContainerGracePeriod time.Duration 18 } 19 20 func NewContainerCollector( 21 containerRepository db.ContainerRepository, 22 missingContainerGracePeriod time.Duration, 23 hijackContainerGracePeriod time.Duration, 24 ) *containerCollector { 25 return &containerCollector{ 26 containerRepository: containerRepository, 27 missingContainerGracePeriod: missingContainerGracePeriod, 28 hijackContainerGracePeriod: hijackContainerGracePeriod, 29 } 30 } 31 32 func (c *containerCollector) Run(ctx context.Context) error { 33 logger := lagerctx.FromContext(ctx).Session("container-collector") 34 35 logger.Debug("start") 36 defer logger.Debug("done") 37 38 start := time.Now() 39 defer func() { 40 metric.ContainerCollectorDuration{ 41 Duration: time.Since(start), 42 }.Emit(logger) 43 }() 44 45 var errs error 46 47 err := c.cleanupOrphanedContainers(logger.Session("orphaned-containers")) 48 if err != nil { 49 errs = multierror.Append(errs, err) 50 logger.Error("failed-to-clean-up-orphaned-containers", err) 51 } 52 53 err = c.markFailedContainersAsDestroying(logger.Session("failed-containers")) 54 if err != nil { 55 errs = multierror.Append(errs, err) 56 logger.Error("failed-to-clean-up-failed-containers", err) 57 } 58 59 _, err = c.containerRepository.RemoveMissingContainers(c.missingContainerGracePeriod) 60 if err != nil { 61 errs = multierror.Append(errs, err) 62 logger.Error("failed-to-clean-up-missing-containers", err) 63 } 64 65 return errs 66 } 67 68 func (c *containerCollector) markFailedContainersAsDestroying(logger lager.Logger) error { 69 70 numFailedContainers, err := c.containerRepository.DestroyFailedContainers() 71 if err != nil { 72 logger.Error("failed-to-find-failed-containers-for-deletion", err) 73 return err 74 } 75 76 if numFailedContainers > 0 { 77 logger.Debug("found-failed-containers-for-deletion", lager.Data{ 78 "number": numFailedContainers, 79 }) 80 } 81 82 metric.FailedContainersToBeGarbageCollected{ 83 Containers: numFailedContainers, 84 }.Emit(logger) 85 86 return nil 87 } 88 89 func (c *containerCollector) cleanupOrphanedContainers(logger lager.Logger) error { 90 91 creatingContainers, createdContainers, destroyingContainers, err := c.containerRepository.FindOrphanedContainers() 92 if err != nil { 93 logger.Error("failed-to-get-orphaned-containers-for-deletion", err) 94 return err 95 } 96 97 if len(creatingContainers) > 0 || len(createdContainers) > 0 || len(destroyingContainers) > 0 { 98 logger.Debug("found-orphaned-containers-for-deletion", lager.Data{ 99 "creating-containers-num": len(creatingContainers), 100 "created-containers-num": len(createdContainers), 101 "destroying-containers-num": len(destroyingContainers), 102 }) 103 } 104 105 metric.CreatingContainersToBeGarbageCollected{ 106 Containers: len(creatingContainers), 107 }.Emit(logger) 108 109 metric.CreatedContainersToBeGarbageCollected{ 110 Containers: len(createdContainers), 111 }.Emit(logger) 112 113 metric.DestroyingContainersToBeGarbageCollected{ 114 Containers: len(destroyingContainers), 115 }.Emit(logger) 116 117 for _, createdContainer := range createdContainers { 118 119 if time.Since(createdContainer.LastHijack()) > c.hijackContainerGracePeriod { 120 _, err := createdContainer.Destroying() 121 if err != nil { 122 logger.Error("failed-to-transition", err, lager.Data{"container": createdContainer.Handle()}) 123 continue 124 } 125 } 126 } 127 128 return nil 129 }