github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/engine/dockerprune/docker_pruner.go (about)

     1  package dockerprune
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sort"
     7  	"strings"
     8  	"time"
     9  
    10  	"github.com/docker/go-units"
    11  
    12  	"github.com/docker/docker/api/types"
    13  	"github.com/docker/docker/api/types/filters"
    14  
    15  	"github.com/tilt-dev/tilt/internal/container"
    16  
    17  	"github.com/tilt-dev/tilt/pkg/model"
    18  
    19  	"github.com/tilt-dev/tilt/internal/engine/buildcontrol"
    20  
    21  	"github.com/tilt-dev/tilt/internal/docker"
    22  	"github.com/tilt-dev/tilt/internal/sliceutils"
    23  	"github.com/tilt-dev/tilt/internal/store"
    24  	"github.com/tilt-dev/tilt/pkg/logger"
    25  )
    26  
    27  var gcEnabledSelector = fmt.Sprintf("%s=true", docker.GCEnabledLabel)
    28  
    29  type DockerPruner struct {
    30  	dCli docker.Client
    31  
    32  	disabledForTesting bool
    33  	disabledOnSetup    bool
    34  
    35  	lastPruneBuildCount int
    36  	lastPruneTime       time.Time
    37  }
    38  
    39  var _ store.Subscriber = &DockerPruner{}
    40  var _ store.SetUpper = &DockerPruner{}
    41  
    42  func NewDockerPruner(dCli docker.Client) *DockerPruner {
    43  	return &DockerPruner{dCli: dCli}
    44  }
    45  
    46  func (dp *DockerPruner) DisabledForTesting(disabled bool) {
    47  	dp.disabledForTesting = disabled
    48  }
    49  
    50  func (dp *DockerPruner) SetUp(ctx context.Context, _ store.RStore) error {
    51  	err := dp.dCli.CheckConnected()
    52  	if err != nil {
    53  		// If Docker is not responding at all, other parts of the system will log this.
    54  		dp.disabledOnSetup = true
    55  		return nil
    56  	}
    57  
    58  	if err := dp.sufficientVersionError(); err != nil {
    59  		logger.Get(ctx).Infof(
    60  			"[Docker Prune] Docker API version too low for Tilt to run Docker Prune:\n\t%v", err,
    61  		)
    62  		dp.disabledOnSetup = true
    63  		return nil
    64  	}
    65  	return nil
    66  }
    67  
    68  // OnChange determines if any Tilt-built Docker images should be pruned based on settings and invokes the pruning
    69  // process if necessary.
    70  //
    71  // Care should be taken when modifying this method to not introduce expensive operations unless necessary, as this
    72  // is invoked for EVERY store action change batch. Because of this, the store (un)locking is done somewhat manually,
    73  // so care must be taken to avoid locking issues.
    74  func (dp *DockerPruner) OnChange(ctx context.Context, st store.RStore, summary store.ChangeSummary) error {
    75  	if dp.disabledForTesting || dp.disabledOnSetup || summary.IsLogOnly() {
    76  		return nil
    77  	}
    78  
    79  	state := st.RLockState()
    80  	settings := state.DockerPruneSettings
    81  	// Exit early if possible if any of the following is true:
    82  	// 	* Pruning is disabled entirely
    83  	// 	* Engine is currently building something
    84  	// 	* There are NO `docker_build`s in the Tiltfile
    85  	// 	* Something is queued for building
    86  	if !settings.Enabled || len(state.CurrentBuildSet) > 0 || !state.HasBuild() || buildcontrol.NextManifestNameToBuild(state) != "" {
    87  		st.RUnlockState()
    88  		return nil
    89  	}
    90  
    91  	// Prune as soon after startup as we can (waiting until we've built SOMETHING)
    92  	curBuildCount := state.CompletedBuildCount
    93  	shouldPrune := dp.lastPruneTime.IsZero() && curBuildCount > 0
    94  	// "Prune every X builds" takes precedence over "prune every Y hours"
    95  	if settings.NumBuilds != 0 {
    96  		buildsSince := curBuildCount - dp.lastPruneBuildCount
    97  		if buildsSince >= settings.NumBuilds {
    98  			shouldPrune = true
    99  		}
   100  	} else {
   101  		interval := settings.Interval
   102  		if interval == 0 {
   103  			interval = model.DockerPruneDefaultInterval
   104  		}
   105  		if time.Since(dp.lastPruneTime) >= interval {
   106  			shouldPrune = true
   107  		}
   108  	}
   109  
   110  	if shouldPrune {
   111  		// N.B. Only determine the ref selectors if we're actually going to prune - OnChange is called for every batch
   112  		// 	of store events and this is a comparatively expensive operation (lots of regex), but 99% of the time this
   113  		// 	is called, no pruning is going to happen, so avoid burning CPU cycles unnecessarily
   114  		imgSelectors := model.LocalRefSelectorsForManifests(state.Manifests(), state.Clusters)
   115  		st.RUnlockState()
   116  		dp.PruneAndRecordState(ctx, settings.MaxAge, settings.KeepRecent, imgSelectors, curBuildCount)
   117  		return nil
   118  	}
   119  
   120  	st.RUnlockState()
   121  	return nil
   122  }
   123  
   124  func (dp *DockerPruner) PruneAndRecordState(ctx context.Context, maxAge time.Duration, keepRecent int, imgSelectors []container.RefSelector, curBuildCount int) {
   125  	dp.Prune(ctx, maxAge, keepRecent, imgSelectors)
   126  	dp.lastPruneTime = time.Now()
   127  	dp.lastPruneBuildCount = curBuildCount
   128  }
   129  
   130  func (dp *DockerPruner) Prune(ctx context.Context, maxAge time.Duration, keepRecent int, imgSelectors []container.RefSelector) {
   131  	// For future: dispatch event with output/errors to be recorded
   132  	//   in engineState.TiltSystemState on store (analogous to TiltfileState)
   133  	err := dp.prune(ctx, maxAge, keepRecent, imgSelectors)
   134  	if err != nil {
   135  		logger.Get(ctx).Infof("[Docker Prune] error running docker prune: %v", err)
   136  	}
   137  }
   138  
   139  func (dp *DockerPruner) prune(ctx context.Context, maxAge time.Duration, keepRecent int, imgSelectors []container.RefSelector) error {
   140  	l := logger.Get(ctx)
   141  	if err := dp.sufficientVersionError(); err != nil {
   142  		l.Debugf("[Docker Prune] skipping Docker prune, Docker API version too low:\t%v", err)
   143  		return nil
   144  	}
   145  
   146  	f := filters.NewArgs(
   147  		filters.Arg("label", gcEnabledSelector),
   148  		filters.Arg("until", maxAge.String()),
   149  	)
   150  
   151  	// PRUNE CONTAINERS
   152  	containerReport, err := dp.dCli.ContainersPrune(ctx, f)
   153  	if err != nil {
   154  		return err
   155  	}
   156  	prettyPrintContainersPruneReport(containerReport, l)
   157  
   158  	// PRUNE IMAGES
   159  	imageReport, err := dp.deleteOldImages(ctx, maxAge, keepRecent, imgSelectors)
   160  	if err != nil {
   161  		return err
   162  	}
   163  	prettyPrintImagesPruneReport(imageReport, l)
   164  
   165  	// PRUNE BUILD CACHE
   166  	opts := types.BuildCachePruneOptions{Filters: f}
   167  	cacheReport, err := dp.dCli.BuildCachePrune(ctx, opts)
   168  	if err != nil {
   169  		if !strings.Contains(err.Error(), `"build prune" requires API version`) {
   170  			return err
   171  		}
   172  		l.Debugf("[Docker Prune] skipping build cache prune, Docker API version too low:\t%s", err)
   173  	} else {
   174  		prettyPrintCachePruneReport(cacheReport, l)
   175  	}
   176  
   177  	return nil
   178  }
   179  
   180  func (dp *DockerPruner) inspectImages(ctx context.Context, imgs []types.ImageSummary) []types.ImageInspect {
   181  	result := []types.ImageInspect{}
   182  	for _, imgSummary := range imgs {
   183  		inspect, _, err := dp.dCli.ImageInspectWithRaw(ctx, imgSummary.ID)
   184  		if err != nil {
   185  			logger.Get(ctx).Debugf("[Docker Prune] error inspecting image '%s': %v", imgSummary.ID, err)
   186  			continue
   187  		}
   188  		result = append(result, inspect)
   189  	}
   190  	return result
   191  }
   192  
   193  // Return all image objects that exceed the max age threshold.
   194  func (dp *DockerPruner) filterImageInspectsByMaxAge(ctx context.Context, inspects []types.ImageInspect, maxAge time.Duration, selectors []container.RefSelector) []types.ImageInspect {
   195  	result := []types.ImageInspect{}
   196  	for _, inspect := range inspects {
   197  		namedRefs, err := container.ParseNamedMulti(inspect.RepoTags)
   198  		if err != nil {
   199  			logger.Get(ctx).Debugf("[Docker Prune] error parsing repo tags for '%s': %v", inspect.ID, err)
   200  			continue
   201  		}
   202  
   203  		// LastTagTime indicates the last time the image was built, which is more
   204  		// meaningful to us than when the image was created.
   205  		if time.Since(inspect.Metadata.LastTagTime) >= maxAge && container.AnyMatch(namedRefs, selectors) {
   206  			if len(inspect.RepoTags) > 1 {
   207  				logger.Get(ctx).Debugf("[Docker Prune] cannot prune image %s (tags: %s); `docker image remove --force` "+
   208  					"required to remove an image with multiple tags (Docker throws error: "+
   209  					"\"image is referenced in one or more repositories\")",
   210  					inspect.ID, strings.Join(inspect.RepoTags, ", "))
   211  				continue
   212  			}
   213  			result = append(result, inspect)
   214  		}
   215  	}
   216  	return result
   217  }
   218  
   219  // Return all image objects that aren't in the N
   220  // most recently used for each tag.
   221  func (dp *DockerPruner) filterOutMostRecentInspects(ctx context.Context, inspects []types.ImageInspect, keepRecent int, selectors []container.RefSelector) []types.ImageInspect {
   222  	// First, sort the images in order from most recent to least recent.
   223  	recentFirst := append([]types.ImageInspect{}, inspects...)
   224  	sort.SliceStable(recentFirst, func(i, j int) bool {
   225  		// LastTagTime indicates the last time the image was built, which is more
   226  		// meaningful to us than when the image was created.
   227  		return recentFirst[i].Metadata.LastTagTime.After(recentFirst[j].Metadata.LastTagTime)
   228  	})
   229  
   230  	// Next, aggregate the images by which selector they match.
   231  	imgsBySelector := make(map[container.RefSelector][]types.ImageInspect)
   232  	for _, inspect := range recentFirst {
   233  		namedRefs, err := container.ParseNamedMulti(inspect.RepoTags)
   234  		if err != nil {
   235  			logger.Get(ctx).Debugf("[Docker Prune] error parsing repo tags for '%s': %v", inspect.ID, err)
   236  			continue
   237  		}
   238  
   239  		for _, sel := range selectors {
   240  			if sel.MatchesAny(namedRefs) {
   241  				imgsBySelector[sel] = append(imgsBySelector[sel], inspect)
   242  				break
   243  			}
   244  		}
   245  	}
   246  
   247  	// Finally, keep the N most recent for each tag.
   248  	idsToKeep := make(map[string]bool)
   249  	for _, list := range imgsBySelector {
   250  		for i := 0; i < keepRecent && i < len(list); i++ {
   251  			idsToKeep[list[i].ID] = true
   252  		}
   253  	}
   254  
   255  	result := []types.ImageInspect{}
   256  	for _, inspect := range inspects {
   257  		if !idsToKeep[inspect.ID] {
   258  			result = append(result, inspect)
   259  		}
   260  	}
   261  	return result
   262  }
   263  
   264  func (dp *DockerPruner) deleteOldImages(ctx context.Context, maxAge time.Duration, keepRecent int, selectors []container.RefSelector) (types.ImagesPruneReport, error) {
   265  	opts := types.ImageListOptions{
   266  		Filters: filters.NewArgs(
   267  			filters.Arg("label", gcEnabledSelector),
   268  		),
   269  	}
   270  	imgs, err := dp.dCli.ImageList(ctx, opts)
   271  	if err != nil {
   272  		return types.ImagesPruneReport{}, err
   273  	}
   274  
   275  	inspects := dp.inspectImages(ctx, imgs)
   276  	inspects = dp.filterImageInspectsByMaxAge(ctx, inspects, maxAge, selectors)
   277  	toDelete := dp.filterOutMostRecentInspects(ctx, inspects, keepRecent, selectors)
   278  
   279  	rmOpts := types.ImageRemoveOptions{PruneChildren: true}
   280  	var responseItems []types.ImageDeleteResponseItem
   281  	var reclaimedBytes uint64
   282  
   283  	for _, inspect := range toDelete {
   284  		items, err := dp.dCli.ImageRemove(ctx, inspect.ID, rmOpts)
   285  		if err != nil {
   286  			// No good way to detect in-use images from `inspect` output, so just ignore those errors
   287  			if !strings.Contains(err.Error(), "image is being used by running container") {
   288  				logger.Get(ctx).Debugf("[Docker Prune] error removing image '%s': %v", inspect.ID, err)
   289  			}
   290  			continue
   291  		}
   292  		responseItems = append(responseItems, items...)
   293  		reclaimedBytes += uint64(inspect.Size)
   294  	}
   295  
   296  	return types.ImagesPruneReport{
   297  		ImagesDeleted:  responseItems,
   298  		SpaceReclaimed: reclaimedBytes,
   299  	}, nil
   300  }
   301  
   302  func (dp *DockerPruner) sufficientVersionError() error {
   303  	return dp.dCli.NewVersionError("1.30", "image | container prune with filter: label")
   304  }
   305  
   306  func prettyPrintImagesPruneReport(report types.ImagesPruneReport, l logger.Logger) {
   307  	if len(report.ImagesDeleted) == 0 && !l.Level().ShouldDisplay(logger.VerboseLvl) {
   308  		return
   309  	}
   310  
   311  	l.Infof("[Docker Prune] removed %d images, reclaimed %s",
   312  		len(report.ImagesDeleted), humanSize(report.SpaceReclaimed))
   313  	if len(report.ImagesDeleted) > 0 {
   314  		for _, img := range report.ImagesDeleted {
   315  			l.Debugf("\t- %s", prettyStringImgDeleteItem(img))
   316  		}
   317  	}
   318  }
   319  
   320  func prettyStringImgDeleteItem(img types.ImageDeleteResponseItem) string {
   321  	if img.Deleted != "" {
   322  		return fmt.Sprintf("deleted: %s", img.Deleted)
   323  	}
   324  	if img.Untagged != "" {
   325  		return fmt.Sprintf("untagged: %s", img.Untagged)
   326  	}
   327  	return ""
   328  }
   329  
   330  func prettyPrintCachePruneReport(report *types.BuildCachePruneReport, l logger.Logger) {
   331  	if len(report.CachesDeleted) == 0 && !l.Level().ShouldDisplay(logger.VerboseLvl) {
   332  		return
   333  	}
   334  
   335  	l.Infof("[Docker Prune] removed %d caches, reclaimed %s",
   336  		len(report.CachesDeleted), humanSize(report.SpaceReclaimed))
   337  	if len(report.CachesDeleted) > 0 {
   338  		l.Debugf("%s", sliceutils.BulletedIndentedStringList(report.CachesDeleted))
   339  	}
   340  }
   341  
   342  func prettyPrintContainersPruneReport(report types.ContainersPruneReport, l logger.Logger) {
   343  	if len(report.ContainersDeleted) == 0 && !l.Level().ShouldDisplay(logger.VerboseLvl) {
   344  		return
   345  	}
   346  
   347  	l.Infof("[Docker Prune] removed %d containers, reclaimed %s",
   348  		len(report.ContainersDeleted), humanSize(report.SpaceReclaimed))
   349  	if len(report.ContainersDeleted) > 0 {
   350  		l.Debugf(sliceutils.BulletedIndentedStringList(report.ContainersDeleted))
   351  	}
   352  }
   353  
   354  func humanSize(bytes uint64) string {
   355  	return units.HumanSize(float64(bytes))
   356  }