github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/engine/dockerprune/docker_pruner.go (about) 1 package dockerprune 2 3 import ( 4 "context" 5 "fmt" 6 "sort" 7 "strings" 8 "time" 9 10 "github.com/docker/go-units" 11 12 "github.com/docker/docker/api/types" 13 "github.com/docker/docker/api/types/filters" 14 15 "github.com/tilt-dev/tilt/internal/container" 16 17 "github.com/tilt-dev/tilt/pkg/model" 18 19 "github.com/tilt-dev/tilt/internal/engine/buildcontrol" 20 21 "github.com/tilt-dev/tilt/internal/docker" 22 "github.com/tilt-dev/tilt/internal/sliceutils" 23 "github.com/tilt-dev/tilt/internal/store" 24 "github.com/tilt-dev/tilt/pkg/logger" 25 ) 26 27 var gcEnabledSelector = fmt.Sprintf("%s=true", docker.GCEnabledLabel) 28 29 type DockerPruner struct { 30 dCli docker.Client 31 32 disabledForTesting bool 33 disabledOnSetup bool 34 35 lastPruneBuildCount int 36 lastPruneTime time.Time 37 } 38 39 var _ store.Subscriber = &DockerPruner{} 40 var _ store.SetUpper = &DockerPruner{} 41 42 func NewDockerPruner(dCli docker.Client) *DockerPruner { 43 return &DockerPruner{dCli: dCli} 44 } 45 46 func (dp *DockerPruner) DisabledForTesting(disabled bool) { 47 dp.disabledForTesting = disabled 48 } 49 50 func (dp *DockerPruner) SetUp(ctx context.Context, _ store.RStore) error { 51 err := dp.dCli.CheckConnected() 52 if err != nil { 53 // If Docker is not responding at all, other parts of the system will log this. 54 dp.disabledOnSetup = true 55 return nil 56 } 57 58 if err := dp.sufficientVersionError(); err != nil { 59 logger.Get(ctx).Infof( 60 "[Docker Prune] Docker API version too low for Tilt to run Docker Prune:\n\t%v", err, 61 ) 62 dp.disabledOnSetup = true 63 return nil 64 } 65 return nil 66 } 67 68 // OnChange determines if any Tilt-built Docker images should be pruned based on settings and invokes the pruning 69 // process if necessary. 70 // 71 // Care should be taken when modifying this method to not introduce expensive operations unless necessary, as this 72 // is invoked for EVERY store action change batch. Because of this, the store (un)locking is done somewhat manually, 73 // so care must be taken to avoid locking issues. 74 func (dp *DockerPruner) OnChange(ctx context.Context, st store.RStore, summary store.ChangeSummary) error { 75 if dp.disabledForTesting || dp.disabledOnSetup || summary.IsLogOnly() { 76 return nil 77 } 78 79 state := st.RLockState() 80 settings := state.DockerPruneSettings 81 // Exit early if possible if any of the following is true: 82 // * Pruning is disabled entirely 83 // * Engine is currently building something 84 // * There are NO `docker_build`s in the Tiltfile 85 // * Something is queued for building 86 if !settings.Enabled || len(state.CurrentBuildSet) > 0 || !state.HasBuild() || buildcontrol.NextManifestNameToBuild(state) != "" { 87 st.RUnlockState() 88 return nil 89 } 90 91 // Prune as soon after startup as we can (waiting until we've built SOMETHING) 92 curBuildCount := state.CompletedBuildCount 93 shouldPrune := dp.lastPruneTime.IsZero() && curBuildCount > 0 94 // "Prune every X builds" takes precedence over "prune every Y hours" 95 if settings.NumBuilds != 0 { 96 buildsSince := curBuildCount - dp.lastPruneBuildCount 97 if buildsSince >= settings.NumBuilds { 98 shouldPrune = true 99 } 100 } else { 101 interval := settings.Interval 102 if interval == 0 { 103 interval = model.DockerPruneDefaultInterval 104 } 105 if time.Since(dp.lastPruneTime) >= interval { 106 shouldPrune = true 107 } 108 } 109 110 if shouldPrune { 111 // N.B. Only determine the ref selectors if we're actually going to prune - OnChange is called for every batch 112 // of store events and this is a comparatively expensive operation (lots of regex), but 99% of the time this 113 // is called, no pruning is going to happen, so avoid burning CPU cycles unnecessarily 114 imgSelectors := model.LocalRefSelectorsForManifests(state.Manifests(), state.Clusters) 115 st.RUnlockState() 116 dp.PruneAndRecordState(ctx, settings.MaxAge, settings.KeepRecent, imgSelectors, curBuildCount) 117 return nil 118 } 119 120 st.RUnlockState() 121 return nil 122 } 123 124 func (dp *DockerPruner) PruneAndRecordState(ctx context.Context, maxAge time.Duration, keepRecent int, imgSelectors []container.RefSelector, curBuildCount int) { 125 dp.Prune(ctx, maxAge, keepRecent, imgSelectors) 126 dp.lastPruneTime = time.Now() 127 dp.lastPruneBuildCount = curBuildCount 128 } 129 130 func (dp *DockerPruner) Prune(ctx context.Context, maxAge time.Duration, keepRecent int, imgSelectors []container.RefSelector) { 131 // For future: dispatch event with output/errors to be recorded 132 // in engineState.TiltSystemState on store (analogous to TiltfileState) 133 err := dp.prune(ctx, maxAge, keepRecent, imgSelectors) 134 if err != nil { 135 logger.Get(ctx).Infof("[Docker Prune] error running docker prune: %v", err) 136 } 137 } 138 139 func (dp *DockerPruner) prune(ctx context.Context, maxAge time.Duration, keepRecent int, imgSelectors []container.RefSelector) error { 140 l := logger.Get(ctx) 141 if err := dp.sufficientVersionError(); err != nil { 142 l.Debugf("[Docker Prune] skipping Docker prune, Docker API version too low:\t%v", err) 143 return nil 144 } 145 146 f := filters.NewArgs( 147 filters.Arg("label", gcEnabledSelector), 148 filters.Arg("until", maxAge.String()), 149 ) 150 151 // PRUNE CONTAINERS 152 containerReport, err := dp.dCli.ContainersPrune(ctx, f) 153 if err != nil { 154 return err 155 } 156 prettyPrintContainersPruneReport(containerReport, l) 157 158 // PRUNE IMAGES 159 imageReport, err := dp.deleteOldImages(ctx, maxAge, keepRecent, imgSelectors) 160 if err != nil { 161 return err 162 } 163 prettyPrintImagesPruneReport(imageReport, l) 164 165 // PRUNE BUILD CACHE 166 opts := types.BuildCachePruneOptions{Filters: f} 167 cacheReport, err := dp.dCli.BuildCachePrune(ctx, opts) 168 if err != nil { 169 if !strings.Contains(err.Error(), `"build prune" requires API version`) { 170 return err 171 } 172 l.Debugf("[Docker Prune] skipping build cache prune, Docker API version too low:\t%s", err) 173 } else { 174 prettyPrintCachePruneReport(cacheReport, l) 175 } 176 177 return nil 178 } 179 180 func (dp *DockerPruner) inspectImages(ctx context.Context, imgs []types.ImageSummary) []types.ImageInspect { 181 result := []types.ImageInspect{} 182 for _, imgSummary := range imgs { 183 inspect, _, err := dp.dCli.ImageInspectWithRaw(ctx, imgSummary.ID) 184 if err != nil { 185 logger.Get(ctx).Debugf("[Docker Prune] error inspecting image '%s': %v", imgSummary.ID, err) 186 continue 187 } 188 result = append(result, inspect) 189 } 190 return result 191 } 192 193 // Return all image objects that exceed the max age threshold. 194 func (dp *DockerPruner) filterImageInspectsByMaxAge(ctx context.Context, inspects []types.ImageInspect, maxAge time.Duration, selectors []container.RefSelector) []types.ImageInspect { 195 result := []types.ImageInspect{} 196 for _, inspect := range inspects { 197 namedRefs, err := container.ParseNamedMulti(inspect.RepoTags) 198 if err != nil { 199 logger.Get(ctx).Debugf("[Docker Prune] error parsing repo tags for '%s': %v", inspect.ID, err) 200 continue 201 } 202 203 // LastTagTime indicates the last time the image was built, which is more 204 // meaningful to us than when the image was created. 205 if time.Since(inspect.Metadata.LastTagTime) >= maxAge && container.AnyMatch(namedRefs, selectors) { 206 if len(inspect.RepoTags) > 1 { 207 logger.Get(ctx).Debugf("[Docker Prune] cannot prune image %s (tags: %s); `docker image remove --force` "+ 208 "required to remove an image with multiple tags (Docker throws error: "+ 209 "\"image is referenced in one or more repositories\")", 210 inspect.ID, strings.Join(inspect.RepoTags, ", ")) 211 continue 212 } 213 result = append(result, inspect) 214 } 215 } 216 return result 217 } 218 219 // Return all image objects that aren't in the N 220 // most recently used for each tag. 221 func (dp *DockerPruner) filterOutMostRecentInspects(ctx context.Context, inspects []types.ImageInspect, keepRecent int, selectors []container.RefSelector) []types.ImageInspect { 222 // First, sort the images in order from most recent to least recent. 223 recentFirst := append([]types.ImageInspect{}, inspects...) 224 sort.SliceStable(recentFirst, func(i, j int) bool { 225 // LastTagTime indicates the last time the image was built, which is more 226 // meaningful to us than when the image was created. 227 return recentFirst[i].Metadata.LastTagTime.After(recentFirst[j].Metadata.LastTagTime) 228 }) 229 230 // Next, aggregate the images by which selector they match. 231 imgsBySelector := make(map[container.RefSelector][]types.ImageInspect) 232 for _, inspect := range recentFirst { 233 namedRefs, err := container.ParseNamedMulti(inspect.RepoTags) 234 if err != nil { 235 logger.Get(ctx).Debugf("[Docker Prune] error parsing repo tags for '%s': %v", inspect.ID, err) 236 continue 237 } 238 239 for _, sel := range selectors { 240 if sel.MatchesAny(namedRefs) { 241 imgsBySelector[sel] = append(imgsBySelector[sel], inspect) 242 break 243 } 244 } 245 } 246 247 // Finally, keep the N most recent for each tag. 248 idsToKeep := make(map[string]bool) 249 for _, list := range imgsBySelector { 250 for i := 0; i < keepRecent && i < len(list); i++ { 251 idsToKeep[list[i].ID] = true 252 } 253 } 254 255 result := []types.ImageInspect{} 256 for _, inspect := range inspects { 257 if !idsToKeep[inspect.ID] { 258 result = append(result, inspect) 259 } 260 } 261 return result 262 } 263 264 func (dp *DockerPruner) deleteOldImages(ctx context.Context, maxAge time.Duration, keepRecent int, selectors []container.RefSelector) (types.ImagesPruneReport, error) { 265 opts := types.ImageListOptions{ 266 Filters: filters.NewArgs( 267 filters.Arg("label", gcEnabledSelector), 268 ), 269 } 270 imgs, err := dp.dCli.ImageList(ctx, opts) 271 if err != nil { 272 return types.ImagesPruneReport{}, err 273 } 274 275 inspects := dp.inspectImages(ctx, imgs) 276 inspects = dp.filterImageInspectsByMaxAge(ctx, inspects, maxAge, selectors) 277 toDelete := dp.filterOutMostRecentInspects(ctx, inspects, keepRecent, selectors) 278 279 rmOpts := types.ImageRemoveOptions{PruneChildren: true} 280 var responseItems []types.ImageDeleteResponseItem 281 var reclaimedBytes uint64 282 283 for _, inspect := range toDelete { 284 items, err := dp.dCli.ImageRemove(ctx, inspect.ID, rmOpts) 285 if err != nil { 286 // No good way to detect in-use images from `inspect` output, so just ignore those errors 287 if !strings.Contains(err.Error(), "image is being used by running container") { 288 logger.Get(ctx).Debugf("[Docker Prune] error removing image '%s': %v", inspect.ID, err) 289 } 290 continue 291 } 292 responseItems = append(responseItems, items...) 293 reclaimedBytes += uint64(inspect.Size) 294 } 295 296 return types.ImagesPruneReport{ 297 ImagesDeleted: responseItems, 298 SpaceReclaimed: reclaimedBytes, 299 }, nil 300 } 301 302 func (dp *DockerPruner) sufficientVersionError() error { 303 return dp.dCli.NewVersionError("1.30", "image | container prune with filter: label") 304 } 305 306 func prettyPrintImagesPruneReport(report types.ImagesPruneReport, l logger.Logger) { 307 if len(report.ImagesDeleted) == 0 && !l.Level().ShouldDisplay(logger.VerboseLvl) { 308 return 309 } 310 311 l.Infof("[Docker Prune] removed %d images, reclaimed %s", 312 len(report.ImagesDeleted), humanSize(report.SpaceReclaimed)) 313 if len(report.ImagesDeleted) > 0 { 314 for _, img := range report.ImagesDeleted { 315 l.Debugf("\t- %s", prettyStringImgDeleteItem(img)) 316 } 317 } 318 } 319 320 func prettyStringImgDeleteItem(img types.ImageDeleteResponseItem) string { 321 if img.Deleted != "" { 322 return fmt.Sprintf("deleted: %s", img.Deleted) 323 } 324 if img.Untagged != "" { 325 return fmt.Sprintf("untagged: %s", img.Untagged) 326 } 327 return "" 328 } 329 330 func prettyPrintCachePruneReport(report *types.BuildCachePruneReport, l logger.Logger) { 331 if len(report.CachesDeleted) == 0 && !l.Level().ShouldDisplay(logger.VerboseLvl) { 332 return 333 } 334 335 l.Infof("[Docker Prune] removed %d caches, reclaimed %s", 336 len(report.CachesDeleted), humanSize(report.SpaceReclaimed)) 337 if len(report.CachesDeleted) > 0 { 338 l.Debugf("%s", sliceutils.BulletedIndentedStringList(report.CachesDeleted)) 339 } 340 } 341 342 func prettyPrintContainersPruneReport(report types.ContainersPruneReport, l logger.Logger) { 343 if len(report.ContainersDeleted) == 0 && !l.Level().ShouldDisplay(logger.VerboseLvl) { 344 return 345 } 346 347 l.Infof("[Docker Prune] removed %d containers, reclaimed %s", 348 len(report.ContainersDeleted), humanSize(report.SpaceReclaimed)) 349 if len(report.ContainersDeleted) > 0 { 350 l.Debugf(sliceutils.BulletedIndentedStringList(report.ContainersDeleted)) 351 } 352 } 353 354 func humanSize(bytes uint64) string { 355 return units.HumanSize(float64(bytes)) 356 }