github.com/tonistiigi/docker@v0.10.1-0.20240229224939-974013b0dc6a/distribution/xfer/download.go (about) 1 package xfer // import "github.com/docker/docker/distribution/xfer" 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "io" 8 "time" 9 10 "github.com/containerd/log" 11 "github.com/docker/distribution" 12 "github.com/docker/docker/image" 13 "github.com/docker/docker/layer" 14 "github.com/docker/docker/pkg/archive" 15 "github.com/docker/docker/pkg/ioutils" 16 "github.com/docker/docker/pkg/progress" 17 ) 18 19 const maxDownloadAttempts = 5 20 21 // LayerDownloadManager figures out which layers need to be downloaded, then 22 // registers and downloads those, taking into account dependencies between 23 // layers. 24 type LayerDownloadManager struct { 25 layerStore layer.Store 26 tm *transferManager 27 waitDuration time.Duration 28 maxDownloadAttempts int 29 } 30 31 // SetConcurrency sets the max concurrent downloads for each pull 32 func (ldm *LayerDownloadManager) SetConcurrency(concurrency int) { 33 ldm.tm.setConcurrency(concurrency) 34 } 35 36 // NewLayerDownloadManager returns a new LayerDownloadManager. 37 func NewLayerDownloadManager(layerStore layer.Store, concurrencyLimit int, options ...DownloadOption) *LayerDownloadManager { 38 manager := LayerDownloadManager{ 39 layerStore: layerStore, 40 tm: newTransferManager(concurrencyLimit), 41 waitDuration: time.Second, 42 maxDownloadAttempts: maxDownloadAttempts, 43 } 44 for _, option := range options { 45 option(&manager) 46 } 47 return &manager 48 } 49 50 // DownloadOption set options for the LayerDownloadManager. 51 type DownloadOption func(*LayerDownloadManager) 52 53 // WithMaxDownloadAttempts configures the maximum number of download 54 // attempts for a download manager. 55 func WithMaxDownloadAttempts(max int) DownloadOption { 56 return func(dlm *LayerDownloadManager) { 57 dlm.maxDownloadAttempts = max 58 } 59 } 60 61 type downloadTransfer struct { 62 transfer 63 64 layerStore layer.Store 65 layer layer.Layer 66 err error 67 } 68 69 // result returns the layer resulting from the download, if the download 70 // and registration were successful. 71 func (d *downloadTransfer) result() (layer.Layer, error) { 72 return d.layer, d.err 73 } 74 75 // A DownloadDescriptor references a layer that may need to be downloaded. 76 type DownloadDescriptor interface { 77 // Key returns the key used to deduplicate downloads. 78 Key() string 79 // ID returns the ID for display purposes. 80 ID() string 81 // DiffID should return the DiffID for this layer, or an error 82 // if it is unknown (for example, if it has not been downloaded 83 // before). 84 DiffID() (layer.DiffID, error) 85 // Download is called to perform the download. 86 Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) 87 // Close is called when the download manager is finished with this 88 // descriptor and will not call Download again or read from the reader 89 // that Download returned. 90 Close() 91 } 92 93 // DigestRegisterer can be implemented by a DownloadDescriptor, and provides a 94 // Registered method which gets called after a downloaded layer is registered. 95 // This allows the user of the download manager to know the DiffID of each 96 // registered layer. This method is called if a cast to DigestRegisterer is 97 // successful. 98 type DigestRegisterer interface { 99 // TODO existing implementations in distribution and builder-next swallow errors 100 // when registering the diffID. Consider changing the Registered signature 101 // to return the error. 102 103 Registered(diffID layer.DiffID) 104 } 105 106 // Download is a blocking function which ensures the requested layers are 107 // present in the layer store. It uses the string returned by the Key method to 108 // deduplicate downloads. If a given layer is not already known to present in 109 // the layer store, and the key is not used by an in-progress download, the 110 // Download method is called to get the layer tar data. Layers are then 111 // registered in the appropriate order. The caller must call the returned 112 // release function once it is done with the returned RootFS object. 113 func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS image.RootFS, layers []DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) { 114 var ( 115 topLayer layer.Layer 116 topDownload *downloadTransfer 117 watcher *watcher 118 missingLayer bool 119 transferKey = "" 120 downloadsByKey = make(map[string]*downloadTransfer) 121 ) 122 123 rootFS := initialRootFS 124 for _, descriptor := range layers { 125 key := descriptor.Key() 126 transferKey += key 127 128 if !missingLayer { 129 missingLayer = true 130 diffID, err := descriptor.DiffID() 131 if err == nil { 132 getRootFS := rootFS 133 getRootFS.Append(diffID) 134 l, err := ldm.layerStore.Get(getRootFS.ChainID()) 135 if err == nil { 136 // Layer already exists. 137 log.G(ctx).Debugf("Layer already exists: %s", descriptor.ID()) 138 progress.Update(progressOutput, descriptor.ID(), "Already exists") 139 if topLayer != nil { 140 layer.ReleaseAndLog(ldm.layerStore, topLayer) 141 } 142 topLayer = l 143 missingLayer = false 144 rootFS.Append(diffID) 145 // Register this repository as a source of this layer. 146 if withRegistered, ok := descriptor.(DigestRegisterer); ok { // As layerstore may set the driver 147 withRegistered.Registered(diffID) 148 } 149 continue 150 } 151 } 152 } 153 154 // Does this layer have the same data as a previous layer in 155 // the stack? If so, avoid downloading it more than once. 156 var topDownloadUncasted transfer 157 if existingDownload, ok := downloadsByKey[key]; ok { 158 xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload) 159 defer topDownload.transfer.release(watcher) 160 topDownloadUncasted, watcher = ldm.tm.transfer(transferKey, xferFunc, progressOutput) 161 topDownload = topDownloadUncasted.(*downloadTransfer) 162 continue 163 } 164 165 // Layer is not known to exist - download and register it. 166 progress.Update(progressOutput, descriptor.ID(), "Pulling fs layer") 167 168 var xferFunc doFunc 169 if topDownload != nil { 170 xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload) 171 defer topDownload.transfer.release(watcher) 172 } else { 173 xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil) 174 } 175 topDownloadUncasted, watcher = ldm.tm.transfer(transferKey, xferFunc, progressOutput) 176 topDownload = topDownloadUncasted.(*downloadTransfer) 177 downloadsByKey[key] = topDownload 178 } 179 180 if topDownload == nil { 181 return rootFS, func() { 182 if topLayer != nil { 183 layer.ReleaseAndLog(ldm.layerStore, topLayer) 184 } 185 }, nil 186 } 187 188 // Won't be using the list built up so far - will generate it 189 // from downloaded layers instead. 190 rootFS.DiffIDs = []layer.DiffID{} 191 192 defer func() { 193 if topLayer != nil { 194 layer.ReleaseAndLog(ldm.layerStore, topLayer) 195 } 196 }() 197 198 select { 199 case <-ctx.Done(): 200 topDownload.transfer.release(watcher) 201 return rootFS, func() {}, ctx.Err() 202 case <-topDownload.done(): 203 break 204 } 205 206 l, err := topDownload.result() 207 if err != nil { 208 topDownload.transfer.release(watcher) 209 return rootFS, func() {}, err 210 } 211 212 // Must do this exactly len(layers) times, so we don't include the 213 // base layer on Windows. 214 for range layers { 215 if l == nil { 216 topDownload.transfer.release(watcher) 217 return rootFS, func() {}, errors.New("internal error: too few parent layers") 218 } 219 rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...) 220 l = l.Parent() 221 } 222 return rootFS, func() { topDownload.transfer.release(watcher) }, err 223 } 224 225 // makeDownloadFunc returns a function that performs the layer download and 226 // registration. If parentDownload is non-nil, it waits for that download to 227 // complete before the registration step, and registers the downloaded data 228 // on top of parentDownload's resulting layer. Otherwise, it registers the 229 // layer on top of the ChainID given by parentLayer. 230 func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer) doFunc { 231 return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) transfer { 232 d := &downloadTransfer{ 233 transfer: newTransfer(), 234 layerStore: ldm.layerStore, 235 } 236 237 go func() { 238 defer func() { 239 close(progressChan) 240 }() 241 242 progressOutput := progress.ChanOutput(progressChan) 243 244 select { 245 case <-start: 246 default: 247 progress.Update(progressOutput, descriptor.ID(), "Waiting") 248 <-start 249 } 250 251 if parentDownload != nil { 252 // Did the parent download already fail or get 253 // cancelled? 254 select { 255 case <-parentDownload.done(): 256 _, err := parentDownload.result() 257 if err != nil { 258 d.err = err 259 return 260 } 261 default: 262 } 263 } 264 265 var ( 266 downloadReader io.ReadCloser 267 size int64 268 err error 269 attempt int = 1 270 ) 271 272 defer descriptor.Close() 273 274 for { 275 downloadReader, size, err = descriptor.Download(d.transfer.context(), progressOutput) 276 if err == nil { 277 break 278 } 279 280 // If an error was returned because the context 281 // was cancelled, we shouldn't retry. 282 select { 283 case <-d.transfer.context().Done(): 284 d.err = err 285 return 286 default: 287 } 288 289 if _, isDNR := err.(DoNotRetry); isDNR || attempt >= ldm.maxDownloadAttempts { 290 log.G(context.TODO()).Errorf("Download failed after %d attempts: %v", attempt, err) 291 d.err = err 292 return 293 } 294 295 log.G(context.TODO()).Infof("Download failed, retrying (%d/%d): %v", attempt, ldm.maxDownloadAttempts, err) 296 delay := attempt * 5 297 ticker := time.NewTicker(ldm.waitDuration) 298 attempt++ 299 300 selectLoop: 301 for { 302 progress.Updatef(progressOutput, descriptor.ID(), "Retrying in %d second%s", delay, (map[bool]string{true: "s"})[delay != 1]) 303 select { 304 case <-ticker.C: 305 delay-- 306 if delay == 0 { 307 ticker.Stop() 308 break selectLoop 309 } 310 case <-d.transfer.context().Done(): 311 ticker.Stop() 312 d.err = errors.New("download cancelled during retry delay") 313 return 314 } 315 } 316 } 317 318 close(inactive) 319 320 if parentDownload != nil { 321 select { 322 case <-d.transfer.context().Done(): 323 d.err = errors.New("layer registration cancelled") 324 downloadReader.Close() 325 return 326 case <-parentDownload.done(): 327 } 328 329 l, err := parentDownload.result() 330 if err != nil { 331 d.err = err 332 downloadReader.Close() 333 return 334 } 335 parentLayer = l.ChainID() 336 } 337 338 reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(d.transfer.context(), downloadReader), progressOutput, size, descriptor.ID(), "Extracting") 339 defer reader.Close() 340 341 inflatedLayerData, err := archive.DecompressStream(reader) 342 if err != nil { 343 d.err = fmt.Errorf("could not get decompression stream: %v", err) 344 return 345 } 346 defer inflatedLayerData.Close() 347 348 var src distribution.Descriptor 349 if fs, ok := descriptor.(distribution.Describable); ok { 350 src = fs.Descriptor() 351 } 352 if ds, ok := d.layerStore.(layer.DescribableStore); ok { 353 d.layer, err = ds.RegisterWithDescriptor(inflatedLayerData, parentLayer, src) 354 } else { 355 d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer) 356 } 357 if err != nil { 358 select { 359 case <-d.transfer.context().Done(): 360 d.err = errors.New("layer registration cancelled") 361 default: 362 d.err = fmt.Errorf("failed to register layer: %v", err) 363 } 364 return 365 } 366 367 progress.Update(progressOutput, descriptor.ID(), "Pull complete") 368 369 if withRegistered, ok := descriptor.(DigestRegisterer); ok { 370 withRegistered.Registered(d.layer.DiffID()) 371 } 372 373 // Doesn't actually need to be its own goroutine, but 374 // done like this so we can defer close(c). 375 go func() { 376 <-d.transfer.released() 377 if d.layer != nil { 378 layer.ReleaseAndLog(d.layerStore, d.layer) 379 } 380 }() 381 }() 382 383 return d 384 } 385 } 386 387 // makeDownloadFuncFromDownload returns a function that performs the layer 388 // registration when the layer data is coming from an existing download. It 389 // waits for sourceDownload and parentDownload to complete, and then 390 // reregisters the data from sourceDownload's top layer on top of 391 // parentDownload. This function does not log progress output because it would 392 // interfere with the progress reporting for sourceDownload, which has the same 393 // Key. 394 func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer) doFunc { 395 return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) transfer { 396 d := &downloadTransfer{ 397 transfer: newTransfer(), 398 layerStore: ldm.layerStore, 399 } 400 401 go func() { 402 defer func() { 403 close(progressChan) 404 }() 405 406 <-start 407 408 close(inactive) 409 410 select { 411 case <-d.transfer.context().Done(): 412 d.err = errors.New("layer registration cancelled") 413 return 414 case <-parentDownload.done(): 415 } 416 417 l, err := parentDownload.result() 418 if err != nil { 419 d.err = err 420 return 421 } 422 parentLayer := l.ChainID() 423 424 // sourceDownload should have already finished if 425 // parentDownload finished, but wait for it explicitly 426 // to be sure. 427 select { 428 case <-d.transfer.context().Done(): 429 d.err = errors.New("layer registration cancelled") 430 return 431 case <-sourceDownload.done(): 432 } 433 434 l, err = sourceDownload.result() 435 if err != nil { 436 d.err = err 437 return 438 } 439 440 layerReader, err := l.TarStream() 441 if err != nil { 442 d.err = err 443 return 444 } 445 defer layerReader.Close() 446 447 var src distribution.Descriptor 448 if fs, ok := l.(distribution.Describable); ok { 449 src = fs.Descriptor() 450 } 451 if ds, ok := d.layerStore.(layer.DescribableStore); ok { 452 d.layer, err = ds.RegisterWithDescriptor(layerReader, parentLayer, src) 453 } else { 454 d.layer, err = d.layerStore.Register(layerReader, parentLayer) 455 } 456 if err != nil { 457 d.err = fmt.Errorf("failed to register layer: %v", err) 458 return 459 } 460 461 if withRegistered, ok := descriptor.(DigestRegisterer); ok { 462 withRegistered.Registered(d.layer.DiffID()) 463 } 464 465 // Doesn't actually need to be its own goroutine, but 466 // done like this so we can defer close(c). 467 go func() { 468 <-d.transfer.released() 469 if d.layer != nil { 470 layer.ReleaseAndLog(d.layerStore, d.layer) 471 } 472 }() 473 }() 474 475 return d 476 } 477 }