gitee.com/bomy/docker.git@v1.13.1/distribution/xfer/download.go (about) 1 package xfer 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "time" 8 9 "github.com/Sirupsen/logrus" 10 "github.com/docker/distribution" 11 "github.com/docker/docker/image" 12 "github.com/docker/docker/layer" 13 "github.com/docker/docker/pkg/archive" 14 "github.com/docker/docker/pkg/ioutils" 15 "github.com/docker/docker/pkg/progress" 16 "golang.org/x/net/context" 17 ) 18 19 const maxDownloadAttempts = 5 20 21 // LayerDownloadManager figures out which layers need to be downloaded, then 22 // registers and downloads those, taking into account dependencies between 23 // layers. 24 type LayerDownloadManager struct { 25 layerStore layer.Store 26 tm TransferManager 27 } 28 29 // SetConcurrency set the max concurrent downloads for each pull 30 func (ldm *LayerDownloadManager) SetConcurrency(concurrency int) { 31 ldm.tm.SetConcurrency(concurrency) 32 } 33 34 // NewLayerDownloadManager returns a new LayerDownloadManager. 35 func NewLayerDownloadManager(layerStore layer.Store, concurrencyLimit int) *LayerDownloadManager { 36 return &LayerDownloadManager{ 37 layerStore: layerStore, 38 tm: NewTransferManager(concurrencyLimit), 39 } 40 } 41 42 type downloadTransfer struct { 43 Transfer 44 45 layerStore layer.Store 46 layer layer.Layer 47 err error 48 } 49 50 // result returns the layer resulting from the download, if the download 51 // and registration were successful. 52 func (d *downloadTransfer) result() (layer.Layer, error) { 53 return d.layer, d.err 54 } 55 56 // A DownloadDescriptor references a layer that may need to be downloaded. 57 type DownloadDescriptor interface { 58 // Key returns the key used to deduplicate downloads. 59 Key() string 60 // ID returns the ID for display purposes. 61 ID() string 62 // DiffID should return the DiffID for this layer, or an error 63 // if it is unknown (for example, if it has not been downloaded 64 // before). 65 DiffID() (layer.DiffID, error) 66 // Download is called to perform the download. 67 Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) 68 // Close is called when the download manager is finished with this 69 // descriptor and will not call Download again or read from the reader 70 // that Download returned. 71 Close() 72 } 73 74 // DownloadDescriptorWithRegistered is a DownloadDescriptor that has an 75 // additional Registered method which gets called after a downloaded layer is 76 // registered. This allows the user of the download manager to know the DiffID 77 // of each registered layer. This method is called if a cast to 78 // DownloadDescriptorWithRegistered is successful. 79 type DownloadDescriptorWithRegistered interface { 80 DownloadDescriptor 81 Registered(diffID layer.DiffID) 82 } 83 84 // Download is a blocking function which ensures the requested layers are 85 // present in the layer store. It uses the string returned by the Key method to 86 // deduplicate downloads. If a given layer is not already known to present in 87 // the layer store, and the key is not used by an in-progress download, the 88 // Download method is called to get the layer tar data. Layers are then 89 // registered in the appropriate order. The caller must call the returned 90 // release function once it is is done with the returned RootFS object. 91 func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS image.RootFS, layers []DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) { 92 var ( 93 topLayer layer.Layer 94 topDownload *downloadTransfer 95 watcher *Watcher 96 missingLayer bool 97 transferKey = "" 98 downloadsByKey = make(map[string]*downloadTransfer) 99 ) 100 101 rootFS := initialRootFS 102 for _, descriptor := range layers { 103 key := descriptor.Key() 104 transferKey += key 105 106 if !missingLayer { 107 missingLayer = true 108 diffID, err := descriptor.DiffID() 109 if err == nil { 110 getRootFS := rootFS 111 getRootFS.Append(diffID) 112 l, err := ldm.layerStore.Get(getRootFS.ChainID()) 113 if err == nil { 114 // Layer already exists. 115 logrus.Debugf("Layer already exists: %s", descriptor.ID()) 116 progress.Update(progressOutput, descriptor.ID(), "Already exists") 117 if topLayer != nil { 118 layer.ReleaseAndLog(ldm.layerStore, topLayer) 119 } 120 topLayer = l 121 missingLayer = false 122 rootFS.Append(diffID) 123 continue 124 } 125 } 126 } 127 128 // Does this layer have the same data as a previous layer in 129 // the stack? If so, avoid downloading it more than once. 130 var topDownloadUncasted Transfer 131 if existingDownload, ok := downloadsByKey[key]; ok { 132 xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload) 133 defer topDownload.Transfer.Release(watcher) 134 topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput) 135 topDownload = topDownloadUncasted.(*downloadTransfer) 136 continue 137 } 138 139 // Layer is not known to exist - download and register it. 140 progress.Update(progressOutput, descriptor.ID(), "Pulling fs layer") 141 142 var xferFunc DoFunc 143 if topDownload != nil { 144 xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload) 145 defer topDownload.Transfer.Release(watcher) 146 } else { 147 xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil) 148 } 149 topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput) 150 topDownload = topDownloadUncasted.(*downloadTransfer) 151 downloadsByKey[key] = topDownload 152 } 153 154 if topDownload == nil { 155 return rootFS, func() { 156 if topLayer != nil { 157 layer.ReleaseAndLog(ldm.layerStore, topLayer) 158 } 159 }, nil 160 } 161 162 // Won't be using the list built up so far - will generate it 163 // from downloaded layers instead. 164 rootFS.DiffIDs = []layer.DiffID{} 165 166 defer func() { 167 if topLayer != nil { 168 layer.ReleaseAndLog(ldm.layerStore, topLayer) 169 } 170 }() 171 172 select { 173 case <-ctx.Done(): 174 topDownload.Transfer.Release(watcher) 175 return rootFS, func() {}, ctx.Err() 176 case <-topDownload.Done(): 177 break 178 } 179 180 l, err := topDownload.result() 181 if err != nil { 182 topDownload.Transfer.Release(watcher) 183 return rootFS, func() {}, err 184 } 185 186 // Must do this exactly len(layers) times, so we don't include the 187 // base layer on Windows. 188 for range layers { 189 if l == nil { 190 topDownload.Transfer.Release(watcher) 191 return rootFS, func() {}, errors.New("internal error: too few parent layers") 192 } 193 rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...) 194 l = l.Parent() 195 } 196 return rootFS, func() { topDownload.Transfer.Release(watcher) }, err 197 } 198 199 // makeDownloadFunc returns a function that performs the layer download and 200 // registration. If parentDownload is non-nil, it waits for that download to 201 // complete before the registration step, and registers the downloaded data 202 // on top of parentDownload's resulting layer. Otherwise, it registers the 203 // layer on top of the ChainID given by parentLayer. 204 func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer) DoFunc { 205 return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer { 206 d := &downloadTransfer{ 207 Transfer: NewTransfer(), 208 layerStore: ldm.layerStore, 209 } 210 211 go func() { 212 defer func() { 213 close(progressChan) 214 }() 215 216 progressOutput := progress.ChanOutput(progressChan) 217 218 select { 219 case <-start: 220 default: 221 progress.Update(progressOutput, descriptor.ID(), "Waiting") 222 <-start 223 } 224 225 if parentDownload != nil { 226 // Did the parent download already fail or get 227 // cancelled? 228 select { 229 case <-parentDownload.Done(): 230 _, err := parentDownload.result() 231 if err != nil { 232 d.err = err 233 return 234 } 235 default: 236 } 237 } 238 239 var ( 240 downloadReader io.ReadCloser 241 size int64 242 err error 243 retries int 244 ) 245 246 defer descriptor.Close() 247 248 for { 249 downloadReader, size, err = descriptor.Download(d.Transfer.Context(), progressOutput) 250 if err == nil { 251 break 252 } 253 254 // If an error was returned because the context 255 // was cancelled, we shouldn't retry. 256 select { 257 case <-d.Transfer.Context().Done(): 258 d.err = err 259 return 260 default: 261 } 262 263 retries++ 264 if _, isDNR := err.(DoNotRetry); isDNR || retries == maxDownloadAttempts { 265 logrus.Errorf("Download failed: %v", err) 266 d.err = err 267 return 268 } 269 270 logrus.Errorf("Download failed, retrying: %v", err) 271 delay := retries * 5 272 ticker := time.NewTicker(time.Second) 273 274 selectLoop: 275 for { 276 progress.Updatef(progressOutput, descriptor.ID(), "Retrying in %d second%s", delay, (map[bool]string{true: "s"})[delay != 1]) 277 select { 278 case <-ticker.C: 279 delay-- 280 if delay == 0 { 281 ticker.Stop() 282 break selectLoop 283 } 284 case <-d.Transfer.Context().Done(): 285 ticker.Stop() 286 d.err = errors.New("download cancelled during retry delay") 287 return 288 } 289 290 } 291 } 292 293 close(inactive) 294 295 if parentDownload != nil { 296 select { 297 case <-d.Transfer.Context().Done(): 298 d.err = errors.New("layer registration cancelled") 299 downloadReader.Close() 300 return 301 case <-parentDownload.Done(): 302 } 303 304 l, err := parentDownload.result() 305 if err != nil { 306 d.err = err 307 downloadReader.Close() 308 return 309 } 310 parentLayer = l.ChainID() 311 } 312 313 reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(d.Transfer.Context(), downloadReader), progressOutput, size, descriptor.ID(), "Extracting") 314 defer reader.Close() 315 316 inflatedLayerData, err := archive.DecompressStream(reader) 317 if err != nil { 318 d.err = fmt.Errorf("could not get decompression stream: %v", err) 319 return 320 } 321 322 var src distribution.Descriptor 323 if fs, ok := descriptor.(distribution.Describable); ok { 324 src = fs.Descriptor() 325 } 326 if ds, ok := d.layerStore.(layer.DescribableStore); ok { 327 d.layer, err = ds.RegisterWithDescriptor(inflatedLayerData, parentLayer, src) 328 } else { 329 d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer) 330 } 331 if err != nil { 332 select { 333 case <-d.Transfer.Context().Done(): 334 d.err = errors.New("layer registration cancelled") 335 default: 336 d.err = fmt.Errorf("failed to register layer: %v", err) 337 } 338 return 339 } 340 341 progress.Update(progressOutput, descriptor.ID(), "Pull complete") 342 withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered) 343 if hasRegistered { 344 withRegistered.Registered(d.layer.DiffID()) 345 } 346 347 // Doesn't actually need to be its own goroutine, but 348 // done like this so we can defer close(c). 349 go func() { 350 <-d.Transfer.Released() 351 if d.layer != nil { 352 layer.ReleaseAndLog(d.layerStore, d.layer) 353 } 354 }() 355 }() 356 357 return d 358 } 359 } 360 361 // makeDownloadFuncFromDownload returns a function that performs the layer 362 // registration when the layer data is coming from an existing download. It 363 // waits for sourceDownload and parentDownload to complete, and then 364 // reregisters the data from sourceDownload's top layer on top of 365 // parentDownload. This function does not log progress output because it would 366 // interfere with the progress reporting for sourceDownload, which has the same 367 // Key. 368 func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer) DoFunc { 369 return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer { 370 d := &downloadTransfer{ 371 Transfer: NewTransfer(), 372 layerStore: ldm.layerStore, 373 } 374 375 go func() { 376 defer func() { 377 close(progressChan) 378 }() 379 380 <-start 381 382 close(inactive) 383 384 select { 385 case <-d.Transfer.Context().Done(): 386 d.err = errors.New("layer registration cancelled") 387 return 388 case <-parentDownload.Done(): 389 } 390 391 l, err := parentDownload.result() 392 if err != nil { 393 d.err = err 394 return 395 } 396 parentLayer := l.ChainID() 397 398 // sourceDownload should have already finished if 399 // parentDownload finished, but wait for it explicitly 400 // to be sure. 401 select { 402 case <-d.Transfer.Context().Done(): 403 d.err = errors.New("layer registration cancelled") 404 return 405 case <-sourceDownload.Done(): 406 } 407 408 l, err = sourceDownload.result() 409 if err != nil { 410 d.err = err 411 return 412 } 413 414 layerReader, err := l.TarStream() 415 if err != nil { 416 d.err = err 417 return 418 } 419 defer layerReader.Close() 420 421 var src distribution.Descriptor 422 if fs, ok := l.(distribution.Describable); ok { 423 src = fs.Descriptor() 424 } 425 if ds, ok := d.layerStore.(layer.DescribableStore); ok { 426 d.layer, err = ds.RegisterWithDescriptor(layerReader, parentLayer, src) 427 } else { 428 d.layer, err = d.layerStore.Register(layerReader, parentLayer) 429 } 430 if err != nil { 431 d.err = fmt.Errorf("failed to register layer: %v", err) 432 return 433 } 434 435 withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered) 436 if hasRegistered { 437 withRegistered.Registered(d.layer.DiffID()) 438 } 439 440 // Doesn't actually need to be its own goroutine, but 441 // done like this so we can defer close(c). 442 go func() { 443 <-d.Transfer.Released() 444 if d.layer != nil { 445 layer.ReleaseAndLog(d.layerStore, d.layer) 446 } 447 }() 448 }() 449 450 return d 451 } 452 }