github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/distribution/xfer/download.go (about)

     1  package xfer // import "github.com/demonoid81/moby/distribution/xfer"
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"io"
     8  	"runtime"
     9  	"time"
    10  
    11  	"github.com/docker/distribution"
    12  	"github.com/demonoid81/moby/image"
    13  	"github.com/demonoid81/moby/layer"
    14  	"github.com/demonoid81/moby/pkg/archive"
    15  	"github.com/demonoid81/moby/pkg/ioutils"
    16  	"github.com/demonoid81/moby/pkg/progress"
    17  	"github.com/demonoid81/moby/pkg/system"
    18  	"github.com/sirupsen/logrus"
    19  )
    20  
    21  const maxDownloadAttempts = 5
    22  
    23  // LayerDownloadManager figures out which layers need to be downloaded, then
    24  // registers and downloads those, taking into account dependencies between
    25  // layers.
    26  type LayerDownloadManager struct {
    27  	layerStores         map[string]layer.Store
    28  	tm                  TransferManager
    29  	waitDuration        time.Duration
    30  	maxDownloadAttempts int
    31  }
    32  
    33  // SetConcurrency sets the max concurrent downloads for each pull
    34  func (ldm *LayerDownloadManager) SetConcurrency(concurrency int) {
    35  	ldm.tm.SetConcurrency(concurrency)
    36  }
    37  
    38  // NewLayerDownloadManager returns a new LayerDownloadManager.
    39  func NewLayerDownloadManager(layerStores map[string]layer.Store, concurrencyLimit int, options ...func(*LayerDownloadManager)) *LayerDownloadManager {
    40  	manager := LayerDownloadManager{
    41  		layerStores:         layerStores,
    42  		tm:                  NewTransferManager(concurrencyLimit),
    43  		waitDuration:        time.Second,
    44  		maxDownloadAttempts: maxDownloadAttempts,
    45  	}
    46  	for _, option := range options {
    47  		option(&manager)
    48  	}
    49  	return &manager
    50  }
    51  
    52  // WithMaxDownloadAttempts configures the maximum number of download
    53  // attempts for a download manager.
    54  func WithMaxDownloadAttempts(max int) func(*LayerDownloadManager) {
    55  	return func(dlm *LayerDownloadManager) {
    56  		dlm.maxDownloadAttempts = max
    57  	}
    58  }
    59  
    60  type downloadTransfer struct {
    61  	Transfer
    62  
    63  	layerStore layer.Store
    64  	layer      layer.Layer
    65  	err        error
    66  }
    67  
    68  // result returns the layer resulting from the download, if the download
    69  // and registration were successful.
    70  func (d *downloadTransfer) result() (layer.Layer, error) {
    71  	return d.layer, d.err
    72  }
    73  
    74  // A DownloadDescriptor references a layer that may need to be downloaded.
    75  type DownloadDescriptor interface {
    76  	// Key returns the key used to deduplicate downloads.
    77  	Key() string
    78  	// ID returns the ID for display purposes.
    79  	ID() string
    80  	// DiffID should return the DiffID for this layer, or an error
    81  	// if it is unknown (for example, if it has not been downloaded
    82  	// before).
    83  	DiffID() (layer.DiffID, error)
    84  	// Download is called to perform the download.
    85  	Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error)
    86  	// Close is called when the download manager is finished with this
    87  	// descriptor and will not call Download again or read from the reader
    88  	// that Download returned.
    89  	Close()
    90  }
    91  
    92  // DownloadDescriptorWithRegistered is a DownloadDescriptor that has an
    93  // additional Registered method which gets called after a downloaded layer is
    94  // registered. This allows the user of the download manager to know the DiffID
    95  // of each registered layer. This method is called if a cast to
    96  // DownloadDescriptorWithRegistered is successful.
    97  type DownloadDescriptorWithRegistered interface {
    98  	DownloadDescriptor
    99  	Registered(diffID layer.DiffID)
   100  }
   101  
   102  // Download is a blocking function which ensures the requested layers are
   103  // present in the layer store. It uses the string returned by the Key method to
   104  // deduplicate downloads. If a given layer is not already known to present in
   105  // the layer store, and the key is not used by an in-progress download, the
   106  // Download method is called to get the layer tar data. Layers are then
   107  // registered in the appropriate order.  The caller must call the returned
   108  // release function once it is done with the returned RootFS object.
   109  func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS image.RootFS, os string, layers []DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) {
   110  	var (
   111  		topLayer       layer.Layer
   112  		topDownload    *downloadTransfer
   113  		watcher        *Watcher
   114  		missingLayer   bool
   115  		transferKey    = ""
   116  		downloadsByKey = make(map[string]*downloadTransfer)
   117  	)
   118  
   119  	// Assume that the operating system is the host OS if blank, and validate it
   120  	// to ensure we don't cause a panic by an invalid index into the layerstores.
   121  	if os == "" {
   122  		os = runtime.GOOS
   123  	}
   124  	if !system.IsOSSupported(os) {
   125  		return image.RootFS{}, nil, system.ErrNotSupportedOperatingSystem
   126  	}
   127  
   128  	rootFS := initialRootFS
   129  	for _, descriptor := range layers {
   130  		key := descriptor.Key()
   131  		transferKey += key
   132  
   133  		if !missingLayer {
   134  			missingLayer = true
   135  			diffID, err := descriptor.DiffID()
   136  			if err == nil {
   137  				getRootFS := rootFS
   138  				getRootFS.Append(diffID)
   139  				l, err := ldm.layerStores[os].Get(getRootFS.ChainID())
   140  				if err == nil {
   141  					// Layer already exists.
   142  					logrus.Debugf("Layer already exists: %s", descriptor.ID())
   143  					progress.Update(progressOutput, descriptor.ID(), "Already exists")
   144  					if topLayer != nil {
   145  						layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
   146  					}
   147  					topLayer = l
   148  					missingLayer = false
   149  					rootFS.Append(diffID)
   150  					// Register this repository as a source of this layer.
   151  					withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
   152  					if hasRegistered { // As layerstore may set the driver
   153  						withRegistered.Registered(diffID)
   154  					}
   155  					continue
   156  				}
   157  			}
   158  		}
   159  
   160  		// Does this layer have the same data as a previous layer in
   161  		// the stack? If so, avoid downloading it more than once.
   162  		var topDownloadUncasted Transfer
   163  		if existingDownload, ok := downloadsByKey[key]; ok {
   164  			xferFunc := ldm.makeDownloadFuncFromDownload(descriptor, existingDownload, topDownload, os)
   165  			defer topDownload.Transfer.Release(watcher)
   166  			topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
   167  			topDownload = topDownloadUncasted.(*downloadTransfer)
   168  			continue
   169  		}
   170  
   171  		// Layer is not known to exist - download and register it.
   172  		progress.Update(progressOutput, descriptor.ID(), "Pulling fs layer")
   173  
   174  		var xferFunc DoFunc
   175  		if topDownload != nil {
   176  			xferFunc = ldm.makeDownloadFunc(descriptor, "", topDownload, os)
   177  			defer topDownload.Transfer.Release(watcher)
   178  		} else {
   179  			xferFunc = ldm.makeDownloadFunc(descriptor, rootFS.ChainID(), nil, os)
   180  		}
   181  		topDownloadUncasted, watcher = ldm.tm.Transfer(transferKey, xferFunc, progressOutput)
   182  		topDownload = topDownloadUncasted.(*downloadTransfer)
   183  		downloadsByKey[key] = topDownload
   184  	}
   185  
   186  	if topDownload == nil {
   187  		return rootFS, func() {
   188  			if topLayer != nil {
   189  				layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
   190  			}
   191  		}, nil
   192  	}
   193  
   194  	// Won't be using the list built up so far - will generate it
   195  	// from downloaded layers instead.
   196  	rootFS.DiffIDs = []layer.DiffID{}
   197  
   198  	defer func() {
   199  		if topLayer != nil {
   200  			layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
   201  		}
   202  	}()
   203  
   204  	select {
   205  	case <-ctx.Done():
   206  		topDownload.Transfer.Release(watcher)
   207  		return rootFS, func() {}, ctx.Err()
   208  	case <-topDownload.Done():
   209  		break
   210  	}
   211  
   212  	l, err := topDownload.result()
   213  	if err != nil {
   214  		topDownload.Transfer.Release(watcher)
   215  		return rootFS, func() {}, err
   216  	}
   217  
   218  	// Must do this exactly len(layers) times, so we don't include the
   219  	// base layer on Windows.
   220  	for range layers {
   221  		if l == nil {
   222  			topDownload.Transfer.Release(watcher)
   223  			return rootFS, func() {}, errors.New("internal error: too few parent layers")
   224  		}
   225  		rootFS.DiffIDs = append([]layer.DiffID{l.DiffID()}, rootFS.DiffIDs...)
   226  		l = l.Parent()
   227  	}
   228  	return rootFS, func() { topDownload.Transfer.Release(watcher) }, err
   229  }
   230  
   231  // makeDownloadFunc returns a function that performs the layer download and
   232  // registration. If parentDownload is non-nil, it waits for that download to
   233  // complete before the registration step, and registers the downloaded data
   234  // on top of parentDownload's resulting layer. Otherwise, it registers the
   235  // layer on top of the ChainID given by parentLayer.
   236  func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, parentLayer layer.ChainID, parentDownload *downloadTransfer, os string) DoFunc {
   237  	return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
   238  		d := &downloadTransfer{
   239  			Transfer:   NewTransfer(),
   240  			layerStore: ldm.layerStores[os],
   241  		}
   242  
   243  		go func() {
   244  			defer func() {
   245  				close(progressChan)
   246  			}()
   247  
   248  			progressOutput := progress.ChanOutput(progressChan)
   249  
   250  			select {
   251  			case <-start:
   252  			default:
   253  				progress.Update(progressOutput, descriptor.ID(), "Waiting")
   254  				<-start
   255  			}
   256  
   257  			if parentDownload != nil {
   258  				// Did the parent download already fail or get
   259  				// cancelled?
   260  				select {
   261  				case <-parentDownload.Done():
   262  					_, err := parentDownload.result()
   263  					if err != nil {
   264  						d.err = err
   265  						return
   266  					}
   267  				default:
   268  				}
   269  			}
   270  
   271  			var (
   272  				downloadReader io.ReadCloser
   273  				size           int64
   274  				err            error
   275  				retries        int
   276  			)
   277  
   278  			defer descriptor.Close()
   279  
   280  			for {
   281  				downloadReader, size, err = descriptor.Download(d.Transfer.Context(), progressOutput)
   282  				if err == nil {
   283  					break
   284  				}
   285  
   286  				// If an error was returned because the context
   287  				// was cancelled, we shouldn't retry.
   288  				select {
   289  				case <-d.Transfer.Context().Done():
   290  					d.err = err
   291  					return
   292  				default:
   293  				}
   294  
   295  				retries++
   296  				if _, isDNR := err.(DoNotRetry); isDNR || retries > ldm.maxDownloadAttempts {
   297  					logrus.Errorf("Download failed after %d attempts: %v", retries, err)
   298  					d.err = err
   299  					return
   300  				}
   301  
   302  				logrus.Infof("Download failed, retrying (%d/%d): %v", retries, ldm.maxDownloadAttempts, err)
   303  				delay := retries * 5
   304  				ticker := time.NewTicker(ldm.waitDuration)
   305  
   306  			selectLoop:
   307  				for {
   308  					progress.Updatef(progressOutput, descriptor.ID(), "Retrying in %d second%s", delay, (map[bool]string{true: "s"})[delay != 1])
   309  					select {
   310  					case <-ticker.C:
   311  						delay--
   312  						if delay == 0 {
   313  							ticker.Stop()
   314  							break selectLoop
   315  						}
   316  					case <-d.Transfer.Context().Done():
   317  						ticker.Stop()
   318  						d.err = errors.New("download cancelled during retry delay")
   319  						return
   320  					}
   321  
   322  				}
   323  			}
   324  
   325  			close(inactive)
   326  
   327  			if parentDownload != nil {
   328  				select {
   329  				case <-d.Transfer.Context().Done():
   330  					d.err = errors.New("layer registration cancelled")
   331  					downloadReader.Close()
   332  					return
   333  				case <-parentDownload.Done():
   334  				}
   335  
   336  				l, err := parentDownload.result()
   337  				if err != nil {
   338  					d.err = err
   339  					downloadReader.Close()
   340  					return
   341  				}
   342  				parentLayer = l.ChainID()
   343  			}
   344  
   345  			reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(d.Transfer.Context(), downloadReader), progressOutput, size, descriptor.ID(), "Extracting")
   346  			defer reader.Close()
   347  
   348  			inflatedLayerData, err := archive.DecompressStream(reader)
   349  			if err != nil {
   350  				d.err = fmt.Errorf("could not get decompression stream: %v", err)
   351  				return
   352  			}
   353  
   354  			var src distribution.Descriptor
   355  			if fs, ok := descriptor.(distribution.Describable); ok {
   356  				src = fs.Descriptor()
   357  			}
   358  			if ds, ok := d.layerStore.(layer.DescribableStore); ok {
   359  				d.layer, err = ds.RegisterWithDescriptor(inflatedLayerData, parentLayer, src)
   360  			} else {
   361  				d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer)
   362  			}
   363  			if err != nil {
   364  				select {
   365  				case <-d.Transfer.Context().Done():
   366  					d.err = errors.New("layer registration cancelled")
   367  				default:
   368  					d.err = fmt.Errorf("failed to register layer: %v", err)
   369  				}
   370  				return
   371  			}
   372  
   373  			progress.Update(progressOutput, descriptor.ID(), "Pull complete")
   374  			withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
   375  			if hasRegistered {
   376  				withRegistered.Registered(d.layer.DiffID())
   377  			}
   378  
   379  			// Doesn't actually need to be its own goroutine, but
   380  			// done like this so we can defer close(c).
   381  			go func() {
   382  				<-d.Transfer.Released()
   383  				if d.layer != nil {
   384  					layer.ReleaseAndLog(d.layerStore, d.layer)
   385  				}
   386  			}()
   387  		}()
   388  
   389  		return d
   390  	}
   391  }
   392  
   393  // makeDownloadFuncFromDownload returns a function that performs the layer
   394  // registration when the layer data is coming from an existing download. It
   395  // waits for sourceDownload and parentDownload to complete, and then
   396  // reregisters the data from sourceDownload's top layer on top of
   397  // parentDownload. This function does not log progress output because it would
   398  // interfere with the progress reporting for sourceDownload, which has the same
   399  // Key.
   400  func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor DownloadDescriptor, sourceDownload *downloadTransfer, parentDownload *downloadTransfer, os string) DoFunc {
   401  	return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
   402  		d := &downloadTransfer{
   403  			Transfer:   NewTransfer(),
   404  			layerStore: ldm.layerStores[os],
   405  		}
   406  
   407  		go func() {
   408  			defer func() {
   409  				close(progressChan)
   410  			}()
   411  
   412  			<-start
   413  
   414  			close(inactive)
   415  
   416  			select {
   417  			case <-d.Transfer.Context().Done():
   418  				d.err = errors.New("layer registration cancelled")
   419  				return
   420  			case <-parentDownload.Done():
   421  			}
   422  
   423  			l, err := parentDownload.result()
   424  			if err != nil {
   425  				d.err = err
   426  				return
   427  			}
   428  			parentLayer := l.ChainID()
   429  
   430  			// sourceDownload should have already finished if
   431  			// parentDownload finished, but wait for it explicitly
   432  			// to be sure.
   433  			select {
   434  			case <-d.Transfer.Context().Done():
   435  				d.err = errors.New("layer registration cancelled")
   436  				return
   437  			case <-sourceDownload.Done():
   438  			}
   439  
   440  			l, err = sourceDownload.result()
   441  			if err != nil {
   442  				d.err = err
   443  				return
   444  			}
   445  
   446  			layerReader, err := l.TarStream()
   447  			if err != nil {
   448  				d.err = err
   449  				return
   450  			}
   451  			defer layerReader.Close()
   452  
   453  			var src distribution.Descriptor
   454  			if fs, ok := l.(distribution.Describable); ok {
   455  				src = fs.Descriptor()
   456  			}
   457  			if ds, ok := d.layerStore.(layer.DescribableStore); ok {
   458  				d.layer, err = ds.RegisterWithDescriptor(layerReader, parentLayer, src)
   459  			} else {
   460  				d.layer, err = d.layerStore.Register(layerReader, parentLayer)
   461  			}
   462  			if err != nil {
   463  				d.err = fmt.Errorf("failed to register layer: %v", err)
   464  				return
   465  			}
   466  
   467  			withRegistered, hasRegistered := descriptor.(DownloadDescriptorWithRegistered)
   468  			if hasRegistered {
   469  				withRegistered.Registered(d.layer.DiffID())
   470  			}
   471  
   472  			// Doesn't actually need to be its own goroutine, but
   473  			// done like this so we can defer close(c).
   474  			go func() {
   475  				<-d.Transfer.Released()
   476  				if d.layer != nil {
   477  					layer.ReleaseAndLog(d.layerStore, d.layer)
   478  				}
   479  			}()
   480  		}()
   481  
   482  		return d
   483  	}
   484  }