github.com/jingleWang/moby@v1.13.1/distribution/push_v2.go (about)

     1  package distribution
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"runtime"
     8  	"sort"
     9  	"strings"
    10  	"sync"
    11  
    12  	"golang.org/x/net/context"
    13  
    14  	"github.com/Sirupsen/logrus"
    15  	"github.com/docker/distribution"
    16  	"github.com/docker/distribution/digest"
    17  	"github.com/docker/distribution/manifest/schema1"
    18  	"github.com/docker/distribution/manifest/schema2"
    19  	distreference "github.com/docker/distribution/reference"
    20  	"github.com/docker/distribution/registry/client"
    21  	"github.com/docker/docker/distribution/metadata"
    22  	"github.com/docker/docker/distribution/xfer"
    23  	"github.com/docker/docker/layer"
    24  	"github.com/docker/docker/pkg/ioutils"
    25  	"github.com/docker/docker/pkg/progress"
    26  	"github.com/docker/docker/pkg/stringid"
    27  	"github.com/docker/docker/reference"
    28  	"github.com/docker/docker/registry"
    29  )
    30  
    31  const (
    32  	smallLayerMaximumSize  = 100 * (1 << 10) // 100KB
    33  	middleLayerMaximumSize = 10 * (1 << 20)  // 10MB
    34  )
    35  
    36  // PushResult contains the tag, manifest digest, and manifest size from the
    37  // push. It's used to signal this information to the trust code in the client
    38  // so it can sign the manifest if necessary.
    39  type PushResult struct {
    40  	Tag    string
    41  	Digest digest.Digest
    42  	Size   int
    43  }
    44  
    45  type v2Pusher struct {
    46  	v2MetadataService metadata.V2MetadataService
    47  	ref               reference.Named
    48  	endpoint          registry.APIEndpoint
    49  	repoInfo          *registry.RepositoryInfo
    50  	config            *ImagePushConfig
    51  	repo              distribution.Repository
    52  
    53  	// pushState is state built by the Upload functions.
    54  	pushState pushState
    55  }
    56  
    57  type pushState struct {
    58  	sync.Mutex
    59  	// remoteLayers is the set of layers known to exist on the remote side.
    60  	// This avoids redundant queries when pushing multiple tags that
    61  	// involve the same layers. It is also used to fill in digest and size
    62  	// information when building the manifest.
    63  	remoteLayers map[layer.DiffID]distribution.Descriptor
    64  	// confirmedV2 is set to true if we confirm we're talking to a v2
    65  	// registry. This is used to limit fallbacks to the v1 protocol.
    66  	confirmedV2 bool
    67  }
    68  
    69  func (p *v2Pusher) Push(ctx context.Context) (err error) {
    70  	p.pushState.remoteLayers = make(map[layer.DiffID]distribution.Descriptor)
    71  
    72  	p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
    73  	if err != nil {
    74  		logrus.Debugf("Error getting v2 registry: %v", err)
    75  		return err
    76  	}
    77  
    78  	if err = p.pushV2Repository(ctx); err != nil {
    79  		if continueOnError(err) {
    80  			return fallbackError{
    81  				err:         err,
    82  				confirmedV2: p.pushState.confirmedV2,
    83  				transportOK: true,
    84  			}
    85  		}
    86  	}
    87  	return err
    88  }
    89  
    90  func (p *v2Pusher) pushV2Repository(ctx context.Context) (err error) {
    91  	if namedTagged, isNamedTagged := p.ref.(reference.NamedTagged); isNamedTagged {
    92  		imageID, err := p.config.ReferenceStore.Get(p.ref)
    93  		if err != nil {
    94  			return fmt.Errorf("tag does not exist: %s", p.ref.String())
    95  		}
    96  
    97  		return p.pushV2Tag(ctx, namedTagged, imageID)
    98  	}
    99  
   100  	if !reference.IsNameOnly(p.ref) {
   101  		return errors.New("cannot push a digest reference")
   102  	}
   103  
   104  	// Pull all tags
   105  	pushed := 0
   106  	for _, association := range p.config.ReferenceStore.ReferencesByName(p.ref) {
   107  		if namedTagged, isNamedTagged := association.Ref.(reference.NamedTagged); isNamedTagged {
   108  			pushed++
   109  			if err := p.pushV2Tag(ctx, namedTagged, association.ID); err != nil {
   110  				return err
   111  			}
   112  		}
   113  	}
   114  
   115  	if pushed == 0 {
   116  		return fmt.Errorf("no tags to push for %s", p.repoInfo.Name())
   117  	}
   118  
   119  	return nil
   120  }
   121  
   122  func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id digest.Digest) error {
   123  	logrus.Debugf("Pushing repository: %s", ref.String())
   124  
   125  	imgConfig, err := p.config.ImageStore.Get(id)
   126  	if err != nil {
   127  		return fmt.Errorf("could not find image from tag %s: %v", ref.String(), err)
   128  	}
   129  
   130  	rootfs, err := p.config.ImageStore.RootFSFromConfig(imgConfig)
   131  	if err != nil {
   132  		return fmt.Errorf("unable to get rootfs for image %s: %s", ref.String(), err)
   133  	}
   134  
   135  	l, err := p.config.LayerStore.Get(rootfs.ChainID())
   136  	if err != nil {
   137  		return fmt.Errorf("failed to get top layer from image: %v", err)
   138  	}
   139  	defer l.Release()
   140  
   141  	hmacKey, err := metadata.ComputeV2MetadataHMACKey(p.config.AuthConfig)
   142  	if err != nil {
   143  		return fmt.Errorf("failed to compute hmac key of auth config: %v", err)
   144  	}
   145  
   146  	var descriptors []xfer.UploadDescriptor
   147  
   148  	descriptorTemplate := v2PushDescriptor{
   149  		v2MetadataService: p.v2MetadataService,
   150  		hmacKey:           hmacKey,
   151  		repoInfo:          p.repoInfo,
   152  		ref:               p.ref,
   153  		repo:              p.repo,
   154  		pushState:         &p.pushState,
   155  	}
   156  
   157  	// Loop bounds condition is to avoid pushing the base layer on Windows.
   158  	for i := 0; i < len(rootfs.DiffIDs); i++ {
   159  		descriptor := descriptorTemplate
   160  		descriptor.layer = l
   161  		descriptor.checkedDigests = make(map[digest.Digest]struct{})
   162  		descriptors = append(descriptors, &descriptor)
   163  
   164  		l = l.Parent()
   165  	}
   166  
   167  	if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil {
   168  		return err
   169  	}
   170  
   171  	// Try schema2 first
   172  	builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), p.config.ConfigMediaType, imgConfig)
   173  	manifest, err := manifestFromBuilder(ctx, builder, descriptors)
   174  	if err != nil {
   175  		return err
   176  	}
   177  
   178  	manSvc, err := p.repo.Manifests(ctx)
   179  	if err != nil {
   180  		return err
   181  	}
   182  
   183  	putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())}
   184  	if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
   185  		if runtime.GOOS == "windows" || p.config.TrustKey == nil || p.config.RequireSchema2 {
   186  			logrus.Warnf("failed to upload schema2 manifest: %v", err)
   187  			return err
   188  		}
   189  
   190  		logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
   191  
   192  		manifestRef, err := distreference.WithTag(p.repo.Named(), ref.Tag())
   193  		if err != nil {
   194  			return err
   195  		}
   196  		builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, imgConfig)
   197  		manifest, err = manifestFromBuilder(ctx, builder, descriptors)
   198  		if err != nil {
   199  			return err
   200  		}
   201  
   202  		if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
   203  			return err
   204  		}
   205  	}
   206  
   207  	var canonicalManifest []byte
   208  
   209  	switch v := manifest.(type) {
   210  	case *schema1.SignedManifest:
   211  		canonicalManifest = v.Canonical
   212  	case *schema2.DeserializedManifest:
   213  		_, canonicalManifest, err = v.Payload()
   214  		if err != nil {
   215  			return err
   216  		}
   217  	}
   218  
   219  	manifestDigest := digest.FromBytes(canonicalManifest)
   220  	progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest))
   221  
   222  	if err := addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
   223  		return err
   224  	}
   225  
   226  	// Signal digest to the trust client so it can sign the
   227  	// push, if appropriate.
   228  	progress.Aux(p.config.ProgressOutput, PushResult{Tag: ref.Tag(), Digest: manifestDigest, Size: len(canonicalManifest)})
   229  
   230  	return nil
   231  }
   232  
   233  func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuilder, descriptors []xfer.UploadDescriptor) (distribution.Manifest, error) {
   234  	// descriptors is in reverse order; iterate backwards to get references
   235  	// appended in the right order.
   236  	for i := len(descriptors) - 1; i >= 0; i-- {
   237  		if err := builder.AppendReference(descriptors[i].(*v2PushDescriptor)); err != nil {
   238  			return nil, err
   239  		}
   240  	}
   241  
   242  	return builder.Build(ctx)
   243  }
   244  
   245  type v2PushDescriptor struct {
   246  	layer             PushLayer
   247  	v2MetadataService metadata.V2MetadataService
   248  	hmacKey           []byte
   249  	repoInfo          reference.Named
   250  	ref               reference.Named
   251  	repo              distribution.Repository
   252  	pushState         *pushState
   253  	remoteDescriptor  distribution.Descriptor
   254  	// a set of digests whose presence has been checked in a target repository
   255  	checkedDigests map[digest.Digest]struct{}
   256  }
   257  
   258  func (pd *v2PushDescriptor) Key() string {
   259  	return "v2push:" + pd.ref.FullName() + " " + pd.layer.DiffID().String()
   260  }
   261  
   262  func (pd *v2PushDescriptor) ID() string {
   263  	return stringid.TruncateID(pd.layer.DiffID().String())
   264  }
   265  
   266  func (pd *v2PushDescriptor) DiffID() layer.DiffID {
   267  	return pd.layer.DiffID()
   268  }
   269  
   270  func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) {
   271  	if fs, ok := pd.layer.(distribution.Describable); ok {
   272  		if d := fs.Descriptor(); len(d.URLs) > 0 {
   273  			progress.Update(progressOutput, pd.ID(), "Skipped foreign layer")
   274  			return d, nil
   275  		}
   276  	}
   277  
   278  	diffID := pd.DiffID()
   279  
   280  	pd.pushState.Lock()
   281  	if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok {
   282  		// it is already known that the push is not needed and
   283  		// therefore doing a stat is unnecessary
   284  		pd.pushState.Unlock()
   285  		progress.Update(progressOutput, pd.ID(), "Layer already exists")
   286  		return descriptor, nil
   287  	}
   288  	pd.pushState.Unlock()
   289  
   290  	maxMountAttempts, maxExistenceChecks, checkOtherRepositories := getMaxMountAndExistenceCheckAttempts(pd.layer)
   291  
   292  	// Do we have any metadata associated with this layer's DiffID?
   293  	v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID)
   294  	if err == nil {
   295  		// check for blob existence in the target repository if we have a mapping with it
   296  		descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, false, 1, v2Metadata)
   297  		if exists || err != nil {
   298  			return descriptor, err
   299  		}
   300  	}
   301  
   302  	// if digest was empty or not saved, or if blob does not exist on the remote repository,
   303  	// then push the blob.
   304  	bs := pd.repo.Blobs(ctx)
   305  
   306  	var layerUpload distribution.BlobWriter
   307  
   308  	// Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload
   309  	candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata)
   310  	for _, mountCandidate := range candidates {
   311  		logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository)
   312  		createOpts := []distribution.BlobCreateOption{}
   313  
   314  		if len(mountCandidate.SourceRepository) > 0 {
   315  			namedRef, err := reference.WithName(mountCandidate.SourceRepository)
   316  			if err != nil {
   317  				logrus.Errorf("failed to parse source repository reference %v: %v", namedRef.String(), err)
   318  				pd.v2MetadataService.Remove(mountCandidate)
   319  				continue
   320  			}
   321  
   322  			// TODO (brianbland): We need to construct a reference where the Name is
   323  			// only the full remote name, so clean this up when distribution has a
   324  			// richer reference package
   325  			remoteRef, err := distreference.WithName(namedRef.RemoteName())
   326  			if err != nil {
   327  				logrus.Errorf("failed to make remote reference out of %q: %v", namedRef.RemoteName(), namedRef.RemoteName())
   328  				continue
   329  			}
   330  
   331  			canonicalRef, err := distreference.WithDigest(distreference.TrimNamed(remoteRef), mountCandidate.Digest)
   332  			if err != nil {
   333  				logrus.Errorf("failed to make canonical reference: %v", err)
   334  				continue
   335  			}
   336  
   337  			createOpts = append(createOpts, client.WithMountFrom(canonicalRef))
   338  		}
   339  
   340  		// send the layer
   341  		lu, err := bs.Create(ctx, createOpts...)
   342  		switch err := err.(type) {
   343  		case nil:
   344  			// noop
   345  		case distribution.ErrBlobMounted:
   346  			progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())
   347  
   348  			err.Descriptor.MediaType = schema2.MediaTypeLayer
   349  
   350  			pd.pushState.Lock()
   351  			pd.pushState.confirmedV2 = true
   352  			pd.pushState.remoteLayers[diffID] = err.Descriptor
   353  			pd.pushState.Unlock()
   354  
   355  			// Cache mapping from this layer's DiffID to the blobsum
   356  			if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
   357  				Digest:           err.Descriptor.Digest,
   358  				SourceRepository: pd.repoInfo.FullName(),
   359  			}); err != nil {
   360  				return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
   361  			}
   362  			return err.Descriptor, nil
   363  		default:
   364  			logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err)
   365  		}
   366  
   367  		if len(mountCandidate.SourceRepository) > 0 &&
   368  			(metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) ||
   369  				len(mountCandidate.HMAC) == 0) {
   370  			cause := "blob mount failure"
   371  			if err != nil {
   372  				cause = fmt.Sprintf("an error: %v", err.Error())
   373  			}
   374  			logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause)
   375  			pd.v2MetadataService.Remove(mountCandidate)
   376  		}
   377  
   378  		if lu != nil {
   379  			// cancel previous upload
   380  			cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload)
   381  			layerUpload = lu
   382  		}
   383  	}
   384  
   385  	if maxExistenceChecks-len(pd.checkedDigests) > 0 {
   386  		// do additional layer existence checks with other known digests if any
   387  		descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, checkOtherRepositories, maxExistenceChecks-len(pd.checkedDigests), v2Metadata)
   388  		if exists || err != nil {
   389  			return descriptor, err
   390  		}
   391  	}
   392  
   393  	logrus.Debugf("Pushing layer: %s", diffID)
   394  	if layerUpload == nil {
   395  		layerUpload, err = bs.Create(ctx)
   396  		if err != nil {
   397  			return distribution.Descriptor{}, retryOnError(err)
   398  		}
   399  	}
   400  	defer layerUpload.Close()
   401  
   402  	// upload the blob
   403  	desc, err := pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload)
   404  	if err != nil {
   405  		return desc, err
   406  	}
   407  
   408  	return desc, nil
   409  }
   410  
   411  func (pd *v2PushDescriptor) SetRemoteDescriptor(descriptor distribution.Descriptor) {
   412  	pd.remoteDescriptor = descriptor
   413  }
   414  
   415  func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor {
   416  	return pd.remoteDescriptor
   417  }
   418  
   419  func (pd *v2PushDescriptor) uploadUsingSession(
   420  	ctx context.Context,
   421  	progressOutput progress.Output,
   422  	diffID layer.DiffID,
   423  	layerUpload distribution.BlobWriter,
   424  ) (distribution.Descriptor, error) {
   425  	var reader io.ReadCloser
   426  
   427  	contentReader, err := pd.layer.Open()
   428  	size, _ := pd.layer.Size()
   429  
   430  	reader = progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, contentReader), progressOutput, size, pd.ID(), "Pushing")
   431  
   432  	switch m := pd.layer.MediaType(); m {
   433  	case schema2.MediaTypeUncompressedLayer:
   434  		compressedReader, compressionDone := compress(reader)
   435  		defer func(closer io.Closer) {
   436  			closer.Close()
   437  			<-compressionDone
   438  		}(reader)
   439  		reader = compressedReader
   440  	case schema2.MediaTypeLayer:
   441  	default:
   442  		reader.Close()
   443  		return distribution.Descriptor{}, fmt.Errorf("unsupported layer media type %s", m)
   444  	}
   445  
   446  	digester := digest.Canonical.New()
   447  	tee := io.TeeReader(reader, digester.Hash())
   448  
   449  	nn, err := layerUpload.ReadFrom(tee)
   450  	reader.Close()
   451  	if err != nil {
   452  		return distribution.Descriptor{}, retryOnError(err)
   453  	}
   454  
   455  	pushDigest := digester.Digest()
   456  	if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil {
   457  		return distribution.Descriptor{}, retryOnError(err)
   458  	}
   459  
   460  	logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn)
   461  	progress.Update(progressOutput, pd.ID(), "Pushed")
   462  
   463  	// Cache mapping from this layer's DiffID to the blobsum
   464  	if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
   465  		Digest:           pushDigest,
   466  		SourceRepository: pd.repoInfo.FullName(),
   467  	}); err != nil {
   468  		return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
   469  	}
   470  
   471  	desc := distribution.Descriptor{
   472  		Digest:    pushDigest,
   473  		MediaType: schema2.MediaTypeLayer,
   474  		Size:      nn,
   475  	}
   476  
   477  	pd.pushState.Lock()
   478  	// If Commit succeeded, that's an indication that the remote registry speaks the v2 protocol.
   479  	pd.pushState.confirmedV2 = true
   480  	pd.pushState.remoteLayers[diffID] = desc
   481  	pd.pushState.Unlock()
   482  
   483  	return desc, nil
   484  }
   485  
   486  // layerAlreadyExists checks if the registry already knows about any of the metadata passed in the "metadata"
   487  // slice. If it finds one that the registry knows about, it returns the known digest and "true". If
   488  // "checkOtherRepositories" is true, stat will be performed also with digests mapped to any other repository
   489  // (not just the target one).
   490  func (pd *v2PushDescriptor) layerAlreadyExists(
   491  	ctx context.Context,
   492  	progressOutput progress.Output,
   493  	diffID layer.DiffID,
   494  	checkOtherRepositories bool,
   495  	maxExistenceCheckAttempts int,
   496  	v2Metadata []metadata.V2Metadata,
   497  ) (desc distribution.Descriptor, exists bool, err error) {
   498  	// filter the metadata
   499  	candidates := []metadata.V2Metadata{}
   500  	for _, meta := range v2Metadata {
   501  		if len(meta.SourceRepository) > 0 && !checkOtherRepositories && meta.SourceRepository != pd.repoInfo.FullName() {
   502  			continue
   503  		}
   504  		candidates = append(candidates, meta)
   505  	}
   506  	// sort the candidates by similarity
   507  	sortV2MetadataByLikenessAndAge(pd.repoInfo, pd.hmacKey, candidates)
   508  
   509  	digestToMetadata := make(map[digest.Digest]*metadata.V2Metadata)
   510  	// an array of unique blob digests ordered from the best mount candidates to worst
   511  	layerDigests := []digest.Digest{}
   512  	for i := 0; i < len(candidates); i++ {
   513  		if len(layerDigests) >= maxExistenceCheckAttempts {
   514  			break
   515  		}
   516  		meta := &candidates[i]
   517  		if _, exists := digestToMetadata[meta.Digest]; exists {
   518  			// keep reference just to the first mapping (the best mount candidate)
   519  			continue
   520  		}
   521  		if _, exists := pd.checkedDigests[meta.Digest]; exists {
   522  			// existence of this digest has already been tested
   523  			continue
   524  		}
   525  		digestToMetadata[meta.Digest] = meta
   526  		layerDigests = append(layerDigests, meta.Digest)
   527  	}
   528  
   529  attempts:
   530  	for _, dgst := range layerDigests {
   531  		meta := digestToMetadata[dgst]
   532  		logrus.Debugf("Checking for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.FullName())
   533  		desc, err = pd.repo.Blobs(ctx).Stat(ctx, dgst)
   534  		pd.checkedDigests[meta.Digest] = struct{}{}
   535  		switch err {
   536  		case nil:
   537  			if m, ok := digestToMetadata[desc.Digest]; !ok || m.SourceRepository != pd.repoInfo.FullName() || !metadata.CheckV2MetadataHMAC(m, pd.hmacKey) {
   538  				// cache mapping from this layer's DiffID to the blobsum
   539  				if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
   540  					Digest:           desc.Digest,
   541  					SourceRepository: pd.repoInfo.FullName(),
   542  				}); err != nil {
   543  					return distribution.Descriptor{}, false, xfer.DoNotRetry{Err: err}
   544  				}
   545  			}
   546  			desc.MediaType = schema2.MediaTypeLayer
   547  			exists = true
   548  			break attempts
   549  		case distribution.ErrBlobUnknown:
   550  			if meta.SourceRepository == pd.repoInfo.FullName() {
   551  				// remove the mapping to the target repository
   552  				pd.v2MetadataService.Remove(*meta)
   553  			}
   554  		default:
   555  			logrus.WithError(err).Debugf("Failed to check for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.FullName())
   556  		}
   557  	}
   558  
   559  	if exists {
   560  		progress.Update(progressOutput, pd.ID(), "Layer already exists")
   561  		pd.pushState.Lock()
   562  		pd.pushState.remoteLayers[diffID] = desc
   563  		pd.pushState.Unlock()
   564  	}
   565  
   566  	return desc, exists, nil
   567  }
   568  
   569  // getMaxMountAndExistenceCheckAttempts returns a maximum number of cross repository mount attempts from
   570  // source repositories of target registry, maximum number of layer existence checks performed on the target
   571  // repository and whether the check shall be done also with digests mapped to different repositories. The
   572  // decision is based on layer size. The smaller the layer, the fewer attempts shall be made because the cost
   573  // of upload does not outweigh a latency.
   574  func getMaxMountAndExistenceCheckAttempts(layer PushLayer) (maxMountAttempts, maxExistenceCheckAttempts int, checkOtherRepositories bool) {
   575  	size, err := layer.Size()
   576  	switch {
   577  	// big blob
   578  	case size > middleLayerMaximumSize:
   579  		// 1st attempt to mount the blob few times
   580  		// 2nd few existence checks with digests associated to any repository
   581  		// then fallback to upload
   582  		return 4, 3, true
   583  
   584  	// middle sized blobs; if we could not get the size, assume we deal with middle sized blob
   585  	case size > smallLayerMaximumSize, err != nil:
   586  		// 1st attempt to mount blobs of average size few times
   587  		// 2nd try at most 1 existence check if there's an existing mapping to the target repository
   588  		// then fallback to upload
   589  		return 3, 1, false
   590  
   591  	// small blobs, do a minimum number of checks
   592  	default:
   593  		return 1, 1, false
   594  	}
   595  }
   596  
   597  // getRepositoryMountCandidates returns an array of v2 metadata items belonging to the given registry. The
   598  // array is sorted from youngest to oldest. If requireReigstryMatch is true, the resulting array will contain
   599  // only metadata entries having registry part of SourceRepository matching the part of repoInfo.
   600  func getRepositoryMountCandidates(
   601  	repoInfo reference.Named,
   602  	hmacKey []byte,
   603  	max int,
   604  	v2Metadata []metadata.V2Metadata,
   605  ) []metadata.V2Metadata {
   606  	candidates := []metadata.V2Metadata{}
   607  	for _, meta := range v2Metadata {
   608  		sourceRepo, err := reference.ParseNamed(meta.SourceRepository)
   609  		if err != nil || repoInfo.Hostname() != sourceRepo.Hostname() {
   610  			continue
   611  		}
   612  		// target repository is not a viable candidate
   613  		if meta.SourceRepository == repoInfo.FullName() {
   614  			continue
   615  		}
   616  		candidates = append(candidates, meta)
   617  	}
   618  
   619  	sortV2MetadataByLikenessAndAge(repoInfo, hmacKey, candidates)
   620  	if max >= 0 && len(candidates) > max {
   621  		// select the youngest metadata
   622  		candidates = candidates[:max]
   623  	}
   624  
   625  	return candidates
   626  }
   627  
   628  // byLikeness is a sorting container for v2 metadata candidates for cross repository mount. The
   629  // candidate "a" is preferred over "b":
   630  //
   631  //  1. if it was hashed using the same AuthConfig as the one used to authenticate to target repository and the
   632  //     "b" was not
   633  //  2. if a number of its repository path components exactly matching path components of target repository is higher
   634  type byLikeness struct {
   635  	arr            []metadata.V2Metadata
   636  	hmacKey        []byte
   637  	pathComponents []string
   638  }
   639  
   640  func (bla byLikeness) Less(i, j int) bool {
   641  	aMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[i], bla.hmacKey)
   642  	bMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[j], bla.hmacKey)
   643  	if aMacMatch != bMacMatch {
   644  		return aMacMatch
   645  	}
   646  	aMatch := numOfMatchingPathComponents(bla.arr[i].SourceRepository, bla.pathComponents)
   647  	bMatch := numOfMatchingPathComponents(bla.arr[j].SourceRepository, bla.pathComponents)
   648  	return aMatch > bMatch
   649  }
   650  func (bla byLikeness) Swap(i, j int) {
   651  	bla.arr[i], bla.arr[j] = bla.arr[j], bla.arr[i]
   652  }
   653  func (bla byLikeness) Len() int { return len(bla.arr) }
   654  
   655  func sortV2MetadataByLikenessAndAge(repoInfo reference.Named, hmacKey []byte, marr []metadata.V2Metadata) {
   656  	// reverse the metadata array to shift the newest entries to the beginning
   657  	for i := 0; i < len(marr)/2; i++ {
   658  		marr[i], marr[len(marr)-i-1] = marr[len(marr)-i-1], marr[i]
   659  	}
   660  	// keep equal entries ordered from the youngest to the oldest
   661  	sort.Stable(byLikeness{
   662  		arr:            marr,
   663  		hmacKey:        hmacKey,
   664  		pathComponents: getPathComponents(repoInfo.FullName()),
   665  	})
   666  }
   667  
   668  // numOfMatchingPathComponents returns a number of path components in "pth" that exactly match "matchComponents".
   669  func numOfMatchingPathComponents(pth string, matchComponents []string) int {
   670  	pthComponents := getPathComponents(pth)
   671  	i := 0
   672  	for ; i < len(pthComponents) && i < len(matchComponents); i++ {
   673  		if matchComponents[i] != pthComponents[i] {
   674  			return i
   675  		}
   676  	}
   677  	return i
   678  }
   679  
   680  func getPathComponents(path string) []string {
   681  	// make sure to add docker.io/ prefix to the path
   682  	named, err := reference.ParseNamed(path)
   683  	if err == nil {
   684  		path = named.FullName()
   685  	}
   686  	return strings.Split(path, "/")
   687  }
   688  
   689  func cancelLayerUpload(ctx context.Context, dgst digest.Digest, layerUpload distribution.BlobWriter) {
   690  	if layerUpload != nil {
   691  		logrus.Debugf("cancelling upload of blob %s", dgst)
   692  		err := layerUpload.Cancel(ctx)
   693  		if err != nil {
   694  			logrus.Warnf("failed to cancel upload: %v", err)
   695  		}
   696  	}
   697  }