github.com/kaisenlinux/docker.io@v0.0.0-20230510090727-ea55db55fac7/engine/distribution/push_v2.go (about)

     1  package distribution // import "github.com/docker/docker/distribution"
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"runtime"
     9  	"sort"
    10  	"strings"
    11  	"sync"
    12  
    13  	"github.com/docker/distribution"
    14  	"github.com/docker/distribution/manifest/schema1"
    15  	"github.com/docker/distribution/manifest/schema2"
    16  	"github.com/docker/distribution/reference"
    17  	"github.com/docker/distribution/registry/api/errcode"
    18  	"github.com/docker/distribution/registry/client"
    19  	apitypes "github.com/docker/docker/api/types"
    20  	"github.com/docker/docker/distribution/metadata"
    21  	"github.com/docker/docker/distribution/xfer"
    22  	"github.com/docker/docker/layer"
    23  	"github.com/docker/docker/pkg/ioutils"
    24  	"github.com/docker/docker/pkg/progress"
    25  	"github.com/docker/docker/pkg/stringid"
    26  	"github.com/docker/docker/registry"
    27  	digest "github.com/opencontainers/go-digest"
    28  	"github.com/pkg/errors"
    29  	"github.com/sirupsen/logrus"
    30  )
    31  
    32  const (
    33  	smallLayerMaximumSize  = 100 * (1 << 10) // 100KB
    34  	middleLayerMaximumSize = 10 * (1 << 20)  // 10MB
    35  )
    36  
    37  type v2Pusher struct {
    38  	v2MetadataService metadata.V2MetadataService
    39  	ref               reference.Named
    40  	endpoint          registry.APIEndpoint
    41  	repoInfo          *registry.RepositoryInfo
    42  	config            *ImagePushConfig
    43  	repo              distribution.Repository
    44  
    45  	// pushState is state built by the Upload functions.
    46  	pushState pushState
    47  }
    48  
    49  type pushState struct {
    50  	sync.Mutex
    51  	// remoteLayers is the set of layers known to exist on the remote side.
    52  	// This avoids redundant queries when pushing multiple tags that
    53  	// involve the same layers. It is also used to fill in digest and size
    54  	// information when building the manifest.
    55  	remoteLayers map[layer.DiffID]distribution.Descriptor
    56  	// confirmedV2 is set to true if we confirm we're talking to a v2
    57  	// registry. This is used to limit fallbacks to the v1 protocol.
    58  	confirmedV2 bool
    59  	hasAuthInfo bool
    60  }
    61  
    62  func (p *v2Pusher) Push(ctx context.Context) (err error) {
    63  	p.pushState.remoteLayers = make(map[layer.DiffID]distribution.Descriptor)
    64  
    65  	p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull")
    66  	p.pushState.hasAuthInfo = p.config.AuthConfig.RegistryToken != "" || (p.config.AuthConfig.Username != "" && p.config.AuthConfig.Password != "")
    67  	if err != nil {
    68  		logrus.Debugf("Error getting v2 registry: %v", err)
    69  		return err
    70  	}
    71  
    72  	if err = p.pushV2Repository(ctx); err != nil {
    73  		if continueOnError(err, p.endpoint.Mirror) {
    74  			return fallbackError{
    75  				err:         err,
    76  				confirmedV2: p.pushState.confirmedV2,
    77  				transportOK: true,
    78  			}
    79  		}
    80  	}
    81  	return err
    82  }
    83  
    84  func (p *v2Pusher) pushV2Repository(ctx context.Context) (err error) {
    85  	if namedTagged, isNamedTagged := p.ref.(reference.NamedTagged); isNamedTagged {
    86  		imageID, err := p.config.ReferenceStore.Get(p.ref)
    87  		if err != nil {
    88  			return fmt.Errorf("tag does not exist: %s", reference.FamiliarString(p.ref))
    89  		}
    90  
    91  		return p.pushV2Tag(ctx, namedTagged, imageID)
    92  	}
    93  
    94  	if !reference.IsNameOnly(p.ref) {
    95  		return errors.New("cannot push a digest reference")
    96  	}
    97  
    98  	// Push all tags
    99  	pushed := 0
   100  	for _, association := range p.config.ReferenceStore.ReferencesByName(p.ref) {
   101  		if namedTagged, isNamedTagged := association.Ref.(reference.NamedTagged); isNamedTagged {
   102  			pushed++
   103  			if err := p.pushV2Tag(ctx, namedTagged, association.ID); err != nil {
   104  				return err
   105  			}
   106  		}
   107  	}
   108  
   109  	if pushed == 0 {
   110  		return fmt.Errorf("no tags to push for %s", reference.FamiliarName(p.repoInfo.Name))
   111  	}
   112  
   113  	return nil
   114  }
   115  
   116  func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id digest.Digest) error {
   117  	logrus.Debugf("Pushing repository: %s", reference.FamiliarString(ref))
   118  
   119  	imgConfig, err := p.config.ImageStore.Get(ctx, id)
   120  	if err != nil {
   121  		return fmt.Errorf("could not find image from tag %s: %v", reference.FamiliarString(ref), err)
   122  	}
   123  
   124  	rootfs, err := p.config.ImageStore.RootFSFromConfig(imgConfig)
   125  	if err != nil {
   126  		return fmt.Errorf("unable to get rootfs for image %s: %s", reference.FamiliarString(ref), err)
   127  	}
   128  
   129  	platform, err := p.config.ImageStore.PlatformFromConfig(imgConfig)
   130  	if err != nil {
   131  		return fmt.Errorf("unable to get platform for image %s: %s", reference.FamiliarString(ref), err)
   132  	}
   133  
   134  	l, err := p.config.LayerStores[platform.OS].Get(rootfs.ChainID())
   135  	if err != nil {
   136  		return fmt.Errorf("failed to get top layer from image: %v", err)
   137  	}
   138  	defer l.Release()
   139  
   140  	hmacKey, err := metadata.ComputeV2MetadataHMACKey(p.config.AuthConfig)
   141  	if err != nil {
   142  		return fmt.Errorf("failed to compute hmac key of auth config: %v", err)
   143  	}
   144  
   145  	var descriptors []xfer.UploadDescriptor
   146  
   147  	descriptorTemplate := v2PushDescriptor{
   148  		v2MetadataService: p.v2MetadataService,
   149  		hmacKey:           hmacKey,
   150  		repoInfo:          p.repoInfo.Name,
   151  		ref:               p.ref,
   152  		endpoint:          p.endpoint,
   153  		repo:              p.repo,
   154  		pushState:         &p.pushState,
   155  	}
   156  
   157  	// Loop bounds condition is to avoid pushing the base layer on Windows.
   158  	for range rootfs.DiffIDs {
   159  		descriptor := descriptorTemplate
   160  		descriptor.layer = l
   161  		descriptor.checkedDigests = make(map[digest.Digest]struct{})
   162  		descriptors = append(descriptors, &descriptor)
   163  
   164  		l = l.Parent()
   165  	}
   166  
   167  	if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil {
   168  		return err
   169  	}
   170  
   171  	// Try schema2 first
   172  	builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), p.config.ConfigMediaType, imgConfig)
   173  	manifest, err := manifestFromBuilder(ctx, builder, descriptors)
   174  	if err != nil {
   175  		return err
   176  	}
   177  
   178  	manSvc, err := p.repo.Manifests(ctx)
   179  	if err != nil {
   180  		return err
   181  	}
   182  
   183  	putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())}
   184  	if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
   185  		if runtime.GOOS == "windows" || p.config.TrustKey == nil || p.config.RequireSchema2 {
   186  			logrus.Warnf("failed to upload schema2 manifest: %v", err)
   187  			return err
   188  		}
   189  
   190  		// This is a temporary environment variables used in CI to allow pushing
   191  		// manifest v2 schema 1 images to test-registries used for testing *pulling*
   192  		// these images.
   193  		if os.Getenv("DOCKER_ALLOW_SCHEMA1_PUSH_DONOTUSE") == "" {
   194  			if err.Error() == "tag invalid" {
   195  				msg := "[DEPRECATED] support for pushing manifest v2 schema1 images has been removed. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/"
   196  				logrus.WithError(err).Error(msg)
   197  				return errors.Wrap(err, msg)
   198  			}
   199  			return err
   200  		}
   201  
   202  		logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
   203  
   204  		// Note: this fallback is deprecated, see log messages below
   205  		manifestRef, err := reference.WithTag(p.repo.Named(), ref.Tag())
   206  		if err != nil {
   207  			return err
   208  		}
   209  		builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, imgConfig)
   210  		manifest, err = manifestFromBuilder(ctx, builder, descriptors)
   211  		if err != nil {
   212  			return err
   213  		}
   214  
   215  		if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
   216  			return err
   217  		}
   218  
   219  		// schema2 failed but schema1 succeeded
   220  		msg := fmt.Sprintf("[DEPRECATION NOTICE] support for pushing manifest v2 schema1 images will be removed in an upcoming release. Please contact admins of the %s registry NOW to avoid future disruption. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", reference.Domain(ref))
   221  		logrus.Warn(msg)
   222  		progress.Message(p.config.ProgressOutput, "", msg)
   223  	}
   224  
   225  	var canonicalManifest []byte
   226  
   227  	switch v := manifest.(type) {
   228  	case *schema1.SignedManifest:
   229  		canonicalManifest = v.Canonical
   230  	case *schema2.DeserializedManifest:
   231  		_, canonicalManifest, err = v.Payload()
   232  		if err != nil {
   233  			return err
   234  		}
   235  	}
   236  
   237  	manifestDigest := digest.FromBytes(canonicalManifest)
   238  	progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest))
   239  
   240  	if err := addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
   241  		return err
   242  	}
   243  
   244  	// Signal digest to the trust client so it can sign the
   245  	// push, if appropriate.
   246  	progress.Aux(p.config.ProgressOutput, apitypes.PushResult{Tag: ref.Tag(), Digest: manifestDigest.String(), Size: len(canonicalManifest)})
   247  
   248  	return nil
   249  }
   250  
   251  func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuilder, descriptors []xfer.UploadDescriptor) (distribution.Manifest, error) {
   252  	// descriptors is in reverse order; iterate backwards to get references
   253  	// appended in the right order.
   254  	for i := len(descriptors) - 1; i >= 0; i-- {
   255  		if err := builder.AppendReference(descriptors[i].(*v2PushDescriptor)); err != nil {
   256  			return nil, err
   257  		}
   258  	}
   259  
   260  	return builder.Build(ctx)
   261  }
   262  
   263  type v2PushDescriptor struct {
   264  	layer             PushLayer
   265  	v2MetadataService metadata.V2MetadataService
   266  	hmacKey           []byte
   267  	repoInfo          reference.Named
   268  	ref               reference.Named
   269  	endpoint          registry.APIEndpoint
   270  	repo              distribution.Repository
   271  	pushState         *pushState
   272  	remoteDescriptor  distribution.Descriptor
   273  	// a set of digests whose presence has been checked in a target repository
   274  	checkedDigests map[digest.Digest]struct{}
   275  }
   276  
   277  func (pd *v2PushDescriptor) Key() string {
   278  	return "v2push:" + pd.ref.Name() + " " + pd.layer.DiffID().String()
   279  }
   280  
   281  func (pd *v2PushDescriptor) ID() string {
   282  	return stringid.TruncateID(pd.layer.DiffID().String())
   283  }
   284  
   285  func (pd *v2PushDescriptor) DiffID() layer.DiffID {
   286  	return pd.layer.DiffID()
   287  }
   288  
   289  func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) {
   290  	// Skip foreign layers unless this registry allows nondistributable artifacts.
   291  	if !pd.endpoint.AllowNondistributableArtifacts {
   292  		if fs, ok := pd.layer.(distribution.Describable); ok {
   293  			if d := fs.Descriptor(); len(d.URLs) > 0 {
   294  				progress.Update(progressOutput, pd.ID(), "Skipped foreign layer")
   295  				return d, nil
   296  			}
   297  		}
   298  	}
   299  
   300  	diffID := pd.DiffID()
   301  
   302  	pd.pushState.Lock()
   303  	if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok {
   304  		// it is already known that the push is not needed and
   305  		// therefore doing a stat is unnecessary
   306  		pd.pushState.Unlock()
   307  		progress.Update(progressOutput, pd.ID(), "Layer already exists")
   308  		return descriptor, nil
   309  	}
   310  	pd.pushState.Unlock()
   311  
   312  	maxMountAttempts, maxExistenceChecks, checkOtherRepositories := getMaxMountAndExistenceCheckAttempts(pd.layer)
   313  
   314  	// Do we have any metadata associated with this layer's DiffID?
   315  	v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID)
   316  	if err == nil {
   317  		// check for blob existence in the target repository
   318  		descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, true, 1, v2Metadata)
   319  		if exists || err != nil {
   320  			return descriptor, err
   321  		}
   322  	}
   323  
   324  	// if digest was empty or not saved, or if blob does not exist on the remote repository,
   325  	// then push the blob.
   326  	bs := pd.repo.Blobs(ctx)
   327  
   328  	var layerUpload distribution.BlobWriter
   329  
   330  	// Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload
   331  	candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata)
   332  	isUnauthorizedError := false
   333  	for _, mc := range candidates {
   334  		mountCandidate := mc
   335  		logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository)
   336  		createOpts := []distribution.BlobCreateOption{}
   337  
   338  		if len(mountCandidate.SourceRepository) > 0 {
   339  			namedRef, err := reference.ParseNormalizedNamed(mountCandidate.SourceRepository)
   340  			if err != nil {
   341  				logrus.Errorf("failed to parse source repository reference %v: %v", reference.FamiliarString(namedRef), err)
   342  				pd.v2MetadataService.Remove(mountCandidate)
   343  				continue
   344  			}
   345  
   346  			// Candidates are always under same domain, create remote reference
   347  			// with only path to set mount from with
   348  			remoteRef, err := reference.WithName(reference.Path(namedRef))
   349  			if err != nil {
   350  				logrus.Errorf("failed to make remote reference out of %q: %v", reference.Path(namedRef), err)
   351  				continue
   352  			}
   353  
   354  			canonicalRef, err := reference.WithDigest(reference.TrimNamed(remoteRef), mountCandidate.Digest)
   355  			if err != nil {
   356  				logrus.Errorf("failed to make canonical reference: %v", err)
   357  				continue
   358  			}
   359  
   360  			createOpts = append(createOpts, client.WithMountFrom(canonicalRef))
   361  		}
   362  
   363  		// send the layer
   364  		lu, err := bs.Create(ctx, createOpts...)
   365  		switch err := err.(type) {
   366  		case nil:
   367  			// noop
   368  		case distribution.ErrBlobMounted:
   369  			progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())
   370  
   371  			err.Descriptor.MediaType = schema2.MediaTypeLayer
   372  
   373  			pd.pushState.Lock()
   374  			pd.pushState.confirmedV2 = true
   375  			pd.pushState.remoteLayers[diffID] = err.Descriptor
   376  			pd.pushState.Unlock()
   377  
   378  			// Cache mapping from this layer's DiffID to the blobsum
   379  			if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
   380  				Digest:           err.Descriptor.Digest,
   381  				SourceRepository: pd.repoInfo.Name(),
   382  			}); err != nil {
   383  				return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
   384  			}
   385  			return err.Descriptor, nil
   386  		case errcode.Errors:
   387  			for _, e := range err {
   388  				switch e := e.(type) {
   389  				case errcode.Error:
   390  					if e.Code == errcode.ErrorCodeUnauthorized {
   391  						// when unauthorized error that indicate user don't has right to push layer to register
   392  						logrus.Debugln("failed to push layer to registry because unauthorized error")
   393  						isUnauthorizedError = true
   394  					}
   395  				default:
   396  				}
   397  			}
   398  		default:
   399  			logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err)
   400  		}
   401  
   402  		// when error is unauthorizedError and user don't hasAuthInfo that's the case user don't has right to push layer to register
   403  		// and he hasn't login either, in this case candidate cache should be removed
   404  		if len(mountCandidate.SourceRepository) > 0 &&
   405  			!(isUnauthorizedError && !pd.pushState.hasAuthInfo) &&
   406  			(metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) ||
   407  				len(mountCandidate.HMAC) == 0) {
   408  			cause := "blob mount failure"
   409  			if err != nil {
   410  				cause = fmt.Sprintf("an error: %v", err.Error())
   411  			}
   412  			logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause)
   413  			pd.v2MetadataService.Remove(mountCandidate)
   414  		}
   415  
   416  		if lu != nil {
   417  			// cancel previous upload
   418  			cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload)
   419  			layerUpload = lu
   420  		}
   421  	}
   422  
   423  	if maxExistenceChecks-len(pd.checkedDigests) > 0 {
   424  		// do additional layer existence checks with other known digests if any
   425  		descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, checkOtherRepositories, maxExistenceChecks-len(pd.checkedDigests), v2Metadata)
   426  		if exists || err != nil {
   427  			return descriptor, err
   428  		}
   429  	}
   430  
   431  	logrus.Debugf("Pushing layer: %s", diffID)
   432  	if layerUpload == nil {
   433  		layerUpload, err = bs.Create(ctx)
   434  		if err != nil {
   435  			return distribution.Descriptor{}, retryOnError(err)
   436  		}
   437  	}
   438  	defer layerUpload.Close()
   439  	// upload the blob
   440  	return pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload)
   441  }
   442  
   443  func (pd *v2PushDescriptor) SetRemoteDescriptor(descriptor distribution.Descriptor) {
   444  	pd.remoteDescriptor = descriptor
   445  }
   446  
   447  func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor {
   448  	return pd.remoteDescriptor
   449  }
   450  
   451  func (pd *v2PushDescriptor) uploadUsingSession(
   452  	ctx context.Context,
   453  	progressOutput progress.Output,
   454  	diffID layer.DiffID,
   455  	layerUpload distribution.BlobWriter,
   456  ) (distribution.Descriptor, error) {
   457  	var reader io.ReadCloser
   458  
   459  	contentReader, err := pd.layer.Open()
   460  	if err != nil {
   461  		return distribution.Descriptor{}, retryOnError(err)
   462  	}
   463  
   464  	size, _ := pd.layer.Size()
   465  
   466  	reader = progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, contentReader), progressOutput, size, pd.ID(), "Pushing")
   467  
   468  	switch m := pd.layer.MediaType(); m {
   469  	case schema2.MediaTypeUncompressedLayer:
   470  		compressedReader, compressionDone := compress(reader)
   471  		defer func(closer io.Closer) {
   472  			closer.Close()
   473  			<-compressionDone
   474  		}(reader)
   475  		reader = compressedReader
   476  	case schema2.MediaTypeLayer:
   477  	default:
   478  		reader.Close()
   479  		return distribution.Descriptor{}, fmt.Errorf("unsupported layer media type %s", m)
   480  	}
   481  
   482  	digester := digest.Canonical.Digester()
   483  	tee := io.TeeReader(reader, digester.Hash())
   484  
   485  	nn, err := layerUpload.ReadFrom(tee)
   486  	reader.Close()
   487  	if err != nil {
   488  		return distribution.Descriptor{}, retryOnError(err)
   489  	}
   490  
   491  	pushDigest := digester.Digest()
   492  	if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil {
   493  		return distribution.Descriptor{}, retryOnError(err)
   494  	}
   495  
   496  	logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn)
   497  	progress.Update(progressOutput, pd.ID(), "Pushed")
   498  
   499  	// Cache mapping from this layer's DiffID to the blobsum
   500  	if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
   501  		Digest:           pushDigest,
   502  		SourceRepository: pd.repoInfo.Name(),
   503  	}); err != nil {
   504  		return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
   505  	}
   506  
   507  	desc := distribution.Descriptor{
   508  		Digest:    pushDigest,
   509  		MediaType: schema2.MediaTypeLayer,
   510  		Size:      nn,
   511  	}
   512  
   513  	pd.pushState.Lock()
   514  	// If Commit succeeded, that's an indication that the remote registry speaks the v2 protocol.
   515  	pd.pushState.confirmedV2 = true
   516  	pd.pushState.remoteLayers[diffID] = desc
   517  	pd.pushState.Unlock()
   518  
   519  	return desc, nil
   520  }
   521  
   522  // layerAlreadyExists checks if the registry already knows about any of the metadata passed in the "metadata"
   523  // slice. If it finds one that the registry knows about, it returns the known digest and "true". If
   524  // "checkOtherRepositories" is true, stat will be performed also with digests mapped to any other repository
   525  // (not just the target one).
   526  func (pd *v2PushDescriptor) layerAlreadyExists(
   527  	ctx context.Context,
   528  	progressOutput progress.Output,
   529  	diffID layer.DiffID,
   530  	checkOtherRepositories bool,
   531  	maxExistenceCheckAttempts int,
   532  	v2Metadata []metadata.V2Metadata,
   533  ) (desc distribution.Descriptor, exists bool, err error) {
   534  	// filter the metadata
   535  	candidates := []metadata.V2Metadata{}
   536  	for _, meta := range v2Metadata {
   537  		if len(meta.SourceRepository) > 0 && !checkOtherRepositories && meta.SourceRepository != pd.repoInfo.Name() {
   538  			continue
   539  		}
   540  		candidates = append(candidates, meta)
   541  	}
   542  	// sort the candidates by similarity
   543  	sortV2MetadataByLikenessAndAge(pd.repoInfo, pd.hmacKey, candidates)
   544  
   545  	digestToMetadata := make(map[digest.Digest]*metadata.V2Metadata)
   546  	// an array of unique blob digests ordered from the best mount candidates to worst
   547  	layerDigests := []digest.Digest{}
   548  	for i := 0; i < len(candidates); i++ {
   549  		if len(layerDigests) >= maxExistenceCheckAttempts {
   550  			break
   551  		}
   552  		meta := &candidates[i]
   553  		if _, exists := digestToMetadata[meta.Digest]; exists {
   554  			// keep reference just to the first mapping (the best mount candidate)
   555  			continue
   556  		}
   557  		if _, exists := pd.checkedDigests[meta.Digest]; exists {
   558  			// existence of this digest has already been tested
   559  			continue
   560  		}
   561  		digestToMetadata[meta.Digest] = meta
   562  		layerDigests = append(layerDigests, meta.Digest)
   563  	}
   564  
   565  attempts:
   566  	for _, dgst := range layerDigests {
   567  		meta := digestToMetadata[dgst]
   568  		logrus.Debugf("Checking for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.Name())
   569  		desc, err = pd.repo.Blobs(ctx).Stat(ctx, dgst)
   570  		pd.checkedDigests[meta.Digest] = struct{}{}
   571  		switch err {
   572  		case nil:
   573  			if m, ok := digestToMetadata[desc.Digest]; !ok || m.SourceRepository != pd.repoInfo.Name() || !metadata.CheckV2MetadataHMAC(m, pd.hmacKey) {
   574  				// cache mapping from this layer's DiffID to the blobsum
   575  				if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
   576  					Digest:           desc.Digest,
   577  					SourceRepository: pd.repoInfo.Name(),
   578  				}); err != nil {
   579  					return distribution.Descriptor{}, false, xfer.DoNotRetry{Err: err}
   580  				}
   581  			}
   582  			desc.MediaType = schema2.MediaTypeLayer
   583  			exists = true
   584  			break attempts
   585  		case distribution.ErrBlobUnknown:
   586  			if meta.SourceRepository == pd.repoInfo.Name() {
   587  				// remove the mapping to the target repository
   588  				pd.v2MetadataService.Remove(*meta)
   589  			}
   590  		default:
   591  			logrus.WithError(err).Debugf("Failed to check for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.Name())
   592  		}
   593  	}
   594  
   595  	if exists {
   596  		progress.Update(progressOutput, pd.ID(), "Layer already exists")
   597  		pd.pushState.Lock()
   598  		pd.pushState.remoteLayers[diffID] = desc
   599  		pd.pushState.Unlock()
   600  	}
   601  
   602  	return desc, exists, nil
   603  }
   604  
   605  // getMaxMountAndExistenceCheckAttempts returns a maximum number of cross repository mount attempts from
   606  // source repositories of target registry, maximum number of layer existence checks performed on the target
   607  // repository and whether the check shall be done also with digests mapped to different repositories. The
   608  // decision is based on layer size. The smaller the layer, the fewer attempts shall be made because the cost
   609  // of upload does not outweigh a latency.
   610  func getMaxMountAndExistenceCheckAttempts(layer PushLayer) (maxMountAttempts, maxExistenceCheckAttempts int, checkOtherRepositories bool) {
   611  	size, err := layer.Size()
   612  	switch {
   613  	// big blob
   614  	case size > middleLayerMaximumSize:
   615  		// 1st attempt to mount the blob few times
   616  		// 2nd few existence checks with digests associated to any repository
   617  		// then fallback to upload
   618  		return 4, 3, true
   619  
   620  	// middle sized blobs; if we could not get the size, assume we deal with middle sized blob
   621  	case size > smallLayerMaximumSize, err != nil:
   622  		// 1st attempt to mount blobs of average size few times
   623  		// 2nd try at most 1 existence check if there's an existing mapping to the target repository
   624  		// then fallback to upload
   625  		return 3, 1, false
   626  
   627  	// small blobs, do a minimum number of checks
   628  	default:
   629  		return 1, 1, false
   630  	}
   631  }
   632  
   633  // getRepositoryMountCandidates returns an array of v2 metadata items belonging to the given registry. The
   634  // array is sorted from youngest to oldest. If requireRegistryMatch is true, the resulting array will contain
   635  // only metadata entries having registry part of SourceRepository matching the part of repoInfo.
   636  func getRepositoryMountCandidates(
   637  	repoInfo reference.Named,
   638  	hmacKey []byte,
   639  	max int,
   640  	v2Metadata []metadata.V2Metadata,
   641  ) []metadata.V2Metadata {
   642  	candidates := []metadata.V2Metadata{}
   643  	for _, meta := range v2Metadata {
   644  		sourceRepo, err := reference.ParseNamed(meta.SourceRepository)
   645  		if err != nil || reference.Domain(repoInfo) != reference.Domain(sourceRepo) {
   646  			continue
   647  		}
   648  		// target repository is not a viable candidate
   649  		if meta.SourceRepository == repoInfo.Name() {
   650  			continue
   651  		}
   652  		candidates = append(candidates, meta)
   653  	}
   654  
   655  	sortV2MetadataByLikenessAndAge(repoInfo, hmacKey, candidates)
   656  	if max >= 0 && len(candidates) > max {
   657  		// select the youngest metadata
   658  		candidates = candidates[:max]
   659  	}
   660  
   661  	return candidates
   662  }
   663  
   664  // byLikeness is a sorting container for v2 metadata candidates for cross repository mount. The
   665  // candidate "a" is preferred over "b":
   666  //
   667  //  1. if it was hashed using the same AuthConfig as the one used to authenticate to target repository and the
   668  //     "b" was not
   669  //  2. if a number of its repository path components exactly matching path components of target repository is higher
   670  type byLikeness struct {
   671  	arr            []metadata.V2Metadata
   672  	hmacKey        []byte
   673  	pathComponents []string
   674  }
   675  
   676  func (bla byLikeness) Less(i, j int) bool {
   677  	aMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[i], bla.hmacKey)
   678  	bMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[j], bla.hmacKey)
   679  	if aMacMatch != bMacMatch {
   680  		return aMacMatch
   681  	}
   682  	aMatch := numOfMatchingPathComponents(bla.arr[i].SourceRepository, bla.pathComponents)
   683  	bMatch := numOfMatchingPathComponents(bla.arr[j].SourceRepository, bla.pathComponents)
   684  	return aMatch > bMatch
   685  }
   686  func (bla byLikeness) Swap(i, j int) {
   687  	bla.arr[i], bla.arr[j] = bla.arr[j], bla.arr[i]
   688  }
   689  func (bla byLikeness) Len() int { return len(bla.arr) }
   690  
   691  func sortV2MetadataByLikenessAndAge(repoInfo reference.Named, hmacKey []byte, marr []metadata.V2Metadata) {
   692  	// reverse the metadata array to shift the newest entries to the beginning
   693  	for i := 0; i < len(marr)/2; i++ {
   694  		marr[i], marr[len(marr)-i-1] = marr[len(marr)-i-1], marr[i]
   695  	}
   696  	// keep equal entries ordered from the youngest to the oldest
   697  	sort.Stable(byLikeness{
   698  		arr:            marr,
   699  		hmacKey:        hmacKey,
   700  		pathComponents: getPathComponents(repoInfo.Name()),
   701  	})
   702  }
   703  
   704  // numOfMatchingPathComponents returns a number of path components in "pth" that exactly match "matchComponents".
   705  func numOfMatchingPathComponents(pth string, matchComponents []string) int {
   706  	pthComponents := getPathComponents(pth)
   707  	i := 0
   708  	for ; i < len(pthComponents) && i < len(matchComponents); i++ {
   709  		if matchComponents[i] != pthComponents[i] {
   710  			return i
   711  		}
   712  	}
   713  	return i
   714  }
   715  
   716  func getPathComponents(path string) []string {
   717  	return strings.Split(path, "/")
   718  }
   719  
   720  func cancelLayerUpload(ctx context.Context, dgst digest.Digest, layerUpload distribution.BlobWriter) {
   721  	if layerUpload != nil {
   722  		logrus.Debugf("cancelling upload of blob %s", dgst)
   723  		err := layerUpload.Cancel(ctx)
   724  		if err != nil {
   725  			logrus.Warnf("failed to cancel upload: %v", err)
   726  		}
   727  	}
   728  }