github.com/rish1988/moby@v25.0.2+incompatible/daemon/containerd/image_push.go (about)

     1  package containerd
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"strings"
     8  	"sync"
     9  
    10  	"github.com/containerd/containerd/content"
    11  	cerrdefs "github.com/containerd/containerd/errdefs"
    12  	"github.com/containerd/containerd/images"
    13  	containerdimages "github.com/containerd/containerd/images"
    14  	containerdlabels "github.com/containerd/containerd/labels"
    15  	"github.com/containerd/containerd/platforms"
    16  	"github.com/containerd/containerd/remotes"
    17  	"github.com/containerd/containerd/remotes/docker"
    18  	"github.com/containerd/log"
    19  	"github.com/distribution/reference"
    20  	"github.com/docker/docker/api/types/events"
    21  	"github.com/docker/docker/api/types/registry"
    22  	"github.com/docker/docker/errdefs"
    23  	"github.com/docker/docker/internal/compatcontext"
    24  	"github.com/docker/docker/pkg/progress"
    25  	"github.com/docker/docker/pkg/streamformatter"
    26  	"github.com/opencontainers/go-digest"
    27  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    28  	"github.com/pkg/errors"
    29  	"golang.org/x/sync/semaphore"
    30  )
    31  
    32  // PushImage initiates a push operation of the image pointed to by sourceRef.
    33  // If reference is untagged, all tags from the reference repository are pushed.
    34  // Image manifest (or index) is pushed as is, which will probably fail if you
    35  // don't have all content referenced by the index.
    36  // Cross-repo mounts will be attempted for non-existing blobs.
    37  //
    38  // It will also add distribution source labels to the pushed content
    39  // pointing to the new target repository. This will allow subsequent pushes
    40  // to perform cross-repo mounts of the shared content when pushing to a different
    41  // repository on the same registry.
    42  func (i *ImageService) PushImage(ctx context.Context, sourceRef reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) (retErr error) {
    43  	out := streamformatter.NewJSONProgressOutput(outStream, false)
    44  	progress.Messagef(out, "", "The push refers to repository [%s]", sourceRef.Name())
    45  
    46  	if _, tagged := sourceRef.(reference.Tagged); !tagged {
    47  		if _, digested := sourceRef.(reference.Digested); !digested {
    48  			// Image is not tagged nor digested, that means all tags push was requested.
    49  
    50  			// Find all images with the same repository.
    51  			imgs, err := i.getAllImagesWithRepository(ctx, sourceRef)
    52  			if err != nil {
    53  				return err
    54  			}
    55  
    56  			if len(imgs) == 0 {
    57  				return fmt.Errorf("An image does not exist locally with the tag: %s", reference.FamiliarName(sourceRef))
    58  			}
    59  
    60  			for _, img := range imgs {
    61  				named, err := reference.ParseNamed(img.Name)
    62  				if err != nil {
    63  					// This shouldn't happen, but log a warning just in case.
    64  					log.G(ctx).WithFields(log.Fields{
    65  						"image":     img.Name,
    66  						"sourceRef": sourceRef,
    67  					}).Warn("refusing to push an invalid tag")
    68  					continue
    69  				}
    70  
    71  				if err := i.pushRef(ctx, named, metaHeaders, authConfig, out); err != nil {
    72  					return err
    73  				}
    74  			}
    75  
    76  			return nil
    77  		}
    78  	}
    79  
    80  	return i.pushRef(ctx, sourceRef, metaHeaders, authConfig, out)
    81  }
    82  
    83  func (i *ImageService) pushRef(ctx context.Context, targetRef reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, out progress.Output) (retErr error) {
    84  	leasedCtx, release, err := i.client.WithLease(ctx)
    85  	if err != nil {
    86  		return err
    87  	}
    88  	defer func() {
    89  		if err := release(compatcontext.WithoutCancel(leasedCtx)); err != nil {
    90  			log.G(ctx).WithField("image", targetRef).WithError(err).Warn("failed to release lease created for push")
    91  		}
    92  	}()
    93  
    94  	img, err := i.client.ImageService().Get(ctx, targetRef.String())
    95  	if err != nil {
    96  		if cerrdefs.IsNotFound(err) {
    97  			return errdefs.NotFound(fmt.Errorf("tag does not exist: %s", reference.FamiliarString(targetRef)))
    98  		}
    99  		return errdefs.NotFound(err)
   100  	}
   101  
   102  	target := img.Target
   103  	store := i.client.ContentStore()
   104  
   105  	resolver, tracker := i.newResolverFromAuthConfig(ctx, authConfig, targetRef)
   106  	pp := pushProgress{Tracker: tracker}
   107  	jobsQueue := newJobs()
   108  	finishProgress := jobsQueue.showProgress(ctx, out, combinedProgress([]progressUpdater{
   109  		&pp,
   110  		pullProgress{showExists: false, store: store},
   111  	}))
   112  	defer func() {
   113  		finishProgress()
   114  		if retErr == nil {
   115  			if tagged, ok := targetRef.(reference.Tagged); ok {
   116  				progress.Messagef(out, "", "%s: digest: %s size: %d", tagged.Tag(), target.Digest, img.Target.Size)
   117  			}
   118  		}
   119  	}()
   120  
   121  	var limiter *semaphore.Weighted = nil // TODO: Respect max concurrent downloads/uploads
   122  
   123  	mountableBlobs, err := findMissingMountable(ctx, store, jobsQueue, target, targetRef, limiter)
   124  	if err != nil {
   125  		return err
   126  	}
   127  
   128  	// Create a store which fakes the local existence of possibly mountable blobs.
   129  	// Otherwise they can't be pushed at all.
   130  	realStore := store
   131  	wrapped := wrapWithFakeMountableBlobs(store, mountableBlobs)
   132  	store = wrapped
   133  
   134  	pusher, err := resolver.Pusher(ctx, targetRef.String())
   135  	if err != nil {
   136  		return err
   137  	}
   138  
   139  	addLayerJobs := containerdimages.HandlerFunc(
   140  		func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
   141  			switch {
   142  			case containerdimages.IsIndexType(desc.MediaType),
   143  				containerdimages.IsManifestType(desc.MediaType),
   144  				containerdimages.IsConfigType(desc.MediaType):
   145  			default:
   146  				jobsQueue.Add(desc)
   147  			}
   148  
   149  			return nil, nil
   150  		},
   151  	)
   152  
   153  	handlerWrapper := func(h images.Handler) images.Handler {
   154  		return containerdimages.Handlers(addLayerJobs, h)
   155  	}
   156  
   157  	err = remotes.PushContent(ctx, pusher, target, store, limiter, platforms.All, handlerWrapper)
   158  	if err != nil {
   159  		if containerdimages.IsIndexType(target.MediaType) && cerrdefs.IsNotFound(err) {
   160  			return errdefs.NotFound(fmt.Errorf(
   161  				"missing content: %w\n"+
   162  					"Note: You're trying to push a manifest list/index which "+
   163  					"references multiple platform specific manifests, but not all of them are available locally "+
   164  					"or available to the remote repository.\n"+
   165  					"Make sure you have all the referenced content and try again.",
   166  				err))
   167  		}
   168  		return err
   169  	}
   170  
   171  	appendDistributionSourceLabel(ctx, realStore, targetRef, target)
   172  
   173  	i.LogImageEvent(reference.FamiliarString(targetRef), reference.FamiliarName(targetRef), events.ActionPush)
   174  
   175  	return nil
   176  }
   177  
   178  func appendDistributionSourceLabel(ctx context.Context, realStore content.Store, targetRef reference.Named, target ocispec.Descriptor) {
   179  	appendSource, err := docker.AppendDistributionSourceLabel(realStore, targetRef.String())
   180  	if err != nil {
   181  		// This shouldn't happen at this point because the reference would have to be invalid
   182  		// and if it was, then it would error out earlier.
   183  		log.G(ctx).WithError(err).Warn("failed to create an handler that appends distribution source label to pushed content")
   184  		return
   185  	}
   186  
   187  	handler := presentChildrenHandler(realStore, appendSource)
   188  	if err := containerdimages.Dispatch(ctx, handler, nil, target); err != nil {
   189  		// Shouldn't happen, but even if it would fail, then make it only a warning
   190  		// because it doesn't affect the pushed data.
   191  		log.G(ctx).WithError(err).Warn("failed to append distribution source labels to pushed content")
   192  	}
   193  }
   194  
   195  // findMissingMountable will walk the target descriptor recursively and return
   196  // missing contents with their distribution source which could potentially
   197  // be cross-repo mounted.
   198  func findMissingMountable(ctx context.Context, store content.Store, queue *jobs,
   199  	target ocispec.Descriptor, targetRef reference.Named, limiter *semaphore.Weighted,
   200  ) (map[digest.Digest]distributionSource, error) {
   201  	mountableBlobs := map[digest.Digest]distributionSource{}
   202  	var mutex sync.Mutex
   203  
   204  	sources, err := getDigestSources(ctx, store, target.Digest)
   205  	if err != nil {
   206  		if !errdefs.IsNotFound(err) {
   207  			return nil, err
   208  		}
   209  		log.G(ctx).WithField("target", target).Debug("distribution source label not found")
   210  		return mountableBlobs, nil
   211  	}
   212  
   213  	handler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
   214  		_, err := store.Info(ctx, desc.Digest)
   215  		if err != nil {
   216  			if !cerrdefs.IsNotFound(err) {
   217  				return nil, errdefs.System(errors.Wrapf(err, "failed to get metadata of content %s", desc.Digest.String()))
   218  			}
   219  
   220  			for _, source := range sources {
   221  				if canBeMounted(desc.MediaType, targetRef, source) {
   222  					mutex.Lock()
   223  					mountableBlobs[desc.Digest] = source
   224  					mutex.Unlock()
   225  					queue.Add(desc)
   226  					break
   227  				}
   228  			}
   229  			return nil, nil
   230  		}
   231  
   232  		return containerdimages.Children(ctx, store, desc)
   233  	}
   234  
   235  	err = containerdimages.Dispatch(ctx, containerdimages.HandlerFunc(handler), limiter, target)
   236  	if err != nil {
   237  		return nil, err
   238  	}
   239  
   240  	return mountableBlobs, nil
   241  }
   242  
   243  func getDigestSources(ctx context.Context, store content.Manager, digest digest.Digest) ([]distributionSource, error) {
   244  	info, err := store.Info(ctx, digest)
   245  	if err != nil {
   246  		if cerrdefs.IsNotFound(err) {
   247  			return nil, errdefs.NotFound(err)
   248  		}
   249  		return nil, errdefs.System(err)
   250  	}
   251  
   252  	sources := extractDistributionSources(info.Labels)
   253  	if sources == nil {
   254  		return nil, errdefs.NotFound(fmt.Errorf("label %q is not attached to %s", containerdlabels.LabelDistributionSource, digest.String()))
   255  	}
   256  
   257  	return sources, nil
   258  }
   259  
   260  func extractDistributionSources(labels map[string]string) []distributionSource {
   261  	var sources []distributionSource
   262  
   263  	// Check if this blob has a distributionSource label
   264  	// if yes, read it as source
   265  	for k, v := range labels {
   266  		if reg := strings.TrimPrefix(k, containerdlabels.LabelDistributionSource); reg != k {
   267  			for _, repo := range strings.Split(v, ",") {
   268  				ref, err := reference.ParseNamed(reg + "/" + repo)
   269  				if err != nil {
   270  					continue
   271  				}
   272  
   273  				sources = append(sources, distributionSource{
   274  					registryRef: ref,
   275  				})
   276  			}
   277  		}
   278  	}
   279  
   280  	return sources
   281  }
   282  
   283  type distributionSource struct {
   284  	registryRef reference.Named
   285  }
   286  
   287  // ToAnnotation returns key and value
   288  func (source distributionSource) ToAnnotation() (string, string) {
   289  	domain := reference.Domain(source.registryRef)
   290  	v := reference.Path(source.registryRef)
   291  	return containerdlabels.LabelDistributionSource + domain, v
   292  }
   293  
   294  func (source distributionSource) GetReference(dgst digest.Digest) (reference.Named, error) {
   295  	return reference.WithDigest(source.registryRef, dgst)
   296  }
   297  
   298  // canBeMounted returns if the content with given media type can be cross-repo
   299  // mounted when pushing it to a remote reference ref.
   300  func canBeMounted(mediaType string, targetRef reference.Named, source distributionSource) bool {
   301  	if containerdimages.IsManifestType(mediaType) {
   302  		return false
   303  	}
   304  	if containerdimages.IsIndexType(mediaType) {
   305  		return false
   306  	}
   307  
   308  	reg := reference.Domain(targetRef)
   309  	// Remove :port suffix from domain
   310  	// containerd distribution source label doesn't store port
   311  	if portIdx := strings.LastIndex(reg, ":"); portIdx != -1 {
   312  		reg = reg[:portIdx]
   313  	}
   314  
   315  	// If the source registry is the same as the one we are pushing to
   316  	// then the cross-repo mount will work.
   317  	return reg == reference.Domain(source.registryRef)
   318  }