github.com/rish1988/moby@v25.0.2+incompatible/daemon/containerd/image_push.go (about) 1 package containerd 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "strings" 8 "sync" 9 10 "github.com/containerd/containerd/content" 11 cerrdefs "github.com/containerd/containerd/errdefs" 12 "github.com/containerd/containerd/images" 13 containerdimages "github.com/containerd/containerd/images" 14 containerdlabels "github.com/containerd/containerd/labels" 15 "github.com/containerd/containerd/platforms" 16 "github.com/containerd/containerd/remotes" 17 "github.com/containerd/containerd/remotes/docker" 18 "github.com/containerd/log" 19 "github.com/distribution/reference" 20 "github.com/docker/docker/api/types/events" 21 "github.com/docker/docker/api/types/registry" 22 "github.com/docker/docker/errdefs" 23 "github.com/docker/docker/internal/compatcontext" 24 "github.com/docker/docker/pkg/progress" 25 "github.com/docker/docker/pkg/streamformatter" 26 "github.com/opencontainers/go-digest" 27 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 28 "github.com/pkg/errors" 29 "golang.org/x/sync/semaphore" 30 ) 31 32 // PushImage initiates a push operation of the image pointed to by sourceRef. 33 // If reference is untagged, all tags from the reference repository are pushed. 34 // Image manifest (or index) is pushed as is, which will probably fail if you 35 // don't have all content referenced by the index. 36 // Cross-repo mounts will be attempted for non-existing blobs. 37 // 38 // It will also add distribution source labels to the pushed content 39 // pointing to the new target repository. This will allow subsequent pushes 40 // to perform cross-repo mounts of the shared content when pushing to a different 41 // repository on the same registry. 42 func (i *ImageService) PushImage(ctx context.Context, sourceRef reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) (retErr error) { 43 out := streamformatter.NewJSONProgressOutput(outStream, false) 44 progress.Messagef(out, "", "The push refers to repository [%s]", sourceRef.Name()) 45 46 if _, tagged := sourceRef.(reference.Tagged); !tagged { 47 if _, digested := sourceRef.(reference.Digested); !digested { 48 // Image is not tagged nor digested, that means all tags push was requested. 49 50 // Find all images with the same repository. 51 imgs, err := i.getAllImagesWithRepository(ctx, sourceRef) 52 if err != nil { 53 return err 54 } 55 56 if len(imgs) == 0 { 57 return fmt.Errorf("An image does not exist locally with the tag: %s", reference.FamiliarName(sourceRef)) 58 } 59 60 for _, img := range imgs { 61 named, err := reference.ParseNamed(img.Name) 62 if err != nil { 63 // This shouldn't happen, but log a warning just in case. 64 log.G(ctx).WithFields(log.Fields{ 65 "image": img.Name, 66 "sourceRef": sourceRef, 67 }).Warn("refusing to push an invalid tag") 68 continue 69 } 70 71 if err := i.pushRef(ctx, named, metaHeaders, authConfig, out); err != nil { 72 return err 73 } 74 } 75 76 return nil 77 } 78 } 79 80 return i.pushRef(ctx, sourceRef, metaHeaders, authConfig, out) 81 } 82 83 func (i *ImageService) pushRef(ctx context.Context, targetRef reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, out progress.Output) (retErr error) { 84 leasedCtx, release, err := i.client.WithLease(ctx) 85 if err != nil { 86 return err 87 } 88 defer func() { 89 if err := release(compatcontext.WithoutCancel(leasedCtx)); err != nil { 90 log.G(ctx).WithField("image", targetRef).WithError(err).Warn("failed to release lease created for push") 91 } 92 }() 93 94 img, err := i.client.ImageService().Get(ctx, targetRef.String()) 95 if err != nil { 96 if cerrdefs.IsNotFound(err) { 97 return errdefs.NotFound(fmt.Errorf("tag does not exist: %s", reference.FamiliarString(targetRef))) 98 } 99 return errdefs.NotFound(err) 100 } 101 102 target := img.Target 103 store := i.client.ContentStore() 104 105 resolver, tracker := i.newResolverFromAuthConfig(ctx, authConfig, targetRef) 106 pp := pushProgress{Tracker: tracker} 107 jobsQueue := newJobs() 108 finishProgress := jobsQueue.showProgress(ctx, out, combinedProgress([]progressUpdater{ 109 &pp, 110 pullProgress{showExists: false, store: store}, 111 })) 112 defer func() { 113 finishProgress() 114 if retErr == nil { 115 if tagged, ok := targetRef.(reference.Tagged); ok { 116 progress.Messagef(out, "", "%s: digest: %s size: %d", tagged.Tag(), target.Digest, img.Target.Size) 117 } 118 } 119 }() 120 121 var limiter *semaphore.Weighted = nil // TODO: Respect max concurrent downloads/uploads 122 123 mountableBlobs, err := findMissingMountable(ctx, store, jobsQueue, target, targetRef, limiter) 124 if err != nil { 125 return err 126 } 127 128 // Create a store which fakes the local existence of possibly mountable blobs. 129 // Otherwise they can't be pushed at all. 130 realStore := store 131 wrapped := wrapWithFakeMountableBlobs(store, mountableBlobs) 132 store = wrapped 133 134 pusher, err := resolver.Pusher(ctx, targetRef.String()) 135 if err != nil { 136 return err 137 } 138 139 addLayerJobs := containerdimages.HandlerFunc( 140 func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 141 switch { 142 case containerdimages.IsIndexType(desc.MediaType), 143 containerdimages.IsManifestType(desc.MediaType), 144 containerdimages.IsConfigType(desc.MediaType): 145 default: 146 jobsQueue.Add(desc) 147 } 148 149 return nil, nil 150 }, 151 ) 152 153 handlerWrapper := func(h images.Handler) images.Handler { 154 return containerdimages.Handlers(addLayerJobs, h) 155 } 156 157 err = remotes.PushContent(ctx, pusher, target, store, limiter, platforms.All, handlerWrapper) 158 if err != nil { 159 if containerdimages.IsIndexType(target.MediaType) && cerrdefs.IsNotFound(err) { 160 return errdefs.NotFound(fmt.Errorf( 161 "missing content: %w\n"+ 162 "Note: You're trying to push a manifest list/index which "+ 163 "references multiple platform specific manifests, but not all of them are available locally "+ 164 "or available to the remote repository.\n"+ 165 "Make sure you have all the referenced content and try again.", 166 err)) 167 } 168 return err 169 } 170 171 appendDistributionSourceLabel(ctx, realStore, targetRef, target) 172 173 i.LogImageEvent(reference.FamiliarString(targetRef), reference.FamiliarName(targetRef), events.ActionPush) 174 175 return nil 176 } 177 178 func appendDistributionSourceLabel(ctx context.Context, realStore content.Store, targetRef reference.Named, target ocispec.Descriptor) { 179 appendSource, err := docker.AppendDistributionSourceLabel(realStore, targetRef.String()) 180 if err != nil { 181 // This shouldn't happen at this point because the reference would have to be invalid 182 // and if it was, then it would error out earlier. 183 log.G(ctx).WithError(err).Warn("failed to create an handler that appends distribution source label to pushed content") 184 return 185 } 186 187 handler := presentChildrenHandler(realStore, appendSource) 188 if err := containerdimages.Dispatch(ctx, handler, nil, target); err != nil { 189 // Shouldn't happen, but even if it would fail, then make it only a warning 190 // because it doesn't affect the pushed data. 191 log.G(ctx).WithError(err).Warn("failed to append distribution source labels to pushed content") 192 } 193 } 194 195 // findMissingMountable will walk the target descriptor recursively and return 196 // missing contents with their distribution source which could potentially 197 // be cross-repo mounted. 198 func findMissingMountable(ctx context.Context, store content.Store, queue *jobs, 199 target ocispec.Descriptor, targetRef reference.Named, limiter *semaphore.Weighted, 200 ) (map[digest.Digest]distributionSource, error) { 201 mountableBlobs := map[digest.Digest]distributionSource{} 202 var mutex sync.Mutex 203 204 sources, err := getDigestSources(ctx, store, target.Digest) 205 if err != nil { 206 if !errdefs.IsNotFound(err) { 207 return nil, err 208 } 209 log.G(ctx).WithField("target", target).Debug("distribution source label not found") 210 return mountableBlobs, nil 211 } 212 213 handler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 214 _, err := store.Info(ctx, desc.Digest) 215 if err != nil { 216 if !cerrdefs.IsNotFound(err) { 217 return nil, errdefs.System(errors.Wrapf(err, "failed to get metadata of content %s", desc.Digest.String())) 218 } 219 220 for _, source := range sources { 221 if canBeMounted(desc.MediaType, targetRef, source) { 222 mutex.Lock() 223 mountableBlobs[desc.Digest] = source 224 mutex.Unlock() 225 queue.Add(desc) 226 break 227 } 228 } 229 return nil, nil 230 } 231 232 return containerdimages.Children(ctx, store, desc) 233 } 234 235 err = containerdimages.Dispatch(ctx, containerdimages.HandlerFunc(handler), limiter, target) 236 if err != nil { 237 return nil, err 238 } 239 240 return mountableBlobs, nil 241 } 242 243 func getDigestSources(ctx context.Context, store content.Manager, digest digest.Digest) ([]distributionSource, error) { 244 info, err := store.Info(ctx, digest) 245 if err != nil { 246 if cerrdefs.IsNotFound(err) { 247 return nil, errdefs.NotFound(err) 248 } 249 return nil, errdefs.System(err) 250 } 251 252 sources := extractDistributionSources(info.Labels) 253 if sources == nil { 254 return nil, errdefs.NotFound(fmt.Errorf("label %q is not attached to %s", containerdlabels.LabelDistributionSource, digest.String())) 255 } 256 257 return sources, nil 258 } 259 260 func extractDistributionSources(labels map[string]string) []distributionSource { 261 var sources []distributionSource 262 263 // Check if this blob has a distributionSource label 264 // if yes, read it as source 265 for k, v := range labels { 266 if reg := strings.TrimPrefix(k, containerdlabels.LabelDistributionSource); reg != k { 267 for _, repo := range strings.Split(v, ",") { 268 ref, err := reference.ParseNamed(reg + "/" + repo) 269 if err != nil { 270 continue 271 } 272 273 sources = append(sources, distributionSource{ 274 registryRef: ref, 275 }) 276 } 277 } 278 } 279 280 return sources 281 } 282 283 type distributionSource struct { 284 registryRef reference.Named 285 } 286 287 // ToAnnotation returns key and value 288 func (source distributionSource) ToAnnotation() (string, string) { 289 domain := reference.Domain(source.registryRef) 290 v := reference.Path(source.registryRef) 291 return containerdlabels.LabelDistributionSource + domain, v 292 } 293 294 func (source distributionSource) GetReference(dgst digest.Digest) (reference.Named, error) { 295 return reference.WithDigest(source.registryRef, dgst) 296 } 297 298 // canBeMounted returns if the content with given media type can be cross-repo 299 // mounted when pushing it to a remote reference ref. 300 func canBeMounted(mediaType string, targetRef reference.Named, source distributionSource) bool { 301 if containerdimages.IsManifestType(mediaType) { 302 return false 303 } 304 if containerdimages.IsIndexType(mediaType) { 305 return false 306 } 307 308 reg := reference.Domain(targetRef) 309 // Remove :port suffix from domain 310 // containerd distribution source label doesn't store port 311 if portIdx := strings.LastIndex(reg, ":"); portIdx != -1 { 312 reg = reg[:portIdx] 313 } 314 315 // If the source registry is the same as the one we are pushing to 316 // then the cross-repo mount will work. 317 return reg == reference.Domain(source.registryRef) 318 }