github.com/kaisenlinux/docker.io@v0.0.0-20230510090727-ea55db55fac7/engine/distribution/push_v2.go (about) 1 package distribution // import "github.com/docker/docker/distribution" 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "os" 8 "runtime" 9 "sort" 10 "strings" 11 "sync" 12 13 "github.com/docker/distribution" 14 "github.com/docker/distribution/manifest/schema1" 15 "github.com/docker/distribution/manifest/schema2" 16 "github.com/docker/distribution/reference" 17 "github.com/docker/distribution/registry/api/errcode" 18 "github.com/docker/distribution/registry/client" 19 apitypes "github.com/docker/docker/api/types" 20 "github.com/docker/docker/distribution/metadata" 21 "github.com/docker/docker/distribution/xfer" 22 "github.com/docker/docker/layer" 23 "github.com/docker/docker/pkg/ioutils" 24 "github.com/docker/docker/pkg/progress" 25 "github.com/docker/docker/pkg/stringid" 26 "github.com/docker/docker/registry" 27 digest "github.com/opencontainers/go-digest" 28 "github.com/pkg/errors" 29 "github.com/sirupsen/logrus" 30 ) 31 32 const ( 33 smallLayerMaximumSize = 100 * (1 << 10) // 100KB 34 middleLayerMaximumSize = 10 * (1 << 20) // 10MB 35 ) 36 37 type v2Pusher struct { 38 v2MetadataService metadata.V2MetadataService 39 ref reference.Named 40 endpoint registry.APIEndpoint 41 repoInfo *registry.RepositoryInfo 42 config *ImagePushConfig 43 repo distribution.Repository 44 45 // pushState is state built by the Upload functions. 46 pushState pushState 47 } 48 49 type pushState struct { 50 sync.Mutex 51 // remoteLayers is the set of layers known to exist on the remote side. 52 // This avoids redundant queries when pushing multiple tags that 53 // involve the same layers. It is also used to fill in digest and size 54 // information when building the manifest. 55 remoteLayers map[layer.DiffID]distribution.Descriptor 56 // confirmedV2 is set to true if we confirm we're talking to a v2 57 // registry. This is used to limit fallbacks to the v1 protocol. 58 confirmedV2 bool 59 hasAuthInfo bool 60 } 61 62 func (p *v2Pusher) Push(ctx context.Context) (err error) { 63 p.pushState.remoteLayers = make(map[layer.DiffID]distribution.Descriptor) 64 65 p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull") 66 p.pushState.hasAuthInfo = p.config.AuthConfig.RegistryToken != "" || (p.config.AuthConfig.Username != "" && p.config.AuthConfig.Password != "") 67 if err != nil { 68 logrus.Debugf("Error getting v2 registry: %v", err) 69 return err 70 } 71 72 if err = p.pushV2Repository(ctx); err != nil { 73 if continueOnError(err, p.endpoint.Mirror) { 74 return fallbackError{ 75 err: err, 76 confirmedV2: p.pushState.confirmedV2, 77 transportOK: true, 78 } 79 } 80 } 81 return err 82 } 83 84 func (p *v2Pusher) pushV2Repository(ctx context.Context) (err error) { 85 if namedTagged, isNamedTagged := p.ref.(reference.NamedTagged); isNamedTagged { 86 imageID, err := p.config.ReferenceStore.Get(p.ref) 87 if err != nil { 88 return fmt.Errorf("tag does not exist: %s", reference.FamiliarString(p.ref)) 89 } 90 91 return p.pushV2Tag(ctx, namedTagged, imageID) 92 } 93 94 if !reference.IsNameOnly(p.ref) { 95 return errors.New("cannot push a digest reference") 96 } 97 98 // Push all tags 99 pushed := 0 100 for _, association := range p.config.ReferenceStore.ReferencesByName(p.ref) { 101 if namedTagged, isNamedTagged := association.Ref.(reference.NamedTagged); isNamedTagged { 102 pushed++ 103 if err := p.pushV2Tag(ctx, namedTagged, association.ID); err != nil { 104 return err 105 } 106 } 107 } 108 109 if pushed == 0 { 110 return fmt.Errorf("no tags to push for %s", reference.FamiliarName(p.repoInfo.Name)) 111 } 112 113 return nil 114 } 115 116 func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id digest.Digest) error { 117 logrus.Debugf("Pushing repository: %s", reference.FamiliarString(ref)) 118 119 imgConfig, err := p.config.ImageStore.Get(ctx, id) 120 if err != nil { 121 return fmt.Errorf("could not find image from tag %s: %v", reference.FamiliarString(ref), err) 122 } 123 124 rootfs, err := p.config.ImageStore.RootFSFromConfig(imgConfig) 125 if err != nil { 126 return fmt.Errorf("unable to get rootfs for image %s: %s", reference.FamiliarString(ref), err) 127 } 128 129 platform, err := p.config.ImageStore.PlatformFromConfig(imgConfig) 130 if err != nil { 131 return fmt.Errorf("unable to get platform for image %s: %s", reference.FamiliarString(ref), err) 132 } 133 134 l, err := p.config.LayerStores[platform.OS].Get(rootfs.ChainID()) 135 if err != nil { 136 return fmt.Errorf("failed to get top layer from image: %v", err) 137 } 138 defer l.Release() 139 140 hmacKey, err := metadata.ComputeV2MetadataHMACKey(p.config.AuthConfig) 141 if err != nil { 142 return fmt.Errorf("failed to compute hmac key of auth config: %v", err) 143 } 144 145 var descriptors []xfer.UploadDescriptor 146 147 descriptorTemplate := v2PushDescriptor{ 148 v2MetadataService: p.v2MetadataService, 149 hmacKey: hmacKey, 150 repoInfo: p.repoInfo.Name, 151 ref: p.ref, 152 endpoint: p.endpoint, 153 repo: p.repo, 154 pushState: &p.pushState, 155 } 156 157 // Loop bounds condition is to avoid pushing the base layer on Windows. 158 for range rootfs.DiffIDs { 159 descriptor := descriptorTemplate 160 descriptor.layer = l 161 descriptor.checkedDigests = make(map[digest.Digest]struct{}) 162 descriptors = append(descriptors, &descriptor) 163 164 l = l.Parent() 165 } 166 167 if err := p.config.UploadManager.Upload(ctx, descriptors, p.config.ProgressOutput); err != nil { 168 return err 169 } 170 171 // Try schema2 first 172 builder := schema2.NewManifestBuilder(p.repo.Blobs(ctx), p.config.ConfigMediaType, imgConfig) 173 manifest, err := manifestFromBuilder(ctx, builder, descriptors) 174 if err != nil { 175 return err 176 } 177 178 manSvc, err := p.repo.Manifests(ctx) 179 if err != nil { 180 return err 181 } 182 183 putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())} 184 if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil { 185 if runtime.GOOS == "windows" || p.config.TrustKey == nil || p.config.RequireSchema2 { 186 logrus.Warnf("failed to upload schema2 manifest: %v", err) 187 return err 188 } 189 190 // This is a temporary environment variables used in CI to allow pushing 191 // manifest v2 schema 1 images to test-registries used for testing *pulling* 192 // these images. 193 if os.Getenv("DOCKER_ALLOW_SCHEMA1_PUSH_DONOTUSE") == "" { 194 if err.Error() == "tag invalid" { 195 msg := "[DEPRECATED] support for pushing manifest v2 schema1 images has been removed. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/" 196 logrus.WithError(err).Error(msg) 197 return errors.Wrap(err, msg) 198 } 199 return err 200 } 201 202 logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err) 203 204 // Note: this fallback is deprecated, see log messages below 205 manifestRef, err := reference.WithTag(p.repo.Named(), ref.Tag()) 206 if err != nil { 207 return err 208 } 209 builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, imgConfig) 210 manifest, err = manifestFromBuilder(ctx, builder, descriptors) 211 if err != nil { 212 return err 213 } 214 215 if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil { 216 return err 217 } 218 219 // schema2 failed but schema1 succeeded 220 msg := fmt.Sprintf("[DEPRECATION NOTICE] support for pushing manifest v2 schema1 images will be removed in an upcoming release. Please contact admins of the %s registry NOW to avoid future disruption. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", reference.Domain(ref)) 221 logrus.Warn(msg) 222 progress.Message(p.config.ProgressOutput, "", msg) 223 } 224 225 var canonicalManifest []byte 226 227 switch v := manifest.(type) { 228 case *schema1.SignedManifest: 229 canonicalManifest = v.Canonical 230 case *schema2.DeserializedManifest: 231 _, canonicalManifest, err = v.Payload() 232 if err != nil { 233 return err 234 } 235 } 236 237 manifestDigest := digest.FromBytes(canonicalManifest) 238 progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", ref.Tag(), manifestDigest, len(canonicalManifest)) 239 240 if err := addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil { 241 return err 242 } 243 244 // Signal digest to the trust client so it can sign the 245 // push, if appropriate. 246 progress.Aux(p.config.ProgressOutput, apitypes.PushResult{Tag: ref.Tag(), Digest: manifestDigest.String(), Size: len(canonicalManifest)}) 247 248 return nil 249 } 250 251 func manifestFromBuilder(ctx context.Context, builder distribution.ManifestBuilder, descriptors []xfer.UploadDescriptor) (distribution.Manifest, error) { 252 // descriptors is in reverse order; iterate backwards to get references 253 // appended in the right order. 254 for i := len(descriptors) - 1; i >= 0; i-- { 255 if err := builder.AppendReference(descriptors[i].(*v2PushDescriptor)); err != nil { 256 return nil, err 257 } 258 } 259 260 return builder.Build(ctx) 261 } 262 263 type v2PushDescriptor struct { 264 layer PushLayer 265 v2MetadataService metadata.V2MetadataService 266 hmacKey []byte 267 repoInfo reference.Named 268 ref reference.Named 269 endpoint registry.APIEndpoint 270 repo distribution.Repository 271 pushState *pushState 272 remoteDescriptor distribution.Descriptor 273 // a set of digests whose presence has been checked in a target repository 274 checkedDigests map[digest.Digest]struct{} 275 } 276 277 func (pd *v2PushDescriptor) Key() string { 278 return "v2push:" + pd.ref.Name() + " " + pd.layer.DiffID().String() 279 } 280 281 func (pd *v2PushDescriptor) ID() string { 282 return stringid.TruncateID(pd.layer.DiffID().String()) 283 } 284 285 func (pd *v2PushDescriptor) DiffID() layer.DiffID { 286 return pd.layer.DiffID() 287 } 288 289 func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { 290 // Skip foreign layers unless this registry allows nondistributable artifacts. 291 if !pd.endpoint.AllowNondistributableArtifacts { 292 if fs, ok := pd.layer.(distribution.Describable); ok { 293 if d := fs.Descriptor(); len(d.URLs) > 0 { 294 progress.Update(progressOutput, pd.ID(), "Skipped foreign layer") 295 return d, nil 296 } 297 } 298 } 299 300 diffID := pd.DiffID() 301 302 pd.pushState.Lock() 303 if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok { 304 // it is already known that the push is not needed and 305 // therefore doing a stat is unnecessary 306 pd.pushState.Unlock() 307 progress.Update(progressOutput, pd.ID(), "Layer already exists") 308 return descriptor, nil 309 } 310 pd.pushState.Unlock() 311 312 maxMountAttempts, maxExistenceChecks, checkOtherRepositories := getMaxMountAndExistenceCheckAttempts(pd.layer) 313 314 // Do we have any metadata associated with this layer's DiffID? 315 v2Metadata, err := pd.v2MetadataService.GetMetadata(diffID) 316 if err == nil { 317 // check for blob existence in the target repository 318 descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, true, 1, v2Metadata) 319 if exists || err != nil { 320 return descriptor, err 321 } 322 } 323 324 // if digest was empty or not saved, or if blob does not exist on the remote repository, 325 // then push the blob. 326 bs := pd.repo.Blobs(ctx) 327 328 var layerUpload distribution.BlobWriter 329 330 // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload 331 candidates := getRepositoryMountCandidates(pd.repoInfo, pd.hmacKey, maxMountAttempts, v2Metadata) 332 isUnauthorizedError := false 333 for _, mc := range candidates { 334 mountCandidate := mc 335 logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository) 336 createOpts := []distribution.BlobCreateOption{} 337 338 if len(mountCandidate.SourceRepository) > 0 { 339 namedRef, err := reference.ParseNormalizedNamed(mountCandidate.SourceRepository) 340 if err != nil { 341 logrus.Errorf("failed to parse source repository reference %v: %v", reference.FamiliarString(namedRef), err) 342 pd.v2MetadataService.Remove(mountCandidate) 343 continue 344 } 345 346 // Candidates are always under same domain, create remote reference 347 // with only path to set mount from with 348 remoteRef, err := reference.WithName(reference.Path(namedRef)) 349 if err != nil { 350 logrus.Errorf("failed to make remote reference out of %q: %v", reference.Path(namedRef), err) 351 continue 352 } 353 354 canonicalRef, err := reference.WithDigest(reference.TrimNamed(remoteRef), mountCandidate.Digest) 355 if err != nil { 356 logrus.Errorf("failed to make canonical reference: %v", err) 357 continue 358 } 359 360 createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) 361 } 362 363 // send the layer 364 lu, err := bs.Create(ctx, createOpts...) 365 switch err := err.(type) { 366 case nil: 367 // noop 368 case distribution.ErrBlobMounted: 369 progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) 370 371 err.Descriptor.MediaType = schema2.MediaTypeLayer 372 373 pd.pushState.Lock() 374 pd.pushState.confirmedV2 = true 375 pd.pushState.remoteLayers[diffID] = err.Descriptor 376 pd.pushState.Unlock() 377 378 // Cache mapping from this layer's DiffID to the blobsum 379 if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{ 380 Digest: err.Descriptor.Digest, 381 SourceRepository: pd.repoInfo.Name(), 382 }); err != nil { 383 return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} 384 } 385 return err.Descriptor, nil 386 case errcode.Errors: 387 for _, e := range err { 388 switch e := e.(type) { 389 case errcode.Error: 390 if e.Code == errcode.ErrorCodeUnauthorized { 391 // when unauthorized error that indicate user don't has right to push layer to register 392 logrus.Debugln("failed to push layer to registry because unauthorized error") 393 isUnauthorizedError = true 394 } 395 default: 396 } 397 } 398 default: 399 logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err) 400 } 401 402 // when error is unauthorizedError and user don't hasAuthInfo that's the case user don't has right to push layer to register 403 // and he hasn't login either, in this case candidate cache should be removed 404 if len(mountCandidate.SourceRepository) > 0 && 405 !(isUnauthorizedError && !pd.pushState.hasAuthInfo) && 406 (metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) || 407 len(mountCandidate.HMAC) == 0) { 408 cause := "blob mount failure" 409 if err != nil { 410 cause = fmt.Sprintf("an error: %v", err.Error()) 411 } 412 logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause) 413 pd.v2MetadataService.Remove(mountCandidate) 414 } 415 416 if lu != nil { 417 // cancel previous upload 418 cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload) 419 layerUpload = lu 420 } 421 } 422 423 if maxExistenceChecks-len(pd.checkedDigests) > 0 { 424 // do additional layer existence checks with other known digests if any 425 descriptor, exists, err := pd.layerAlreadyExists(ctx, progressOutput, diffID, checkOtherRepositories, maxExistenceChecks-len(pd.checkedDigests), v2Metadata) 426 if exists || err != nil { 427 return descriptor, err 428 } 429 } 430 431 logrus.Debugf("Pushing layer: %s", diffID) 432 if layerUpload == nil { 433 layerUpload, err = bs.Create(ctx) 434 if err != nil { 435 return distribution.Descriptor{}, retryOnError(err) 436 } 437 } 438 defer layerUpload.Close() 439 // upload the blob 440 return pd.uploadUsingSession(ctx, progressOutput, diffID, layerUpload) 441 } 442 443 func (pd *v2PushDescriptor) SetRemoteDescriptor(descriptor distribution.Descriptor) { 444 pd.remoteDescriptor = descriptor 445 } 446 447 func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor { 448 return pd.remoteDescriptor 449 } 450 451 func (pd *v2PushDescriptor) uploadUsingSession( 452 ctx context.Context, 453 progressOutput progress.Output, 454 diffID layer.DiffID, 455 layerUpload distribution.BlobWriter, 456 ) (distribution.Descriptor, error) { 457 var reader io.ReadCloser 458 459 contentReader, err := pd.layer.Open() 460 if err != nil { 461 return distribution.Descriptor{}, retryOnError(err) 462 } 463 464 size, _ := pd.layer.Size() 465 466 reader = progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, contentReader), progressOutput, size, pd.ID(), "Pushing") 467 468 switch m := pd.layer.MediaType(); m { 469 case schema2.MediaTypeUncompressedLayer: 470 compressedReader, compressionDone := compress(reader) 471 defer func(closer io.Closer) { 472 closer.Close() 473 <-compressionDone 474 }(reader) 475 reader = compressedReader 476 case schema2.MediaTypeLayer: 477 default: 478 reader.Close() 479 return distribution.Descriptor{}, fmt.Errorf("unsupported layer media type %s", m) 480 } 481 482 digester := digest.Canonical.Digester() 483 tee := io.TeeReader(reader, digester.Hash()) 484 485 nn, err := layerUpload.ReadFrom(tee) 486 reader.Close() 487 if err != nil { 488 return distribution.Descriptor{}, retryOnError(err) 489 } 490 491 pushDigest := digester.Digest() 492 if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil { 493 return distribution.Descriptor{}, retryOnError(err) 494 } 495 496 logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn) 497 progress.Update(progressOutput, pd.ID(), "Pushed") 498 499 // Cache mapping from this layer's DiffID to the blobsum 500 if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{ 501 Digest: pushDigest, 502 SourceRepository: pd.repoInfo.Name(), 503 }); err != nil { 504 return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} 505 } 506 507 desc := distribution.Descriptor{ 508 Digest: pushDigest, 509 MediaType: schema2.MediaTypeLayer, 510 Size: nn, 511 } 512 513 pd.pushState.Lock() 514 // If Commit succeeded, that's an indication that the remote registry speaks the v2 protocol. 515 pd.pushState.confirmedV2 = true 516 pd.pushState.remoteLayers[diffID] = desc 517 pd.pushState.Unlock() 518 519 return desc, nil 520 } 521 522 // layerAlreadyExists checks if the registry already knows about any of the metadata passed in the "metadata" 523 // slice. If it finds one that the registry knows about, it returns the known digest and "true". If 524 // "checkOtherRepositories" is true, stat will be performed also with digests mapped to any other repository 525 // (not just the target one). 526 func (pd *v2PushDescriptor) layerAlreadyExists( 527 ctx context.Context, 528 progressOutput progress.Output, 529 diffID layer.DiffID, 530 checkOtherRepositories bool, 531 maxExistenceCheckAttempts int, 532 v2Metadata []metadata.V2Metadata, 533 ) (desc distribution.Descriptor, exists bool, err error) { 534 // filter the metadata 535 candidates := []metadata.V2Metadata{} 536 for _, meta := range v2Metadata { 537 if len(meta.SourceRepository) > 0 && !checkOtherRepositories && meta.SourceRepository != pd.repoInfo.Name() { 538 continue 539 } 540 candidates = append(candidates, meta) 541 } 542 // sort the candidates by similarity 543 sortV2MetadataByLikenessAndAge(pd.repoInfo, pd.hmacKey, candidates) 544 545 digestToMetadata := make(map[digest.Digest]*metadata.V2Metadata) 546 // an array of unique blob digests ordered from the best mount candidates to worst 547 layerDigests := []digest.Digest{} 548 for i := 0; i < len(candidates); i++ { 549 if len(layerDigests) >= maxExistenceCheckAttempts { 550 break 551 } 552 meta := &candidates[i] 553 if _, exists := digestToMetadata[meta.Digest]; exists { 554 // keep reference just to the first mapping (the best mount candidate) 555 continue 556 } 557 if _, exists := pd.checkedDigests[meta.Digest]; exists { 558 // existence of this digest has already been tested 559 continue 560 } 561 digestToMetadata[meta.Digest] = meta 562 layerDigests = append(layerDigests, meta.Digest) 563 } 564 565 attempts: 566 for _, dgst := range layerDigests { 567 meta := digestToMetadata[dgst] 568 logrus.Debugf("Checking for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.Name()) 569 desc, err = pd.repo.Blobs(ctx).Stat(ctx, dgst) 570 pd.checkedDigests[meta.Digest] = struct{}{} 571 switch err { 572 case nil: 573 if m, ok := digestToMetadata[desc.Digest]; !ok || m.SourceRepository != pd.repoInfo.Name() || !metadata.CheckV2MetadataHMAC(m, pd.hmacKey) { 574 // cache mapping from this layer's DiffID to the blobsum 575 if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{ 576 Digest: desc.Digest, 577 SourceRepository: pd.repoInfo.Name(), 578 }); err != nil { 579 return distribution.Descriptor{}, false, xfer.DoNotRetry{Err: err} 580 } 581 } 582 desc.MediaType = schema2.MediaTypeLayer 583 exists = true 584 break attempts 585 case distribution.ErrBlobUnknown: 586 if meta.SourceRepository == pd.repoInfo.Name() { 587 // remove the mapping to the target repository 588 pd.v2MetadataService.Remove(*meta) 589 } 590 default: 591 logrus.WithError(err).Debugf("Failed to check for presence of layer %s (%s) in %s", diffID, dgst, pd.repoInfo.Name()) 592 } 593 } 594 595 if exists { 596 progress.Update(progressOutput, pd.ID(), "Layer already exists") 597 pd.pushState.Lock() 598 pd.pushState.remoteLayers[diffID] = desc 599 pd.pushState.Unlock() 600 } 601 602 return desc, exists, nil 603 } 604 605 // getMaxMountAndExistenceCheckAttempts returns a maximum number of cross repository mount attempts from 606 // source repositories of target registry, maximum number of layer existence checks performed on the target 607 // repository and whether the check shall be done also with digests mapped to different repositories. The 608 // decision is based on layer size. The smaller the layer, the fewer attempts shall be made because the cost 609 // of upload does not outweigh a latency. 610 func getMaxMountAndExistenceCheckAttempts(layer PushLayer) (maxMountAttempts, maxExistenceCheckAttempts int, checkOtherRepositories bool) { 611 size, err := layer.Size() 612 switch { 613 // big blob 614 case size > middleLayerMaximumSize: 615 // 1st attempt to mount the blob few times 616 // 2nd few existence checks with digests associated to any repository 617 // then fallback to upload 618 return 4, 3, true 619 620 // middle sized blobs; if we could not get the size, assume we deal with middle sized blob 621 case size > smallLayerMaximumSize, err != nil: 622 // 1st attempt to mount blobs of average size few times 623 // 2nd try at most 1 existence check if there's an existing mapping to the target repository 624 // then fallback to upload 625 return 3, 1, false 626 627 // small blobs, do a minimum number of checks 628 default: 629 return 1, 1, false 630 } 631 } 632 633 // getRepositoryMountCandidates returns an array of v2 metadata items belonging to the given registry. The 634 // array is sorted from youngest to oldest. If requireRegistryMatch is true, the resulting array will contain 635 // only metadata entries having registry part of SourceRepository matching the part of repoInfo. 636 func getRepositoryMountCandidates( 637 repoInfo reference.Named, 638 hmacKey []byte, 639 max int, 640 v2Metadata []metadata.V2Metadata, 641 ) []metadata.V2Metadata { 642 candidates := []metadata.V2Metadata{} 643 for _, meta := range v2Metadata { 644 sourceRepo, err := reference.ParseNamed(meta.SourceRepository) 645 if err != nil || reference.Domain(repoInfo) != reference.Domain(sourceRepo) { 646 continue 647 } 648 // target repository is not a viable candidate 649 if meta.SourceRepository == repoInfo.Name() { 650 continue 651 } 652 candidates = append(candidates, meta) 653 } 654 655 sortV2MetadataByLikenessAndAge(repoInfo, hmacKey, candidates) 656 if max >= 0 && len(candidates) > max { 657 // select the youngest metadata 658 candidates = candidates[:max] 659 } 660 661 return candidates 662 } 663 664 // byLikeness is a sorting container for v2 metadata candidates for cross repository mount. The 665 // candidate "a" is preferred over "b": 666 // 667 // 1. if it was hashed using the same AuthConfig as the one used to authenticate to target repository and the 668 // "b" was not 669 // 2. if a number of its repository path components exactly matching path components of target repository is higher 670 type byLikeness struct { 671 arr []metadata.V2Metadata 672 hmacKey []byte 673 pathComponents []string 674 } 675 676 func (bla byLikeness) Less(i, j int) bool { 677 aMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[i], bla.hmacKey) 678 bMacMatch := metadata.CheckV2MetadataHMAC(&bla.arr[j], bla.hmacKey) 679 if aMacMatch != bMacMatch { 680 return aMacMatch 681 } 682 aMatch := numOfMatchingPathComponents(bla.arr[i].SourceRepository, bla.pathComponents) 683 bMatch := numOfMatchingPathComponents(bla.arr[j].SourceRepository, bla.pathComponents) 684 return aMatch > bMatch 685 } 686 func (bla byLikeness) Swap(i, j int) { 687 bla.arr[i], bla.arr[j] = bla.arr[j], bla.arr[i] 688 } 689 func (bla byLikeness) Len() int { return len(bla.arr) } 690 691 func sortV2MetadataByLikenessAndAge(repoInfo reference.Named, hmacKey []byte, marr []metadata.V2Metadata) { 692 // reverse the metadata array to shift the newest entries to the beginning 693 for i := 0; i < len(marr)/2; i++ { 694 marr[i], marr[len(marr)-i-1] = marr[len(marr)-i-1], marr[i] 695 } 696 // keep equal entries ordered from the youngest to the oldest 697 sort.Stable(byLikeness{ 698 arr: marr, 699 hmacKey: hmacKey, 700 pathComponents: getPathComponents(repoInfo.Name()), 701 }) 702 } 703 704 // numOfMatchingPathComponents returns a number of path components in "pth" that exactly match "matchComponents". 705 func numOfMatchingPathComponents(pth string, matchComponents []string) int { 706 pthComponents := getPathComponents(pth) 707 i := 0 708 for ; i < len(pthComponents) && i < len(matchComponents); i++ { 709 if matchComponents[i] != pthComponents[i] { 710 return i 711 } 712 } 713 return i 714 } 715 716 func getPathComponents(path string) []string { 717 return strings.Split(path, "/") 718 } 719 720 func cancelLayerUpload(ctx context.Context, dgst digest.Digest, layerUpload distribution.BlobWriter) { 721 if layerUpload != nil { 722 logrus.Debugf("cancelling upload of blob %s", dgst) 723 err := layerUpload.Cancel(ctx) 724 if err != nil { 725 logrus.Warnf("failed to cancel upload: %v", err) 726 } 727 } 728 }