oras.land/oras-go/v2@v2.5.1-0.20240520045656-aef90e4d04c4/copy.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "context" 20 "errors" 21 "fmt" 22 "io" 23 24 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 25 "golang.org/x/sync/semaphore" 26 "oras.land/oras-go/v2/content" 27 "oras.land/oras-go/v2/errdef" 28 "oras.land/oras-go/v2/internal/cas" 29 "oras.land/oras-go/v2/internal/descriptor" 30 "oras.land/oras-go/v2/internal/platform" 31 "oras.land/oras-go/v2/internal/registryutil" 32 "oras.land/oras-go/v2/internal/status" 33 "oras.land/oras-go/v2/internal/syncutil" 34 "oras.land/oras-go/v2/registry" 35 ) 36 37 // defaultConcurrency is the default value of CopyGraphOptions.Concurrency. 38 const defaultConcurrency int = 3 // This value is consistent with dockerd and containerd. 39 40 // SkipNode signals to stop copying a node. When returned from PreCopy the blob must exist in the target. 41 // This can be used to signal that a blob has been made available in the target repository by "Mount()" or some other technique. 42 var SkipNode = errors.New("skip node") 43 44 // DefaultCopyOptions provides the default CopyOptions. 45 var DefaultCopyOptions CopyOptions = CopyOptions{ 46 CopyGraphOptions: DefaultCopyGraphOptions, 47 } 48 49 // CopyOptions contains parameters for [oras.Copy]. 50 type CopyOptions struct { 51 CopyGraphOptions 52 // MapRoot maps the resolved root node to a desired root node for copy. 53 // When MapRoot is provided, the descriptor resolved from the source 54 // reference will be passed to MapRoot, and the mapped descriptor will be 55 // used as the root node for copy. 56 MapRoot func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) 57 } 58 59 // WithTargetPlatform configures opts.MapRoot to select the manifest whose 60 // platform matches the given platform. When MapRoot is provided, the platform 61 // selection will be applied on the mapped root node. 62 // - If the given platform is nil, no platform selection will be applied. 63 // - If the root node is a manifest, it will remain the same if platform 64 // matches, otherwise ErrNotFound will be returned. 65 // - If the root node is a manifest list, it will be mapped to the first 66 // matching manifest if exists, otherwise ErrNotFound will be returned. 67 // - Otherwise ErrUnsupported will be returned. 68 func (opts *CopyOptions) WithTargetPlatform(p *ocispec.Platform) { 69 if p == nil { 70 return 71 } 72 mapRoot := opts.MapRoot 73 opts.MapRoot = func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (desc ocispec.Descriptor, err error) { 74 if mapRoot != nil { 75 if root, err = mapRoot(ctx, src, root); err != nil { 76 return ocispec.Descriptor{}, err 77 } 78 } 79 return platform.SelectManifest(ctx, src, root, p) 80 } 81 } 82 83 // defaultCopyMaxMetadataBytes is the default value of 84 // CopyGraphOptions.MaxMetadataBytes. 85 const defaultCopyMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 86 87 // DefaultCopyGraphOptions provides the default CopyGraphOptions. 88 var DefaultCopyGraphOptions CopyGraphOptions 89 90 // CopyGraphOptions contains parameters for [oras.CopyGraph]. 91 type CopyGraphOptions struct { 92 // Concurrency limits the maximum number of concurrent copy tasks. 93 // If less than or equal to 0, a default (currently 3) is used. 94 Concurrency int 95 // MaxMetadataBytes limits the maximum size of the metadata that can be 96 // cached in the memory. 97 // If less than or equal to 0, a default (currently 4 MiB) is used. 98 MaxMetadataBytes int64 99 // PreCopy handles the current descriptor before it is copied. PreCopy can 100 // return a SkipNode to signal that desc should be skipped when it already 101 // exists in the target. 102 PreCopy func(ctx context.Context, desc ocispec.Descriptor) error 103 // PostCopy handles the current descriptor after it is copied. 104 PostCopy func(ctx context.Context, desc ocispec.Descriptor) error 105 // OnCopySkipped will be called when the sub-DAG rooted by the current node 106 // is skipped. 107 OnCopySkipped func(ctx context.Context, desc ocispec.Descriptor) error 108 // MountFrom returns the candidate repositories that desc may be mounted from. 109 // The OCI references will be tried in turn. If mounting fails on all of them, 110 // then it falls back to a copy. 111 MountFrom func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) 112 // OnMounted will be invoked when desc is mounted. 113 OnMounted func(ctx context.Context, desc ocispec.Descriptor) error 114 // FindSuccessors finds the successors of the current node. 115 // fetcher provides cached access to the source storage, and is suitable 116 // for fetching non-leaf nodes like manifests. Since anything fetched from 117 // fetcher will be cached in the memory, it is recommended to use original 118 // source storage to fetch large blobs. 119 // If FindSuccessors is nil, content.Successors will be used. 120 FindSuccessors func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) 121 } 122 123 // Copy copies a rooted directed acyclic graph (DAG), such as an artifact, 124 // from the source Target to the destination Target. 125 // 126 // The root node (e.g. a tagged manifest of the artifact) is identified by the 127 // source reference. 128 // The destination reference will be the same as the source reference if the 129 // destination reference is left blank. 130 // 131 // Returns the descriptor of the root node on successful copy. 132 func Copy(ctx context.Context, src ReadOnlyTarget, srcRef string, dst Target, dstRef string, opts CopyOptions) (ocispec.Descriptor, error) { 133 if src == nil { 134 return ocispec.Descriptor{}, errors.New("nil source target") 135 } 136 if dst == nil { 137 return ocispec.Descriptor{}, errors.New("nil destination target") 138 } 139 if dstRef == "" { 140 dstRef = srcRef 141 } 142 143 // use caching proxy on non-leaf nodes 144 if opts.MaxMetadataBytes <= 0 { 145 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 146 } 147 proxy := cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 148 root, err := resolveRoot(ctx, src, srcRef, proxy) 149 if err != nil { 150 return ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", srcRef, err) 151 } 152 153 if opts.MapRoot != nil { 154 proxy.StopCaching = true 155 root, err = opts.MapRoot(ctx, proxy, root) 156 if err != nil { 157 return ocispec.Descriptor{}, err 158 } 159 proxy.StopCaching = false 160 } 161 162 if err := prepareCopy(ctx, dst, dstRef, proxy, root, &opts); err != nil { 163 return ocispec.Descriptor{}, err 164 } 165 166 if err := copyGraph(ctx, src, dst, root, proxy, nil, nil, opts.CopyGraphOptions); err != nil { 167 return ocispec.Descriptor{}, err 168 } 169 170 return root, nil 171 } 172 173 // CopyGraph copies a rooted directed acyclic graph (DAG), such as an artifact, 174 // from the source CAS to the destination CAS. 175 // The root node (e.g. a manifest of the artifact) is identified by a descriptor. 176 func CopyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, root ocispec.Descriptor, opts CopyGraphOptions) error { 177 return copyGraph(ctx, src, dst, root, nil, nil, nil, opts) 178 } 179 180 // copyGraph copies a rooted directed acyclic graph (DAG) from the source CAS to 181 // the destination CAS with specified caching, concurrency limiter and tracker. 182 func copyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, root ocispec.Descriptor, 183 proxy *cas.Proxy, limiter *semaphore.Weighted, tracker *status.Tracker, opts CopyGraphOptions) error { 184 if proxy == nil { 185 // use caching proxy on non-leaf nodes 186 if opts.MaxMetadataBytes <= 0 { 187 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 188 } 189 proxy = cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 190 } 191 if limiter == nil { 192 // if Concurrency is not set or invalid, use the default concurrency 193 if opts.Concurrency <= 0 { 194 opts.Concurrency = defaultConcurrency 195 } 196 limiter = semaphore.NewWeighted(int64(opts.Concurrency)) 197 } 198 if tracker == nil { 199 // track content status 200 tracker = status.NewTracker() 201 } 202 // if FindSuccessors is not provided, use the default one 203 if opts.FindSuccessors == nil { 204 opts.FindSuccessors = content.Successors 205 } 206 207 // traverse the graph 208 var fn syncutil.GoFunc[ocispec.Descriptor] 209 fn = func(ctx context.Context, region *syncutil.LimitedRegion, desc ocispec.Descriptor) (err error) { 210 // skip the descriptor if other go routine is working on it 211 done, committed := tracker.TryCommit(desc) 212 if !committed { 213 return nil 214 } 215 defer func() { 216 if err == nil { 217 // mark the content as done on success 218 close(done) 219 } 220 }() 221 222 // skip if a rooted sub-DAG exists 223 exists, err := dst.Exists(ctx, desc) 224 if err != nil { 225 return err 226 } 227 if exists { 228 if opts.OnCopySkipped != nil { 229 if err := opts.OnCopySkipped(ctx, desc); err != nil { 230 return err 231 } 232 } 233 return nil 234 } 235 236 // find successors while non-leaf nodes will be fetched and cached 237 successors, err := opts.FindSuccessors(ctx, proxy, desc) 238 if err != nil { 239 return err 240 } 241 successors = removeForeignLayers(successors) 242 243 if len(successors) != 0 { 244 // for non-leaf nodes, process successors and wait for them to complete 245 region.End() 246 if err := syncutil.Go(ctx, limiter, fn, successors...); err != nil { 247 return err 248 } 249 for _, node := range successors { 250 done, committed := tracker.TryCommit(node) 251 if committed { 252 return fmt.Errorf("%s: %s: successor not committed", desc.Digest, node.Digest) 253 } 254 select { 255 case <-done: 256 case <-ctx.Done(): 257 return ctx.Err() 258 } 259 } 260 if err := region.Start(); err != nil { 261 return err 262 } 263 } 264 265 exists, err = proxy.Cache.Exists(ctx, desc) 266 if err != nil { 267 return err 268 } 269 if exists { 270 return copyNode(ctx, proxy.Cache, dst, desc, opts) 271 } 272 return mountOrCopyNode(ctx, src, dst, desc, opts) 273 } 274 275 return syncutil.Go(ctx, limiter, fn, root) 276 } 277 278 // mountOrCopyNode tries to mount the node, if not falls back to copying. 279 func mountOrCopyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor, opts CopyGraphOptions) error { 280 // Need MountFrom and it must be a blob 281 if opts.MountFrom == nil || descriptor.IsManifest(desc) { 282 return copyNode(ctx, src, dst, desc, opts) 283 } 284 285 mounter, ok := dst.(registry.Mounter) 286 if !ok { 287 // mounting is not supported by the destination 288 return copyNode(ctx, src, dst, desc, opts) 289 } 290 291 sourceRepositories, err := opts.MountFrom(ctx, desc) 292 if err != nil { 293 // Technically this error is not fatal, we can still attempt to copy the node 294 // But for consistency with the other callbacks we bail out. 295 return err 296 } 297 298 if len(sourceRepositories) == 0 { 299 return copyNode(ctx, src, dst, desc, opts) 300 } 301 302 skipSource := errors.New("skip source") 303 for i, sourceRepository := range sourceRepositories { 304 // try mounting this source repository 305 var mountFailed bool 306 getContent := func() (io.ReadCloser, error) { 307 // the invocation of getContent indicates that mounting has failed 308 mountFailed = true 309 310 if i < len(sourceRepositories)-1 { 311 // If this is not the last one, skip this source and try next one 312 // We want to return an error that we will test for from mounter.Mount() 313 return nil, skipSource 314 } 315 // this is the last iteration so we need to actually get the content and do the copy 316 // but first call the PreCopy function 317 if opts.PreCopy != nil { 318 if err := opts.PreCopy(ctx, desc); err != nil { 319 return nil, err 320 } 321 } 322 return src.Fetch(ctx, desc) 323 } 324 325 // Mount or copy 326 if err := mounter.Mount(ctx, desc, sourceRepository, getContent); err != nil && !errors.Is(err, skipSource) { 327 return err 328 } 329 330 if !mountFailed { 331 // mounted, success 332 if opts.OnMounted != nil { 333 if err := opts.OnMounted(ctx, desc); err != nil { 334 return err 335 } 336 } 337 return nil 338 } 339 } 340 341 // we copied it 342 if opts.PostCopy != nil { 343 if err := opts.PostCopy(ctx, desc); err != nil { 344 return err 345 } 346 } 347 348 return nil 349 } 350 351 // doCopyNode copies a single content from the source CAS to the destination CAS. 352 func doCopyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor) error { 353 rc, err := src.Fetch(ctx, desc) 354 if err != nil { 355 return err 356 } 357 defer rc.Close() 358 err = dst.Push(ctx, desc, rc) 359 if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 360 return err 361 } 362 return nil 363 } 364 365 // copyNode copies a single content from the source CAS to the destination CAS, 366 // and apply the given options. 367 func copyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor, opts CopyGraphOptions) error { 368 if opts.PreCopy != nil { 369 if err := opts.PreCopy(ctx, desc); err != nil { 370 if err == SkipNode { 371 return nil 372 } 373 return err 374 } 375 } 376 377 if err := doCopyNode(ctx, src, dst, desc); err != nil { 378 return err 379 } 380 381 if opts.PostCopy != nil { 382 return opts.PostCopy(ctx, desc) 383 } 384 return nil 385 } 386 387 // copyCachedNodeWithReference copies a single content with a reference from the 388 // source cache to the destination ReferencePusher. 389 func copyCachedNodeWithReference(ctx context.Context, src *cas.Proxy, dst registry.ReferencePusher, desc ocispec.Descriptor, dstRef string) error { 390 rc, err := src.FetchCached(ctx, desc) 391 if err != nil { 392 return err 393 } 394 defer rc.Close() 395 396 err = dst.PushReference(ctx, desc, rc, dstRef) 397 if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 398 return err 399 } 400 return nil 401 } 402 403 // resolveRoot resolves the source reference to the root node. 404 func resolveRoot(ctx context.Context, src ReadOnlyTarget, srcRef string, proxy *cas.Proxy) (ocispec.Descriptor, error) { 405 refFetcher, ok := src.(registry.ReferenceFetcher) 406 if !ok { 407 return src.Resolve(ctx, srcRef) 408 } 409 410 // optimize performance for ReferenceFetcher targets 411 refProxy := ®istryutil.Proxy{ 412 ReferenceFetcher: refFetcher, 413 Proxy: proxy, 414 } 415 root, rc, err := refProxy.FetchReference(ctx, srcRef) 416 if err != nil { 417 return ocispec.Descriptor{}, err 418 } 419 defer rc.Close() 420 // cache root if it is a non-leaf node 421 fetcher := content.FetcherFunc(func(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { 422 if content.Equal(target, root) { 423 return rc, nil 424 } 425 return nil, errors.New("fetching only root node expected") 426 }) 427 if _, err = content.Successors(ctx, fetcher, root); err != nil { 428 return ocispec.Descriptor{}, err 429 } 430 431 // TODO: optimize special case where root is a leaf node (i.e. a blob) 432 // and dst is a ReferencePusher. 433 return root, nil 434 } 435 436 // prepareCopy prepares the hooks for copy. 437 func prepareCopy(ctx context.Context, dst Target, dstRef string, proxy *cas.Proxy, root ocispec.Descriptor, opts *CopyOptions) error { 438 if refPusher, ok := dst.(registry.ReferencePusher); ok { 439 // optimize performance for ReferencePusher targets 440 preCopy := opts.PreCopy 441 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 442 if preCopy != nil { 443 if err := preCopy(ctx, desc); err != nil { 444 return err 445 } 446 } 447 if !content.Equal(desc, root) { 448 // for non-root node, do nothing 449 return nil 450 } 451 452 // for root node, prepare optimized copy 453 if err := copyCachedNodeWithReference(ctx, proxy, refPusher, desc, dstRef); err != nil { 454 return err 455 } 456 if opts.PostCopy != nil { 457 if err := opts.PostCopy(ctx, desc); err != nil { 458 return err 459 } 460 } 461 // skip the regular copy workflow 462 return SkipNode 463 } 464 } else { 465 postCopy := opts.PostCopy 466 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 467 if content.Equal(desc, root) { 468 // for root node, tag it after copying it 469 if err := dst.Tag(ctx, root, dstRef); err != nil { 470 return err 471 } 472 } 473 if postCopy != nil { 474 return postCopy(ctx, desc) 475 } 476 return nil 477 } 478 } 479 480 onCopySkipped := opts.OnCopySkipped 481 opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { 482 if !content.Equal(desc, root) { 483 if onCopySkipped != nil { 484 return onCopySkipped(ctx, desc) 485 } 486 return nil 487 } 488 489 // enforce tagging when the skipped node is root 490 if refPusher, ok := dst.(registry.ReferencePusher); ok { 491 // NOTE: refPusher tags the node by copying it with the reference, 492 // so onCopySkipped shouldn't be invoked in this case 493 return copyCachedNodeWithReference(ctx, proxy, refPusher, desc, dstRef) 494 } 495 496 // invoke onCopySkipped before tagging 497 if onCopySkipped != nil { 498 if err := onCopySkipped(ctx, desc); err != nil { 499 return err 500 } 501 } 502 return dst.Tag(ctx, root, dstRef) 503 } 504 505 return nil 506 } 507 508 // removeForeignLayers in-place removes all foreign layers in the given slice. 509 func removeForeignLayers(descs []ocispec.Descriptor) []ocispec.Descriptor { 510 var j int 511 for i, desc := range descs { 512 if !descriptor.IsForeignLayer(desc) { 513 if i != j { 514 descs[j] = desc 515 } 516 j++ 517 } 518 } 519 return descs[:j] 520 }