github.com/dtroyer-salad/og2/v2@v2.0.0-20240412154159-c47231610877/copy.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "context" 20 "errors" 21 "fmt" 22 "io" 23 "os" 24 "strconv" 25 26 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 27 "golang.org/x/sync/semaphore" 28 "oras.land/oras-go/v2/content" 29 "oras.land/oras-go/v2/errdef" 30 "oras.land/oras-go/v2/internal/cas" 31 "oras.land/oras-go/v2/internal/descriptor" 32 "oras.land/oras-go/v2/internal/platform" 33 "oras.land/oras-go/v2/internal/registryutil" 34 "oras.land/oras-go/v2/internal/spec" 35 "oras.land/oras-go/v2/internal/status" 36 "oras.land/oras-go/v2/internal/syncutil" 37 "oras.land/oras-go/v2/registry" 38 ) 39 40 // defaultConcurrency is the default value of CopyGraphOptions.Concurrency. 41 const defaultConcurrency int = 3 // This value is consistent with dockerd and containerd. 42 43 // SkipNode signals to stop copying a node. When returned from PreCopy the blob must exist in the target. 44 // This can be used to signal that a blob has been made available in the target repository by "Mount()" or some other technique. 45 var SkipNode = errors.New("skip node") 46 47 // DefaultCopyOptions provides the default CopyOptions. 48 var DefaultCopyOptions CopyOptions = CopyOptions{ 49 CopyGraphOptions: DefaultCopyGraphOptions, 50 } 51 52 // CopyOptions contains parameters for [oras.Copy]. 53 type CopyOptions struct { 54 CopyGraphOptions 55 // MapRoot maps the resolved root node to a desired root node for copy. 56 // When MapRoot is provided, the descriptor resolved from the source 57 // reference will be passed to MapRoot, and the mapped descriptor will be 58 // used as the root node for copy. 59 MapRoot func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) 60 } 61 62 // WithTargetPlatform configures opts.MapRoot to select the manifest whose 63 // platform matches the given platform. When MapRoot is provided, the platform 64 // selection will be applied on the mapped root node. 65 // - If the given platform is nil, no platform selection will be applied. 66 // - If the root node is a manifest, it will remain the same if platform 67 // matches, otherwise ErrNotFound will be returned. 68 // - If the root node is a manifest list, it will be mapped to the first 69 // matching manifest if exists, otherwise ErrNotFound will be returned. 70 // - Otherwise ErrUnsupported will be returned. 71 func (opts *CopyOptions) WithTargetPlatform(p *ocispec.Platform) { 72 if p == nil { 73 return 74 } 75 mapRoot := opts.MapRoot 76 opts.MapRoot = func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (desc ocispec.Descriptor, err error) { 77 if mapRoot != nil { 78 if root, err = mapRoot(ctx, src, root); err != nil { 79 return ocispec.Descriptor{}, err 80 } 81 } 82 return platform.SelectManifest(ctx, src, root, p) 83 } 84 } 85 86 // defaultCopyMaxMetadataBytes is the default value of 87 // CopyGraphOptions.MaxMetadataBytes. 88 const defaultCopyMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 89 90 // DefaultCopyGraphOptions provides the default CopyGraphOptions. 91 var DefaultCopyGraphOptions CopyGraphOptions 92 93 // CopyGraphOptions contains parameters for [oras.CopyGraph]. 94 type CopyGraphOptions struct { 95 // Concurrency limits the maximum number of concurrent copy tasks. 96 // If less than or equal to 0, a default (currently 3) is used. 97 Concurrency int 98 // MaxMetadataBytes limits the maximum size of the metadata that can be 99 // cached in the memory. 100 // If less than or equal to 0, a default (currently 4 MiB) is used. 101 MaxMetadataBytes int64 102 // PreCopy handles the current descriptor before it is copied. PreCopy can 103 // return a SkipNode to signal that desc should be skipped when it already 104 // exists in the target. 105 PreCopy func(ctx context.Context, desc ocispec.Descriptor) error 106 // PostCopy handles the current descriptor after it is copied. 107 PostCopy func(ctx context.Context, desc ocispec.Descriptor) error 108 // OnCopySkipped will be called when the sub-DAG rooted by the current node 109 // is skipped. 110 OnCopySkipped func(ctx context.Context, desc ocispec.Descriptor) error 111 // MountFrom returns the candidate repositories that desc may be mounted from. 112 // The OCI references will be tried in turn. If mounting fails on all of them, 113 // then it falls back to a copy. 114 MountFrom func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) 115 // OnMounted will be invoked when desc is mounted. 116 OnMounted func(ctx context.Context, desc ocispec.Descriptor) error 117 // FindSuccessors finds the successors of the current node. 118 // fetcher provides cached access to the source storage, and is suitable 119 // for fetching non-leaf nodes like manifests. Since anything fetched from 120 // fetcher will be cached in the memory, it is recommended to use original 121 // source storage to fetch large blobs. 122 // If FindSuccessors is nil, content.Successors will be used. 123 FindSuccessors func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) 124 } 125 126 // Copy copies a rooted directed acyclic graph (DAG) with the tagged root node 127 // in the source Target to the destination Target. 128 // The destination reference will be the same as the source reference if the 129 // destination reference is left blank. 130 // 131 // Returns the descriptor of the root node on successful copy. 132 func Copy(ctx context.Context, src ReadOnlyTarget, srcRef string, dst Target, dstRef string, opts CopyOptions) (ocispec.Descriptor, error) { 133 if src == nil { 134 return ocispec.Descriptor{}, errors.New("nil source target") 135 } 136 if dst == nil { 137 return ocispec.Descriptor{}, errors.New("nil destination target") 138 } 139 if dstRef == "" { 140 dstRef = srcRef 141 } 142 143 // use caching proxy on non-leaf nodes 144 if opts.MaxMetadataBytes <= 0 { 145 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 146 } 147 proxy := cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 148 root, err := resolveRoot(ctx, src, srcRef, proxy) 149 if err != nil { 150 return ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", srcRef, err) 151 } 152 153 if opts.MapRoot != nil { 154 proxy.StopCaching = true 155 root, err = opts.MapRoot(ctx, proxy, root) 156 if err != nil { 157 return ocispec.Descriptor{}, err 158 } 159 proxy.StopCaching = false 160 } 161 162 if err := prepareCopy(ctx, dst, dstRef, proxy, root, &opts); err != nil { 163 return ocispec.Descriptor{}, err 164 } 165 166 if err := copyGraph(ctx, src, dst, root, proxy, nil, nil, opts.CopyGraphOptions); err != nil { 167 return ocispec.Descriptor{}, err 168 } 169 170 return root, nil 171 } 172 173 // CopyGraph copies a rooted directed acyclic graph (DAG) from the source CAS to 174 // the destination CAS. 175 func CopyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, root ocispec.Descriptor, opts CopyGraphOptions) error { 176 return copyGraph(ctx, src, dst, root, nil, nil, nil, opts) 177 } 178 179 // copyGraph copies a rooted directed acyclic graph (DAG) from the source CAS to 180 // the destination CAS with specified caching, concurrency limiter and tracker. 181 func copyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, root ocispec.Descriptor, 182 proxy *cas.Proxy, limiter *semaphore.Weighted, tracker *status.Tracker, opts CopyGraphOptions) error { 183 if proxy == nil { 184 // use caching proxy on non-leaf nodes 185 if opts.MaxMetadataBytes <= 0 { 186 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 187 } 188 proxy = cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 189 } 190 if limiter == nil { 191 // if Concurrency is not set or invalid, use the default concurrency 192 if opts.Concurrency <= 0 { 193 opts.Concurrency = defaultConcurrency 194 } 195 limiter = semaphore.NewWeighted(int64(opts.Concurrency)) 196 } 197 if tracker == nil { 198 // track content status 199 tracker = status.NewTracker() 200 } 201 // if FindSuccessors is not provided, use the default one 202 if opts.FindSuccessors == nil { 203 opts.FindSuccessors = content.Successors 204 } 205 206 // traverse the graph 207 var fn syncutil.GoFunc[ocispec.Descriptor] 208 fn = func(ctx context.Context, region *syncutil.LimitedRegion, desc ocispec.Descriptor) (err error) { 209 // skip the descriptor if other go routine is working on it 210 done, committed := tracker.TryCommit(desc) 211 if !committed { 212 return nil 213 } 214 defer func() { 215 if err == nil { 216 // mark the content as done on success 217 close(done) 218 } 219 }() 220 221 // skip if a rooted sub-DAG exists 222 exists, err := dst.Exists(ctx, desc) 223 if err != nil { 224 return err 225 } 226 if exists { 227 if opts.OnCopySkipped != nil { 228 if err := opts.OnCopySkipped(ctx, desc); err != nil { 229 return err 230 } 231 } 232 return nil 233 } 234 235 // find successors while non-leaf nodes will be fetched and cached 236 successors, err := opts.FindSuccessors(ctx, proxy, desc) 237 if err != nil { 238 return err 239 } 240 successors = removeForeignLayers(successors) 241 242 if len(successors) != 0 { 243 // for non-leaf nodes, process successors and wait for them to complete 244 region.End() 245 if err := syncutil.Go(ctx, limiter, fn, successors...); err != nil { 246 return err 247 } 248 for _, node := range successors { 249 done, committed := tracker.TryCommit(node) 250 if committed { 251 return fmt.Errorf("%s: %s: successor not committed", desc.Digest, node.Digest) 252 } 253 select { 254 case <-done: 255 case <-ctx.Done(): 256 return ctx.Err() 257 } 258 } 259 if err := region.Start(); err != nil { 260 return err 261 } 262 } 263 264 exists, err = proxy.Cache.Exists(ctx, desc) 265 if err != nil { 266 return err 267 } 268 if exists { 269 return copyNode(ctx, proxy.Cache, dst, desc, opts) 270 } 271 return mountOrCopyNode(ctx, src, dst, desc, opts) 272 } 273 274 return syncutil.Go(ctx, limiter, fn, root) 275 } 276 277 // Quick way to get all of the blobs associated with an image 278 func GetSuccessors(ctx context.Context, src ReadOnlyTarget, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 279 proxy := cas.NewProxyWithLimit(src, cas.NewMemory(), defaultCopyMaxMetadataBytes) 280 successors, err := content.Successors(ctx, proxy, desc) 281 if err != nil { 282 return nil, err 283 } 284 // if len(successors) != 0 { 285 // for _, node := range successors { 286 // fmt.Printf(" names: %\n", node.Digest) 287 // } 288 // } 289 return successors, nil 290 } 291 292 // mountOrCopyNode tries to mount the node, if not falls back to copying. 293 func mountOrCopyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor, opts CopyGraphOptions) error { 294 // Need MountFrom and it must be a blob 295 if opts.MountFrom == nil || descriptor.IsManifest(desc) { 296 return copyNode(ctx, src, dst, desc, opts) 297 } 298 299 mounter, ok := dst.(registry.Mounter) 300 if !ok { 301 // mounting is not supported by the destination 302 return copyNode(ctx, src, dst, desc, opts) 303 } 304 305 sourceRepositories, err := opts.MountFrom(ctx, desc) 306 if err != nil { 307 // Technically this error is not fatal, we can still attempt to copy the node 308 // But for consistency with the other callbacks we bail out. 309 return err 310 } 311 312 if len(sourceRepositories) == 0 { 313 return copyNode(ctx, src, dst, desc, opts) 314 } 315 316 skipSource := errors.New("skip source") 317 for i, sourceRepository := range sourceRepositories { 318 // try mounting this source repository 319 var mountFailed bool 320 getContent := func() (io.ReadCloser, error) { 321 // the invocation of getContent indicates that mounting has failed 322 mountFailed = true 323 324 if i < len(sourceRepositories)-1 { 325 // If this is not the last one, skip this source and try next one 326 // We want to return an error that we will test for from mounter.Mount() 327 return nil, skipSource 328 } 329 // this is the last iteration so we need to actually get the content and do the copy 330 // but first call the PreCopy function 331 if opts.PreCopy != nil { 332 if err := opts.PreCopy(ctx, desc); err != nil { 333 return nil, err 334 } 335 } 336 return src.Fetch(ctx, desc) 337 } 338 339 // Mount or copy 340 if err := mounter.Mount(ctx, desc, sourceRepository, getContent); err != nil && !errors.Is(err, skipSource) { 341 return err 342 } 343 344 if !mountFailed { 345 // mounted, success 346 if opts.OnMounted != nil { 347 if err := opts.OnMounted(ctx, desc); err != nil { 348 return err 349 } 350 } 351 return nil 352 } 353 } 354 355 // we copied it 356 if opts.PostCopy != nil { 357 if err := opts.PostCopy(ctx, desc); err != nil { 358 return err 359 } 360 } 361 362 return nil 363 } 364 365 // doCopyNode copies a single content from the source CAS to the destination CAS. 366 func doCopyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor) error { 367 // We don't resume on files without Annotations 368 if desc.Annotations != nil { 369 // Get ingest path from dst if it is a local store 370 var ingestFile string 371 var ingestFileSize int64 372 if st, ok := dst.(content.Resumer); ok { 373 // Look for matching partial ingest files 374 ingestFile = st.IngestFile(desc.Digest.Encoded()) 375 if ingestFile != "" { 376 // Found existing ingest file 377 f, err := os.Stat(ingestFile) 378 if err != nil { 379 // Can't find it??? Shouldn't happen... 380 ingestFile = "" 381 } else { 382 ingestFileSize = f.Size() 383 desc.Annotations[spec.AnnotationResumeOffset] = strconv.FormatInt(ingestFileSize, 10) 384 desc.Annotations[spec.AnnotationResumeFilename] = ingestFile 385 desc.Annotations[spec.AnnotationResumeDownload] = "true" 386 } 387 } 388 } 389 } 390 391 rc, err := src.Fetch(ctx, desc) 392 if err != nil { 393 return err 394 } 395 defer rc.Close() 396 err = dst.Push(ctx, desc, rc) 397 if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 398 return err 399 } 400 return nil 401 } 402 403 // copyNode copies a single content from the source CAS to the destination CAS, 404 // and apply the given options. 405 func copyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor, opts CopyGraphOptions) error { 406 if opts.PreCopy != nil { 407 if err := opts.PreCopy(ctx, desc); err != nil { 408 if err == SkipNode { 409 return nil 410 } 411 return err 412 } 413 } 414 415 if err := doCopyNode(ctx, src, dst, desc); err != nil { 416 return err 417 } 418 419 if opts.PostCopy != nil { 420 return opts.PostCopy(ctx, desc) 421 } 422 return nil 423 } 424 425 // copyCachedNodeWithReference copies a single content with a reference from the 426 // source cache to the destination ReferencePusher. 427 func copyCachedNodeWithReference(ctx context.Context, src *cas.Proxy, dst registry.ReferencePusher, desc ocispec.Descriptor, dstRef string) error { 428 rc, err := src.FetchCached(ctx, desc) 429 if err != nil { 430 return err 431 } 432 defer rc.Close() 433 434 err = dst.PushReference(ctx, desc, rc, dstRef) 435 if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 436 return err 437 } 438 return nil 439 } 440 441 // resolveRoot resolves the source reference to the root node. 442 func resolveRoot(ctx context.Context, src ReadOnlyTarget, srcRef string, proxy *cas.Proxy) (ocispec.Descriptor, error) { 443 refFetcher, ok := src.(registry.ReferenceFetcher) 444 if !ok { 445 return src.Resolve(ctx, srcRef) 446 } 447 448 // optimize performance for ReferenceFetcher targets 449 refProxy := ®istryutil.Proxy{ 450 ReferenceFetcher: refFetcher, 451 Proxy: proxy, 452 } 453 root, rc, err := refProxy.FetchReference(ctx, srcRef) 454 if err != nil { 455 return ocispec.Descriptor{}, err 456 } 457 defer rc.Close() 458 // cache root if it is a non-leaf node 459 fetcher := content.FetcherFunc(func(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { 460 if content.Equal(target, root) { 461 return rc, nil 462 } 463 return nil, errors.New("fetching only root node expected") 464 }) 465 if _, err = content.Successors(ctx, fetcher, root); err != nil { 466 return ocispec.Descriptor{}, err 467 } 468 469 // TODO: optimize special case where root is a leaf node (i.e. a blob) 470 // and dst is a ReferencePusher. 471 return root, nil 472 } 473 474 // prepareCopy prepares the hooks for copy. 475 func prepareCopy(ctx context.Context, dst Target, dstRef string, proxy *cas.Proxy, root ocispec.Descriptor, opts *CopyOptions) error { 476 if refPusher, ok := dst.(registry.ReferencePusher); ok { 477 // optimize performance for ReferencePusher targets 478 preCopy := opts.PreCopy 479 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 480 if preCopy != nil { 481 if err := preCopy(ctx, desc); err != nil { 482 return err 483 } 484 } 485 if !content.Equal(desc, root) { 486 // for non-root node, do nothing 487 return nil 488 } 489 490 // for root node, prepare optimized copy 491 if err := copyCachedNodeWithReference(ctx, proxy, refPusher, desc, dstRef); err != nil { 492 return err 493 } 494 if opts.PostCopy != nil { 495 if err := opts.PostCopy(ctx, desc); err != nil { 496 return err 497 } 498 } 499 // skip the regular copy workflow 500 return SkipNode 501 } 502 } else { 503 postCopy := opts.PostCopy 504 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 505 if content.Equal(desc, root) { 506 // for root node, tag it after copying it 507 if err := dst.Tag(ctx, root, dstRef); err != nil { 508 return err 509 } 510 } 511 if postCopy != nil { 512 return postCopy(ctx, desc) 513 } 514 return nil 515 } 516 } 517 518 onCopySkipped := opts.OnCopySkipped 519 opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { 520 if !content.Equal(desc, root) { 521 if onCopySkipped != nil { 522 return onCopySkipped(ctx, desc) 523 } 524 return nil 525 } 526 527 // enforce tagging when the skipped node is root 528 if refPusher, ok := dst.(registry.ReferencePusher); ok { 529 // NOTE: refPusher tags the node by copying it with the reference, 530 // so onCopySkipped shouldn't be invoked in this case 531 return copyCachedNodeWithReference(ctx, proxy, refPusher, desc, dstRef) 532 } 533 534 // invoke onCopySkipped before tagging 535 if onCopySkipped != nil { 536 if err := onCopySkipped(ctx, desc); err != nil { 537 return err 538 } 539 } 540 return dst.Tag(ctx, root, dstRef) 541 } 542 543 return nil 544 } 545 546 // removeForeignLayers in-place removes all foreign layers in the given slice. 547 func removeForeignLayers(descs []ocispec.Descriptor) []ocispec.Descriptor { 548 var j int 549 for i, desc := range descs { 550 if !descriptor.IsForeignLayer(desc) { 551 if i != j { 552 descs[j] = desc 553 } 554 j++ 555 } 556 } 557 return descs[:j] 558 }