github.com/opcr-io/oras-go/v2@v2.0.0-20231122155130-eb4260d8a0ae/copy.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "context" 20 "errors" 21 "fmt" 22 "io" 23 24 "github.com/opcr-io/oras-go/v2/content" 25 "github.com/opcr-io/oras-go/v2/errdef" 26 "github.com/opcr-io/oras-go/v2/internal/cas" 27 "github.com/opcr-io/oras-go/v2/internal/descriptor" 28 "github.com/opcr-io/oras-go/v2/internal/platform" 29 "github.com/opcr-io/oras-go/v2/internal/registryutil" 30 "github.com/opcr-io/oras-go/v2/internal/status" 31 "github.com/opcr-io/oras-go/v2/internal/syncutil" 32 "github.com/opcr-io/oras-go/v2/registry" 33 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 34 "golang.org/x/sync/semaphore" 35 ) 36 37 // defaultConcurrency is the default value of CopyGraphOptions.Concurrency. 38 const defaultConcurrency int = 3 // This value is consistent with dockerd and containerd. 39 40 // errSkipDesc signals copyNode() to stop processing a descriptor. 41 var errSkipDesc = errors.New("skip descriptor") 42 43 // DefaultCopyOptions provides the default CopyOptions. 44 var DefaultCopyOptions CopyOptions = CopyOptions{ 45 CopyGraphOptions: DefaultCopyGraphOptions, 46 } 47 48 // CopyOptions contains parameters for [oras.Copy]. 49 type CopyOptions struct { 50 CopyGraphOptions 51 // MapRoot maps the resolved root node to a desired root node for copy. 52 // When MapRoot is provided, the descriptor resolved from the source 53 // reference will be passed to MapRoot, and the mapped descriptor will be 54 // used as the root node for copy. 55 MapRoot func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) 56 } 57 58 // WithTargetPlatform configures opts.MapRoot to select the manifest whose 59 // platform matches the given platform. When MapRoot is provided, the platform 60 // selection will be applied on the mapped root node. 61 // - If the given platform is nil, no platform selection will be applied. 62 // - If the root node is a manifest, it will remain the same if platform 63 // matches, otherwise ErrNotFound will be returned. 64 // - If the root node is a manifest list, it will be mapped to the first 65 // matching manifest if exists, otherwise ErrNotFound will be returned. 66 // - Otherwise ErrUnsupported will be returned. 67 func (opts *CopyOptions) WithTargetPlatform(p *ocispec.Platform) { 68 if p == nil { 69 return 70 } 71 mapRoot := opts.MapRoot 72 opts.MapRoot = func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (desc ocispec.Descriptor, err error) { 73 if mapRoot != nil { 74 if root, err = mapRoot(ctx, src, root); err != nil { 75 return ocispec.Descriptor{}, err 76 } 77 } 78 return platform.SelectManifest(ctx, src, root, p) 79 } 80 } 81 82 // defaultCopyMaxMetadataBytes is the default value of 83 // CopyGraphOptions.MaxMetadataBytes. 84 const defaultCopyMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 85 86 // DefaultCopyGraphOptions provides the default CopyGraphOptions. 87 var DefaultCopyGraphOptions CopyGraphOptions 88 89 // CopyGraphOptions contains parameters for [oras.CopyGraph]. 90 type CopyGraphOptions struct { 91 // Concurrency limits the maximum number of concurrent copy tasks. 92 // If less than or equal to 0, a default (currently 3) is used. 93 Concurrency int 94 // MaxMetadataBytes limits the maximum size of the metadata that can be 95 // cached in the memory. 96 // If less than or equal to 0, a default (currently 4 MiB) is used. 97 MaxMetadataBytes int64 98 // PreCopy handles the current descriptor before copying it. 99 PreCopy func(ctx context.Context, desc ocispec.Descriptor) error 100 // PostCopy handles the current descriptor after copying it. 101 PostCopy func(ctx context.Context, desc ocispec.Descriptor) error 102 // OnCopySkipped will be called when the sub-DAG rooted by the current node 103 // is skipped. 104 OnCopySkipped func(ctx context.Context, desc ocispec.Descriptor) error 105 // FindSuccessors finds the successors of the current node. 106 // fetcher provides cached access to the source storage, and is suitable 107 // for fetching non-leaf nodes like manifests. Since anything fetched from 108 // fetcher will be cached in the memory, it is recommended to use original 109 // source storage to fetch large blobs. 110 // If FindSuccessors is nil, content.Successors will be used. 111 FindSuccessors func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) 112 } 113 114 // Copy copies a rooted directed acyclic graph (DAG) with the tagged root node 115 // in the source Target to the destination Target. 116 // The destination reference will be the same as the source reference if the 117 // destination reference is left blank. 118 // 119 // Returns the descriptor of the root node on successful copy. 120 func Copy(ctx context.Context, src ReadOnlyTarget, srcRef string, dst Target, dstRef string, opts CopyOptions) (ocispec.Descriptor, error) { 121 if src == nil { 122 return ocispec.Descriptor{}, errors.New("nil source target") 123 } 124 if dst == nil { 125 return ocispec.Descriptor{}, errors.New("nil destination target") 126 } 127 if dstRef == "" { 128 dstRef = srcRef 129 } 130 131 // use caching proxy on non-leaf nodes 132 if opts.MaxMetadataBytes <= 0 { 133 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 134 } 135 proxy := cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 136 root, err := resolveRoot(ctx, src, srcRef, proxy) 137 if err != nil { 138 return ocispec.Descriptor{}, fmt.Errorf("failed to resolve %s: %w", srcRef, err) 139 } 140 141 if opts.MapRoot != nil { 142 proxy.StopCaching = true 143 root, err = opts.MapRoot(ctx, proxy, root) 144 if err != nil { 145 return ocispec.Descriptor{}, err 146 } 147 proxy.StopCaching = false 148 } 149 150 if err := prepareCopy(ctx, dst, dstRef, proxy, root, &opts); err != nil { 151 return ocispec.Descriptor{}, err 152 } 153 154 if err := copyGraph(ctx, src, dst, root, proxy, nil, nil, opts.CopyGraphOptions); err != nil { 155 return ocispec.Descriptor{}, err 156 } 157 158 return root, nil 159 } 160 161 // CopyGraph copies a rooted directed acyclic graph (DAG) from the source CAS to 162 // the destination CAS. 163 func CopyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, root ocispec.Descriptor, opts CopyGraphOptions) error { 164 return copyGraph(ctx, src, dst, root, nil, nil, nil, opts) 165 } 166 167 // copyGraph copies a rooted directed acyclic graph (DAG) from the source CAS to 168 // the destination CAS with specified caching, concurrency limiter and tracker. 169 func copyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, root ocispec.Descriptor, 170 proxy *cas.Proxy, limiter *semaphore.Weighted, tracker *status.Tracker, opts CopyGraphOptions) error { 171 if proxy == nil { 172 // use caching proxy on non-leaf nodes 173 if opts.MaxMetadataBytes <= 0 { 174 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 175 } 176 proxy = cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 177 } 178 if limiter == nil { 179 // if Concurrency is not set or invalid, use the default concurrency 180 if opts.Concurrency <= 0 { 181 opts.Concurrency = defaultConcurrency 182 } 183 limiter = semaphore.NewWeighted(int64(opts.Concurrency)) 184 } 185 if tracker == nil { 186 // track content status 187 tracker = status.NewTracker() 188 } 189 // if FindSuccessors is not provided, use the default one 190 if opts.FindSuccessors == nil { 191 opts.FindSuccessors = content.Successors 192 } 193 194 // traverse the graph 195 var fn syncutil.GoFunc[ocispec.Descriptor] 196 fn = func(ctx context.Context, region *syncutil.LimitedRegion, desc ocispec.Descriptor) (err error) { 197 // skip the descriptor if other go routine is working on it 198 done, committed := tracker.TryCommit(desc) 199 if !committed { 200 return nil 201 } 202 defer func() { 203 if err == nil { 204 // mark the content as done on success 205 close(done) 206 } 207 }() 208 209 // skip if a rooted sub-DAG exists 210 exists, err := dst.Exists(ctx, desc) 211 if err != nil { 212 return err 213 } 214 if exists { 215 if opts.OnCopySkipped != nil { 216 if err := opts.OnCopySkipped(ctx, desc); err != nil { 217 return err 218 } 219 } 220 return nil 221 } 222 223 // find successors while non-leaf nodes will be fetched and cached 224 successors, err := opts.FindSuccessors(ctx, proxy, desc) 225 if err != nil { 226 return err 227 } 228 successors = removeForeignLayers(successors) 229 230 if len(successors) != 0 { 231 // for non-leaf nodes, process successors and wait for them to complete 232 region.End() 233 if err := syncutil.Go(ctx, limiter, fn, successors...); err != nil { 234 return err 235 } 236 for _, node := range successors { 237 done, committed := tracker.TryCommit(node) 238 if committed { 239 return fmt.Errorf("%s: %s: successor not committed", desc.Digest, node.Digest) 240 } 241 select { 242 case <-done: 243 case <-ctx.Done(): 244 return ctx.Err() 245 } 246 } 247 if err := region.Start(); err != nil { 248 return err 249 } 250 } 251 252 exists, err = proxy.Cache.Exists(ctx, desc) 253 if err != nil { 254 return err 255 } 256 if exists { 257 return copyNode(ctx, proxy.Cache, dst, desc, opts) 258 } 259 return copyNode(ctx, src, dst, desc, opts) 260 } 261 262 return syncutil.Go(ctx, limiter, fn, root) 263 } 264 265 // doCopyNode copies a single content from the source CAS to the destination CAS. 266 func doCopyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor) error { 267 rc, err := src.Fetch(ctx, desc) 268 if err != nil { 269 return err 270 } 271 defer rc.Close() 272 err = dst.Push(ctx, desc, rc) 273 if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 274 return err 275 } 276 return nil 277 } 278 279 // copyNode copies a single content from the source CAS to the destination CAS, 280 // and apply the given options. 281 func copyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor, opts CopyGraphOptions) error { 282 if opts.PreCopy != nil { 283 if err := opts.PreCopy(ctx, desc); err != nil { 284 if err == errSkipDesc { 285 return nil 286 } 287 return err 288 } 289 } 290 291 if err := doCopyNode(ctx, src, dst, desc); err != nil { 292 return err 293 } 294 295 if opts.PostCopy != nil { 296 return opts.PostCopy(ctx, desc) 297 } 298 return nil 299 } 300 301 // copyCachedNodeWithReference copies a single content with a reference from the 302 // source cache to the destination ReferencePusher. 303 func copyCachedNodeWithReference(ctx context.Context, src *cas.Proxy, dst registry.ReferencePusher, desc ocispec.Descriptor, dstRef string) error { 304 rc, err := src.FetchCached(ctx, desc) 305 if err != nil { 306 return err 307 } 308 defer rc.Close() 309 310 err = dst.PushReference(ctx, desc, rc, dstRef) 311 if err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 312 return err 313 } 314 return nil 315 } 316 317 // resolveRoot resolves the source reference to the root node. 318 func resolveRoot(ctx context.Context, src ReadOnlyTarget, srcRef string, proxy *cas.Proxy) (ocispec.Descriptor, error) { 319 refFetcher, ok := src.(registry.ReferenceFetcher) 320 if !ok { 321 return src.Resolve(ctx, srcRef) 322 } 323 324 // optimize performance for ReferenceFetcher targets 325 refProxy := ®istryutil.Proxy{ 326 ReferenceFetcher: refFetcher, 327 Proxy: proxy, 328 } 329 root, rc, err := refProxy.FetchReference(ctx, srcRef) 330 if err != nil { 331 return ocispec.Descriptor{}, err 332 } 333 defer rc.Close() 334 // cache root if it is a non-leaf node 335 fetcher := content.FetcherFunc(func(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { 336 if content.Equal(target, root) { 337 return rc, nil 338 } 339 return nil, errors.New("fetching only root node expected") 340 }) 341 if _, err = content.Successors(ctx, fetcher, root); err != nil { 342 return ocispec.Descriptor{}, err 343 } 344 345 // TODO: optimize special case where root is a leaf node (i.e. a blob) 346 // and dst is a ReferencePusher. 347 return root, nil 348 } 349 350 // prepareCopy prepares the hooks for copy. 351 func prepareCopy(ctx context.Context, dst Target, dstRef string, proxy *cas.Proxy, root ocispec.Descriptor, opts *CopyOptions) error { 352 if refPusher, ok := dst.(registry.ReferencePusher); ok { 353 // optimize performance for ReferencePusher targets 354 preCopy := opts.PreCopy 355 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 356 if preCopy != nil { 357 if err := preCopy(ctx, desc); err != nil { 358 return err 359 } 360 } 361 if !content.Equal(desc, root) { 362 // for non-root node, do nothing 363 return nil 364 } 365 366 // for root node, prepare optimized copy 367 if err := copyCachedNodeWithReference(ctx, proxy, refPusher, desc, dstRef); err != nil { 368 return err 369 } 370 if opts.PostCopy != nil { 371 if err := opts.PostCopy(ctx, desc); err != nil { 372 return err 373 } 374 } 375 // skip the regular copy workflow 376 return errSkipDesc 377 } 378 } else { 379 postCopy := opts.PostCopy 380 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 381 if content.Equal(desc, root) { 382 // for root node, tag it after copying it 383 if err := dst.Tag(ctx, root, dstRef); err != nil { 384 return err 385 } 386 } 387 if postCopy != nil { 388 return postCopy(ctx, desc) 389 } 390 return nil 391 } 392 } 393 394 onCopySkipped := opts.OnCopySkipped 395 opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { 396 if onCopySkipped != nil { 397 if err := onCopySkipped(ctx, desc); err != nil { 398 return err 399 } 400 } 401 if !content.Equal(desc, root) { 402 return nil 403 } 404 // enforce tagging when root is skipped 405 if refPusher, ok := dst.(registry.ReferencePusher); ok { 406 return copyCachedNodeWithReference(ctx, proxy, refPusher, desc, dstRef) 407 } 408 return dst.Tag(ctx, root, dstRef) 409 } 410 411 return nil 412 } 413 414 // removeForeignLayers in-place removes all foreign layers in the given slice. 415 func removeForeignLayers(descs []ocispec.Descriptor) []ocispec.Descriptor { 416 var j int 417 for i, desc := range descs { 418 if !descriptor.IsForeignLayer(desc) { 419 if i != j { 420 descs[j] = desc 421 } 422 j++ 423 } 424 } 425 return descs[:j] 426 }