github.com/opcr-io/oras-go/v2@v2.0.0-20231122155130-eb4260d8a0ae/content.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "bytes" 20 "context" 21 "errors" 22 "fmt" 23 "io" 24 25 "github.com/opcr-io/oras-go/v2/content" 26 "github.com/opcr-io/oras-go/v2/errdef" 27 "github.com/opcr-io/oras-go/v2/internal/cas" 28 "github.com/opcr-io/oras-go/v2/internal/docker" 29 "github.com/opcr-io/oras-go/v2/internal/interfaces" 30 "github.com/opcr-io/oras-go/v2/internal/platform" 31 "github.com/opcr-io/oras-go/v2/internal/registryutil" 32 "github.com/opcr-io/oras-go/v2/internal/syncutil" 33 "github.com/opcr-io/oras-go/v2/registry" 34 "github.com/opcr-io/oras-go/v2/registry/remote/auth" 35 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 36 ) 37 38 const ( 39 // defaultTagConcurrency is the default concurrency of tagging. 40 defaultTagConcurrency int = 5 // This value is consistent with dockerd 41 42 // defaultTagNMaxMetadataBytes is the default value of 43 // TagNOptions.MaxMetadataBytes. 44 defaultTagNMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 45 46 // defaultResolveMaxMetadataBytes is the default value of 47 // ResolveOptions.MaxMetadataBytes. 48 defaultResolveMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 49 50 // defaultMaxBytes is the default value of FetchBytesOptions.MaxBytes. 51 defaultMaxBytes int64 = 4 * 1024 * 1024 // 4 MiB 52 ) 53 54 // DefaultTagNOptions provides the default TagNOptions. 55 var DefaultTagNOptions TagNOptions 56 57 // TagNOptions contains parameters for [oras.TagN]. 58 type TagNOptions struct { 59 // Concurrency limits the maximum number of concurrent tag tasks. 60 // If less than or equal to 0, a default (currently 5) is used. 61 Concurrency int 62 63 // MaxMetadataBytes limits the maximum size of metadata that can be cached 64 // in the memory. 65 // If less than or equal to 0, a default (currently 4 MiB) is used. 66 MaxMetadataBytes int64 67 } 68 69 // TagN tags the descriptor identified by srcReference with dstReferences. 70 func TagN(ctx context.Context, target Target, srcReference string, dstReferences []string, opts TagNOptions) (ocispec.Descriptor, error) { 71 switch len(dstReferences) { 72 case 0: 73 return ocispec.Descriptor{}, fmt.Errorf("dstReferences cannot be empty: %w", errdef.ErrMissingReference) 74 case 1: 75 return Tag(ctx, target, srcReference, dstReferences[0]) 76 } 77 78 if opts.Concurrency <= 0 { 79 opts.Concurrency = defaultTagConcurrency 80 } 81 if opts.MaxMetadataBytes <= 0 { 82 opts.MaxMetadataBytes = defaultTagNMaxMetadataBytes 83 } 84 85 _, isRefFetcher := target.(registry.ReferenceFetcher) 86 _, isRefPusher := target.(registry.ReferencePusher) 87 if isRefFetcher && isRefPusher { 88 if repo, ok := target.(interfaces.ReferenceParser); ok { 89 // add scope hints to minimize the number of auth requests 90 ref, err := repo.ParseReference(srcReference) 91 if err != nil { 92 return ocispec.Descriptor{}, err 93 } 94 ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull, auth.ActionPush) 95 } 96 97 desc, contentBytes, err := FetchBytes(ctx, target, srcReference, FetchBytesOptions{ 98 MaxBytes: opts.MaxMetadataBytes, 99 }) 100 if err != nil { 101 if errors.Is(err, errdef.ErrSizeExceedsLimit) { 102 err = fmt.Errorf( 103 "content size %v exceeds MaxMetadataBytes %v: %w", 104 desc.Size, 105 opts.MaxMetadataBytes, 106 errdef.ErrSizeExceedsLimit) 107 } 108 return ocispec.Descriptor{}, err 109 } 110 111 if err := tagBytesN(ctx, target, desc, contentBytes, dstReferences, TagBytesNOptions{ 112 Concurrency: opts.Concurrency, 113 }); err != nil { 114 return ocispec.Descriptor{}, err 115 } 116 return desc, nil 117 } 118 119 desc, err := target.Resolve(ctx, srcReference) 120 if err != nil { 121 return ocispec.Descriptor{}, err 122 } 123 eg, egCtx := syncutil.LimitGroup(ctx, opts.Concurrency) 124 for _, dstRef := range dstReferences { 125 eg.Go(func(dst string) func() error { 126 return func() error { 127 if err := target.Tag(egCtx, desc, dst); err != nil { 128 return fmt.Errorf("failed to tag %s as %s: %w", srcReference, dst, err) 129 } 130 return nil 131 } 132 }(dstRef)) 133 } 134 135 if err := eg.Wait(); err != nil { 136 return ocispec.Descriptor{}, err 137 } 138 return desc, nil 139 } 140 141 // Tag tags the descriptor identified by src with dst. 142 func Tag(ctx context.Context, target Target, src, dst string) (ocispec.Descriptor, error) { 143 refFetcher, okFetch := target.(registry.ReferenceFetcher) 144 refPusher, okPush := target.(registry.ReferencePusher) 145 if okFetch && okPush { 146 if repo, ok := target.(interfaces.ReferenceParser); ok { 147 // add scope hints to minimize the number of auth requests 148 ref, err := repo.ParseReference(src) 149 if err != nil { 150 return ocispec.Descriptor{}, err 151 } 152 ctx = registryutil.WithScopeHint(ctx, ref, auth.ActionPull, auth.ActionPush) 153 } 154 desc, rc, err := refFetcher.FetchReference(ctx, src) 155 if err != nil { 156 return ocispec.Descriptor{}, err 157 } 158 defer rc.Close() 159 if err := refPusher.PushReference(ctx, desc, rc, dst); err != nil { 160 return ocispec.Descriptor{}, err 161 } 162 return desc, nil 163 } 164 165 desc, err := target.Resolve(ctx, src) 166 if err != nil { 167 return ocispec.Descriptor{}, err 168 } 169 if err := target.Tag(ctx, desc, dst); err != nil { 170 return ocispec.Descriptor{}, err 171 } 172 return desc, nil 173 } 174 175 // DefaultResolveOptions provides the default ResolveOptions. 176 var DefaultResolveOptions ResolveOptions 177 178 // ResolveOptions contains parameters for [oras.Resolve]. 179 type ResolveOptions struct { 180 // TargetPlatform ensures the resolved content matches the target platform 181 // if the node is a manifest, or selects the first resolved content that 182 // matches the target platform if the node is a manifest list. 183 TargetPlatform *ocispec.Platform 184 185 // MaxMetadataBytes limits the maximum size of metadata that can be cached 186 // in the memory. 187 // If less than or equal to 0, a default (currently 4 MiB) is used. 188 MaxMetadataBytes int64 189 } 190 191 // Resolve resolves a descriptor with provided reference from the target. 192 func Resolve(ctx context.Context, target ReadOnlyTarget, reference string, opts ResolveOptions) (ocispec.Descriptor, error) { 193 if opts.TargetPlatform == nil { 194 return target.Resolve(ctx, reference) 195 } 196 return resolve(ctx, target, nil, reference, opts) 197 } 198 199 // resolve resolves a descriptor with provided reference from the target, with 200 // specified caching. 201 func resolve(ctx context.Context, target ReadOnlyTarget, proxy *cas.Proxy, reference string, opts ResolveOptions) (ocispec.Descriptor, error) { 202 if opts.MaxMetadataBytes <= 0 { 203 opts.MaxMetadataBytes = defaultResolveMaxMetadataBytes 204 } 205 206 if refFetcher, ok := target.(registry.ReferenceFetcher); ok { 207 // optimize performance for ReferenceFetcher targets 208 desc, rc, err := refFetcher.FetchReference(ctx, reference) 209 if err != nil { 210 return ocispec.Descriptor{}, err 211 } 212 defer rc.Close() 213 214 switch desc.MediaType { 215 case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex, 216 docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: 217 // cache the fetched content 218 if desc.Size > opts.MaxMetadataBytes { 219 return ocispec.Descriptor{}, fmt.Errorf( 220 "content size %v exceeds MaxMetadataBytes %v: %w", 221 desc.Size, 222 opts.MaxMetadataBytes, 223 errdef.ErrSizeExceedsLimit) 224 } 225 if proxy == nil { 226 proxy = cas.NewProxyWithLimit(target, cas.NewMemory(), opts.MaxMetadataBytes) 227 } 228 if err := proxy.Cache.Push(ctx, desc, rc); err != nil { 229 return ocispec.Descriptor{}, err 230 } 231 // stop caching as SelectManifest may fetch a config blob 232 proxy.StopCaching = true 233 return platform.SelectManifest(ctx, proxy, desc, opts.TargetPlatform) 234 default: 235 return ocispec.Descriptor{}, fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrUnsupported) 236 } 237 } 238 239 desc, err := target.Resolve(ctx, reference) 240 if err != nil { 241 return ocispec.Descriptor{}, err 242 } 243 return platform.SelectManifest(ctx, target, desc, opts.TargetPlatform) 244 } 245 246 // DefaultFetchOptions provides the default FetchOptions. 247 var DefaultFetchOptions FetchOptions 248 249 // FetchOptions contains parameters for [oras.Fetch]. 250 type FetchOptions struct { 251 // ResolveOptions contains parameters for resolving reference. 252 ResolveOptions 253 } 254 255 // Fetch fetches the content identified by the reference. 256 func Fetch(ctx context.Context, target ReadOnlyTarget, reference string, opts FetchOptions) (ocispec.Descriptor, io.ReadCloser, error) { 257 if opts.TargetPlatform == nil { 258 if refFetcher, ok := target.(registry.ReferenceFetcher); ok { 259 return refFetcher.FetchReference(ctx, reference) 260 } 261 262 desc, err := target.Resolve(ctx, reference) 263 if err != nil { 264 return ocispec.Descriptor{}, nil, err 265 } 266 rc, err := target.Fetch(ctx, desc) 267 if err != nil { 268 return ocispec.Descriptor{}, nil, err 269 } 270 return desc, rc, nil 271 } 272 273 if opts.MaxMetadataBytes <= 0 { 274 opts.MaxMetadataBytes = defaultResolveMaxMetadataBytes 275 } 276 proxy := cas.NewProxyWithLimit(target, cas.NewMemory(), opts.MaxMetadataBytes) 277 desc, err := resolve(ctx, target, proxy, reference, opts.ResolveOptions) 278 if err != nil { 279 return ocispec.Descriptor{}, nil, err 280 } 281 // if the content exists in cache, fetch it from cache 282 // otherwise fetch without caching 283 proxy.StopCaching = true 284 rc, err := proxy.Fetch(ctx, desc) 285 if err != nil { 286 return ocispec.Descriptor{}, nil, err 287 } 288 return desc, rc, nil 289 } 290 291 // DefaultFetchBytesOptions provides the default FetchBytesOptions. 292 var DefaultFetchBytesOptions FetchBytesOptions 293 294 // FetchBytesOptions contains parameters for [oras.FetchBytes]. 295 type FetchBytesOptions struct { 296 // FetchOptions contains parameters for fetching content. 297 FetchOptions 298 // MaxBytes limits the maximum size of the fetched content bytes. 299 // If less than or equal to 0, a default (currently 4 MiB) is used. 300 MaxBytes int64 301 } 302 303 // FetchBytes fetches the content bytes identified by the reference. 304 func FetchBytes(ctx context.Context, target ReadOnlyTarget, reference string, opts FetchBytesOptions) (ocispec.Descriptor, []byte, error) { 305 if opts.MaxBytes <= 0 { 306 opts.MaxBytes = defaultMaxBytes 307 } 308 309 desc, rc, err := Fetch(ctx, target, reference, opts.FetchOptions) 310 if err != nil { 311 return ocispec.Descriptor{}, nil, err 312 } 313 defer rc.Close() 314 315 if desc.Size > opts.MaxBytes { 316 return ocispec.Descriptor{}, nil, fmt.Errorf( 317 "content size %v exceeds MaxBytes %v: %w", 318 desc.Size, 319 opts.MaxBytes, 320 errdef.ErrSizeExceedsLimit) 321 } 322 bytes, err := content.ReadAll(rc, desc) 323 if err != nil { 324 return ocispec.Descriptor{}, nil, err 325 } 326 327 return desc, bytes, nil 328 } 329 330 // PushBytes describes the contentBytes using the given mediaType and pushes it. 331 // If mediaType is not specified, "application/octet-stream" is used. 332 func PushBytes(ctx context.Context, pusher content.Pusher, mediaType string, contentBytes []byte) (ocispec.Descriptor, error) { 333 desc := content.NewDescriptorFromBytes(mediaType, contentBytes) 334 r := bytes.NewReader(contentBytes) 335 if err := pusher.Push(ctx, desc, r); err != nil { 336 return ocispec.Descriptor{}, err 337 } 338 339 return desc, nil 340 } 341 342 // DefaultTagBytesNOptions provides the default TagBytesNOptions. 343 var DefaultTagBytesNOptions TagBytesNOptions 344 345 // TagBytesNOptions contains parameters for [oras.TagBytesN]. 346 type TagBytesNOptions struct { 347 // Concurrency limits the maximum number of concurrent tag tasks. 348 // If less than or equal to 0, a default (currently 5) is used. 349 Concurrency int 350 } 351 352 // TagBytesN describes the contentBytes using the given mediaType, pushes it, 353 // and tag it with the given references. 354 // If mediaType is not specified, "application/octet-stream" is used. 355 func TagBytesN(ctx context.Context, target Target, mediaType string, contentBytes []byte, references []string, opts TagBytesNOptions) (ocispec.Descriptor, error) { 356 if len(references) == 0 { 357 return PushBytes(ctx, target, mediaType, contentBytes) 358 } 359 360 desc := content.NewDescriptorFromBytes(mediaType, contentBytes) 361 if opts.Concurrency <= 0 { 362 opts.Concurrency = defaultTagConcurrency 363 } 364 365 if err := tagBytesN(ctx, target, desc, contentBytes, references, opts); err != nil { 366 return ocispec.Descriptor{}, err 367 } 368 return desc, nil 369 } 370 371 // tagBytesN pushes the contentBytes using the given desc, and tag it with the 372 // given references. 373 func tagBytesN(ctx context.Context, target Target, desc ocispec.Descriptor, contentBytes []byte, references []string, opts TagBytesNOptions) error { 374 eg, egCtx := syncutil.LimitGroup(ctx, opts.Concurrency) 375 if refPusher, ok := target.(registry.ReferencePusher); ok { 376 for _, reference := range references { 377 eg.Go(func(ref string) func() error { 378 return func() error { 379 r := bytes.NewReader(contentBytes) 380 if err := refPusher.PushReference(egCtx, desc, r, ref); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 381 return fmt.Errorf("failed to tag %s: %w", ref, err) 382 } 383 return nil 384 } 385 }(reference)) 386 } 387 } else { 388 r := bytes.NewReader(contentBytes) 389 if err := target.Push(ctx, desc, r); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 390 return fmt.Errorf("failed to push content: %w", err) 391 } 392 for _, reference := range references { 393 eg.Go(func(ref string) func() error { 394 return func() error { 395 if err := target.Tag(egCtx, desc, ref); err != nil { 396 return fmt.Errorf("failed to tag %s: %w", ref, err) 397 } 398 return nil 399 } 400 }(reference)) 401 } 402 } 403 404 return eg.Wait() 405 } 406 407 // TagBytes describes the contentBytes using the given mediaType, pushes it, 408 // and tag it with the given reference. 409 // If mediaType is not specified, "application/octet-stream" is used. 410 func TagBytes(ctx context.Context, target Target, mediaType string, contentBytes []byte, reference string) (ocispec.Descriptor, error) { 411 return TagBytesN(ctx, target, mediaType, contentBytes, []string{reference}, DefaultTagBytesNOptions) 412 }