github.com/dtroyer-salad/og2/v2@v2.0.0-20240412154159-c47231610877/content.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "bytes" 20 "context" 21 "errors" 22 "fmt" 23 "io" 24 25 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 26 "oras.land/oras-go/v2/content" 27 "oras.land/oras-go/v2/errdef" 28 "oras.land/oras-go/v2/internal/cas" 29 "oras.land/oras-go/v2/internal/docker" 30 "oras.land/oras-go/v2/internal/interfaces" 31 "oras.land/oras-go/v2/internal/platform" 32 "oras.land/oras-go/v2/internal/syncutil" 33 "oras.land/oras-go/v2/registry" 34 "oras.land/oras-go/v2/registry/remote/auth" 35 ) 36 37 const ( 38 // defaultTagConcurrency is the default concurrency of tagging. 39 defaultTagConcurrency int = 5 // This value is consistent with dockerd 40 41 // defaultTagNMaxMetadataBytes is the default value of 42 // TagNOptions.MaxMetadataBytes. 43 defaultTagNMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 44 45 // defaultResolveMaxMetadataBytes is the default value of 46 // ResolveOptions.MaxMetadataBytes. 47 defaultResolveMaxMetadataBytes int64 = 4 * 1024 * 1024 // 4 MiB 48 49 // defaultMaxBytes is the default value of FetchBytesOptions.MaxBytes. 50 defaultMaxBytes int64 = 4 * 1024 * 1024 // 4 MiB 51 ) 52 53 // DefaultTagNOptions provides the default TagNOptions. 54 var DefaultTagNOptions TagNOptions 55 56 // TagNOptions contains parameters for [oras.TagN]. 57 type TagNOptions struct { 58 // Concurrency limits the maximum number of concurrent tag tasks. 59 // If less than or equal to 0, a default (currently 5) is used. 60 Concurrency int 61 62 // MaxMetadataBytes limits the maximum size of metadata that can be cached 63 // in the memory. 64 // If less than or equal to 0, a default (currently 4 MiB) is used. 65 MaxMetadataBytes int64 66 } 67 68 // TagN tags the descriptor identified by srcReference with dstReferences. 69 func TagN(ctx context.Context, target Target, srcReference string, dstReferences []string, opts TagNOptions) (ocispec.Descriptor, error) { 70 switch len(dstReferences) { 71 case 0: 72 return ocispec.Descriptor{}, fmt.Errorf("dstReferences cannot be empty: %w", errdef.ErrMissingReference) 73 case 1: 74 return Tag(ctx, target, srcReference, dstReferences[0]) 75 } 76 77 if opts.Concurrency <= 0 { 78 opts.Concurrency = defaultTagConcurrency 79 } 80 if opts.MaxMetadataBytes <= 0 { 81 opts.MaxMetadataBytes = defaultTagNMaxMetadataBytes 82 } 83 84 _, isRefFetcher := target.(registry.ReferenceFetcher) 85 _, isRefPusher := target.(registry.ReferencePusher) 86 if isRefFetcher && isRefPusher { 87 if repo, ok := target.(interfaces.ReferenceParser); ok { 88 // add scope hints to minimize the number of auth requests 89 ref, err := repo.ParseReference(srcReference) 90 if err != nil { 91 return ocispec.Descriptor{}, err 92 } 93 ctx = auth.AppendRepositoryScope(ctx, ref, auth.ActionPull, auth.ActionPush) 94 } 95 96 desc, contentBytes, err := FetchBytes(ctx, target, srcReference, FetchBytesOptions{ 97 MaxBytes: opts.MaxMetadataBytes, 98 }) 99 if err != nil { 100 if errors.Is(err, errdef.ErrSizeExceedsLimit) { 101 err = fmt.Errorf( 102 "content size %v exceeds MaxMetadataBytes %v: %w", 103 desc.Size, 104 opts.MaxMetadataBytes, 105 errdef.ErrSizeExceedsLimit) 106 } 107 return ocispec.Descriptor{}, err 108 } 109 110 if err := tagBytesN(ctx, target, desc, contentBytes, dstReferences, TagBytesNOptions{ 111 Concurrency: opts.Concurrency, 112 }); err != nil { 113 return ocispec.Descriptor{}, err 114 } 115 return desc, nil 116 } 117 118 desc, err := target.Resolve(ctx, srcReference) 119 if err != nil { 120 return ocispec.Descriptor{}, err 121 } 122 eg, egCtx := syncutil.LimitGroup(ctx, opts.Concurrency) 123 for _, dstRef := range dstReferences { 124 eg.Go(func(dst string) func() error { 125 return func() error { 126 if err := target.Tag(egCtx, desc, dst); err != nil { 127 return fmt.Errorf("failed to tag %s as %s: %w", srcReference, dst, err) 128 } 129 return nil 130 } 131 }(dstRef)) 132 } 133 134 if err := eg.Wait(); err != nil { 135 return ocispec.Descriptor{}, err 136 } 137 return desc, nil 138 } 139 140 // Tag tags the descriptor identified by src with dst. 141 func Tag(ctx context.Context, target Target, src, dst string) (ocispec.Descriptor, error) { 142 refFetcher, okFetch := target.(registry.ReferenceFetcher) 143 refPusher, okPush := target.(registry.ReferencePusher) 144 if okFetch && okPush { 145 if repo, ok := target.(interfaces.ReferenceParser); ok { 146 // add scope hints to minimize the number of auth requests 147 ref, err := repo.ParseReference(src) 148 if err != nil { 149 return ocispec.Descriptor{}, err 150 } 151 ctx = auth.AppendRepositoryScope(ctx, ref, auth.ActionPull, auth.ActionPush) 152 } 153 desc, rc, err := refFetcher.FetchReference(ctx, src) 154 if err != nil { 155 return ocispec.Descriptor{}, err 156 } 157 defer rc.Close() 158 if err := refPusher.PushReference(ctx, desc, rc, dst); err != nil { 159 return ocispec.Descriptor{}, err 160 } 161 return desc, nil 162 } 163 164 desc, err := target.Resolve(ctx, src) 165 if err != nil { 166 return ocispec.Descriptor{}, err 167 } 168 if err := target.Tag(ctx, desc, dst); err != nil { 169 return ocispec.Descriptor{}, err 170 } 171 return desc, nil 172 } 173 174 // DefaultResolveOptions provides the default ResolveOptions. 175 var DefaultResolveOptions ResolveOptions 176 177 // ResolveOptions contains parameters for [oras.Resolve]. 178 type ResolveOptions struct { 179 // TargetPlatform ensures the resolved content matches the target platform 180 // if the node is a manifest, or selects the first resolved content that 181 // matches the target platform if the node is a manifest list. 182 TargetPlatform *ocispec.Platform 183 184 // MaxMetadataBytes limits the maximum size of metadata that can be cached 185 // in the memory. 186 // If less than or equal to 0, a default (currently 4 MiB) is used. 187 MaxMetadataBytes int64 188 } 189 190 // Resolve resolves a descriptor with provided reference from the target. 191 func Resolve(ctx context.Context, target ReadOnlyTarget, reference string, opts ResolveOptions) (ocispec.Descriptor, error) { 192 if opts.TargetPlatform == nil { 193 return target.Resolve(ctx, reference) 194 } 195 return resolve(ctx, target, nil, reference, opts) 196 } 197 198 // resolve resolves a descriptor with provided reference from the target, with 199 // specified caching. 200 func resolve(ctx context.Context, target ReadOnlyTarget, proxy *cas.Proxy, reference string, opts ResolveOptions) (ocispec.Descriptor, error) { 201 if opts.MaxMetadataBytes <= 0 { 202 opts.MaxMetadataBytes = defaultResolveMaxMetadataBytes 203 } 204 205 if refFetcher, ok := target.(registry.ReferenceFetcher); ok { 206 // optimize performance for ReferenceFetcher targets 207 desc, rc, err := refFetcher.FetchReference(ctx, reference) 208 if err != nil { 209 return ocispec.Descriptor{}, err 210 } 211 defer rc.Close() 212 213 switch desc.MediaType { 214 case docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex, 215 docker.MediaTypeManifest, ocispec.MediaTypeImageManifest: 216 // cache the fetched content 217 if desc.Size > opts.MaxMetadataBytes { 218 return ocispec.Descriptor{}, fmt.Errorf( 219 "content size %v exceeds MaxMetadataBytes %v: %w", 220 desc.Size, 221 opts.MaxMetadataBytes, 222 errdef.ErrSizeExceedsLimit) 223 } 224 if proxy == nil { 225 proxy = cas.NewProxyWithLimit(target, cas.NewMemory(), opts.MaxMetadataBytes) 226 } 227 if err := proxy.Cache.Push(ctx, desc, rc); err != nil { 228 return ocispec.Descriptor{}, err 229 } 230 // stop caching as SelectManifest may fetch a config blob 231 proxy.StopCaching = true 232 return platform.SelectManifest(ctx, proxy, desc, opts.TargetPlatform) 233 default: 234 return ocispec.Descriptor{}, fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrUnsupported) 235 } 236 } 237 238 desc, err := target.Resolve(ctx, reference) 239 if err != nil { 240 return ocispec.Descriptor{}, err 241 } 242 return platform.SelectManifest(ctx, target, desc, opts.TargetPlatform) 243 } 244 245 // DefaultFetchOptions provides the default FetchOptions. 246 var DefaultFetchOptions FetchOptions 247 248 // FetchOptions contains parameters for [oras.Fetch]. 249 type FetchOptions struct { 250 // ResolveOptions contains parameters for resolving reference. 251 ResolveOptions 252 } 253 254 // Fetch fetches the content identified by the reference. 255 func Fetch(ctx context.Context, target ReadOnlyTarget, reference string, opts FetchOptions) (ocispec.Descriptor, io.ReadCloser, error) { 256 if opts.TargetPlatform == nil { 257 if refFetcher, ok := target.(registry.ReferenceFetcher); ok { 258 return refFetcher.FetchReference(ctx, reference) 259 } 260 261 desc, err := target.Resolve(ctx, reference) 262 if err != nil { 263 return ocispec.Descriptor{}, nil, err 264 } 265 rc, err := target.Fetch(ctx, desc) 266 if err != nil { 267 return ocispec.Descriptor{}, nil, err 268 } 269 return desc, rc, nil 270 } 271 272 if opts.MaxMetadataBytes <= 0 { 273 opts.MaxMetadataBytes = defaultResolveMaxMetadataBytes 274 } 275 proxy := cas.NewProxyWithLimit(target, cas.NewMemory(), opts.MaxMetadataBytes) 276 desc, err := resolve(ctx, target, proxy, reference, opts.ResolveOptions) 277 if err != nil { 278 return ocispec.Descriptor{}, nil, err 279 } 280 // if the content exists in cache, fetch it from cache 281 // otherwise fetch without caching 282 proxy.StopCaching = true 283 rc, err := proxy.Fetch(ctx, desc) 284 if err != nil { 285 return ocispec.Descriptor{}, nil, err 286 } 287 return desc, rc, nil 288 } 289 290 // DefaultFetchBytesOptions provides the default FetchBytesOptions. 291 var DefaultFetchBytesOptions FetchBytesOptions 292 293 // FetchBytesOptions contains parameters for [oras.FetchBytes]. 294 type FetchBytesOptions struct { 295 // FetchOptions contains parameters for fetching content. 296 FetchOptions 297 // MaxBytes limits the maximum size of the fetched content bytes. 298 // If less than or equal to 0, a default (currently 4 MiB) is used. 299 MaxBytes int64 300 } 301 302 // FetchBytes fetches the content bytes identified by the reference. 303 func FetchBytes(ctx context.Context, target ReadOnlyTarget, reference string, opts FetchBytesOptions) (ocispec.Descriptor, []byte, error) { 304 if opts.MaxBytes <= 0 { 305 opts.MaxBytes = defaultMaxBytes 306 } 307 308 desc, rc, err := Fetch(ctx, target, reference, opts.FetchOptions) 309 if err != nil { 310 return ocispec.Descriptor{}, nil, err 311 } 312 defer rc.Close() 313 314 if desc.Size > opts.MaxBytes { 315 return ocispec.Descriptor{}, nil, fmt.Errorf( 316 "content size %v exceeds MaxBytes %v: %w", 317 desc.Size, 318 opts.MaxBytes, 319 errdef.ErrSizeExceedsLimit) 320 } 321 bytes, err := content.ReadAll(rc, desc) 322 if err != nil { 323 return ocispec.Descriptor{}, nil, err 324 } 325 326 return desc, bytes, nil 327 } 328 329 // PushBytes describes the contentBytes using the given mediaType and pushes it. 330 // If mediaType is not specified, "application/octet-stream" is used. 331 func PushBytes(ctx context.Context, pusher content.Pusher, mediaType string, contentBytes []byte) (ocispec.Descriptor, error) { 332 desc := content.NewDescriptorFromBytes(mediaType, contentBytes) 333 r := bytes.NewReader(contentBytes) 334 if err := pusher.Push(ctx, desc, r); err != nil { 335 return ocispec.Descriptor{}, err 336 } 337 338 return desc, nil 339 } 340 341 // DefaultTagBytesNOptions provides the default TagBytesNOptions. 342 var DefaultTagBytesNOptions TagBytesNOptions 343 344 // TagBytesNOptions contains parameters for [oras.TagBytesN]. 345 type TagBytesNOptions struct { 346 // Concurrency limits the maximum number of concurrent tag tasks. 347 // If less than or equal to 0, a default (currently 5) is used. 348 Concurrency int 349 } 350 351 // TagBytesN describes the contentBytes using the given mediaType, pushes it, 352 // and tag it with the given references. 353 // If mediaType is not specified, "application/octet-stream" is used. 354 func TagBytesN(ctx context.Context, target Target, mediaType string, contentBytes []byte, references []string, opts TagBytesNOptions) (ocispec.Descriptor, error) { 355 if len(references) == 0 { 356 return PushBytes(ctx, target, mediaType, contentBytes) 357 } 358 359 desc := content.NewDescriptorFromBytes(mediaType, contentBytes) 360 if opts.Concurrency <= 0 { 361 opts.Concurrency = defaultTagConcurrency 362 } 363 364 if err := tagBytesN(ctx, target, desc, contentBytes, references, opts); err != nil { 365 return ocispec.Descriptor{}, err 366 } 367 return desc, nil 368 } 369 370 // tagBytesN pushes the contentBytes using the given desc, and tag it with the 371 // given references. 372 func tagBytesN(ctx context.Context, target Target, desc ocispec.Descriptor, contentBytes []byte, references []string, opts TagBytesNOptions) error { 373 eg, egCtx := syncutil.LimitGroup(ctx, opts.Concurrency) 374 if refPusher, ok := target.(registry.ReferencePusher); ok { 375 for _, reference := range references { 376 eg.Go(func(ref string) func() error { 377 return func() error { 378 r := bytes.NewReader(contentBytes) 379 if err := refPusher.PushReference(egCtx, desc, r, ref); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 380 return fmt.Errorf("failed to tag %s: %w", ref, err) 381 } 382 return nil 383 } 384 }(reference)) 385 } 386 } else { 387 r := bytes.NewReader(contentBytes) 388 if err := target.Push(ctx, desc, r); err != nil && !errors.Is(err, errdef.ErrAlreadyExists) { 389 return fmt.Errorf("failed to push content: %w", err) 390 } 391 for _, reference := range references { 392 eg.Go(func(ref string) func() error { 393 return func() error { 394 if err := target.Tag(egCtx, desc, ref); err != nil { 395 return fmt.Errorf("failed to tag %s: %w", ref, err) 396 } 397 return nil 398 } 399 }(reference)) 400 } 401 } 402 403 return eg.Wait() 404 } 405 406 // TagBytes describes the contentBytes using the given mediaType, pushes it, 407 // and tag it with the given reference. 408 // If mediaType is not specified, "application/octet-stream" is used. 409 func TagBytes(ctx context.Context, target Target, mediaType string, contentBytes []byte, reference string) (ocispec.Descriptor, error) { 410 return TagBytesN(ctx, target, mediaType, contentBytes, []string{reference}, DefaultTagBytesNOptions) 411 }