github.com/dtroyer-salad/og2/v2@v2.0.0-20240412154159-c47231610877/extendedcopy.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "context" 20 "encoding/json" 21 "errors" 22 "regexp" 23 24 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 25 "golang.org/x/sync/semaphore" 26 "oras.land/oras-go/v2/content" 27 "oras.land/oras-go/v2/internal/cas" 28 "oras.land/oras-go/v2/internal/container/set" 29 "oras.land/oras-go/v2/internal/copyutil" 30 "oras.land/oras-go/v2/internal/descriptor" 31 "oras.land/oras-go/v2/internal/docker" 32 "oras.land/oras-go/v2/internal/spec" 33 "oras.land/oras-go/v2/internal/status" 34 "oras.land/oras-go/v2/internal/syncutil" 35 "oras.land/oras-go/v2/registry" 36 ) 37 38 // DefaultExtendedCopyOptions provides the default ExtendedCopyOptions. 39 var DefaultExtendedCopyOptions ExtendedCopyOptions = ExtendedCopyOptions{ 40 ExtendedCopyGraphOptions: DefaultExtendedCopyGraphOptions, 41 } 42 43 // ExtendedCopyOptions contains parameters for [oras.ExtendedCopy]. 44 type ExtendedCopyOptions struct { 45 ExtendedCopyGraphOptions 46 } 47 48 // DefaultExtendedCopyGraphOptions provides the default ExtendedCopyGraphOptions. 49 var DefaultExtendedCopyGraphOptions ExtendedCopyGraphOptions = ExtendedCopyGraphOptions{ 50 CopyGraphOptions: DefaultCopyGraphOptions, 51 } 52 53 // ExtendedCopyGraphOptions contains parameters for [oras.ExtendedCopyGraph]. 54 type ExtendedCopyGraphOptions struct { 55 CopyGraphOptions 56 // Depth limits the maximum depth of the directed acyclic graph (DAG) that 57 // will be extended-copied. 58 // If Depth is no specified, or the specified value is less than or 59 // equal to 0, the depth limit will be considered as infinity. 60 Depth int 61 // FindPredecessors finds the predecessors of the current node. 62 // If FindPredecessors is nil, src.Predecessors will be adapted and used. 63 FindPredecessors func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) 64 } 65 66 // ExtendedCopy copies the directed acyclic graph (DAG) that are reachable from 67 // the given tagged node from the source GraphTarget to the destination Target. 68 // The destination reference will be the same as the source reference if the 69 // destination reference is left blank. 70 // 71 // Returns the descriptor of the tagged node on successful copy. 72 func ExtendedCopy(ctx context.Context, src ReadOnlyGraphTarget, srcRef string, dst Target, dstRef string, opts ExtendedCopyOptions) (ocispec.Descriptor, error) { 73 if src == nil { 74 return ocispec.Descriptor{}, errors.New("nil source graph target") 75 } 76 if dst == nil { 77 return ocispec.Descriptor{}, errors.New("nil destination target") 78 } 79 if dstRef == "" { 80 dstRef = srcRef 81 } 82 83 node, err := src.Resolve(ctx, srcRef) 84 if err != nil { 85 return ocispec.Descriptor{}, err 86 } 87 88 if err := ExtendedCopyGraph(ctx, src, dst, node, opts.ExtendedCopyGraphOptions); err != nil { 89 return ocispec.Descriptor{}, err 90 } 91 92 if err := dst.Tag(ctx, node, dstRef); err != nil { 93 return ocispec.Descriptor{}, err 94 } 95 96 return node, nil 97 } 98 99 // ExtendedCopyGraph copies the directed acyclic graph (DAG) that are reachable 100 // from the given node from the source GraphStorage to the destination Storage. 101 func ExtendedCopyGraph(ctx context.Context, src content.ReadOnlyGraphStorage, dst content.Storage, node ocispec.Descriptor, opts ExtendedCopyGraphOptions) error { 102 roots, err := findRoots(ctx, src, node, opts) 103 if err != nil { 104 return err 105 } 106 107 // if Concurrency is not set or invalid, use the default concurrency 108 if opts.Concurrency <= 0 { 109 opts.Concurrency = defaultConcurrency 110 } 111 limiter := semaphore.NewWeighted(int64(opts.Concurrency)) 112 // use caching proxy on non-leaf nodes 113 if opts.MaxMetadataBytes <= 0 { 114 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 115 } 116 proxy := cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 117 // track content status 118 tracker := status.NewTracker() 119 120 // copy the sub-DAGs rooted by the root nodes 121 return syncutil.Go(ctx, limiter, func(ctx context.Context, region *syncutil.LimitedRegion, root ocispec.Descriptor) error { 122 // As a root can be a predecessor of other roots, release the limit here 123 // for dispatching, to avoid dead locks where predecessor roots are 124 // handled first and are waiting for its successors to complete. 125 region.End() 126 if err := copyGraph(ctx, src, dst, root, proxy, limiter, tracker, opts.CopyGraphOptions); err != nil { 127 return err 128 } 129 return region.Start() 130 }, roots...) 131 } 132 133 // findRoots finds the root nodes reachable from the given node through a 134 // depth-first search. 135 func findRoots(ctx context.Context, storage content.ReadOnlyGraphStorage, node ocispec.Descriptor, opts ExtendedCopyGraphOptions) ([]ocispec.Descriptor, error) { 136 visited := set.New[descriptor.Descriptor]() 137 rootMap := make(map[descriptor.Descriptor]ocispec.Descriptor) 138 addRoot := func(key descriptor.Descriptor, val ocispec.Descriptor) { 139 if _, exists := rootMap[key]; !exists { 140 rootMap[key] = val 141 } 142 } 143 144 // if FindPredecessors is not provided, use the default one 145 if opts.FindPredecessors == nil { 146 opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 147 return src.Predecessors(ctx, desc) 148 } 149 } 150 151 var stack copyutil.Stack 152 // push the initial node to the stack, set the depth to 0 153 stack.Push(copyutil.NodeInfo{Node: node, Depth: 0}) 154 for { 155 current, ok := stack.Pop() 156 if !ok { 157 // empty stack 158 break 159 } 160 currentNode := current.Node 161 currentKey := descriptor.FromOCI(currentNode) 162 163 if visited.Contains(currentKey) { 164 // skip the current node if it has been visited 165 continue 166 } 167 visited.Add(currentKey) 168 169 // stop finding predecessors if the target depth is reached 170 if opts.Depth > 0 && current.Depth == opts.Depth { 171 addRoot(currentKey, currentNode) 172 continue 173 } 174 175 predecessors, err := opts.FindPredecessors(ctx, storage, currentNode) 176 if err != nil { 177 return nil, err 178 } 179 180 // The current node has no predecessor node, 181 // which means it is a root node of a sub-DAG. 182 if len(predecessors) == 0 { 183 addRoot(currentKey, currentNode) 184 continue 185 } 186 187 // The current node has predecessor nodes, which means it is NOT a root node. 188 // Push the predecessor nodes to the stack and keep finding from there. 189 for _, predecessor := range predecessors { 190 predecessorKey := descriptor.FromOCI(predecessor) 191 if !visited.Contains(predecessorKey) { 192 // push the predecessor node with increased depth 193 stack.Push(copyutil.NodeInfo{Node: predecessor, Depth: current.Depth + 1}) 194 } 195 } 196 } 197 198 roots := make([]ocispec.Descriptor, 0, len(rootMap)) 199 for _, root := range rootMap { 200 roots = append(roots, root) 201 } 202 return roots, nil 203 } 204 205 // FilterAnnotation configures opts.FindPredecessors to filter the predecessors 206 // whose annotation matches a given regex pattern. 207 // 208 // A predecessor is kept if key is in its annotations and the annotation value 209 // matches regex. 210 // If regex is nil, predecessors whose annotations contain key will be kept, 211 // no matter of the annotation value. 212 // 213 // For performance consideration, when using both FilterArtifactType and 214 // FilterAnnotation, it's recommended to call FilterArtifactType first. 215 func (opts *ExtendedCopyGraphOptions) FilterAnnotation(key string, regex *regexp.Regexp) { 216 keep := func(desc ocispec.Descriptor) bool { 217 value, ok := desc.Annotations[key] 218 return ok && (regex == nil || regex.MatchString(value)) 219 } 220 221 fp := opts.FindPredecessors 222 opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 223 var predecessors []ocispec.Descriptor 224 var err error 225 if fp == nil { 226 if rf, ok := src.(registry.ReferrerLister); ok { 227 // if src is a ReferrerLister, use Referrers() for possible memory saving 228 if err := rf.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error { 229 // for each page of the results, filter the referrers 230 for _, r := range referrers { 231 if keep(r) { 232 predecessors = append(predecessors, r) 233 } 234 } 235 return nil 236 }); err != nil { 237 return nil, err 238 } 239 return predecessors, nil 240 } 241 predecessors, err = src.Predecessors(ctx, desc) 242 } else { 243 predecessors, err = fp(ctx, src, desc) 244 } 245 if err != nil { 246 return nil, err 247 } 248 249 // Predecessor descriptors that are not from Referrers API are not 250 // guaranteed to include the annotations of the corresponding manifests. 251 var kept []ocispec.Descriptor 252 for _, p := range predecessors { 253 if p.Annotations == nil { 254 // If the annotations are not present in the descriptors, 255 // fetch it from the manifest content. 256 switch p.MediaType { 257 case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest, 258 docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex, 259 spec.MediaTypeArtifactManifest: 260 annotations, err := fetchAnnotations(ctx, src, p) 261 if err != nil { 262 return nil, err 263 } 264 p.Annotations = annotations 265 } 266 } 267 if keep(p) { 268 kept = append(kept, p) 269 } 270 } 271 return kept, nil 272 } 273 } 274 275 // fetchAnnotations fetches the annotations of the manifest described by desc. 276 func fetchAnnotations(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) (map[string]string, error) { 277 rc, err := src.Fetch(ctx, desc) 278 if err != nil { 279 return nil, err 280 } 281 defer rc.Close() 282 283 var manifest struct { 284 Annotations map[string]string `json:"annotations"` 285 } 286 if err := json.NewDecoder(rc).Decode(&manifest); err != nil { 287 return nil, err 288 } 289 if manifest.Annotations == nil { 290 // to differentiate with nil 291 return make(map[string]string), nil 292 } 293 return manifest.Annotations, nil 294 } 295 296 // FilterArtifactType configures opts.FindPredecessors to filter the 297 // predecessors whose artifact type matches a given regex pattern. 298 // 299 // A predecessor is kept if its artifact type matches regex. 300 // If regex is nil, all predecessors will be kept. 301 // 302 // For performance consideration, when using both FilterArtifactType and 303 // FilterAnnotation, it's recommended to call FilterArtifactType first. 304 func (opts *ExtendedCopyGraphOptions) FilterArtifactType(regex *regexp.Regexp) { 305 if regex == nil { 306 return 307 } 308 keep := func(desc ocispec.Descriptor) bool { 309 return regex.MatchString(desc.ArtifactType) 310 } 311 312 fp := opts.FindPredecessors 313 opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 314 var predecessors []ocispec.Descriptor 315 var err error 316 if fp == nil { 317 if rf, ok := src.(registry.ReferrerLister); ok { 318 // if src is a ReferrerLister, use Referrers() for possible memory saving 319 if err := rf.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error { 320 // for each page of the results, filter the referrers 321 for _, r := range referrers { 322 if keep(r) { 323 predecessors = append(predecessors, r) 324 } 325 } 326 return nil 327 }); err != nil { 328 return nil, err 329 } 330 return predecessors, nil 331 } 332 predecessors, err = src.Predecessors(ctx, desc) 333 } else { 334 predecessors, err = fp(ctx, src, desc) 335 } 336 if err != nil { 337 return nil, err 338 } 339 340 // predecessor descriptors that are not from Referrers API are not 341 // guaranteed to include the artifact type of the corresponding 342 // manifests. 343 var kept []ocispec.Descriptor 344 for _, p := range predecessors { 345 if p.ArtifactType == "" { 346 // if the artifact type is not present in the descriptors, 347 // fetch it from the manifest content. 348 switch p.MediaType { 349 case spec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: 350 artifactType, err := fetchArtifactType(ctx, src, p) 351 if err != nil { 352 return nil, err 353 } 354 p.ArtifactType = artifactType 355 } 356 } 357 if keep(p) { 358 kept = append(kept, p) 359 } 360 } 361 return kept, nil 362 } 363 } 364 365 // fetchArtifactType fetches the artifact type of the manifest described by desc. 366 func fetchArtifactType(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) (string, error) { 367 rc, err := src.Fetch(ctx, desc) 368 if err != nil { 369 return "", err 370 } 371 defer rc.Close() 372 373 switch desc.MediaType { 374 case spec.MediaTypeArtifactManifest: 375 var manifest spec.Artifact 376 if err := json.NewDecoder(rc).Decode(&manifest); err != nil { 377 return "", err 378 } 379 return manifest.ArtifactType, nil 380 case ocispec.MediaTypeImageManifest: 381 var manifest ocispec.Manifest 382 if err := json.NewDecoder(rc).Decode(&manifest); err != nil { 383 return "", err 384 } 385 return manifest.Config.MediaType, nil 386 default: 387 return "", nil 388 } 389 }