github.com/opcr-io/oras-go/v2@v2.0.0-20231122155130-eb4260d8a0ae/extendedcopy.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras 17 18 import ( 19 "context" 20 "encoding/json" 21 "errors" 22 "regexp" 23 24 "github.com/opcr-io/oras-go/v2/content" 25 "github.com/opcr-io/oras-go/v2/internal/cas" 26 "github.com/opcr-io/oras-go/v2/internal/container/set" 27 "github.com/opcr-io/oras-go/v2/internal/copyutil" 28 "github.com/opcr-io/oras-go/v2/internal/descriptor" 29 "github.com/opcr-io/oras-go/v2/internal/docker" 30 "github.com/opcr-io/oras-go/v2/internal/status" 31 "github.com/opcr-io/oras-go/v2/internal/syncutil" 32 "github.com/opcr-io/oras-go/v2/registry" 33 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 34 "golang.org/x/sync/semaphore" 35 ) 36 37 // DefaultExtendedCopyOptions provides the default ExtendedCopyOptions. 38 var DefaultExtendedCopyOptions ExtendedCopyOptions = ExtendedCopyOptions{ 39 ExtendedCopyGraphOptions: DefaultExtendedCopyGraphOptions, 40 } 41 42 // ExtendedCopyOptions contains parameters for [oras.ExtendedCopy]. 43 type ExtendedCopyOptions struct { 44 ExtendedCopyGraphOptions 45 } 46 47 // DefaultExtendedCopyGraphOptions provides the default ExtendedCopyGraphOptions. 48 var DefaultExtendedCopyGraphOptions ExtendedCopyGraphOptions = ExtendedCopyGraphOptions{ 49 CopyGraphOptions: DefaultCopyGraphOptions, 50 } 51 52 // ExtendedCopyGraphOptions contains parameters for [oras.ExtendedCopyGraph]. 53 type ExtendedCopyGraphOptions struct { 54 CopyGraphOptions 55 // Depth limits the maximum depth of the directed acyclic graph (DAG) that 56 // will be extended-copied. 57 // If Depth is no specified, or the specified value is less than or 58 // equal to 0, the depth limit will be considered as infinity. 59 Depth int 60 // FindPredecessors finds the predecessors of the current node. 61 // If FindPredecessors is nil, src.Predecessors will be adapted and used. 62 FindPredecessors func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) 63 } 64 65 // ExtendedCopy copies the directed acyclic graph (DAG) that are reachable from 66 // the given tagged node from the source GraphTarget to the destination Target. 67 // The destination reference will be the same as the source reference if the 68 // destination reference is left blank. 69 // 70 // Returns the descriptor of the tagged node on successful copy. 71 func ExtendedCopy(ctx context.Context, src ReadOnlyGraphTarget, srcRef string, dst Target, dstRef string, opts ExtendedCopyOptions) (ocispec.Descriptor, error) { 72 if src == nil { 73 return ocispec.Descriptor{}, errors.New("nil source graph target") 74 } 75 if dst == nil { 76 return ocispec.Descriptor{}, errors.New("nil destination target") 77 } 78 if dstRef == "" { 79 dstRef = srcRef 80 } 81 82 node, err := src.Resolve(ctx, srcRef) 83 if err != nil { 84 return ocispec.Descriptor{}, err 85 } 86 87 if err := ExtendedCopyGraph(ctx, src, dst, node, opts.ExtendedCopyGraphOptions); err != nil { 88 return ocispec.Descriptor{}, err 89 } 90 91 if err := dst.Tag(ctx, node, dstRef); err != nil { 92 return ocispec.Descriptor{}, err 93 } 94 95 return node, nil 96 } 97 98 // ExtendedCopyGraph copies the directed acyclic graph (DAG) that are reachable 99 // from the given node from the source GraphStorage to the destination Storage. 100 func ExtendedCopyGraph(ctx context.Context, src content.ReadOnlyGraphStorage, dst content.Storage, node ocispec.Descriptor, opts ExtendedCopyGraphOptions) error { 101 roots, err := findRoots(ctx, src, node, opts) 102 if err != nil { 103 return err 104 } 105 106 // if Concurrency is not set or invalid, use the default concurrency 107 if opts.Concurrency <= 0 { 108 opts.Concurrency = defaultConcurrency 109 } 110 limiter := semaphore.NewWeighted(int64(opts.Concurrency)) 111 // use caching proxy on non-leaf nodes 112 if opts.MaxMetadataBytes <= 0 { 113 opts.MaxMetadataBytes = defaultCopyMaxMetadataBytes 114 } 115 proxy := cas.NewProxyWithLimit(src, cas.NewMemory(), opts.MaxMetadataBytes) 116 // track content status 117 tracker := status.NewTracker() 118 119 // copy the sub-DAGs rooted by the root nodes 120 return syncutil.Go(ctx, limiter, func(ctx context.Context, region *syncutil.LimitedRegion, root ocispec.Descriptor) error { 121 // As a root can be a predecessor of other roots, release the limit here 122 // for dispatching, to avoid dead locks where predecessor roots are 123 // handled first and are waiting for its successors to complete. 124 region.End() 125 if err := copyGraph(ctx, src, dst, root, proxy, limiter, tracker, opts.CopyGraphOptions); err != nil { 126 return err 127 } 128 return region.Start() 129 }, roots...) 130 } 131 132 // findRoots finds the root nodes reachable from the given node through a 133 // depth-first search. 134 func findRoots(ctx context.Context, storage content.ReadOnlyGraphStorage, node ocispec.Descriptor, opts ExtendedCopyGraphOptions) ([]ocispec.Descriptor, error) { 135 visited := set.New[descriptor.Descriptor]() 136 rootMap := make(map[descriptor.Descriptor]ocispec.Descriptor) 137 addRoot := func(key descriptor.Descriptor, val ocispec.Descriptor) { 138 if _, exists := rootMap[key]; !exists { 139 rootMap[key] = val 140 } 141 } 142 143 // if FindPredecessors is not provided, use the default one 144 if opts.FindPredecessors == nil { 145 opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 146 return src.Predecessors(ctx, desc) 147 } 148 } 149 150 var stack copyutil.Stack 151 // push the initial node to the stack, set the depth to 0 152 stack.Push(copyutil.NodeInfo{Node: node, Depth: 0}) 153 for { 154 current, ok := stack.Pop() 155 if !ok { 156 // empty stack 157 break 158 } 159 currentNode := current.Node 160 currentKey := descriptor.FromOCI(currentNode) 161 162 if visited.Contains(currentKey) { 163 // skip the current node if it has been visited 164 continue 165 } 166 visited.Add(currentKey) 167 168 // stop finding predecessors if the target depth is reached 169 if opts.Depth > 0 && current.Depth == opts.Depth { 170 addRoot(currentKey, currentNode) 171 continue 172 } 173 174 predecessors, err := opts.FindPredecessors(ctx, storage, currentNode) 175 if err != nil { 176 return nil, err 177 } 178 179 // The current node has no predecessor node, 180 // which means it is a root node of a sub-DAG. 181 if len(predecessors) == 0 { 182 addRoot(currentKey, currentNode) 183 continue 184 } 185 186 // The current node has predecessor nodes, which means it is NOT a root node. 187 // Push the predecessor nodes to the stack and keep finding from there. 188 for _, predecessor := range predecessors { 189 predecessorKey := descriptor.FromOCI(predecessor) 190 if !visited.Contains(predecessorKey) { 191 // push the predecessor node with increased depth 192 stack.Push(copyutil.NodeInfo{Node: predecessor, Depth: current.Depth + 1}) 193 } 194 } 195 } 196 197 roots := make([]ocispec.Descriptor, 0, len(rootMap)) 198 for _, root := range rootMap { 199 roots = append(roots, root) 200 } 201 return roots, nil 202 } 203 204 // FilterAnnotation configures opts.FindPredecessors to filter the predecessors 205 // whose annotation matches a given regex pattern. 206 // 207 // A predecessor is kept if key is in its annotations and the annotation value 208 // matches regex. 209 // If regex is nil, predecessors whose annotations contain key will be kept, 210 // no matter of the annotation value. 211 // 212 // For performance consideration, when using both FilterArtifactType and 213 // FilterAnnotation, it's recommended to call FilterArtifactType first. 214 func (opts *ExtendedCopyGraphOptions) FilterAnnotation(key string, regex *regexp.Regexp) { 215 keep := func(desc ocispec.Descriptor) bool { 216 value, ok := desc.Annotations[key] 217 return ok && (regex == nil || regex.MatchString(value)) 218 } 219 220 fp := opts.FindPredecessors 221 opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 222 var predecessors []ocispec.Descriptor 223 var err error 224 if fp == nil { 225 if rf, ok := src.(registry.ReferrerLister); ok { 226 // if src is a ReferrerLister, use Referrers() for possible memory saving 227 if err := rf.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error { 228 // for each page of the results, filter the referrers 229 for _, r := range referrers { 230 if keep(r) { 231 predecessors = append(predecessors, r) 232 } 233 } 234 return nil 235 }); err != nil { 236 return nil, err 237 } 238 return predecessors, nil 239 } 240 predecessors, err = src.Predecessors(ctx, desc) 241 } else { 242 predecessors, err = fp(ctx, src, desc) 243 } 244 if err != nil { 245 return nil, err 246 } 247 248 // Predecessor descriptors that are not from Referrers API are not 249 // guaranteed to include the annotations of the corresponding manifests. 250 var kept []ocispec.Descriptor 251 for _, p := range predecessors { 252 if p.Annotations == nil { 253 // If the annotations are not present in the descriptors, 254 // fetch it from the manifest content. 255 switch p.MediaType { 256 case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest, 257 docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex: 258 annotations, err := fetchAnnotations(ctx, src, p) 259 if err != nil { 260 return nil, err 261 } 262 p.Annotations = annotations 263 } 264 } 265 if keep(p) { 266 kept = append(kept, p) 267 } 268 } 269 return kept, nil 270 } 271 } 272 273 // fetchAnnotations fetches the annotations of the manifest described by desc. 274 func fetchAnnotations(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) (map[string]string, error) { 275 rc, err := src.Fetch(ctx, desc) 276 if err != nil { 277 return nil, err 278 } 279 defer rc.Close() 280 281 var manifest struct { 282 Annotations map[string]string `json:"annotations"` 283 } 284 if err := json.NewDecoder(rc).Decode(&manifest); err != nil { 285 return nil, err 286 } 287 if manifest.Annotations == nil { 288 // to differentiate with nil 289 return make(map[string]string), nil 290 } 291 return manifest.Annotations, nil 292 } 293 294 // FilterArtifactType configures opts.FindPredecessors to filter the 295 // predecessors whose artifact type matches a given regex pattern. 296 // 297 // A predecessor is kept if its artifact type matches regex. 298 // If regex is nil, all predecessors will be kept. 299 // 300 // For performance consideration, when using both FilterArtifactType and 301 // FilterAnnotation, it's recommended to call FilterArtifactType first. 302 func (opts *ExtendedCopyGraphOptions) FilterArtifactType(regex *regexp.Regexp) { 303 if regex == nil { 304 return 305 } 306 keep := func(desc ocispec.Descriptor) bool { 307 return regex.MatchString(desc.ArtifactType) 308 } 309 310 fp := opts.FindPredecessors 311 opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 312 var predecessors []ocispec.Descriptor 313 var err error 314 if fp == nil { 315 if rf, ok := src.(registry.ReferrerLister); ok { 316 // if src is a ReferrerLister, use Referrers() for possible memory saving 317 if err := rf.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error { 318 // for each page of the results, filter the referrers 319 for _, r := range referrers { 320 if keep(r) { 321 predecessors = append(predecessors, r) 322 } 323 } 324 return nil 325 }); err != nil { 326 return nil, err 327 } 328 return predecessors, nil 329 } 330 predecessors, err = src.Predecessors(ctx, desc) 331 } else { 332 predecessors, err = fp(ctx, src, desc) 333 } 334 if err != nil { 335 return nil, err 336 } 337 338 // predecessor descriptors that are not from Referrers API are not 339 // guaranteed to include the artifact type of the corresponding 340 // manifests. 341 var kept []ocispec.Descriptor 342 for _, p := range predecessors { 343 if p.ArtifactType == "" { 344 // if the artifact type is not present in the descriptors, 345 // fetch it from the manifest content. 346 switch p.MediaType { 347 case ocispec.MediaTypeImageManifest: 348 artifactType, err := fetchArtifactType(ctx, src, p) 349 if err != nil { 350 return nil, err 351 } 352 p.ArtifactType = artifactType 353 } 354 } 355 if keep(p) { 356 kept = append(kept, p) 357 } 358 } 359 return kept, nil 360 } 361 } 362 363 // fetchArtifactType fetches the artifact type of the manifest described by desc. 364 func fetchArtifactType(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) (string, error) { 365 rc, err := src.Fetch(ctx, desc) 366 if err != nil { 367 return "", err 368 } 369 defer rc.Close() 370 371 switch desc.MediaType { 372 case ocispec.MediaTypeImageManifest: 373 var manifest ocispec.Manifest 374 if err := json.NewDecoder(rc).Decode(&manifest); err != nil { 375 return "", err 376 } 377 return manifest.Config.MediaType, nil 378 default: 379 return "", nil 380 } 381 }