oras.land/oras-go/v2@v2.5.1-0.20240520045656-aef90e4d04c4/copy_test.go (about) 1 /* 2 Copyright The ORAS Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package oras_test 17 18 import ( 19 "bytes" 20 "context" 21 _ "crypto/sha256" 22 "encoding/json" 23 "errors" 24 "fmt" 25 "io" 26 "reflect" 27 "sync/atomic" 28 "testing" 29 30 "github.com/opencontainers/go-digest" 31 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 32 "oras.land/oras-go/v2" 33 "oras.land/oras-go/v2/content" 34 "oras.land/oras-go/v2/content/file" 35 "oras.land/oras-go/v2/content/memory" 36 "oras.land/oras-go/v2/errdef" 37 "oras.land/oras-go/v2/internal/cas" 38 "oras.land/oras-go/v2/internal/docker" 39 "oras.land/oras-go/v2/internal/spec" 40 ) 41 42 // storageTracker tracks storage API counts. 43 type storageTracker struct { 44 content.Storage 45 fetch int64 46 push int64 47 exists int64 48 } 49 50 func (t *storageTracker) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { 51 atomic.AddInt64(&t.fetch, 1) 52 return t.Storage.Fetch(ctx, target) 53 } 54 55 func (t *storageTracker) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error { 56 atomic.AddInt64(&t.push, 1) 57 return t.Storage.Push(ctx, expected, content) 58 } 59 60 func (t *storageTracker) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { 61 atomic.AddInt64(&t.exists, 1) 62 return t.Storage.Exists(ctx, target) 63 } 64 65 func TestCopy_FullCopy(t *testing.T) { 66 src := memory.New() 67 dst := memory.New() 68 69 // generate test content 70 var blobs [][]byte 71 var descs []ocispec.Descriptor 72 appendBlob := func(mediaType string, blob []byte) { 73 blobs = append(blobs, blob) 74 descs = append(descs, ocispec.Descriptor{ 75 MediaType: mediaType, 76 Digest: digest.FromBytes(blob), 77 Size: int64(len(blob)), 78 }) 79 } 80 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 81 manifest := ocispec.Manifest{ 82 Config: config, 83 Layers: layers, 84 } 85 manifestJSON, err := json.Marshal(manifest) 86 if err != nil { 87 t.Fatal(err) 88 } 89 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 90 } 91 92 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 93 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 94 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 95 generateManifest(descs[0], descs[1:3]...) // Blob 3 96 97 ctx := context.Background() 98 for i := range blobs { 99 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 100 if err != nil { 101 t.Fatalf("failed to push test content to src: %d: %v", i, err) 102 } 103 } 104 105 root := descs[3] 106 ref := "foobar" 107 err := src.Tag(ctx, root, ref) 108 if err != nil { 109 t.Fatal("fail to tag root node", err) 110 } 111 112 // test copy 113 gotDesc, err := oras.Copy(ctx, src, ref, dst, "", oras.CopyOptions{}) 114 if err != nil { 115 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 116 } 117 if !reflect.DeepEqual(gotDesc, root) { 118 t.Errorf("Copy() = %v, want %v", gotDesc, root) 119 } 120 121 // verify contents 122 for i, desc := range descs { 123 exists, err := dst.Exists(ctx, desc) 124 if err != nil { 125 t.Fatalf("dst.Exists(%d) error = %v", i, err) 126 } 127 if !exists { 128 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 129 } 130 } 131 132 // verify tag 133 gotDesc, err = dst.Resolve(ctx, ref) 134 if err != nil { 135 t.Fatal("dst.Resolve() error =", err) 136 } 137 if !reflect.DeepEqual(gotDesc, root) { 138 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, root) 139 } 140 } 141 142 func TestCopy_ExistedRoot(t *testing.T) { 143 src := memory.New() 144 dst := memory.New() 145 146 // generate test content 147 var blobs [][]byte 148 var descs []ocispec.Descriptor 149 appendBlob := func(mediaType string, blob []byte) { 150 blobs = append(blobs, blob) 151 descs = append(descs, ocispec.Descriptor{ 152 MediaType: mediaType, 153 Digest: digest.FromBytes(blob), 154 Size: int64(len(blob)), 155 }) 156 } 157 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 158 manifest := ocispec.Manifest{ 159 Config: config, 160 Layers: layers, 161 } 162 manifestJSON, err := json.Marshal(manifest) 163 if err != nil { 164 t.Fatal(err) 165 } 166 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 167 } 168 169 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 170 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 171 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 172 generateManifest(descs[0], descs[1:3]...) // Blob 3 173 174 ctx := context.Background() 175 for i := range blobs { 176 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 177 if err != nil { 178 t.Fatalf("failed to push test content to src: %d: %v", i, err) 179 } 180 } 181 182 root := descs[3] 183 ref := "foobar" 184 newTag := "newtag" 185 err := src.Tag(ctx, root, ref) 186 if err != nil { 187 t.Fatal("fail to tag root node", err) 188 } 189 190 var skippedCount int64 191 copyOpts := oras.CopyOptions{ 192 CopyGraphOptions: oras.CopyGraphOptions{ 193 OnCopySkipped: func(ctx context.Context, desc ocispec.Descriptor) error { 194 atomic.AddInt64(&skippedCount, 1) 195 return nil 196 }, 197 }, 198 } 199 200 // copy with src tag 201 gotDesc, err := oras.Copy(ctx, src, ref, dst, "", copyOpts) 202 if err != nil { 203 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 204 } 205 if !reflect.DeepEqual(gotDesc, root) { 206 t.Errorf("Copy() = %v, want %v", gotDesc, root) 207 } 208 // copy with new tag 209 gotDesc, err = oras.Copy(ctx, src, ref, dst, newTag, copyOpts) 210 if err != nil { 211 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 212 } 213 if !reflect.DeepEqual(gotDesc, root) { 214 t.Errorf("Copy() = %v, want %v", gotDesc, root) 215 } 216 217 // verify contents 218 for i, desc := range descs { 219 exists, err := dst.Exists(ctx, desc) 220 if err != nil { 221 t.Fatalf("dst.Exists(%d) error = %v", i, err) 222 } 223 if !exists { 224 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 225 } 226 } 227 228 // verify src tag 229 gotDesc, err = dst.Resolve(ctx, ref) 230 if err != nil { 231 t.Fatal("dst.Resolve() error =", err) 232 } 233 if !reflect.DeepEqual(gotDesc, root) { 234 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, root) 235 } 236 // verify new tag 237 gotDesc, err = dst.Resolve(ctx, newTag) 238 if err != nil { 239 t.Fatal("dst.Resolve() error =", err) 240 } 241 if !reflect.DeepEqual(gotDesc, root) { 242 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, root) 243 } 244 // verify invocation of onCopySkipped() 245 if got, want := skippedCount, int64(1); got != want { 246 t.Errorf("count(OnCopySkipped()) = %v, want %v", got, want) 247 } 248 } 249 250 func TestCopyGraph_FullCopy(t *testing.T) { 251 src := cas.NewMemory() 252 dst := cas.NewMemory() 253 254 // generate test content 255 var blobs [][]byte 256 var descs []ocispec.Descriptor 257 appendBlob := func(mediaType string, blob []byte) { 258 blobs = append(blobs, blob) 259 descs = append(descs, ocispec.Descriptor{ 260 MediaType: mediaType, 261 Digest: digest.FromBytes(blob), 262 Size: int64(len(blob)), 263 }) 264 } 265 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 266 manifest := ocispec.Manifest{ 267 Config: config, 268 Layers: layers, 269 } 270 manifestJSON, err := json.Marshal(manifest) 271 if err != nil { 272 t.Fatal(err) 273 } 274 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 275 } 276 generateIndex := func(manifests ...ocispec.Descriptor) { 277 index := ocispec.Index{ 278 Manifests: manifests, 279 } 280 indexJSON, err := json.Marshal(index) 281 if err != nil { 282 t.Fatal(err) 283 } 284 appendBlob(ocispec.MediaTypeImageIndex, indexJSON) 285 } 286 287 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 288 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 289 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 290 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 291 generateManifest(descs[0], descs[1:3]...) // Blob 4 292 generateManifest(descs[0], descs[3]) // Blob 5 293 generateManifest(descs[0], descs[1:4]...) // Blob 6 294 generateIndex(descs[4:6]...) // Blob 7 295 generateIndex(descs[6]) // Blob 8 296 generateIndex() // Blob 9 297 generateIndex(descs[7:10]...) // Blob 10 298 299 ctx := context.Background() 300 for i := range blobs { 301 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 302 if err != nil { 303 t.Fatalf("failed to push test content to src: %d: %v", i, err) 304 } 305 } 306 307 // test copy graph 308 srcTracker := &storageTracker{Storage: src} 309 dstTracker := &storageTracker{Storage: dst} 310 root := descs[len(descs)-1] 311 if err := oras.CopyGraph(ctx, srcTracker, dstTracker, root, oras.CopyGraphOptions{}); err != nil { 312 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 313 } 314 315 // verify contents 316 contents := dst.Map() 317 if got, want := len(contents), len(blobs); got != want { 318 t.Errorf("len(dst) = %v, wantErr %v", got, want) 319 } 320 for i := range blobs { 321 got, err := content.FetchAll(ctx, dst, descs[i]) 322 if err != nil { 323 t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) 324 continue 325 } 326 if want := blobs[i]; !bytes.Equal(got, want) { 327 t.Errorf("content[%d] = %v, want %v", i, got, want) 328 } 329 } 330 331 // verify API counts 332 if got, want := srcTracker.fetch, int64(len(blobs)); got != want { 333 t.Errorf("count(src.Fetch()) = %v, want %v", got, want) 334 } 335 if got, want := srcTracker.push, int64(0); got != want { 336 t.Errorf("count(src.Push()) = %v, want %v", got, want) 337 } 338 if got, want := srcTracker.exists, int64(0); got != want { 339 t.Errorf("count(src.Exists()) = %v, want %v", got, want) 340 } 341 if got, want := dstTracker.fetch, int64(0); got != want { 342 t.Errorf("count(dst.Fetch()) = %v, want %v", got, want) 343 } 344 if got, want := dstTracker.push, int64(len(blobs)); got != want { 345 t.Errorf("count(dst.Push()) = %v, want %v", got, want) 346 } 347 if got, want := dstTracker.exists, int64(len(blobs)); got != want { 348 t.Errorf("count(dst.Exists()) = %v, want %v", got, want) 349 } 350 } 351 352 func TestCopyGraph_PartialCopy(t *testing.T) { 353 src := cas.NewMemory() 354 dst := cas.NewMemory() 355 356 // generate test content 357 var blobs [][]byte 358 var descs []ocispec.Descriptor 359 appendBlob := func(mediaType string, blob []byte) { 360 blobs = append(blobs, blob) 361 descs = append(descs, ocispec.Descriptor{ 362 MediaType: mediaType, 363 Digest: digest.FromBytes(blob), 364 Size: int64(len(blob)), 365 }) 366 } 367 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 368 manifest := ocispec.Manifest{ 369 Config: config, 370 Layers: layers, 371 } 372 manifestJSON, err := json.Marshal(manifest) 373 if err != nil { 374 t.Fatal(err) 375 } 376 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 377 } 378 generateIndex := func(manifests ...ocispec.Descriptor) { 379 index := ocispec.Index{ 380 Manifests: manifests, 381 } 382 indexJSON, err := json.Marshal(index) 383 if err != nil { 384 t.Fatal(err) 385 } 386 appendBlob(ocispec.MediaTypeImageIndex, indexJSON) 387 } 388 389 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 390 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 391 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 392 generateManifest(descs[0], descs[1:3]...) // Blob 3 393 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 4 394 generateManifest(descs[0], descs[4]) // Blob 5 395 generateIndex(descs[3], descs[5]) // Blob 6 396 397 ctx := context.Background() 398 for i := range blobs { 399 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 400 if err != nil { 401 t.Fatalf("failed to push test content to src: %d: %v", i, err) 402 } 403 } 404 405 // initial copy 406 root := descs[3] 407 if err := oras.CopyGraph(ctx, src, dst, root, oras.CopyGraphOptions{}); err != nil { 408 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 409 } 410 // verify contents 411 contents := dst.Map() 412 if got, want := len(contents), len(blobs[:4]); got != want { 413 t.Fatalf("len(dst) = %v, wantErr %v", got, want) 414 } 415 for i := range blobs[:4] { 416 got, err := content.FetchAll(ctx, dst, descs[i]) 417 if err != nil { 418 t.Fatalf("content[%d] error = %v, wantErr %v", i, err, false) 419 } 420 if want := blobs[i]; !bytes.Equal(got, want) { 421 t.Fatalf("content[%d] = %v, want %v", i, got, want) 422 } 423 } 424 425 // test copy 426 srcTracker := &storageTracker{Storage: src} 427 dstTracker := &storageTracker{Storage: dst} 428 root = descs[len(descs)-1] 429 if err := oras.CopyGraph(ctx, srcTracker, dstTracker, root, oras.CopyGraphOptions{}); err != nil { 430 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 431 } 432 433 // verify contents 434 contents = dst.Map() 435 if got, want := len(contents), len(blobs); got != want { 436 t.Errorf("len(dst) = %v, wantErr %v", got, want) 437 } 438 for i := range blobs { 439 got, err := content.FetchAll(ctx, dst, descs[i]) 440 if err != nil { 441 t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) 442 continue 443 } 444 if want := blobs[i]; !bytes.Equal(got, want) { 445 t.Errorf("content[%d] = %v, want %v", i, got, want) 446 } 447 } 448 449 // verify API counts 450 if got, want := srcTracker.fetch, int64(3); got != want { 451 t.Errorf("count(src.Fetch()) = %v, want %v", got, want) 452 } 453 if got, want := srcTracker.push, int64(0); got != want { 454 t.Errorf("count(src.Push()) = %v, want %v", got, want) 455 } 456 if got, want := srcTracker.exists, int64(0); got != want { 457 t.Errorf("count(src.Exists()) = %v, want %v", got, want) 458 } 459 if got, want := dstTracker.fetch, int64(0); got != want { 460 t.Errorf("count(dst.Fetch()) = %v, want %v", got, want) 461 } 462 if got, want := dstTracker.push, int64(3); got != want { 463 t.Errorf("count(dst.Push()) = %v, want %v", got, want) 464 } 465 if got, want := dstTracker.exists, int64(5); got != want { 466 t.Errorf("count(dst.Exists()) = %v, want %v", got, want) 467 } 468 } 469 470 func TestCopy_WithOptions(t *testing.T) { 471 src := memory.New() 472 473 // generate test content 474 var blobs [][]byte 475 var descs []ocispec.Descriptor 476 appendBlob := func(mediaType string, blob []byte) { 477 blobs = append(blobs, blob) 478 descs = append(descs, ocispec.Descriptor{ 479 MediaType: mediaType, 480 Digest: digest.FromBytes(blob), 481 Size: int64(len(blob)), 482 }) 483 } 484 appendManifest := func(arc, os string, mediaType string, blob []byte) { 485 blobs = append(blobs, blob) 486 descs = append(descs, ocispec.Descriptor{ 487 MediaType: mediaType, 488 Digest: digest.FromBytes(blob), 489 Size: int64(len(blob)), 490 Platform: &ocispec.Platform{ 491 Architecture: arc, 492 OS: os, 493 }, 494 }) 495 } 496 generateManifest := func(arc, os string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { 497 manifest := ocispec.Manifest{ 498 Config: config, 499 Layers: layers, 500 } 501 manifestJSON, err := json.Marshal(manifest) 502 if err != nil { 503 t.Fatal(err) 504 } 505 appendManifest(arc, os, ocispec.MediaTypeImageManifest, manifestJSON) 506 } 507 generateIndex := func(manifests ...ocispec.Descriptor) { 508 index := ocispec.Index{ 509 Manifests: manifests, 510 } 511 indexJSON, err := json.Marshal(index) 512 if err != nil { 513 t.Fatal(err) 514 } 515 appendBlob(ocispec.MediaTypeImageIndex, indexJSON) 516 } 517 518 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 519 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 520 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 521 generateManifest("test-arc-1", "test-os-1", descs[0], descs[1:3]...) // Blob 3 522 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 4 523 generateManifest("test-arc-2", "test-os-2", descs[0], descs[4]) // Blob 5 524 generateIndex(descs[3], descs[5]) // Blob 6 525 526 ctx := context.Background() 527 for i := range blobs { 528 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 529 if err != nil { 530 t.Fatalf("failed to push test content to src: %d: %v", i, err) 531 } 532 } 533 534 root := descs[6] 535 ref := "foobar" 536 err := src.Tag(ctx, root, ref) 537 if err != nil { 538 t.Fatal("fail to tag root node", err) 539 } 540 541 // test copy with media type filter 542 dst := memory.New() 543 opts := oras.DefaultCopyOptions 544 opts.MapRoot = func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) { 545 if root.MediaType == ocispec.MediaTypeImageIndex { 546 return root, nil 547 } else { 548 return ocispec.Descriptor{}, errdef.ErrNotFound 549 } 550 } 551 gotDesc, err := oras.Copy(ctx, src, ref, dst, "", opts) 552 if err != nil { 553 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 554 } 555 if !reflect.DeepEqual(gotDesc, root) { 556 t.Errorf("Copy() = %v, want %v", gotDesc, root) 557 } 558 559 // verify contents 560 for i, desc := range descs { 561 exists, err := dst.Exists(ctx, desc) 562 if err != nil { 563 t.Fatalf("dst.Exists(%d) error = %v", i, err) 564 } 565 if !exists { 566 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 567 } 568 } 569 570 // verify tag 571 gotDesc, err = dst.Resolve(ctx, ref) 572 if err != nil { 573 t.Fatal("dst.Resolve() error =", err) 574 } 575 if !reflect.DeepEqual(gotDesc, root) { 576 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, root) 577 } 578 579 // test copy with platform filter and hooks 580 dst = memory.New() 581 preCopyCount := int64(0) 582 postCopyCount := int64(0) 583 opts = oras.CopyOptions{ 584 MapRoot: func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) { 585 manifests, err := content.Successors(ctx, src, root) 586 if err != nil { 587 return ocispec.Descriptor{}, errdef.ErrNotFound 588 } 589 590 // platform filter 591 for _, m := range manifests { 592 if m.Platform.Architecture == "test-arc-2" && m.Platform.OS == "test-os-2" { 593 return m, nil 594 } 595 } 596 return ocispec.Descriptor{}, errdef.ErrNotFound 597 }, 598 CopyGraphOptions: oras.CopyGraphOptions{ 599 PreCopy: func(ctx context.Context, desc ocispec.Descriptor) error { 600 atomic.AddInt64(&preCopyCount, 1) 601 return nil 602 }, 603 PostCopy: func(ctx context.Context, desc ocispec.Descriptor) error { 604 atomic.AddInt64(&postCopyCount, 1) 605 return nil 606 }, 607 }, 608 } 609 wantDesc := descs[5] 610 gotDesc, err = oras.Copy(ctx, src, ref, dst, "", opts) 611 if err != nil { 612 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 613 } 614 if !reflect.DeepEqual(gotDesc, wantDesc) { 615 t.Errorf("Copy() = %v, want %v", gotDesc, wantDesc) 616 } 617 618 // verify contents 619 for i, desc := range append([]ocispec.Descriptor{descs[0]}, descs[4:6]...) { 620 exists, err := dst.Exists(ctx, desc) 621 if err != nil { 622 t.Fatalf("dst.Exists(%d) error = %v", i, err) 623 } 624 if !exists { 625 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 626 } 627 } 628 629 // verify tag 630 gotDesc, err = dst.Resolve(ctx, ref) 631 if err != nil { 632 t.Fatal("dst.Resolve() error =", err) 633 } 634 if !reflect.DeepEqual(gotDesc, wantDesc) { 635 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, wantDesc) 636 } 637 638 // verify API counts 639 if got, want := preCopyCount, int64(3); got != want { 640 t.Errorf("count(PreCopy()) = %v, want %v", got, want) 641 } 642 if got, want := postCopyCount, int64(3); got != want { 643 t.Errorf("count(PostCopy()) = %v, want %v", got, want) 644 } 645 646 // test copy with root filter, but no matching node can be found 647 dst = memory.New() 648 opts = oras.CopyOptions{ 649 MapRoot: func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) { 650 if root.MediaType == "test" { 651 return root, nil 652 } else { 653 return ocispec.Descriptor{}, errdef.ErrNotFound 654 } 655 }, 656 CopyGraphOptions: oras.DefaultCopyGraphOptions, 657 } 658 659 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 660 if !errors.Is(err, errdef.ErrNotFound) { 661 t.Fatalf("Copy() error = %v, wantErr %v", err, errdef.ErrNotFound) 662 } 663 664 // test copy with MaxMetadataBytes = 1 665 dst = memory.New() 666 opts = oras.CopyOptions{ 667 CopyGraphOptions: oras.CopyGraphOptions{ 668 MaxMetadataBytes: 1, 669 }, 670 } 671 if _, err := oras.Copy(ctx, src, ref, dst, "", opts); !errors.Is(err, errdef.ErrSizeExceedsLimit) { 672 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, errdef.ErrSizeExceedsLimit) 673 } 674 } 675 676 func TestCopy_WithTargetPlatformOptions(t *testing.T) { 677 src := memory.New() 678 arc_1 := "test-arc-1" 679 os_1 := "test-os-1" 680 variant_1 := "v1" 681 arc_2 := "test-arc-2" 682 os_2 := "test-os-2" 683 variant_2 := "v2" 684 685 // generate test content 686 var blobs [][]byte 687 var descs []ocispec.Descriptor 688 appendBlob := func(mediaType string, blob []byte) { 689 blobs = append(blobs, blob) 690 descs = append(descs, ocispec.Descriptor{ 691 MediaType: mediaType, 692 Digest: digest.FromBytes(blob), 693 Size: int64(len(blob)), 694 }) 695 } 696 appendManifest := func(arc, os, variant string, mediaType string, blob []byte) { 697 blobs = append(blobs, blob) 698 descs = append(descs, ocispec.Descriptor{ 699 MediaType: mediaType, 700 Digest: digest.FromBytes(blob), 701 Size: int64(len(blob)), 702 Platform: &ocispec.Platform{ 703 Architecture: arc, 704 OS: os, 705 Variant: variant, 706 }, 707 }) 708 } 709 generateManifest := func(arc, os, variant string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { 710 manifest := ocispec.Manifest{ 711 Config: config, 712 Layers: layers, 713 } 714 manifestJSON, err := json.Marshal(manifest) 715 if err != nil { 716 t.Fatal(err) 717 } 718 appendManifest(arc, os, variant, ocispec.MediaTypeImageManifest, manifestJSON) 719 } 720 generateIndex := func(manifests ...ocispec.Descriptor) { 721 index := ocispec.Index{ 722 Manifests: manifests, 723 } 724 indexJSON, err := json.Marshal(index) 725 if err != nil { 726 t.Fatal(err) 727 } 728 appendBlob(ocispec.MediaTypeImageIndex, indexJSON) 729 } 730 731 appendBlob(ocispec.MediaTypeImageConfig, []byte(`{"mediaType":"application/vnd.oci.image.config.v1+json", 732 "created":"2022-07-29T08:13:55Z", 733 "author":"test author", 734 "architecture":"test-arc-1", 735 "os":"test-os-1", 736 "variant":"v1"}`)) // Blob 0 737 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 738 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 739 generateManifest(arc_1, os_1, variant_1, descs[0], descs[1:3]...) // Blob 3 740 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello1")) // Blob 4 741 generateManifest(arc_2, os_2, variant_1, descs[0], descs[4]) // Blob 5 742 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello2")) // Blob 6 743 generateManifest(arc_1, os_1, variant_2, descs[0], descs[6]) // Blob 7 744 generateIndex(descs[3], descs[5], descs[7]) // Blob 8 745 746 ctx := context.Background() 747 for i := range blobs { 748 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 749 if err != nil { 750 t.Fatalf("failed to push test content to src: %d: %v", i, err) 751 } 752 } 753 754 root := descs[8] 755 ref := "foobar" 756 err := src.Tag(ctx, root, ref) 757 if err != nil { 758 t.Fatal("fail to tag root node", err) 759 } 760 761 // test copy with platform filter for the image index 762 dst := memory.New() 763 opts := oras.CopyOptions{} 764 targetPlatform := ocispec.Platform{ 765 Architecture: arc_2, 766 OS: os_2, 767 } 768 opts.WithTargetPlatform(&targetPlatform) 769 wantDesc := descs[5] 770 gotDesc, err := oras.Copy(ctx, src, ref, dst, "", opts) 771 if err != nil { 772 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 773 } 774 if !reflect.DeepEqual(gotDesc, wantDesc) { 775 t.Errorf("Copy() = %v, want %v", gotDesc, wantDesc) 776 } 777 778 // verify contents 779 for i, desc := range append([]ocispec.Descriptor{descs[0]}, descs[4:6]...) { 780 exists, err := dst.Exists(ctx, desc) 781 if err != nil { 782 t.Fatalf("dst.Exists(%d) error = %v", i, err) 783 } 784 if !exists { 785 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 786 } 787 } 788 789 // verify tag 790 gotDesc, err = dst.Resolve(ctx, ref) 791 if err != nil { 792 t.Fatal("dst.Resolve() error =", err) 793 } 794 if !reflect.DeepEqual(gotDesc, wantDesc) { 795 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, wantDesc) 796 } 797 798 // test copy with platform filter for the image index, and multiple 799 // manifests match the required platform. Should return the first matching 800 // entry. 801 dst = memory.New() 802 targetPlatform = ocispec.Platform{ 803 Architecture: arc_1, 804 OS: os_1, 805 } 806 opts = oras.CopyOptions{} 807 opts.WithTargetPlatform(&targetPlatform) 808 wantDesc = descs[3] 809 gotDesc, err = oras.Copy(ctx, src, ref, dst, "", opts) 810 if err != nil { 811 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 812 } 813 if !reflect.DeepEqual(gotDesc, wantDesc) { 814 t.Errorf("Copy() = %v, want %v", gotDesc, wantDesc) 815 } 816 817 // verify contents 818 for i, desc := range append([]ocispec.Descriptor{descs[0]}, descs[1:3]...) { 819 exists, err := dst.Exists(ctx, desc) 820 if err != nil { 821 t.Fatalf("dst.Exists(%d) error = %v", i, err) 822 } 823 if !exists { 824 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 825 } 826 } 827 828 // verify tag 829 gotDesc, err = dst.Resolve(ctx, ref) 830 if err != nil { 831 t.Fatal("dst.Resolve() error =", err) 832 } 833 if !reflect.DeepEqual(gotDesc, wantDesc) { 834 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, wantDesc) 835 } 836 837 // test copy with platform filter and existing MapRoot func for the image 838 // index, but there is no matching node. Should return not found error. 839 dst = memory.New() 840 opts = oras.CopyOptions{ 841 MapRoot: func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) { 842 if root.MediaType == ocispec.MediaTypeImageIndex { 843 return root, nil 844 } else { 845 return ocispec.Descriptor{}, errdef.ErrNotFound 846 } 847 }, 848 } 849 targetPlatform = ocispec.Platform{ 850 Architecture: arc_1, 851 OS: os_2, 852 } 853 opts.WithTargetPlatform(&targetPlatform) 854 855 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 856 expected := fmt.Sprintf("%s: %v: no matching manifest was found in the manifest list", root.Digest, errdef.ErrNotFound) 857 if err.Error() != expected { 858 t.Fatalf("Copy() error = %v, wantErr %v", err, expected) 859 } 860 861 // test copy with platform filter for the manifest 862 dst = memory.New() 863 opts = oras.CopyOptions{} 864 targetPlatform = ocispec.Platform{ 865 Architecture: arc_1, 866 OS: os_1, 867 } 868 opts.WithTargetPlatform(&targetPlatform) 869 870 root = descs[7] 871 err = src.Tag(ctx, root, ref) 872 if err != nil { 873 t.Fatal("fail to tag root node", err) 874 } 875 876 wantDesc = descs[7] 877 gotDesc, err = oras.Copy(ctx, src, ref, dst, "", opts) 878 if err != nil { 879 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 880 } 881 if !reflect.DeepEqual(gotDesc, wantDesc) { 882 t.Errorf("Copy() = %v, want %v", gotDesc, wantDesc) 883 } 884 885 // verify contents 886 for i, desc := range append([]ocispec.Descriptor{descs[0]}, descs[6]) { 887 exists, err := dst.Exists(ctx, desc) 888 if err != nil { 889 t.Fatalf("dst.Exists(%d) error = %v", i, err) 890 } 891 if !exists { 892 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 893 } 894 } 895 896 // verify tag 897 gotDesc, err = dst.Resolve(ctx, ref) 898 if err != nil { 899 t.Fatal("dst.Resolve() error =", err) 900 } 901 if !reflect.DeepEqual(gotDesc, wantDesc) { 902 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, wantDesc) 903 } 904 905 // test copy with platform filter for the manifest, but there is no matching 906 // node. Should return not found error. 907 dst = memory.New() 908 opts = oras.CopyOptions{} 909 targetPlatform = ocispec.Platform{ 910 Architecture: arc_1, 911 OS: os_1, 912 Variant: variant_2, 913 } 914 opts.WithTargetPlatform(&targetPlatform) 915 916 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 917 expected = fmt.Sprintf("%s: %v: platform in manifest does not match target platform", root.Digest, errdef.ErrNotFound) 918 if err.Error() != expected { 919 t.Fatalf("Copy() error = %v, wantErr %v", err, expected) 920 } 921 922 // test copy with platform filter, but the node's media type is not 923 // supported. Should return unsupported error 924 dst = memory.New() 925 opts = oras.CopyOptions{} 926 targetPlatform = ocispec.Platform{ 927 Architecture: arc_1, 928 OS: os_1, 929 } 930 opts.WithTargetPlatform(&targetPlatform) 931 932 root = descs[1] 933 err = src.Tag(ctx, root, ref) 934 if err != nil { 935 t.Fatal("fail to tag root node", err) 936 } 937 938 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 939 if !errors.Is(err, errdef.ErrUnsupported) { 940 t.Fatalf("Copy() error = %v, wantErr %v", err, errdef.ErrUnsupported) 941 } 942 943 // generate incorrect test content 944 blobs = nil 945 descs = nil 946 appendBlob(docker.MediaTypeConfig, []byte(`{"mediaType":"application/vnd.oci.image.config.v1+json", 947 "created":"2022-07-29T08:13:55Z", 948 "author":"test author 1", 949 "architecture":"test-arc-1", 950 "os":"test-os-1", 951 "variant":"v1"}`)) // Blob 0 952 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo1")) // Blob 1 953 generateManifest(arc_1, os_1, variant_1, descs[0], descs[1]) // Blob 2 954 generateIndex(descs[2]) // Blob 3 955 956 ctx = context.Background() 957 for i := range blobs { 958 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 959 if err != nil { 960 t.Fatalf("failed to push test content to src: %d: %v", i, err) 961 } 962 } 963 964 dst = memory.New() 965 opts = oras.CopyOptions{} 966 targetPlatform = ocispec.Platform{ 967 Architecture: arc_1, 968 OS: os_1, 969 } 970 opts.WithTargetPlatform(&targetPlatform) 971 972 // test copy with platform filter for the manifest, but the manifest is 973 // invalid by having docker mediaType config in the manifest and oci 974 // mediaType in the image config. Should return error. 975 root = descs[2] 976 err = src.Tag(ctx, root, ref) 977 if err != nil { 978 t.Fatal("fail to tag root node", err) 979 } 980 981 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 982 expected = fmt.Sprintf("fail to recognize platform from unknown config %s: expect %s", docker.MediaTypeConfig, ocispec.MediaTypeImageConfig) 983 if err.Error() != expected { 984 t.Fatalf("Copy() error = %v, wantErr %v", err, expected) 985 } 986 987 // generate test content with null config blob 988 blobs = nil 989 descs = nil 990 appendBlob(ocispec.MediaTypeImageConfig, []byte("null")) // Blob 0 991 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo2")) // Blob 1 992 generateManifest(arc_1, os_1, variant_1, descs[0], descs[1]) // Blob 2 993 generateIndex(descs[2]) // Blob 3 994 995 ctx = context.Background() 996 for i := range blobs { 997 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 998 if err != nil { 999 t.Fatalf("failed to push test content to src: %d: %v", i, err) 1000 } 1001 } 1002 1003 dst = memory.New() 1004 opts = oras.CopyOptions{} 1005 targetPlatform = ocispec.Platform{ 1006 Architecture: arc_1, 1007 OS: os_1, 1008 } 1009 opts.WithTargetPlatform(&targetPlatform) 1010 1011 // test copy with platform filter for the manifest with null config blob 1012 // should return not found error 1013 root = descs[2] 1014 err = src.Tag(ctx, root, ref) 1015 if err != nil { 1016 t.Fatal("fail to tag root node", err) 1017 } 1018 1019 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 1020 expected = fmt.Sprintf("%s: %v: platform in manifest does not match target platform", root.Digest, errdef.ErrNotFound) 1021 if err.Error() != expected { 1022 t.Fatalf("Copy() error = %v, wantErr %v", err, expected) 1023 } 1024 1025 // generate test content with empty config blob 1026 blobs = nil 1027 descs = nil 1028 appendBlob(ocispec.MediaTypeImageConfig, []byte("")) // Blob 0 1029 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo3")) // Blob 1 1030 generateManifest(arc_1, os_1, variant_1, descs[0], descs[1]) // Blob 2 1031 generateIndex(descs[2]) // Blob 3 1032 1033 ctx = context.Background() 1034 for i := range blobs { 1035 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 1036 if err != nil { 1037 t.Fatalf("failed to push test content to src: %d: %v", i, err) 1038 } 1039 } 1040 1041 dst = memory.New() 1042 opts = oras.CopyOptions{} 1043 targetPlatform = ocispec.Platform{ 1044 Architecture: arc_1, 1045 OS: os_1, 1046 } 1047 opts.WithTargetPlatform(&targetPlatform) 1048 1049 // test copy with platform filter for the manifest with empty config blob 1050 // should return not found error 1051 root = descs[2] 1052 err = src.Tag(ctx, root, ref) 1053 if err != nil { 1054 t.Fatal("fail to tag root node", err) 1055 } 1056 1057 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 1058 expected = fmt.Sprintf("%s: %v: platform in manifest does not match target platform", root.Digest, errdef.ErrNotFound) 1059 if err.Error() != expected { 1060 t.Fatalf("Copy() error = %v, wantErr %v", err, expected) 1061 } 1062 1063 // test copy with no platform filter and nil opts.MapRoot 1064 // opts.MapRoot should be nil 1065 opts = oras.CopyOptions{} 1066 opts.WithTargetPlatform(nil) 1067 if opts.MapRoot != nil { 1068 t.Fatal("opts.MapRoot not equal to nil when platform is not provided") 1069 } 1070 1071 // test copy with no platform filter and custom opts.MapRoot 1072 // should return ErrNotFound 1073 opts = oras.CopyOptions{ 1074 MapRoot: func(ctx context.Context, src content.ReadOnlyStorage, root ocispec.Descriptor) (ocispec.Descriptor, error) { 1075 if root.MediaType == "test" { 1076 return root, nil 1077 } else { 1078 return ocispec.Descriptor{}, errdef.ErrNotFound 1079 } 1080 }, 1081 CopyGraphOptions: oras.DefaultCopyGraphOptions, 1082 } 1083 opts.WithTargetPlatform(nil) 1084 1085 _, err = oras.Copy(ctx, src, ref, dst, "", opts) 1086 if !errors.Is(err, errdef.ErrNotFound) { 1087 t.Fatalf("Copy() error = %v, wantErr %v", err, errdef.ErrNotFound) 1088 } 1089 } 1090 1091 func TestCopy_RestoreDuplicates(t *testing.T) { 1092 src := memory.New() 1093 temp := t.TempDir() 1094 dst, err := file.New(temp) 1095 if err != nil { 1096 t.Fatal("file.New() error =", err) 1097 } 1098 defer dst.Close() 1099 1100 // generate test content 1101 var blobs [][]byte 1102 var descs []ocispec.Descriptor 1103 appendBlob := func(mediaType string, blob []byte, title string) { 1104 blobs = append(blobs, blob) 1105 desc := ocispec.Descriptor{ 1106 MediaType: mediaType, 1107 Digest: digest.FromBytes(blob), 1108 Size: int64(len(blob)), 1109 } 1110 if title != "" { 1111 desc.Annotations = map[string]string{ 1112 ocispec.AnnotationTitle: title, 1113 } 1114 } 1115 descs = append(descs, desc) 1116 } 1117 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 1118 manifest := ocispec.Manifest{ 1119 Config: config, 1120 Layers: layers, 1121 } 1122 manifestJSON, err := json.Marshal(manifest) 1123 if err != nil { 1124 t.Fatal(err) 1125 } 1126 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON, "") 1127 } 1128 1129 appendBlob(ocispec.MediaTypeImageConfig, []byte("{}"), "") // Blob 0 1130 // 2 blobs with same content 1131 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello"), "foo.txt") // Blob 1 1132 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello"), "bar.txt") // Blob 2 1133 generateManifest(descs[0], descs[1:3]...) // Blob 3 1134 1135 ctx := context.Background() 1136 for i := range blobs { 1137 exists, err := src.Exists(ctx, descs[i]) 1138 if err != nil { 1139 t.Fatalf("failed to check existence in src: %d: %v", i, err) 1140 } 1141 if exists { 1142 continue 1143 } 1144 if err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])); err != nil { 1145 t.Fatalf("failed to push test content to src: %d: %v", i, err) 1146 } 1147 } 1148 1149 root := descs[3] 1150 ref := "latest" 1151 err = src.Tag(ctx, root, ref) 1152 if err != nil { 1153 t.Fatal("fail to tag root node", err) 1154 } 1155 1156 // test copy 1157 gotDesc, err := oras.Copy(ctx, src, ref, dst, "", oras.CopyOptions{}) 1158 if err != nil { 1159 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 1160 } 1161 if !reflect.DeepEqual(gotDesc, root) { 1162 t.Errorf("Copy() = %v, want %v", gotDesc, root) 1163 } 1164 1165 // verify contents 1166 for i, desc := range descs { 1167 exists, err := dst.Exists(ctx, desc) 1168 if err != nil { 1169 t.Fatalf("dst.Exists(%d) error = %v", i, err) 1170 } 1171 if !exists { 1172 t.Errorf("dst.Exists(%d) = %v, want %v", i, exists, true) 1173 } 1174 } 1175 1176 // verify tag 1177 gotDesc, err = dst.Resolve(ctx, ref) 1178 if err != nil { 1179 t.Fatal("dst.Resolve() error =", err) 1180 } 1181 if !reflect.DeepEqual(gotDesc, root) { 1182 t.Errorf("dst.Resolve() = %v, want %v", gotDesc, root) 1183 } 1184 } 1185 1186 func TestCopy_DiscardDuplicates(t *testing.T) { 1187 src := memory.New() 1188 temp := t.TempDir() 1189 dst, err := file.New(temp) 1190 if err != nil { 1191 t.Fatal("file.New() error =", err) 1192 } 1193 dst.ForceCAS = true 1194 defer dst.Close() 1195 1196 // generate test content 1197 var blobs [][]byte 1198 var descs []ocispec.Descriptor 1199 appendBlob := func(mediaType string, blob []byte, title string) { 1200 blobs = append(blobs, blob) 1201 desc := ocispec.Descriptor{ 1202 MediaType: mediaType, 1203 Digest: digest.FromBytes(blob), 1204 Size: int64(len(blob)), 1205 } 1206 if title != "" { 1207 desc.Annotations = map[string]string{ 1208 ocispec.AnnotationTitle: title, 1209 } 1210 } 1211 descs = append(descs, desc) 1212 } 1213 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 1214 manifest := ocispec.Manifest{ 1215 Config: config, 1216 Layers: layers, 1217 } 1218 manifestJSON, err := json.Marshal(manifest) 1219 if err != nil { 1220 t.Fatal(err) 1221 } 1222 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON, "") 1223 } 1224 1225 appendBlob(ocispec.MediaTypeImageConfig, []byte("{}"), "") // Blob 0 1226 // 2 blobs with same content 1227 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello"), "foo.txt") // Blob 1 1228 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello"), "bar.txt") // Blob 2 1229 generateManifest(descs[0], descs[1:3]...) // Blob 3 1230 1231 ctx := context.Background() 1232 for i := range blobs { 1233 exists, err := src.Exists(ctx, descs[i]) 1234 if err != nil { 1235 t.Fatalf("failed to check existence in src: %d: %v", i, err) 1236 } 1237 if exists { 1238 continue 1239 } 1240 if err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])); err != nil { 1241 t.Fatalf("failed to push test content to src: %d: %v", i, err) 1242 } 1243 } 1244 1245 root := descs[3] 1246 ref := "latest" 1247 err = src.Tag(ctx, root, ref) 1248 if err != nil { 1249 t.Fatal("fail to tag root node", err) 1250 } 1251 1252 // test copy 1253 gotDesc, err := oras.Copy(ctx, src, ref, dst, "", oras.CopyOptions{}) 1254 if err != nil { 1255 t.Fatalf("Copy() error = %v, wantErr %v", err, false) 1256 } 1257 if !reflect.DeepEqual(gotDesc, root) { 1258 t.Errorf("Copy() = %v, want %v", gotDesc, root) 1259 } 1260 1261 // verify only one of foo.txt and bar.txt exists 1262 fooExists, err := dst.Exists(ctx, descs[1]) 1263 if err != nil { 1264 t.Fatalf("dst.Exists(foo) error = %v", err) 1265 } 1266 barExists, err := dst.Exists(ctx, descs[2]) 1267 if err != nil { 1268 t.Fatalf("dst.Exists(bar) error = %v", err) 1269 } 1270 if fooExists == barExists { 1271 t.Error("Only one of foo.txt and bar.txt should exist") 1272 } 1273 } 1274 1275 func TestCopyGraph_WithOptions(t *testing.T) { 1276 src := cas.NewMemory() 1277 dst := cas.NewMemory() 1278 1279 // generate test content 1280 var blobs [][]byte 1281 var descs []ocispec.Descriptor 1282 appendBlob := func(mediaType string, blob []byte) { 1283 blobs = append(blobs, blob) 1284 descs = append(descs, ocispec.Descriptor{ 1285 MediaType: mediaType, 1286 Digest: digest.FromBytes(blob), 1287 Size: int64(len(blob)), 1288 }) 1289 } 1290 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 1291 manifest := ocispec.Manifest{ 1292 Config: config, 1293 Layers: layers, 1294 } 1295 manifestJSON, err := json.Marshal(manifest) 1296 if err != nil { 1297 t.Fatal(err) 1298 } 1299 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 1300 } 1301 generateIndex := func(manifests ...ocispec.Descriptor) { 1302 index := ocispec.Index{ 1303 Manifests: manifests, 1304 } 1305 indexJSON, err := json.Marshal(index) 1306 if err != nil { 1307 t.Fatal(err) 1308 } 1309 appendBlob(ocispec.MediaTypeImageIndex, indexJSON) 1310 } 1311 1312 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 1313 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 1314 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 1315 generateManifest(descs[0], descs[1:3]...) // Blob 3 1316 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 4 1317 generateManifest(descs[0], descs[4]) // Blob 5 1318 generateIndex(descs[3], descs[5]) // Blob 6 1319 1320 ctx := context.Background() 1321 for i := range blobs { 1322 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 1323 if err != nil { 1324 t.Fatalf("failed to push test content to src: %d: %v", i, err) 1325 } 1326 } 1327 1328 // initial copy 1329 root := descs[3] 1330 opts := oras.DefaultCopyGraphOptions 1331 opts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 1332 successors, err := content.Successors(ctx, fetcher, desc) 1333 if err != nil { 1334 return nil, err 1335 } 1336 // filter media type 1337 var filtered []ocispec.Descriptor 1338 for _, s := range successors { 1339 if s.MediaType != ocispec.MediaTypeImageConfig { 1340 filtered = append(filtered, s) 1341 } 1342 } 1343 return filtered, nil 1344 } 1345 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1346 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 1347 } 1348 // verify contents 1349 contents := dst.Map() 1350 if got, want := len(contents), len(blobs[1:4]); got != want { 1351 t.Fatalf("len(dst) = %v, wantErr %v", got, want) 1352 } 1353 for i := 1; i < 4; i++ { 1354 got, err := content.FetchAll(ctx, dst, descs[i]) 1355 if err != nil { 1356 t.Fatalf("content[%d] error = %v, wantErr %v", i, err, false) 1357 } 1358 if want := blobs[i]; !bytes.Equal(got, want) { 1359 t.Fatalf("content[%d] = %v, want %v", i, got, want) 1360 } 1361 } 1362 1363 // test successor descriptors not obtained from src 1364 root = descs[3] 1365 opts = oras.DefaultCopyGraphOptions 1366 opts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 1367 if content.Equal(desc, root) { 1368 return descs[1:3], nil 1369 } 1370 return content.Successors(ctx, fetcher, desc) 1371 } 1372 if err := oras.CopyGraph(ctx, src, cas.NewMemory(), root, opts); err != nil { 1373 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 1374 } 1375 1376 // test partial copy 1377 var preCopyCount int64 1378 var postCopyCount int64 1379 var skippedCount int64 1380 opts = oras.CopyGraphOptions{ 1381 PreCopy: func(ctx context.Context, desc ocispec.Descriptor) error { 1382 atomic.AddInt64(&preCopyCount, 1) 1383 return nil 1384 }, 1385 PostCopy: func(ctx context.Context, desc ocispec.Descriptor) error { 1386 atomic.AddInt64(&postCopyCount, 1) 1387 return nil 1388 }, 1389 OnCopySkipped: func(ctx context.Context, desc ocispec.Descriptor) error { 1390 atomic.AddInt64(&skippedCount, 1) 1391 return nil 1392 }, 1393 } 1394 root = descs[len(descs)-1] 1395 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1396 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 1397 } 1398 1399 // verify contents 1400 contents = dst.Map() 1401 if got, want := len(contents), len(blobs); got != want { 1402 t.Errorf("len(dst) = %v, wantErr %v", got, want) 1403 } 1404 for i := range blobs { 1405 got, err := content.FetchAll(ctx, dst, descs[i]) 1406 if err != nil { 1407 t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) 1408 continue 1409 } 1410 if want := blobs[i]; !bytes.Equal(got, want) { 1411 t.Errorf("content[%d] = %v, want %v", i, got, want) 1412 } 1413 } 1414 1415 // verify API counts 1416 if got, want := preCopyCount, int64(4); got != want { 1417 t.Errorf("count(PreCopy()) = %v, want %v", got, want) 1418 } 1419 if got, want := postCopyCount, int64(4); got != want { 1420 t.Errorf("count(PostCopy()) = %v, want %v", got, want) 1421 } 1422 if got, want := skippedCount, int64(1); got != want { 1423 t.Errorf("count(OnCopySkipped()) = %v, want %v", got, want) 1424 } 1425 1426 // test CopyGraph with MaxMetadataBytes = 1 1427 root = descs[6] 1428 dst = cas.NewMemory() 1429 opts = oras.CopyGraphOptions{ 1430 MaxMetadataBytes: 1, 1431 } 1432 if err := oras.CopyGraph(ctx, src, dst, root, opts); !errors.Is(err, errdef.ErrSizeExceedsLimit) { 1433 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, errdef.ErrSizeExceedsLimit) 1434 } 1435 1436 t.Run("SkipNode", func(t *testing.T) { 1437 // test CopyGraph with PreCopy = 1 1438 root = descs[6] 1439 dst := &countingStorage{storage: cas.NewMemory()} 1440 opts = oras.CopyGraphOptions{ 1441 PreCopy: func(ctx context.Context, desc ocispec.Descriptor) error { 1442 if descs[1].Digest == desc.Digest { 1443 // blob 1 is handled by us (really this would be a Mount but ) 1444 rc, err := src.Fetch(ctx, desc) 1445 if err != nil { 1446 t.Fatalf("Failed to fetch: %v", err) 1447 } 1448 defer rc.Close() 1449 err = dst.storage.Push(ctx, desc, rc) // bypass the counters 1450 if err != nil { 1451 t.Fatalf("Failed to fetch: %v", err) 1452 } 1453 return oras.SkipNode 1454 } 1455 return nil 1456 }, 1457 } 1458 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1459 t.Fatalf("CopyGraph() error = %v", err) 1460 } 1461 1462 if got, expected := dst.numExists.Load(), int64(7); got != expected { 1463 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1464 } 1465 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1466 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1467 } 1468 // 7 (exists) - 1 (skipped) = 6 pushes expected 1469 if got, expected := dst.numPush.Load(), int64(6); got != expected { 1470 // If we get >=7 then SkipNode did not short circuit the push like it is supposed to do. 1471 t.Errorf("count(Push()) = %d, want %d", got, expected) 1472 } 1473 }) 1474 1475 t.Run("MountFrom mounted", func(t *testing.T) { 1476 root = descs[6] 1477 dst := &countingStorage{storage: cas.NewMemory()} 1478 var numMount atomic.Int64 1479 dst.mount = func(ctx context.Context, 1480 desc ocispec.Descriptor, 1481 fromRepo string, 1482 getContent func() (io.ReadCloser, error), 1483 ) error { 1484 numMount.Add(1) 1485 if expected := "source"; fromRepo != expected { 1486 t.Fatalf("fromRepo = %v, want %v", fromRepo, expected) 1487 } 1488 rc, err := src.Fetch(ctx, desc) 1489 if err != nil { 1490 t.Fatalf("Failed to fetch content: %v", err) 1491 } 1492 defer rc.Close() 1493 err = dst.storage.Push(ctx, desc, rc) // bypass the counters 1494 if err != nil { 1495 t.Fatalf("Failed to push content: %v", err) 1496 } 1497 return nil 1498 } 1499 opts = oras.CopyGraphOptions{} 1500 var numPreCopy, numPostCopy, numOnMounted, numMountFrom atomic.Int64 1501 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1502 numPreCopy.Add(1) 1503 return nil 1504 } 1505 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1506 numPostCopy.Add(1) 1507 return nil 1508 } 1509 opts.OnMounted = func(ctx context.Context, d ocispec.Descriptor) error { 1510 numOnMounted.Add(1) 1511 return nil 1512 } 1513 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1514 numMountFrom.Add(1) 1515 return []string{"source"}, nil 1516 } 1517 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1518 t.Fatalf("CopyGraph() error = %v", err) 1519 } 1520 1521 if got, expected := dst.numExists.Load(), int64(7); got != expected { 1522 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1523 } 1524 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1525 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1526 } 1527 // 7 (exists) - 1 (skipped) = 6 pushes expected 1528 if got, expected := dst.numPush.Load(), int64(3); got != expected { 1529 // If we get >=7 then ErrSkipDesc did not short circuit the push like it is supposed to do. 1530 t.Errorf("count(Push()) = %d, want %d", got, expected) 1531 } 1532 if got, expected := numMount.Load(), int64(4); got != expected { 1533 t.Errorf("count(Mount()) = %d, want %d", got, expected) 1534 } 1535 if got, expected := numOnMounted.Load(), int64(4); got != expected { 1536 t.Errorf("count(OnMounted()) = %d, want %d", got, expected) 1537 } 1538 if got, expected := numMountFrom.Load(), int64(4); got != expected { 1539 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1540 } 1541 if got, expected := numPreCopy.Load(), int64(3); got != expected { 1542 t.Errorf("count(PreCopy()) = %d, want %d", got, expected) 1543 } 1544 if got, expected := numPostCopy.Load(), int64(3); got != expected { 1545 t.Errorf("count(PostCopy()) = %d, want %d", got, expected) 1546 } 1547 }) 1548 1549 t.Run("MountFrom copied", func(t *testing.T) { 1550 root = descs[6] 1551 dst := &countingStorage{storage: cas.NewMemory()} 1552 var numMount atomic.Int64 1553 dst.mount = func(ctx context.Context, 1554 desc ocispec.Descriptor, 1555 fromRepo string, 1556 getContent func() (io.ReadCloser, error), 1557 ) error { 1558 numMount.Add(1) 1559 if expected := "source"; fromRepo != expected { 1560 t.Fatalf("fromRepo = %v, want %v", fromRepo, expected) 1561 } 1562 1563 rc, err := getContent() 1564 if err != nil { 1565 t.Fatalf("Failed to fetch content: %v", err) 1566 } 1567 defer rc.Close() 1568 err = dst.storage.Push(ctx, desc, rc) // bypass the counters 1569 if err != nil { 1570 t.Fatalf("Failed to push content: %v", err) 1571 } 1572 return nil 1573 } 1574 opts = oras.CopyGraphOptions{} 1575 var numPreCopy, numPostCopy, numOnMounted, numMountFrom atomic.Int64 1576 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1577 numPreCopy.Add(1) 1578 return nil 1579 } 1580 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1581 numPostCopy.Add(1) 1582 return nil 1583 } 1584 opts.OnMounted = func(ctx context.Context, d ocispec.Descriptor) error { 1585 numOnMounted.Add(1) 1586 return nil 1587 } 1588 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1589 numMountFrom.Add(1) 1590 return []string{"source"}, nil 1591 } 1592 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1593 t.Fatalf("CopyGraph() error = %v", err) 1594 } 1595 1596 if got, expected := dst.numExists.Load(), int64(7); got != expected { 1597 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1598 } 1599 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1600 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1601 } 1602 // 7 (exists) - 1 (skipped) = 6 pushes expected 1603 if got, expected := dst.numPush.Load(), int64(3); got != expected { 1604 // If we get >=7 then ErrSkipDesc did not short circuit the push like it is supposed to do. 1605 t.Errorf("count(Push()) = %d, want %d", got, expected) 1606 } 1607 if got, expected := numMount.Load(), int64(4); got != expected { 1608 t.Errorf("count(Mount()) = %d, want %d", got, expected) 1609 } 1610 if got, expected := numOnMounted.Load(), int64(0); got != expected { 1611 t.Errorf("count(OnMounted()) = %d, want %d", got, expected) 1612 } 1613 if got, expected := numMountFrom.Load(), int64(4); got != expected { 1614 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1615 } 1616 if got, expected := numPreCopy.Load(), int64(7); got != expected { 1617 t.Errorf("count(PreCopy()) = %d, want %d", got, expected) 1618 } 1619 if got, expected := numPostCopy.Load(), int64(7); got != expected { 1620 t.Errorf("count(PostCopy()) = %d, want %d", got, expected) 1621 } 1622 }) 1623 1624 t.Run("MountFrom mounted second try", func(t *testing.T) { 1625 root = descs[6] 1626 dst := &countingStorage{storage: cas.NewMemory()} 1627 var numMount atomic.Int64 1628 dst.mount = func(ctx context.Context, 1629 desc ocispec.Descriptor, 1630 fromRepo string, 1631 getContent func() (io.ReadCloser, error), 1632 ) error { 1633 numMount.Add(1) 1634 switch fromRepo { 1635 case "source": 1636 rc, err := src.Fetch(ctx, desc) 1637 if err != nil { 1638 t.Fatalf("Failed to fetch content: %v", err) 1639 } 1640 defer rc.Close() 1641 err = dst.storage.Push(ctx, desc, rc) // bypass the counters 1642 if err != nil { 1643 t.Fatalf("Failed to push content: %v", err) 1644 } 1645 return nil 1646 case "missing/the/data": 1647 // simulate a registry mount will fail, so it will request the content to start the copy. 1648 rc, err := getContent() 1649 if err != nil { 1650 return fmt.Errorf("getContent failed: %w", err) 1651 } 1652 defer rc.Close() 1653 err = dst.storage.Push(ctx, desc, rc) // bypass the counters 1654 if err != nil { 1655 t.Fatalf("Failed to push content: %v", err) 1656 } 1657 return nil 1658 default: 1659 t.Fatalf("fromRepo = %v, want either %v or %v", fromRepo, "missing/the/data", "source") 1660 return nil 1661 } 1662 } 1663 opts = oras.CopyGraphOptions{} 1664 var numPreCopy, numPostCopy, numOnMounted, numMountFrom atomic.Int64 1665 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1666 numPreCopy.Add(1) 1667 return nil 1668 } 1669 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1670 numPostCopy.Add(1) 1671 return nil 1672 } 1673 opts.OnMounted = func(ctx context.Context, d ocispec.Descriptor) error { 1674 numOnMounted.Add(1) 1675 return nil 1676 } 1677 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1678 numMountFrom.Add(1) 1679 return []string{"missing/the/data", "source"}, nil 1680 } 1681 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1682 t.Fatalf("CopyGraph() error = %v", err) 1683 } 1684 1685 if got, expected := dst.numExists.Load(), int64(7); got != expected { 1686 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1687 } 1688 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1689 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1690 } 1691 // 7 (exists) - 1 (skipped) = 6 pushes expected 1692 if got, expected := dst.numPush.Load(), int64(3); got != expected { 1693 // If we get >=7 then ErrSkipDesc did not short circuit the push like it is supposed to do. 1694 t.Errorf("count(Push()) = %d, want %d", got, expected) 1695 } 1696 if got, expected := numMount.Load(), int64(4*2); got != expected { 1697 t.Errorf("count(Mount()) = %d, want %d", got, expected) 1698 } 1699 if got, expected := numOnMounted.Load(), int64(4); got != expected { 1700 t.Errorf("count(OnMounted()) = %d, want %d", got, expected) 1701 } 1702 if got, expected := numMountFrom.Load(), int64(4); got != expected { 1703 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1704 } 1705 if got, expected := numPreCopy.Load(), int64(3); got != expected { 1706 t.Errorf("count(PreCopy()) = %d, want %d", got, expected) 1707 } 1708 if got, expected := numPostCopy.Load(), int64(3); got != expected { 1709 t.Errorf("count(PostCopy()) = %d, want %d", got, expected) 1710 } 1711 }) 1712 1713 t.Run("MountFrom copied dst not a Mounter", func(t *testing.T) { 1714 root = descs[6] 1715 dst := cas.NewMemory() 1716 opts = oras.CopyGraphOptions{} 1717 var numPreCopy, numPostCopy, numOnMounted, numMountFrom atomic.Int64 1718 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1719 numPreCopy.Add(1) 1720 return nil 1721 } 1722 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1723 numPostCopy.Add(1) 1724 return nil 1725 } 1726 opts.OnMounted = func(ctx context.Context, d ocispec.Descriptor) error { 1727 numOnMounted.Add(1) 1728 return nil 1729 } 1730 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1731 numMountFrom.Add(1) 1732 return []string{"source"}, nil 1733 } 1734 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1735 t.Fatalf("CopyGraph() error = %v", err) 1736 } 1737 1738 if got, expected := numOnMounted.Load(), int64(0); got != expected { 1739 t.Errorf("count(OnMounted()) = %d, want %d", got, expected) 1740 } 1741 if got, expected := numMountFrom.Load(), int64(0); got != expected { 1742 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1743 } 1744 if got, expected := numPreCopy.Load(), int64(7); got != expected { 1745 t.Errorf("count(PreCopy()) = %d, want %d", got, expected) 1746 } 1747 if got, expected := numPostCopy.Load(), int64(7); got != expected { 1748 t.Errorf("count(PostCopy()) = %d, want %d", got, expected) 1749 } 1750 }) 1751 1752 t.Run("MountFrom empty sourceRepositories", func(t *testing.T) { 1753 root = descs[6] 1754 dst := &countingStorage{storage: cas.NewMemory()} 1755 opts = oras.CopyGraphOptions{} 1756 var numMountFrom atomic.Int64 1757 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1758 numMountFrom.Add(1) 1759 return nil, nil 1760 } 1761 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1762 t.Fatalf("CopyGraph() error = %v", err) 1763 } 1764 1765 if got, expected := dst.numExists.Load(), int64(7); got != expected { 1766 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1767 } 1768 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1769 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1770 } 1771 if got, expected := dst.numPush.Load(), int64(7); got != expected { 1772 t.Errorf("count(Push()) = %d, want %d", got, expected) 1773 } 1774 if got, expected := numMountFrom.Load(), int64(4); got != expected { 1775 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1776 } 1777 }) 1778 1779 t.Run("MountFrom error", func(t *testing.T) { 1780 root = descs[3] 1781 dst := &countingStorage{storage: cas.NewMemory()} 1782 opts = oras.CopyGraphOptions{ 1783 // to make the run result deterministic, we limit concurrency to 1 1784 Concurrency: 1, 1785 } 1786 var numMountFrom atomic.Int64 1787 e := errors.New("mountFrom error") 1788 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1789 numMountFrom.Add(1) 1790 return nil, e 1791 } 1792 if err := oras.CopyGraph(ctx, src, dst, root, opts); !errors.Is(err, e) { 1793 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, e) 1794 } 1795 1796 if got, expected := dst.numExists.Load(), int64(2); got != expected { 1797 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1798 } 1799 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1800 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1801 } 1802 if got, expected := dst.numPush.Load(), int64(0); got != expected { 1803 t.Errorf("count(Push()) = %d, want %d", got, expected) 1804 } 1805 if got, expected := numMountFrom.Load(), int64(1); got != expected { 1806 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1807 } 1808 }) 1809 1810 t.Run("MountFrom OnMounted error", func(t *testing.T) { 1811 root = descs[3] 1812 dst := &countingStorage{storage: cas.NewMemory()} 1813 var numMount atomic.Int64 1814 dst.mount = func(ctx context.Context, 1815 desc ocispec.Descriptor, 1816 fromRepo string, 1817 getContent func() (io.ReadCloser, error), 1818 ) error { 1819 numMount.Add(1) 1820 if expected := "source"; fromRepo != expected { 1821 t.Fatalf("fromRepo = %v, want %v", fromRepo, expected) 1822 } 1823 rc, err := src.Fetch(ctx, desc) 1824 if err != nil { 1825 t.Fatalf("Failed to fetch content: %v", err) 1826 } 1827 defer rc.Close() 1828 err = dst.storage.Push(ctx, desc, rc) // bypass the counters 1829 if err != nil { 1830 t.Fatalf("Failed to push content: %v", err) 1831 } 1832 return nil 1833 } 1834 opts = oras.CopyGraphOptions{ 1835 // to make the run result deterministic, we limit concurrency to 1 1836 Concurrency: 1, 1837 } 1838 var numPreCopy, numPostCopy, numOnMounted, numMountFrom atomic.Int64 1839 opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1840 numPreCopy.Add(1) 1841 return nil 1842 } 1843 opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { 1844 numPostCopy.Add(1) 1845 return nil 1846 } 1847 e := errors.New("onMounted error") 1848 opts.OnMounted = func(ctx context.Context, d ocispec.Descriptor) error { 1849 numOnMounted.Add(1) 1850 return e 1851 } 1852 opts.MountFrom = func(ctx context.Context, desc ocispec.Descriptor) ([]string, error) { 1853 numMountFrom.Add(1) 1854 return []string{"source"}, nil 1855 } 1856 if err := oras.CopyGraph(ctx, src, dst, root, opts); !errors.Is(err, e) { 1857 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, e) 1858 } 1859 1860 if got, expected := dst.numExists.Load(), int64(2); got != expected { 1861 t.Errorf("count(Exists()) = %d, want %d", got, expected) 1862 } 1863 if got, expected := dst.numFetch.Load(), int64(0); got != expected { 1864 t.Errorf("count(Fetch()) = %d, want %d", got, expected) 1865 } 1866 if got, expected := dst.numPush.Load(), int64(0); got != expected { 1867 t.Errorf("count(Push()) = %d, want %d", got, expected) 1868 } 1869 if got, expected := numMount.Load(), int64(1); got != expected { 1870 t.Errorf("count(Mount()) = %d, want %d", got, expected) 1871 } 1872 if got, expected := numOnMounted.Load(), int64(1); got != expected { 1873 t.Errorf("count(OnMounted()) = %d, want %d", got, expected) 1874 } 1875 if got, expected := numMountFrom.Load(), int64(1); got != expected { 1876 t.Errorf("count(MountFrom()) = %d, want %d", got, expected) 1877 } 1878 if got, expected := numPreCopy.Load(), int64(0); got != expected { 1879 t.Errorf("count(PreCopy()) = %d, want %d", got, expected) 1880 } 1881 if got, expected := numPostCopy.Load(), int64(0); got != expected { 1882 t.Errorf("count(PostCopy()) = %d, want %d", got, expected) 1883 } 1884 }) 1885 } 1886 1887 // countingStorage counts the calls to its content.Storage methods 1888 type countingStorage struct { 1889 storage content.Storage 1890 mount mountFunc 1891 1892 numExists, numFetch, numPush atomic.Int64 1893 } 1894 1895 func (cs *countingStorage) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { 1896 cs.numExists.Add(1) 1897 return cs.storage.Exists(ctx, target) 1898 } 1899 1900 func (cs *countingStorage) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { 1901 cs.numFetch.Add(1) 1902 return cs.storage.Fetch(ctx, target) 1903 } 1904 1905 func (cs *countingStorage) Push(ctx context.Context, target ocispec.Descriptor, r io.Reader) error { 1906 cs.numPush.Add(1) 1907 return cs.storage.Push(ctx, target, r) 1908 } 1909 1910 type mountFunc func(context.Context, ocispec.Descriptor, string, func() (io.ReadCloser, error)) error 1911 1912 func (cs *countingStorage) Mount(ctx context.Context, 1913 desc ocispec.Descriptor, 1914 fromRepo string, 1915 getContent func() (io.ReadCloser, error), 1916 ) error { 1917 return cs.mount(ctx, desc, fromRepo, getContent) 1918 } 1919 1920 func TestCopyGraph_WithConcurrencyLimit(t *testing.T) { 1921 src := cas.NewMemory() 1922 // generate test content 1923 var blobs [][]byte 1924 var descs []ocispec.Descriptor 1925 appendBlob := func(mediaType string, blob []byte) { 1926 blobs = append(blobs, blob) 1927 descs = append(descs, ocispec.Descriptor{ 1928 MediaType: mediaType, 1929 Digest: digest.FromBytes(blob), 1930 Size: int64(len(blob)), 1931 }) 1932 } 1933 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 1934 manifest := ocispec.Manifest{ 1935 MediaType: ocispec.MediaTypeImageManifest, 1936 Config: config, 1937 Layers: layers, 1938 } 1939 manifestJSON, err := json.Marshal(manifest) 1940 if err != nil { 1941 t.Fatal(err) 1942 } 1943 appendBlob(manifest.MediaType, manifestJSON) 1944 } 1945 generateArtifact := func(subject *ocispec.Descriptor, artifactType string, blobs ...ocispec.Descriptor) { 1946 manifest := spec.Artifact{ 1947 MediaType: spec.MediaTypeArtifactManifest, 1948 Subject: subject, 1949 Blobs: blobs, 1950 ArtifactType: artifactType, 1951 } 1952 manifestJSON, err := json.Marshal(manifest) 1953 if err != nil { 1954 t.Fatal(err) 1955 } 1956 appendBlob(manifest.MediaType, manifestJSON) 1957 } 1958 generateIndex := func(manifests ...ocispec.Descriptor) { 1959 index := ocispec.Index{ 1960 MediaType: ocispec.MediaTypeImageIndex, 1961 Manifests: manifests, 1962 } 1963 indexJSON, err := json.Marshal(index) 1964 if err != nil { 1965 t.Fatal(err) 1966 } 1967 appendBlob(index.MediaType, indexJSON) 1968 } 1969 1970 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 1971 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 1972 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 1973 generateManifest(descs[0], descs[1:3]...) // Blob 3 1974 generateArtifact(&descs[3], "artifact.1") // Blob 4 1975 generateArtifact(&descs[3], "artifact.2") // Blob 5 1976 generateArtifact(&descs[3], "artifact.3") // Blob 6 1977 generateArtifact(&descs[3], "artifact.4") // Blob 7 1978 generateIndex(descs[3:8]...) // Blob 8 1979 1980 ctx := context.Background() 1981 for i := range blobs { 1982 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 1983 if err != nil { 1984 t.Fatalf("failed to push test content to src: %d: %v", i, err) 1985 } 1986 } 1987 1988 // test different concurrency limit 1989 root := descs[len(descs)-1] 1990 directSuccessorsNum := 5 1991 opts := oras.DefaultCopyGraphOptions 1992 for i := 1; i <= directSuccessorsNum; i++ { 1993 dst := cas.NewMemory() 1994 opts.Concurrency = i 1995 if err := oras.CopyGraph(ctx, src, dst, root, opts); err != nil { 1996 t.Fatalf("CopyGraph(concurrency: %d) error = %v, wantErr %v", i, err, false) 1997 } 1998 1999 // verify contents 2000 contents := dst.Map() 2001 if got, want := len(contents), len(blobs); got != want { 2002 t.Errorf("len(dst) = %v, wantErr %v", got, want) 2003 } 2004 for i := range blobs { 2005 got, err := content.FetchAll(ctx, dst, descs[i]) 2006 if err != nil { 2007 t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) 2008 continue 2009 } 2010 if want := blobs[i]; !bytes.Equal(got, want) { 2011 t.Errorf("content[%d] = %v, want %v", i, got, want) 2012 } 2013 } 2014 } 2015 } 2016 2017 func TestCopyGraph_ForeignLayers(t *testing.T) { 2018 src := cas.NewMemory() 2019 dst := cas.NewMemory() 2020 2021 // generate test content 2022 var blobs [][]byte 2023 var descs []ocispec.Descriptor 2024 appendBlob := func(mediaType string, blob []byte) { 2025 desc := ocispec.Descriptor{ 2026 MediaType: mediaType, 2027 Digest: digest.FromBytes(blob), 2028 Size: int64(len(blob)), 2029 } 2030 if mediaType == ocispec.MediaTypeImageLayerNonDistributable { 2031 desc.URLs = append(desc.URLs, "http://127.0.0.1/dummy") 2032 blob = nil 2033 } 2034 descs = append(descs, desc) 2035 blobs = append(blobs, blob) 2036 } 2037 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 2038 manifest := ocispec.Manifest{ 2039 Config: config, 2040 Layers: layers, 2041 } 2042 manifestJSON, err := json.Marshal(manifest) 2043 if err != nil { 2044 t.Fatal(err) 2045 } 2046 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 2047 } 2048 2049 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 2050 appendBlob(ocispec.MediaTypeImageLayerNonDistributable, []byte("hello")) // Blob 1 2051 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 2 2052 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 3 2053 generateManifest(descs[0], descs[1:4]...) // Blob 4 2054 2055 ctx := context.Background() 2056 for i := range blobs { 2057 if blobs[i] == nil { 2058 continue 2059 } 2060 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 2061 if err != nil { 2062 t.Fatalf("failed to push test content to src: %d: %v", i, err) 2063 } 2064 } 2065 2066 // test copy 2067 srcTracker := &storageTracker{Storage: src} 2068 dstTracker := &storageTracker{Storage: dst} 2069 root := descs[len(descs)-1] 2070 if err := oras.CopyGraph(ctx, srcTracker, dstTracker, root, oras.CopyGraphOptions{}); err != nil { 2071 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 2072 } 2073 2074 // verify contents 2075 contents := dst.Map() 2076 if got, want := len(contents), len(blobs)-1; got != want { 2077 t.Errorf("len(dst) = %v, wantErr %v", got, want) 2078 } 2079 for i := range blobs { 2080 if blobs[i] == nil { 2081 continue 2082 } 2083 got, err := content.FetchAll(ctx, dst, descs[i]) 2084 if err != nil { 2085 t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) 2086 continue 2087 } 2088 if want := blobs[i]; !bytes.Equal(got, want) { 2089 t.Errorf("content[%d] = %v, want %v", i, got, want) 2090 } 2091 } 2092 2093 // verify API counts 2094 if got, want := srcTracker.fetch, int64(len(blobs)-1); got != want { 2095 t.Errorf("count(src.Fetch()) = %v, want %v", got, want) 2096 } 2097 if got, want := srcTracker.push, int64(0); got != want { 2098 t.Errorf("count(src.Push()) = %v, want %v", got, want) 2099 } 2100 if got, want := srcTracker.exists, int64(0); got != want { 2101 t.Errorf("count(src.Exists()) = %v, want %v", got, want) 2102 } 2103 if got, want := dstTracker.fetch, int64(0); got != want { 2104 t.Errorf("count(dst.Fetch()) = %v, want %v", got, want) 2105 } 2106 if got, want := dstTracker.push, int64(len(blobs)-1); got != want { 2107 t.Errorf("count(dst.Push()) = %v, want %v", got, want) 2108 } 2109 if got, want := dstTracker.exists, int64(len(blobs)-1); got != want { 2110 t.Errorf("count(dst.Exists()) = %v, want %v", got, want) 2111 } 2112 } 2113 2114 func TestCopyGraph_ForeignLayers_Mixed(t *testing.T) { 2115 src := cas.NewMemory() 2116 dst := cas.NewMemory() 2117 2118 // generate test content 2119 var blobs [][]byte 2120 var descs []ocispec.Descriptor 2121 appendBlob := func(mediaType string, blob []byte) { 2122 desc := ocispec.Descriptor{ 2123 MediaType: mediaType, 2124 Digest: digest.FromBytes(blob), 2125 Size: int64(len(blob)), 2126 } 2127 if mediaType == ocispec.MediaTypeImageLayerNonDistributable { 2128 desc.URLs = append(desc.URLs, "http://127.0.0.1/dummy") 2129 blob = nil 2130 } 2131 descs = append(descs, desc) 2132 blobs = append(blobs, blob) 2133 } 2134 generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { 2135 manifest := ocispec.Manifest{ 2136 Config: config, 2137 Layers: layers, 2138 } 2139 manifestJSON, err := json.Marshal(manifest) 2140 if err != nil { 2141 t.Fatal(err) 2142 } 2143 appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) 2144 } 2145 2146 appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 2147 appendBlob(ocispec.MediaTypeImageLayerNonDistributable, []byte("hello")) // Blob 1 2148 appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 2 2149 appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 3 2150 appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 4 2151 generateManifest(descs[0], descs[1:5]...) // Blob 5 2152 2153 ctx := context.Background() 2154 for i := range blobs { 2155 if blobs[i] == nil { 2156 continue 2157 } 2158 err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i])) 2159 if err != nil { 2160 t.Fatalf("failed to push test content to src: %d: %v", i, err) 2161 } 2162 } 2163 2164 // test copy 2165 srcTracker := &storageTracker{Storage: src} 2166 dstTracker := &storageTracker{Storage: dst} 2167 root := descs[len(descs)-1] 2168 if err := oras.CopyGraph(ctx, srcTracker, dstTracker, root, oras.CopyGraphOptions{ 2169 Concurrency: 1, 2170 }); err != nil { 2171 t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false) 2172 } 2173 2174 // verify contents 2175 contents := dst.Map() 2176 if got, want := len(contents), len(blobs)-1; got != want { 2177 t.Errorf("len(dst) = %v, wantErr %v", got, want) 2178 } 2179 for i := range blobs { 2180 if blobs[i] == nil { 2181 continue 2182 } 2183 got, err := content.FetchAll(ctx, dst, descs[i]) 2184 if err != nil { 2185 t.Errorf("content[%d] error = %v, wantErr %v", i, err, false) 2186 continue 2187 } 2188 if want := blobs[i]; !bytes.Equal(got, want) { 2189 t.Errorf("content[%d] = %v, want %v", i, got, want) 2190 } 2191 } 2192 2193 // verify API counts 2194 if got, want := srcTracker.fetch, int64(len(blobs)-1); got != want { 2195 t.Errorf("count(src.Fetch()) = %v, want %v", got, want) 2196 } 2197 if got, want := srcTracker.push, int64(0); got != want { 2198 t.Errorf("count(src.Push()) = %v, want %v", got, want) 2199 } 2200 if got, want := srcTracker.exists, int64(0); got != want { 2201 t.Errorf("count(src.Exists()) = %v, want %v", got, want) 2202 } 2203 if got, want := dstTracker.fetch, int64(0); got != want { 2204 t.Errorf("count(dst.Fetch()) = %v, want %v", got, want) 2205 } 2206 if got, want := dstTracker.push, int64(len(blobs)-1); got != want { 2207 t.Errorf("count(dst.Push()) = %v, want %v", got, want) 2208 } 2209 if got, want := dstTracker.exists, int64(len(blobs)-1); got != want { 2210 t.Errorf("count(dst.Exists()) = %v, want %v", got, want) 2211 } 2212 }