zotregistry.io/zot@v1.4.4-0.20231124084042-02a8ed785457/pkg/storage/gc/gc.go (about) 1 package gc 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "math/rand" 8 "path" 9 "strings" 10 "time" 11 12 "github.com/docker/distribution/registry/storage/driver" 13 godigest "github.com/opencontainers/go-digest" 14 ispec "github.com/opencontainers/image-spec/specs-go/v1" 15 oras "github.com/oras-project/artifacts-spec/specs-go/v1" 16 17 zerr "zotregistry.io/zot/errors" 18 "zotregistry.io/zot/pkg/api/config" 19 zcommon "zotregistry.io/zot/pkg/common" 20 zlog "zotregistry.io/zot/pkg/log" 21 mTypes "zotregistry.io/zot/pkg/meta/types" 22 "zotregistry.io/zot/pkg/retention" 23 rTypes "zotregistry.io/zot/pkg/retention/types" 24 "zotregistry.io/zot/pkg/scheduler" 25 "zotregistry.io/zot/pkg/storage" 26 common "zotregistry.io/zot/pkg/storage/common" 27 "zotregistry.io/zot/pkg/storage/types" 28 ) 29 30 const ( 31 cosignSignatureTagSuffix = "sig" 32 SBOMTagSuffix = "sbom" 33 ) 34 35 type Options struct { 36 // will garbage collect blobs older than Delay 37 Delay time.Duration 38 39 ImageRetention config.ImageRetention 40 } 41 42 type GarbageCollect struct { 43 imgStore types.ImageStore 44 opts Options 45 metaDB mTypes.MetaDB 46 policyMgr rTypes.PolicyManager 47 auditLog *zlog.Logger 48 log zlog.Logger 49 } 50 51 func NewGarbageCollect(imgStore types.ImageStore, metaDB mTypes.MetaDB, opts Options, 52 auditLog *zlog.Logger, log zlog.Logger, 53 ) GarbageCollect { 54 return GarbageCollect{ 55 imgStore: imgStore, 56 metaDB: metaDB, 57 opts: opts, 58 policyMgr: retention.NewPolicyManager(opts.ImageRetention, log, auditLog), 59 auditLog: auditLog, 60 log: log, 61 } 62 } 63 64 /* 65 CleanImageStorePeriodically runs a periodic garbage collect on the ImageStore provided in constructor, 66 given an interval and a Scheduler. 67 */ 68 func (gc GarbageCollect) CleanImageStorePeriodically(interval time.Duration, sch *scheduler.Scheduler) { 69 generator := &GCTaskGenerator{ 70 imgStore: gc.imgStore, 71 gc: gc, 72 } 73 74 sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) 75 } 76 77 /* 78 CleanRepo executes a garbage collection of any blob found in storage which is not referenced 79 in any manifests referenced in repo's index.json 80 It also gc referrers with missing subject if the Referrer Option is enabled 81 It also gc untagged manifests. 82 */ 83 func (gc GarbageCollect) CleanRepo(ctx context.Context, repo string) error { 84 gc.log.Info().Str("module", "gc"). 85 Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(gc.imgStore.RootDir(), repo))) 86 87 if err := gc.cleanRepo(ctx, repo); err != nil { 88 errMessage := fmt.Sprintf("error while running GC for %s", path.Join(gc.imgStore.RootDir(), repo)) 89 gc.log.Error().Err(err).Str("module", "gc").Msg(errMessage) 90 gc.log.Info().Str("module", "gc"). 91 Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(gc.imgStore.RootDir(), repo))) 92 93 return err 94 } 95 96 gc.log.Info().Str("module", "gc"). 97 Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(gc.imgStore.RootDir(), repo))) 98 99 return nil 100 } 101 102 func (gc GarbageCollect) cleanRepo(ctx context.Context, repo string) error { 103 var lockLatency time.Time 104 105 dir := path.Join(gc.imgStore.RootDir(), repo) 106 if !gc.imgStore.DirExists(dir) { 107 return zerr.ErrRepoNotFound 108 } 109 110 gc.imgStore.Lock(&lockLatency) 111 defer gc.imgStore.Unlock(&lockLatency) 112 113 /* this index (which represents the index.json of this repo) is the root point from which we 114 search for dangling manifests/blobs 115 so this index is passed by reference in all functions that modifies it 116 117 Instead of removing manifests one by one with storage APIs we just remove manifests descriptors 118 from index.Manifests[] list and update repo's index.json afterwards. 119 120 After updating repo's index.json we clean all unreferenced blobs (manifests included). 121 */ 122 index, err := common.GetIndex(gc.imgStore, repo, gc.log) 123 if err != nil { 124 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to read index.json in repo") 125 126 return err 127 } 128 129 // apply tags retention 130 if err := gc.removeTagsPerRetentionPolicy(ctx, repo, &index); err != nil { 131 return err 132 } 133 134 // gc referrers manifests with missing subject and untagged manifests 135 if err := gc.removeManifestsPerRepoPolicy(ctx, repo, &index); err != nil { 136 return err 137 } 138 139 // update repos's index.json in storage 140 if !gc.opts.ImageRetention.DryRun { 141 /* this will update the index.json with manifests deleted above 142 and the manifests blobs will be removed by gc.removeUnreferencedBlobs()*/ 143 if err := gc.imgStore.PutIndexContent(repo, index); err != nil { 144 return err 145 } 146 } 147 148 // gc unreferenced blobs 149 if err := gc.removeUnreferencedBlobs(repo, gc.opts.Delay, gc.log); err != nil { 150 return err 151 } 152 153 return nil 154 } 155 156 func (gc GarbageCollect) removeManifestsPerRepoPolicy(ctx context.Context, repo string, index *ispec.Index) error { 157 var err error 158 159 /* gc all manifests that have a missing subject, stop when neither gc(referrer and untagged) 160 happened in a full loop over index.json. */ 161 var stop bool 162 for !stop { 163 if zcommon.IsContextDone(ctx) { 164 return ctx.Err() 165 } 166 167 var gcedReferrer bool 168 169 var gcedUntagged bool 170 171 if gc.policyMgr.HasDeleteReferrer(repo) { 172 gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests with missing referrers") 173 174 gcedReferrer, err = gc.removeIndexReferrers(repo, index, *index) 175 if err != nil { 176 return err 177 } 178 } 179 180 if gc.policyMgr.HasDeleteUntagged(repo) { 181 referenced := make(map[godigest.Digest]bool, 0) 182 183 /* gather all manifests referenced in multiarch images/by other manifests 184 so that we can skip them in cleanUntaggedManifests */ 185 if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil { 186 return err 187 } 188 189 // apply image retention policy 190 gcedUntagged, err = gc.removeUntaggedManifests(repo, index, referenced) 191 if err != nil { 192 return err 193 } 194 } 195 196 /* if we gced any manifest then loop again and gc manifests with 197 a subject pointing to the last ones which were gced. */ 198 stop = !gcedReferrer && !gcedUntagged 199 } 200 201 return nil 202 } 203 204 /* 205 garbageCollectIndexReferrers will gc all referrers with a missing subject recursively 206 207 rootIndex is indexJson, need to pass it down to garbageCollectReferrer() 208 rootIndex is the place we look for referrers. 209 */ 210 func (gc GarbageCollect) removeIndexReferrers(repo string, rootIndex *ispec.Index, index ispec.Index, 211 ) (bool, error) { 212 var count int 213 214 var err error 215 216 for _, desc := range index.Manifests { 217 switch desc.MediaType { 218 case ispec.MediaTypeImageIndex: 219 indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log) 220 if err != nil { 221 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()). 222 Msg("failed to read multiarch(index) image") 223 224 return false, err 225 } 226 227 gced, err := gc.removeReferrer(repo, rootIndex, desc, indexImage.Subject, indexImage.ArtifactType) 228 if err != nil { 229 return false, err 230 } 231 232 /* if we gc index then no need to continue searching for referrers inside it. 233 they will be gced when the next garbage collect is executed(if they are older than retentionDelay), 234 because manifests part of indexes will still be referenced in index.json */ 235 if gced { 236 return true, nil 237 } 238 239 gced, err = gc.removeIndexReferrers(repo, rootIndex, indexImage) 240 if err != nil { 241 return false, err 242 } 243 244 if gced { 245 count++ 246 } 247 case ispec.MediaTypeImageManifest, oras.MediaTypeArtifactManifest: 248 image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log) 249 if err != nil { 250 gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).Str("digest", desc.Digest.String()). 251 Msg("failed to read manifest image") 252 253 return false, err 254 } 255 256 artifactType := zcommon.GetManifestArtifactType(image) 257 258 gced, err := gc.removeReferrer(repo, rootIndex, desc, image.Subject, artifactType) 259 if err != nil { 260 return false, err 261 } 262 263 if gced { 264 count++ 265 } 266 } 267 } 268 269 return count > 0, err 270 } 271 272 func (gc GarbageCollect) removeReferrer(repo string, index *ispec.Index, manifestDesc ispec.Descriptor, 273 subject *ispec.Descriptor, artifactType string, 274 ) (bool, error) { 275 var gced bool 276 277 var err error 278 279 if subject != nil { 280 // try to find subject in index.json 281 referenced := isManifestReferencedInIndex(index, subject.Digest) 282 283 var signatureType string 284 // check if its notation or cosign signature 285 if artifactType == zcommon.ArtifactTypeNotation { 286 signatureType = storage.NotationType 287 } else if artifactType == zcommon.ArtifactTypeCosign { 288 signatureType = storage.CosignType 289 } 290 291 if !referenced { 292 gced, err = gc.gcManifest(repo, index, manifestDesc, signatureType, subject.Digest, gc.opts.ImageRetention.Delay) 293 if err != nil { 294 return false, err 295 } 296 297 if gced { 298 gc.log.Info().Str("module", "gc"). 299 Str("repository", repo). 300 Str("reference", manifestDesc.Digest.String()). 301 Str("subject", subject.Digest.String()). 302 Str("decision", "delete"). 303 Str("reason", "deleteReferrers").Msg("removed manifest without reference") 304 305 if gc.auditLog != nil { 306 gc.auditLog.Info().Str("module", "gc"). 307 Str("repository", repo). 308 Str("reference", manifestDesc.Digest.String()). 309 Str("subject", subject.Digest.String()). 310 Str("decision", "delete"). 311 Str("reason", "deleteReferrers").Msg("removed manifest without reference") 312 } 313 } 314 } 315 } 316 317 // cosign 318 tag, ok := getDescriptorTag(manifestDesc) 319 if ok { 320 if isCosignTag(tag) { 321 subjectDigest := getSubjectFromCosignTag(tag) 322 referenced := isManifestReferencedInIndex(index, subjectDigest) 323 324 if !referenced { 325 gced, err = gc.gcManifest(repo, index, manifestDesc, storage.CosignType, subjectDigest, gc.opts.Delay) 326 if err != nil { 327 return false, err 328 } 329 330 if gced { 331 gc.log.Info().Str("module", "gc"). 332 Bool("dry-run", gc.opts.ImageRetention.DryRun). 333 Str("repository", repo). 334 Str("reference", tag). 335 Str("subject", subjectDigest.String()). 336 Str("decision", "delete"). 337 Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference") 338 339 if gc.auditLog != nil { 340 gc.auditLog.Info().Str("module", "gc"). 341 Bool("dry-run", gc.opts.ImageRetention.DryRun). 342 Str("repository", repo). 343 Str("reference", tag). 344 Str("subject", subjectDigest.String()). 345 Str("decision", "delete"). 346 Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference") 347 } 348 } 349 } 350 } 351 } 352 353 return gced, nil 354 } 355 356 func (gc GarbageCollect) removeTagsPerRetentionPolicy(ctx context.Context, repo string, index *ispec.Index) error { 357 if !gc.policyMgr.HasTagRetention(repo) { 358 return nil 359 } 360 361 repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo) 362 if err != nil { 363 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("can't retrieve repoMeta for repo") 364 365 return err 366 } 367 368 retainTags := gc.policyMgr.GetRetainedTags(ctx, repoMeta, *index) 369 370 // remove 371 for _, desc := range index.Manifests { 372 if zcommon.IsContextDone(ctx) { 373 return ctx.Err() 374 } 375 376 // check tag 377 tag, ok := getDescriptorTag(desc) 378 if ok && !zcommon.Contains(retainTags, tag) { 379 // remove tags which should not be retained 380 _, err := gc.removeManifest(repo, index, desc, tag, "", "") 381 if err != nil && !errors.Is(err, zerr.ErrManifestNotFound) { 382 return err 383 } 384 } 385 } 386 387 return nil 388 } 389 390 // gcManifest removes a manifest entry from an index and syncs metaDB accordingly if the blob is older than gc.Delay. 391 func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec.Descriptor, 392 signatureType string, subjectDigest godigest.Digest, delay time.Duration, 393 ) (bool, error) { 394 var gced bool 395 396 canGC, err := isBlobOlderThan(gc.imgStore, repo, desc.Digest, delay, gc.log) 397 if err != nil { 398 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()). 399 Str("delay", delay.String()).Msg("failed to check if blob is older than delay") 400 401 return false, err 402 } 403 404 if canGC { 405 if gced, err = gc.removeManifest(repo, index, desc, desc.Digest.String(), signatureType, subjectDigest); err != nil { 406 return false, err 407 } 408 } 409 410 return gced, nil 411 } 412 413 // removeManifest removes a manifest entry from an index and syncs metaDB accordingly. 414 func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, 415 desc ispec.Descriptor, reference string, signatureType string, subjectDigest godigest.Digest, 416 ) (bool, error) { 417 _, err := common.RemoveManifestDescByReference(index, reference, true) 418 if err != nil { 419 if errors.Is(err, zerr.ErrManifestConflict) { 420 return false, nil 421 } 422 423 return false, err 424 } 425 426 if gc.opts.ImageRetention.DryRun { 427 return true, nil 428 } 429 430 // sync metaDB 431 if gc.metaDB != nil { 432 if signatureType != "" { 433 err = gc.metaDB.DeleteSignature(repo, subjectDigest, mTypes.SignatureMetadata{ 434 SignatureDigest: desc.Digest.String(), 435 SignatureType: signatureType, 436 }) 437 if err != nil { 438 gc.log.Error().Err(err).Str("module", "gc").Msg("metadb: unable to remove signature in metaDB") 439 440 return false, err 441 } 442 } else { 443 err := gc.metaDB.RemoveRepoReference(repo, reference, desc.Digest) 444 if err != nil { 445 gc.log.Error().Err(err).Str("module", "gc").Msg("metadb: unable to remove repo reference in metaDB") 446 447 return false, err 448 } 449 } 450 } 451 452 return true, nil 453 } 454 455 func (gc GarbageCollect) removeUntaggedManifests(repo string, index *ispec.Index, 456 referenced map[godigest.Digest]bool, 457 ) (bool, error) { 458 var gced bool 459 460 var err error 461 462 gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests without tags") 463 464 for _, desc := range index.Manifests { 465 // skip manifests referenced in image indexes 466 if _, referenced := referenced[desc.Digest]; referenced { 467 continue 468 } 469 470 // remove untagged images 471 if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex { 472 _, ok := getDescriptorTag(desc) 473 if !ok { 474 gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.ImageRetention.Delay) 475 if err != nil { 476 return false, err 477 } 478 479 if gced { 480 gc.log.Info().Str("module", "gc"). 481 Bool("dry-run", gc.opts.ImageRetention.DryRun). 482 Str("repository", repo). 483 Str("reference", desc.Digest.String()). 484 Str("decision", "delete"). 485 Str("reason", "deleteUntagged").Msg("removed untagged manifest") 486 487 if gc.auditLog != nil { 488 gc.auditLog.Info().Str("module", "gc"). 489 Bool("dry-run", gc.opts.ImageRetention.DryRun). 490 Str("repository", repo). 491 Str("reference", desc.Digest.String()). 492 Str("decision", "delete"). 493 Str("reason", "deleteUntagged").Msg("removed untagged manifest") 494 } 495 } 496 } 497 } 498 } 499 500 return gced, nil 501 } 502 503 // Adds both referenced manifests and referrers from an index. 504 func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, repo string, 505 referenced map[godigest.Digest]bool, 506 ) error { 507 for _, desc := range index.Manifests { 508 switch desc.MediaType { 509 case ispec.MediaTypeImageIndex: 510 indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log) 511 if err != nil { 512 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). 513 Str("digest", desc.Digest.String()).Msg("failed to read multiarch(index) image") 514 515 return err 516 } 517 518 if indexImage.Subject != nil { 519 referenced[desc.Digest] = true 520 } 521 522 for _, indexDesc := range indexImage.Manifests { 523 referenced[indexDesc.Digest] = true 524 } 525 526 if err := gc.identifyManifestsReferencedInIndex(indexImage, repo, referenced); err != nil { 527 return err 528 } 529 case ispec.MediaTypeImageManifest, oras.MediaTypeArtifactManifest: 530 image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log) 531 if err != nil { 532 gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo). 533 Str("digest", desc.Digest.String()).Msg("failed to read manifest image") 534 535 return err 536 } 537 538 if image.Subject != nil { 539 referenced[desc.Digest] = true 540 } 541 } 542 } 543 544 return nil 545 } 546 547 // removeUnreferencedBlobs gc all blobs which are not referenced by any manifest found in repo's index.json. 548 func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duration, log zlog.Logger, 549 ) error { 550 gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("cleaning orphan blobs") 551 552 refBlobs := map[string]bool{} 553 554 index, err := common.GetIndex(gc.imgStore, repo, gc.log) 555 if err != nil { 556 log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to read index.json in repo") 557 558 return err 559 } 560 561 err = gc.addIndexBlobsToReferences(repo, index, refBlobs) 562 if err != nil { 563 log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to get referenced blobs in repo") 564 565 return err 566 } 567 568 allBlobs, err := gc.imgStore.GetAllBlobs(repo) 569 if err != nil { 570 // /blobs/sha256/ may be empty in the case of s3, no need to return err, we want to skip 571 if errors.As(err, &driver.PathNotFoundError{}) { 572 return nil 573 } 574 575 log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to get all blobs") 576 577 return err 578 } 579 580 gcBlobs := make([]godigest.Digest, 0) 581 582 for _, blob := range allBlobs { 583 digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob) 584 if err = digest.Validate(); err != nil { 585 log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob). 586 Msg("unable to parse digest") 587 588 return err 589 } 590 591 if _, ok := refBlobs[digest.String()]; !ok { 592 canGC, err := isBlobOlderThan(gc.imgStore, repo, digest, delay, log) 593 if err != nil { 594 log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob). 595 Msg("unable to determine GC delay") 596 597 return err 598 } 599 600 if canGC { 601 gcBlobs = append(gcBlobs, digest) 602 } 603 } 604 } 605 606 // if we removed all blobs from repo 607 removeRepo := len(gcBlobs) > 0 && len(gcBlobs) == len(allBlobs) 608 609 reaped, err := gc.imgStore.CleanupRepo(repo, gcBlobs, removeRepo) 610 if err != nil { 611 return err 612 } 613 614 log.Info().Str("module", "gc").Str("repository", repo).Int("count", reaped). 615 Msg("garbage collected blobs") 616 617 return nil 618 } 619 620 // used by removeUnreferencedBlobs() 621 // addIndexBlobsToReferences adds referenced blobs found in referenced manifests (index.json) in refblobs map. 622 func (gc GarbageCollect) addIndexBlobsToReferences(repo string, index ispec.Index, refBlobs map[string]bool, 623 ) error { 624 for _, desc := range index.Manifests { 625 switch desc.MediaType { 626 case ispec.MediaTypeImageIndex: 627 if err := gc.addImageIndexBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { 628 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). 629 Str("digest", desc.Digest.String()).Msg("failed to read blobs in multiarch(index) image") 630 631 return err 632 } 633 case ispec.MediaTypeImageManifest: 634 if err := gc.addImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { 635 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). 636 Str("digest", desc.Digest.String()).Msg("failed to read blobs in image manifest") 637 638 return err 639 } 640 case oras.MediaTypeArtifactManifest: 641 if err := gc.addORASImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { 642 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). 643 Str("digest", desc.Digest.String()).Msg("failed to read blobs in ORAS image manifest") 644 645 return err 646 } 647 } 648 } 649 650 return nil 651 } 652 653 func (gc GarbageCollect) addImageIndexBlobsToReferences(repo string, mdigest godigest.Digest, refBlobs map[string]bool, 654 ) error { 655 index, err := common.GetImageIndex(gc.imgStore, repo, mdigest, gc.log) 656 if err != nil { 657 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", mdigest.String()). 658 Msg("failed to read manifest image") 659 660 return err 661 } 662 663 refBlobs[mdigest.String()] = true 664 665 // if there is a Subject, it may not exist yet and that is ok 666 if index.Subject != nil { 667 refBlobs[index.Subject.Digest.String()] = true 668 } 669 670 for _, manifest := range index.Manifests { 671 refBlobs[manifest.Digest.String()] = true 672 } 673 674 return nil 675 } 676 677 func (gc GarbageCollect) addImageManifestBlobsToReferences(repo string, mdigest godigest.Digest, 678 refBlobs map[string]bool, 679 ) error { 680 manifestContent, err := common.GetImageManifest(gc.imgStore, repo, mdigest, gc.log) 681 if err != nil { 682 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). 683 Str("digest", mdigest.String()).Msg("failed to read manifest image") 684 685 return err 686 } 687 688 refBlobs[mdigest.String()] = true 689 refBlobs[manifestContent.Config.Digest.String()] = true 690 691 // if there is a Subject, it may not exist yet and that is ok 692 if manifestContent.Subject != nil { 693 refBlobs[manifestContent.Subject.Digest.String()] = true 694 } 695 696 for _, layer := range manifestContent.Layers { 697 refBlobs[layer.Digest.String()] = true 698 } 699 700 return nil 701 } 702 703 func (gc GarbageCollect) addORASImageManifestBlobsToReferences(repo string, mdigest godigest.Digest, 704 refBlobs map[string]bool, 705 ) error { 706 manifestContent, err := common.GetOrasManifestByDigest(gc.imgStore, repo, mdigest, gc.log) 707 if err != nil { 708 gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). 709 Str("digest", mdigest.String()).Msg("failed to read manifest image") 710 711 return err 712 } 713 714 refBlobs[mdigest.String()] = true 715 716 // if there is a Subject, it may not exist yet and that is ok 717 if manifestContent.Subject != nil { 718 refBlobs[manifestContent.Subject.Digest.String()] = true 719 } 720 721 for _, blob := range manifestContent.Blobs { 722 refBlobs[blob.Digest.String()] = true 723 } 724 725 return nil 726 } 727 728 func isManifestReferencedInIndex(index *ispec.Index, digest godigest.Digest) bool { 729 for _, manifest := range index.Manifests { 730 if manifest.Digest == digest { 731 return true 732 } 733 } 734 735 return false 736 } 737 738 func isBlobOlderThan(imgStore types.ImageStore, repo string, 739 digest godigest.Digest, delay time.Duration, log zlog.Logger, 740 ) (bool, error) { 741 _, _, modtime, err := imgStore.StatBlob(repo, digest) 742 if err != nil { 743 log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", digest.String()). 744 Msg("failed to stat blob") 745 746 return false, err 747 } 748 749 if modtime.Add(delay).After(time.Now()) { 750 return false, nil 751 } 752 753 return true, nil 754 } 755 756 func getSubjectFromCosignTag(tag string) godigest.Digest { 757 alg := strings.Split(tag, "-")[0] 758 encoded := strings.Split(strings.Split(tag, "-")[1], ".sig")[0] 759 760 return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded) 761 } 762 763 func getDescriptorTag(desc ispec.Descriptor) (string, bool) { 764 tag, ok := desc.Annotations[ispec.AnnotationRefName] 765 766 return tag, ok 767 } 768 769 // this function will check if tag is a cosign tag (signature or sbom). 770 func isCosignTag(tag string) bool { 771 if strings.HasPrefix(tag, "sha256-") && 772 (strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) { 773 return true 774 } 775 776 return false 777 } 778 779 /* 780 GCTaskGenerator takes all repositories found in the storage.imagestore 781 782 and it will execute garbage collection for each repository by creating a task 783 for each repository and pushing it to the task scheduler. 784 */ 785 type GCTaskGenerator struct { 786 imgStore types.ImageStore 787 gc GarbageCollect 788 lastRepo string 789 nextRun time.Time 790 done bool 791 rand *rand.Rand 792 } 793 794 func (gen *GCTaskGenerator) getRandomDelay() int { 795 maxDelay := 30 796 797 return gen.rand.Intn(maxDelay) 798 } 799 800 func (gen *GCTaskGenerator) Next() (scheduler.Task, error) { 801 if gen.lastRepo == "" && gen.nextRun.IsZero() { 802 gen.rand = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) //nolint: gosec 803 } 804 805 delay := gen.getRandomDelay() 806 807 gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second) 808 809 repo, err := gen.imgStore.GetNextRepository(gen.lastRepo) 810 if err != nil { 811 return nil, err 812 } 813 814 if repo == "" { 815 gen.done = true 816 817 return nil, nil 818 } 819 820 gen.lastRepo = repo 821 822 return NewGCTask(gen.imgStore, gen.gc, repo), nil 823 } 824 825 func (gen *GCTaskGenerator) IsDone() bool { 826 return gen.done 827 } 828 829 func (gen *GCTaskGenerator) IsReady() bool { 830 return time.Now().After(gen.nextRun) 831 } 832 833 func (gen *GCTaskGenerator) Reset() { 834 gen.lastRepo = "" 835 gen.done = false 836 gen.nextRun = time.Time{} 837 } 838 839 type gcTask struct { 840 imgStore types.ImageStore 841 gc GarbageCollect 842 repo string 843 } 844 845 func NewGCTask(imgStore types.ImageStore, gc GarbageCollect, repo string, 846 ) *gcTask { 847 return &gcTask{imgStore, gc, repo} 848 } 849 850 func (gct *gcTask) DoWork(ctx context.Context) error { 851 // run task 852 return gct.gc.CleanRepo(ctx, gct.repo) //nolint: contextcheck 853 }