zotregistry.io/zot@v1.4.4-0.20231124084042-02a8ed785457/pkg/storage/gc/gc.go (about)

     1  package gc
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"math/rand"
     8  	"path"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/docker/distribution/registry/storage/driver"
    13  	godigest "github.com/opencontainers/go-digest"
    14  	ispec "github.com/opencontainers/image-spec/specs-go/v1"
    15  	oras "github.com/oras-project/artifacts-spec/specs-go/v1"
    16  
    17  	zerr "zotregistry.io/zot/errors"
    18  	"zotregistry.io/zot/pkg/api/config"
    19  	zcommon "zotregistry.io/zot/pkg/common"
    20  	zlog "zotregistry.io/zot/pkg/log"
    21  	mTypes "zotregistry.io/zot/pkg/meta/types"
    22  	"zotregistry.io/zot/pkg/retention"
    23  	rTypes "zotregistry.io/zot/pkg/retention/types"
    24  	"zotregistry.io/zot/pkg/scheduler"
    25  	"zotregistry.io/zot/pkg/storage"
    26  	common "zotregistry.io/zot/pkg/storage/common"
    27  	"zotregistry.io/zot/pkg/storage/types"
    28  )
    29  
    30  const (
    31  	cosignSignatureTagSuffix = "sig"
    32  	SBOMTagSuffix            = "sbom"
    33  )
    34  
    35  type Options struct {
    36  	// will garbage collect blobs older than Delay
    37  	Delay time.Duration
    38  
    39  	ImageRetention config.ImageRetention
    40  }
    41  
    42  type GarbageCollect struct {
    43  	imgStore  types.ImageStore
    44  	opts      Options
    45  	metaDB    mTypes.MetaDB
    46  	policyMgr rTypes.PolicyManager
    47  	auditLog  *zlog.Logger
    48  	log       zlog.Logger
    49  }
    50  
    51  func NewGarbageCollect(imgStore types.ImageStore, metaDB mTypes.MetaDB, opts Options,
    52  	auditLog *zlog.Logger, log zlog.Logger,
    53  ) GarbageCollect {
    54  	return GarbageCollect{
    55  		imgStore:  imgStore,
    56  		metaDB:    metaDB,
    57  		opts:      opts,
    58  		policyMgr: retention.NewPolicyManager(opts.ImageRetention, log, auditLog),
    59  		auditLog:  auditLog,
    60  		log:       log,
    61  	}
    62  }
    63  
    64  /*
    65  CleanImageStorePeriodically runs a periodic garbage collect on the ImageStore provided in constructor,
    66  given an interval and a Scheduler.
    67  */
    68  func (gc GarbageCollect) CleanImageStorePeriodically(interval time.Duration, sch *scheduler.Scheduler) {
    69  	generator := &GCTaskGenerator{
    70  		imgStore: gc.imgStore,
    71  		gc:       gc,
    72  	}
    73  
    74  	sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
    75  }
    76  
    77  /*
    78  CleanRepo executes a garbage collection of any blob found in storage which is not referenced
    79  in any manifests referenced in repo's index.json
    80  It also gc referrers with missing subject if the Referrer Option is enabled
    81  It also gc untagged manifests.
    82  */
    83  func (gc GarbageCollect) CleanRepo(ctx context.Context, repo string) error {
    84  	gc.log.Info().Str("module", "gc").
    85  		Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(gc.imgStore.RootDir(), repo)))
    86  
    87  	if err := gc.cleanRepo(ctx, repo); err != nil {
    88  		errMessage := fmt.Sprintf("error while running GC for %s", path.Join(gc.imgStore.RootDir(), repo))
    89  		gc.log.Error().Err(err).Str("module", "gc").Msg(errMessage)
    90  		gc.log.Info().Str("module", "gc").
    91  			Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(gc.imgStore.RootDir(), repo)))
    92  
    93  		return err
    94  	}
    95  
    96  	gc.log.Info().Str("module", "gc").
    97  		Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(gc.imgStore.RootDir(), repo)))
    98  
    99  	return nil
   100  }
   101  
   102  func (gc GarbageCollect) cleanRepo(ctx context.Context, repo string) error {
   103  	var lockLatency time.Time
   104  
   105  	dir := path.Join(gc.imgStore.RootDir(), repo)
   106  	if !gc.imgStore.DirExists(dir) {
   107  		return zerr.ErrRepoNotFound
   108  	}
   109  
   110  	gc.imgStore.Lock(&lockLatency)
   111  	defer gc.imgStore.Unlock(&lockLatency)
   112  
   113  	/* this index (which represents the index.json of this repo) is the root point from which we
   114  	search for dangling manifests/blobs
   115  	so this index is passed by reference in all functions that modifies it
   116  
   117  	Instead of removing manifests one by one with storage APIs we just remove manifests descriptors
   118  	from index.Manifests[] list and update repo's index.json afterwards.
   119  
   120  	After updating repo's index.json we clean all unreferenced blobs (manifests included).
   121  	*/
   122  	index, err := common.GetIndex(gc.imgStore, repo, gc.log)
   123  	if err != nil {
   124  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to read index.json in repo")
   125  
   126  		return err
   127  	}
   128  
   129  	// apply tags retention
   130  	if err := gc.removeTagsPerRetentionPolicy(ctx, repo, &index); err != nil {
   131  		return err
   132  	}
   133  
   134  	// gc referrers manifests with missing subject and untagged manifests
   135  	if err := gc.removeManifestsPerRepoPolicy(ctx, repo, &index); err != nil {
   136  		return err
   137  	}
   138  
   139  	// update repos's index.json in storage
   140  	if !gc.opts.ImageRetention.DryRun {
   141  		/* this will update the index.json with manifests deleted above
   142  		and the manifests blobs will be removed by gc.removeUnreferencedBlobs()*/
   143  		if err := gc.imgStore.PutIndexContent(repo, index); err != nil {
   144  			return err
   145  		}
   146  	}
   147  
   148  	// gc unreferenced blobs
   149  	if err := gc.removeUnreferencedBlobs(repo, gc.opts.Delay, gc.log); err != nil {
   150  		return err
   151  	}
   152  
   153  	return nil
   154  }
   155  
   156  func (gc GarbageCollect) removeManifestsPerRepoPolicy(ctx context.Context, repo string, index *ispec.Index) error {
   157  	var err error
   158  
   159  	/* gc all manifests that have a missing subject, stop when neither gc(referrer and untagged)
   160  	happened in a full loop over index.json. */
   161  	var stop bool
   162  	for !stop {
   163  		if zcommon.IsContextDone(ctx) {
   164  			return ctx.Err()
   165  		}
   166  
   167  		var gcedReferrer bool
   168  
   169  		var gcedUntagged bool
   170  
   171  		if gc.policyMgr.HasDeleteReferrer(repo) {
   172  			gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests with missing referrers")
   173  
   174  			gcedReferrer, err = gc.removeIndexReferrers(repo, index, *index)
   175  			if err != nil {
   176  				return err
   177  			}
   178  		}
   179  
   180  		if gc.policyMgr.HasDeleteUntagged(repo) {
   181  			referenced := make(map[godigest.Digest]bool, 0)
   182  
   183  			/* gather all manifests referenced in multiarch images/by other manifests
   184  			so that we can skip them in cleanUntaggedManifests */
   185  			if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil {
   186  				return err
   187  			}
   188  
   189  			// apply image retention policy
   190  			gcedUntagged, err = gc.removeUntaggedManifests(repo, index, referenced)
   191  			if err != nil {
   192  				return err
   193  			}
   194  		}
   195  
   196  		/* if we gced any manifest then loop again and gc manifests with
   197  		a subject pointing to the last ones which were gced. */
   198  		stop = !gcedReferrer && !gcedUntagged
   199  	}
   200  
   201  	return nil
   202  }
   203  
   204  /*
   205  garbageCollectIndexReferrers will gc all referrers with a missing subject recursively
   206  
   207  rootIndex is indexJson, need to pass it down to garbageCollectReferrer()
   208  rootIndex is the place we look for referrers.
   209  */
   210  func (gc GarbageCollect) removeIndexReferrers(repo string, rootIndex *ispec.Index, index ispec.Index,
   211  ) (bool, error) {
   212  	var count int
   213  
   214  	var err error
   215  
   216  	for _, desc := range index.Manifests {
   217  		switch desc.MediaType {
   218  		case ispec.MediaTypeImageIndex:
   219  			indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log)
   220  			if err != nil {
   221  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()).
   222  					Msg("failed to read multiarch(index) image")
   223  
   224  				return false, err
   225  			}
   226  
   227  			gced, err := gc.removeReferrer(repo, rootIndex, desc, indexImage.Subject, indexImage.ArtifactType)
   228  			if err != nil {
   229  				return false, err
   230  			}
   231  
   232  			/* if we gc index then no need to continue searching for referrers inside it.
   233  			they will be gced when the next garbage collect is executed(if they are older than retentionDelay),
   234  			 because manifests part of indexes will still be referenced in index.json */
   235  			if gced {
   236  				return true, nil
   237  			}
   238  
   239  			gced, err = gc.removeIndexReferrers(repo, rootIndex, indexImage)
   240  			if err != nil {
   241  				return false, err
   242  			}
   243  
   244  			if gced {
   245  				count++
   246  			}
   247  		case ispec.MediaTypeImageManifest, oras.MediaTypeArtifactManifest:
   248  			image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log)
   249  			if err != nil {
   250  				gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).Str("digest", desc.Digest.String()).
   251  					Msg("failed to read manifest image")
   252  
   253  				return false, err
   254  			}
   255  
   256  			artifactType := zcommon.GetManifestArtifactType(image)
   257  
   258  			gced, err := gc.removeReferrer(repo, rootIndex, desc, image.Subject, artifactType)
   259  			if err != nil {
   260  				return false, err
   261  			}
   262  
   263  			if gced {
   264  				count++
   265  			}
   266  		}
   267  	}
   268  
   269  	return count > 0, err
   270  }
   271  
   272  func (gc GarbageCollect) removeReferrer(repo string, index *ispec.Index, manifestDesc ispec.Descriptor,
   273  	subject *ispec.Descriptor, artifactType string,
   274  ) (bool, error) {
   275  	var gced bool
   276  
   277  	var err error
   278  
   279  	if subject != nil {
   280  		// try to find subject in index.json
   281  		referenced := isManifestReferencedInIndex(index, subject.Digest)
   282  
   283  		var signatureType string
   284  		// check if its notation or cosign signature
   285  		if artifactType == zcommon.ArtifactTypeNotation {
   286  			signatureType = storage.NotationType
   287  		} else if artifactType == zcommon.ArtifactTypeCosign {
   288  			signatureType = storage.CosignType
   289  		}
   290  
   291  		if !referenced {
   292  			gced, err = gc.gcManifest(repo, index, manifestDesc, signatureType, subject.Digest, gc.opts.ImageRetention.Delay)
   293  			if err != nil {
   294  				return false, err
   295  			}
   296  
   297  			if gced {
   298  				gc.log.Info().Str("module", "gc").
   299  					Str("repository", repo).
   300  					Str("reference", manifestDesc.Digest.String()).
   301  					Str("subject", subject.Digest.String()).
   302  					Str("decision", "delete").
   303  					Str("reason", "deleteReferrers").Msg("removed manifest without reference")
   304  
   305  				if gc.auditLog != nil {
   306  					gc.auditLog.Info().Str("module", "gc").
   307  						Str("repository", repo).
   308  						Str("reference", manifestDesc.Digest.String()).
   309  						Str("subject", subject.Digest.String()).
   310  						Str("decision", "delete").
   311  						Str("reason", "deleteReferrers").Msg("removed manifest without reference")
   312  				}
   313  			}
   314  		}
   315  	}
   316  
   317  	// cosign
   318  	tag, ok := getDescriptorTag(manifestDesc)
   319  	if ok {
   320  		if isCosignTag(tag) {
   321  			subjectDigest := getSubjectFromCosignTag(tag)
   322  			referenced := isManifestReferencedInIndex(index, subjectDigest)
   323  
   324  			if !referenced {
   325  				gced, err = gc.gcManifest(repo, index, manifestDesc, storage.CosignType, subjectDigest, gc.opts.Delay)
   326  				if err != nil {
   327  					return false, err
   328  				}
   329  
   330  				if gced {
   331  					gc.log.Info().Str("module", "gc").
   332  						Bool("dry-run", gc.opts.ImageRetention.DryRun).
   333  						Str("repository", repo).
   334  						Str("reference", tag).
   335  						Str("subject", subjectDigest.String()).
   336  						Str("decision", "delete").
   337  						Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference")
   338  
   339  					if gc.auditLog != nil {
   340  						gc.auditLog.Info().Str("module", "gc").
   341  							Bool("dry-run", gc.opts.ImageRetention.DryRun).
   342  							Str("repository", repo).
   343  							Str("reference", tag).
   344  							Str("subject", subjectDigest.String()).
   345  							Str("decision", "delete").
   346  							Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference")
   347  					}
   348  				}
   349  			}
   350  		}
   351  	}
   352  
   353  	return gced, nil
   354  }
   355  
   356  func (gc GarbageCollect) removeTagsPerRetentionPolicy(ctx context.Context, repo string, index *ispec.Index) error {
   357  	if !gc.policyMgr.HasTagRetention(repo) {
   358  		return nil
   359  	}
   360  
   361  	repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo)
   362  	if err != nil {
   363  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("can't retrieve repoMeta for repo")
   364  
   365  		return err
   366  	}
   367  
   368  	retainTags := gc.policyMgr.GetRetainedTags(ctx, repoMeta, *index)
   369  
   370  	// remove
   371  	for _, desc := range index.Manifests {
   372  		if zcommon.IsContextDone(ctx) {
   373  			return ctx.Err()
   374  		}
   375  
   376  		// check tag
   377  		tag, ok := getDescriptorTag(desc)
   378  		if ok && !zcommon.Contains(retainTags, tag) {
   379  			// remove tags which should not be retained
   380  			_, err := gc.removeManifest(repo, index, desc, tag, "", "")
   381  			if err != nil && !errors.Is(err, zerr.ErrManifestNotFound) {
   382  				return err
   383  			}
   384  		}
   385  	}
   386  
   387  	return nil
   388  }
   389  
   390  // gcManifest removes a manifest entry from an index and syncs metaDB accordingly if the blob is older than gc.Delay.
   391  func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec.Descriptor,
   392  	signatureType string, subjectDigest godigest.Digest, delay time.Duration,
   393  ) (bool, error) {
   394  	var gced bool
   395  
   396  	canGC, err := isBlobOlderThan(gc.imgStore, repo, desc.Digest, delay, gc.log)
   397  	if err != nil {
   398  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()).
   399  			Str("delay", delay.String()).Msg("failed to check if blob is older than delay")
   400  
   401  		return false, err
   402  	}
   403  
   404  	if canGC {
   405  		if gced, err = gc.removeManifest(repo, index, desc, desc.Digest.String(), signatureType, subjectDigest); err != nil {
   406  			return false, err
   407  		}
   408  	}
   409  
   410  	return gced, nil
   411  }
   412  
   413  // removeManifest removes a manifest entry from an index and syncs metaDB accordingly.
   414  func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index,
   415  	desc ispec.Descriptor, reference string, signatureType string, subjectDigest godigest.Digest,
   416  ) (bool, error) {
   417  	_, err := common.RemoveManifestDescByReference(index, reference, true)
   418  	if err != nil {
   419  		if errors.Is(err, zerr.ErrManifestConflict) {
   420  			return false, nil
   421  		}
   422  
   423  		return false, err
   424  	}
   425  
   426  	if gc.opts.ImageRetention.DryRun {
   427  		return true, nil
   428  	}
   429  
   430  	// sync metaDB
   431  	if gc.metaDB != nil {
   432  		if signatureType != "" {
   433  			err = gc.metaDB.DeleteSignature(repo, subjectDigest, mTypes.SignatureMetadata{
   434  				SignatureDigest: desc.Digest.String(),
   435  				SignatureType:   signatureType,
   436  			})
   437  			if err != nil {
   438  				gc.log.Error().Err(err).Str("module", "gc").Msg("metadb: unable to remove signature in metaDB")
   439  
   440  				return false, err
   441  			}
   442  		} else {
   443  			err := gc.metaDB.RemoveRepoReference(repo, reference, desc.Digest)
   444  			if err != nil {
   445  				gc.log.Error().Err(err).Str("module", "gc").Msg("metadb: unable to remove repo reference in metaDB")
   446  
   447  				return false, err
   448  			}
   449  		}
   450  	}
   451  
   452  	return true, nil
   453  }
   454  
   455  func (gc GarbageCollect) removeUntaggedManifests(repo string, index *ispec.Index,
   456  	referenced map[godigest.Digest]bool,
   457  ) (bool, error) {
   458  	var gced bool
   459  
   460  	var err error
   461  
   462  	gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests without tags")
   463  
   464  	for _, desc := range index.Manifests {
   465  		// skip manifests referenced in image indexes
   466  		if _, referenced := referenced[desc.Digest]; referenced {
   467  			continue
   468  		}
   469  
   470  		// remove untagged images
   471  		if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex {
   472  			_, ok := getDescriptorTag(desc)
   473  			if !ok {
   474  				gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.ImageRetention.Delay)
   475  				if err != nil {
   476  					return false, err
   477  				}
   478  
   479  				if gced {
   480  					gc.log.Info().Str("module", "gc").
   481  						Bool("dry-run", gc.opts.ImageRetention.DryRun).
   482  						Str("repository", repo).
   483  						Str("reference", desc.Digest.String()).
   484  						Str("decision", "delete").
   485  						Str("reason", "deleteUntagged").Msg("removed untagged manifest")
   486  
   487  					if gc.auditLog != nil {
   488  						gc.auditLog.Info().Str("module", "gc").
   489  							Bool("dry-run", gc.opts.ImageRetention.DryRun).
   490  							Str("repository", repo).
   491  							Str("reference", desc.Digest.String()).
   492  							Str("decision", "delete").
   493  							Str("reason", "deleteUntagged").Msg("removed untagged manifest")
   494  					}
   495  				}
   496  			}
   497  		}
   498  	}
   499  
   500  	return gced, nil
   501  }
   502  
   503  // Adds both referenced manifests and referrers from an index.
   504  func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, repo string,
   505  	referenced map[godigest.Digest]bool,
   506  ) error {
   507  	for _, desc := range index.Manifests {
   508  		switch desc.MediaType {
   509  		case ispec.MediaTypeImageIndex:
   510  			indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log)
   511  			if err != nil {
   512  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   513  					Str("digest", desc.Digest.String()).Msg("failed to read multiarch(index) image")
   514  
   515  				return err
   516  			}
   517  
   518  			if indexImage.Subject != nil {
   519  				referenced[desc.Digest] = true
   520  			}
   521  
   522  			for _, indexDesc := range indexImage.Manifests {
   523  				referenced[indexDesc.Digest] = true
   524  			}
   525  
   526  			if err := gc.identifyManifestsReferencedInIndex(indexImage, repo, referenced); err != nil {
   527  				return err
   528  			}
   529  		case ispec.MediaTypeImageManifest, oras.MediaTypeArtifactManifest:
   530  			image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log)
   531  			if err != nil {
   532  				gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).
   533  					Str("digest", desc.Digest.String()).Msg("failed to read manifest image")
   534  
   535  				return err
   536  			}
   537  
   538  			if image.Subject != nil {
   539  				referenced[desc.Digest] = true
   540  			}
   541  		}
   542  	}
   543  
   544  	return nil
   545  }
   546  
   547  // removeUnreferencedBlobs gc all blobs which are not referenced by any manifest found in repo's index.json.
   548  func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duration, log zlog.Logger,
   549  ) error {
   550  	gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("cleaning orphan blobs")
   551  
   552  	refBlobs := map[string]bool{}
   553  
   554  	index, err := common.GetIndex(gc.imgStore, repo, gc.log)
   555  	if err != nil {
   556  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to read index.json in repo")
   557  
   558  		return err
   559  	}
   560  
   561  	err = gc.addIndexBlobsToReferences(repo, index, refBlobs)
   562  	if err != nil {
   563  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to get referenced blobs in repo")
   564  
   565  		return err
   566  	}
   567  
   568  	allBlobs, err := gc.imgStore.GetAllBlobs(repo)
   569  	if err != nil {
   570  		// /blobs/sha256/ may be empty in the case of s3, no need to return err, we want to skip
   571  		if errors.As(err, &driver.PathNotFoundError{}) {
   572  			return nil
   573  		}
   574  
   575  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to get all blobs")
   576  
   577  		return err
   578  	}
   579  
   580  	gcBlobs := make([]godigest.Digest, 0)
   581  
   582  	for _, blob := range allBlobs {
   583  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob)
   584  		if err = digest.Validate(); err != nil {
   585  			log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob).
   586  				Msg("unable to parse digest")
   587  
   588  			return err
   589  		}
   590  
   591  		if _, ok := refBlobs[digest.String()]; !ok {
   592  			canGC, err := isBlobOlderThan(gc.imgStore, repo, digest, delay, log)
   593  			if err != nil {
   594  				log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob).
   595  					Msg("unable to determine GC delay")
   596  
   597  				return err
   598  			}
   599  
   600  			if canGC {
   601  				gcBlobs = append(gcBlobs, digest)
   602  			}
   603  		}
   604  	}
   605  
   606  	// if we removed all blobs from repo
   607  	removeRepo := len(gcBlobs) > 0 && len(gcBlobs) == len(allBlobs)
   608  
   609  	reaped, err := gc.imgStore.CleanupRepo(repo, gcBlobs, removeRepo)
   610  	if err != nil {
   611  		return err
   612  	}
   613  
   614  	log.Info().Str("module", "gc").Str("repository", repo).Int("count", reaped).
   615  		Msg("garbage collected blobs")
   616  
   617  	return nil
   618  }
   619  
   620  // used by removeUnreferencedBlobs()
   621  // addIndexBlobsToReferences adds referenced blobs found in referenced manifests (index.json) in refblobs map.
   622  func (gc GarbageCollect) addIndexBlobsToReferences(repo string, index ispec.Index, refBlobs map[string]bool,
   623  ) error {
   624  	for _, desc := range index.Manifests {
   625  		switch desc.MediaType {
   626  		case ispec.MediaTypeImageIndex:
   627  			if err := gc.addImageIndexBlobsToReferences(repo, desc.Digest, refBlobs); err != nil {
   628  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   629  					Str("digest", desc.Digest.String()).Msg("failed to read blobs in multiarch(index) image")
   630  
   631  				return err
   632  			}
   633  		case ispec.MediaTypeImageManifest:
   634  			if err := gc.addImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil {
   635  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   636  					Str("digest", desc.Digest.String()).Msg("failed to read blobs in image manifest")
   637  
   638  				return err
   639  			}
   640  		case oras.MediaTypeArtifactManifest:
   641  			if err := gc.addORASImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil {
   642  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   643  					Str("digest", desc.Digest.String()).Msg("failed to read blobs in ORAS image manifest")
   644  
   645  				return err
   646  			}
   647  		}
   648  	}
   649  
   650  	return nil
   651  }
   652  
   653  func (gc GarbageCollect) addImageIndexBlobsToReferences(repo string, mdigest godigest.Digest, refBlobs map[string]bool,
   654  ) error {
   655  	index, err := common.GetImageIndex(gc.imgStore, repo, mdigest, gc.log)
   656  	if err != nil {
   657  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", mdigest.String()).
   658  			Msg("failed to read manifest image")
   659  
   660  		return err
   661  	}
   662  
   663  	refBlobs[mdigest.String()] = true
   664  
   665  	// if there is a Subject, it may not exist yet and that is ok
   666  	if index.Subject != nil {
   667  		refBlobs[index.Subject.Digest.String()] = true
   668  	}
   669  
   670  	for _, manifest := range index.Manifests {
   671  		refBlobs[manifest.Digest.String()] = true
   672  	}
   673  
   674  	return nil
   675  }
   676  
   677  func (gc GarbageCollect) addImageManifestBlobsToReferences(repo string, mdigest godigest.Digest,
   678  	refBlobs map[string]bool,
   679  ) error {
   680  	manifestContent, err := common.GetImageManifest(gc.imgStore, repo, mdigest, gc.log)
   681  	if err != nil {
   682  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   683  			Str("digest", mdigest.String()).Msg("failed to read manifest image")
   684  
   685  		return err
   686  	}
   687  
   688  	refBlobs[mdigest.String()] = true
   689  	refBlobs[manifestContent.Config.Digest.String()] = true
   690  
   691  	// if there is a Subject, it may not exist yet and that is ok
   692  	if manifestContent.Subject != nil {
   693  		refBlobs[manifestContent.Subject.Digest.String()] = true
   694  	}
   695  
   696  	for _, layer := range manifestContent.Layers {
   697  		refBlobs[layer.Digest.String()] = true
   698  	}
   699  
   700  	return nil
   701  }
   702  
   703  func (gc GarbageCollect) addORASImageManifestBlobsToReferences(repo string, mdigest godigest.Digest,
   704  	refBlobs map[string]bool,
   705  ) error {
   706  	manifestContent, err := common.GetOrasManifestByDigest(gc.imgStore, repo, mdigest, gc.log)
   707  	if err != nil {
   708  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   709  			Str("digest", mdigest.String()).Msg("failed to read manifest image")
   710  
   711  		return err
   712  	}
   713  
   714  	refBlobs[mdigest.String()] = true
   715  
   716  	// if there is a Subject, it may not exist yet and that is ok
   717  	if manifestContent.Subject != nil {
   718  		refBlobs[manifestContent.Subject.Digest.String()] = true
   719  	}
   720  
   721  	for _, blob := range manifestContent.Blobs {
   722  		refBlobs[blob.Digest.String()] = true
   723  	}
   724  
   725  	return nil
   726  }
   727  
   728  func isManifestReferencedInIndex(index *ispec.Index, digest godigest.Digest) bool {
   729  	for _, manifest := range index.Manifests {
   730  		if manifest.Digest == digest {
   731  			return true
   732  		}
   733  	}
   734  
   735  	return false
   736  }
   737  
   738  func isBlobOlderThan(imgStore types.ImageStore, repo string,
   739  	digest godigest.Digest, delay time.Duration, log zlog.Logger,
   740  ) (bool, error) {
   741  	_, _, modtime, err := imgStore.StatBlob(repo, digest)
   742  	if err != nil {
   743  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", digest.String()).
   744  			Msg("failed to stat blob")
   745  
   746  		return false, err
   747  	}
   748  
   749  	if modtime.Add(delay).After(time.Now()) {
   750  		return false, nil
   751  	}
   752  
   753  	return true, nil
   754  }
   755  
   756  func getSubjectFromCosignTag(tag string) godigest.Digest {
   757  	alg := strings.Split(tag, "-")[0]
   758  	encoded := strings.Split(strings.Split(tag, "-")[1], ".sig")[0]
   759  
   760  	return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded)
   761  }
   762  
   763  func getDescriptorTag(desc ispec.Descriptor) (string, bool) {
   764  	tag, ok := desc.Annotations[ispec.AnnotationRefName]
   765  
   766  	return tag, ok
   767  }
   768  
   769  // this function will check if tag is a cosign tag (signature or sbom).
   770  func isCosignTag(tag string) bool {
   771  	if strings.HasPrefix(tag, "sha256-") &&
   772  		(strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) {
   773  		return true
   774  	}
   775  
   776  	return false
   777  }
   778  
   779  /*
   780  	GCTaskGenerator takes all repositories found in the storage.imagestore
   781  
   782  and it will execute garbage collection for each repository by creating a task
   783  for each repository and pushing it to the task scheduler.
   784  */
   785  type GCTaskGenerator struct {
   786  	imgStore types.ImageStore
   787  	gc       GarbageCollect
   788  	lastRepo string
   789  	nextRun  time.Time
   790  	done     bool
   791  	rand     *rand.Rand
   792  }
   793  
   794  func (gen *GCTaskGenerator) getRandomDelay() int {
   795  	maxDelay := 30
   796  
   797  	return gen.rand.Intn(maxDelay)
   798  }
   799  
   800  func (gen *GCTaskGenerator) Next() (scheduler.Task, error) {
   801  	if gen.lastRepo == "" && gen.nextRun.IsZero() {
   802  		gen.rand = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) //nolint: gosec
   803  	}
   804  
   805  	delay := gen.getRandomDelay()
   806  
   807  	gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second)
   808  
   809  	repo, err := gen.imgStore.GetNextRepository(gen.lastRepo)
   810  	if err != nil {
   811  		return nil, err
   812  	}
   813  
   814  	if repo == "" {
   815  		gen.done = true
   816  
   817  		return nil, nil
   818  	}
   819  
   820  	gen.lastRepo = repo
   821  
   822  	return NewGCTask(gen.imgStore, gen.gc, repo), nil
   823  }
   824  
   825  func (gen *GCTaskGenerator) IsDone() bool {
   826  	return gen.done
   827  }
   828  
   829  func (gen *GCTaskGenerator) IsReady() bool {
   830  	return time.Now().After(gen.nextRun)
   831  }
   832  
   833  func (gen *GCTaskGenerator) Reset() {
   834  	gen.lastRepo = ""
   835  	gen.done = false
   836  	gen.nextRun = time.Time{}
   837  }
   838  
   839  type gcTask struct {
   840  	imgStore types.ImageStore
   841  	gc       GarbageCollect
   842  	repo     string
   843  }
   844  
   845  func NewGCTask(imgStore types.ImageStore, gc GarbageCollect, repo string,
   846  ) *gcTask {
   847  	return &gcTask{imgStore, gc, repo}
   848  }
   849  
   850  func (gct *gcTask) DoWork(ctx context.Context) error {
   851  	// run task
   852  	return gct.gc.CleanRepo(ctx, gct.repo) //nolint: contextcheck
   853  }