zotregistry.dev/zot@v1.4.4-0.20240314164342-eec277e14d20/pkg/storage/gc/gc.go (about)

     1  package gc
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"math/rand"
     8  	"path"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/docker/distribution/registry/storage/driver"
    13  	godigest "github.com/opencontainers/go-digest"
    14  	ispec "github.com/opencontainers/image-spec/specs-go/v1"
    15  
    16  	zerr "zotregistry.dev/zot/errors"
    17  	"zotregistry.dev/zot/pkg/api/config"
    18  	zcommon "zotregistry.dev/zot/pkg/common"
    19  	zlog "zotregistry.dev/zot/pkg/log"
    20  	mTypes "zotregistry.dev/zot/pkg/meta/types"
    21  	"zotregistry.dev/zot/pkg/retention"
    22  	rTypes "zotregistry.dev/zot/pkg/retention/types"
    23  	"zotregistry.dev/zot/pkg/scheduler"
    24  	"zotregistry.dev/zot/pkg/storage"
    25  	common "zotregistry.dev/zot/pkg/storage/common"
    26  	"zotregistry.dev/zot/pkg/storage/types"
    27  )
    28  
    29  const (
    30  	cosignSignatureTagSuffix = "sig"
    31  	SBOMTagSuffix            = "sbom"
    32  )
    33  
    34  type Options struct {
    35  	// will garbage collect blobs older than Delay
    36  	Delay time.Duration
    37  
    38  	ImageRetention config.ImageRetention
    39  }
    40  
    41  type GarbageCollect struct {
    42  	imgStore  types.ImageStore
    43  	opts      Options
    44  	metaDB    mTypes.MetaDB
    45  	policyMgr rTypes.PolicyManager
    46  	auditLog  *zlog.Logger
    47  	log       zlog.Logger
    48  }
    49  
    50  func NewGarbageCollect(imgStore types.ImageStore, metaDB mTypes.MetaDB, opts Options,
    51  	auditLog *zlog.Logger, log zlog.Logger,
    52  ) GarbageCollect {
    53  	return GarbageCollect{
    54  		imgStore:  imgStore,
    55  		metaDB:    metaDB,
    56  		opts:      opts,
    57  		policyMgr: retention.NewPolicyManager(opts.ImageRetention, log, auditLog),
    58  		auditLog:  auditLog,
    59  		log:       log,
    60  	}
    61  }
    62  
    63  /*
    64  CleanImageStorePeriodically runs a periodic garbage collect on the ImageStore provided in constructor,
    65  given an interval and a Scheduler.
    66  */
    67  func (gc GarbageCollect) CleanImageStorePeriodically(interval time.Duration, sch *scheduler.Scheduler) {
    68  	generator := &GCTaskGenerator{
    69  		imgStore: gc.imgStore,
    70  		gc:       gc,
    71  	}
    72  
    73  	sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
    74  }
    75  
    76  /*
    77  CleanRepo executes a garbage collection of any blob found in storage which is not referenced
    78  in any manifests referenced in repo's index.json
    79  It also gc referrers with missing subject if the Referrer Option is enabled
    80  It also gc untagged manifests.
    81  */
    82  func (gc GarbageCollect) CleanRepo(ctx context.Context, repo string) error {
    83  	gc.log.Info().Str("module", "gc").
    84  		Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(gc.imgStore.RootDir(), repo)))
    85  
    86  	if err := gc.cleanRepo(ctx, repo); err != nil {
    87  		errMessage := fmt.Sprintf("failed to run GC for %s", path.Join(gc.imgStore.RootDir(), repo))
    88  		gc.log.Error().Err(err).Str("module", "gc").Msg(errMessage)
    89  		gc.log.Info().Str("module", "gc").
    90  			Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(gc.imgStore.RootDir(), repo)))
    91  
    92  		return err
    93  	}
    94  
    95  	gc.log.Info().Str("module", "gc").
    96  		Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(gc.imgStore.RootDir(), repo)))
    97  
    98  	return nil
    99  }
   100  
   101  func (gc GarbageCollect) cleanRepo(ctx context.Context, repo string) error {
   102  	var lockLatency time.Time
   103  
   104  	dir := path.Join(gc.imgStore.RootDir(), repo)
   105  	if !gc.imgStore.DirExists(dir) {
   106  		return zerr.ErrRepoNotFound
   107  	}
   108  
   109  	gc.imgStore.Lock(&lockLatency)
   110  	defer gc.imgStore.Unlock(&lockLatency)
   111  
   112  	/* this index (which represents the index.json of this repo) is the root point from which we
   113  	search for dangling manifests/blobs
   114  	so this index is passed by reference in all functions that modifies it
   115  
   116  	Instead of removing manifests one by one with storage APIs we just remove manifests descriptors
   117  	from index.Manifests[] list and update repo's index.json afterwards.
   118  
   119  	After updating repo's index.json we clean all unreferenced blobs (manifests included).
   120  	*/
   121  	index, err := common.GetIndex(gc.imgStore, repo, gc.log)
   122  	if err != nil {
   123  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("failed to read index.json in repo")
   124  
   125  		return err
   126  	}
   127  
   128  	// apply tags retention
   129  	if err := gc.removeTagsPerRetentionPolicy(ctx, repo, &index); err != nil {
   130  		return err
   131  	}
   132  
   133  	// gc referrers manifests with missing subject and untagged manifests
   134  	if err := gc.removeManifestsPerRepoPolicy(ctx, repo, &index); err != nil {
   135  		return err
   136  	}
   137  
   138  	// update repos's index.json in storage
   139  	if !gc.opts.ImageRetention.DryRun {
   140  		/* this will update the index.json with manifests deleted above
   141  		and the manifests blobs will be removed by gc.removeUnreferencedBlobs()*/
   142  		if err := gc.imgStore.PutIndexContent(repo, index); err != nil {
   143  			return err
   144  		}
   145  	}
   146  
   147  	// gc unreferenced blobs
   148  	if err := gc.removeUnreferencedBlobs(repo, gc.opts.Delay, gc.log); err != nil {
   149  		return err
   150  	}
   151  
   152  	return nil
   153  }
   154  
   155  func (gc GarbageCollect) removeManifestsPerRepoPolicy(ctx context.Context, repo string, index *ispec.Index) error {
   156  	var err error
   157  
   158  	/* gc all manifests that have a missing subject, stop when neither gc(referrer and untagged)
   159  	happened in a full loop over index.json. */
   160  	var stop bool
   161  	for !stop {
   162  		if zcommon.IsContextDone(ctx) {
   163  			return ctx.Err()
   164  		}
   165  
   166  		var gcedReferrer bool
   167  
   168  		var gcedUntagged bool
   169  
   170  		if gc.policyMgr.HasDeleteReferrer(repo) {
   171  			gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests with missing referrers")
   172  
   173  			gcedReferrer, err = gc.removeIndexReferrers(repo, index, *index)
   174  			if err != nil {
   175  				return err
   176  			}
   177  		}
   178  
   179  		if gc.policyMgr.HasDeleteUntagged(repo) {
   180  			referenced := make(map[godigest.Digest]bool, 0)
   181  
   182  			/* gather all manifests referenced in multiarch images/by other manifests
   183  			so that we can skip them in cleanUntaggedManifests */
   184  			if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil {
   185  				return err
   186  			}
   187  
   188  			// apply image retention policy
   189  			gcedUntagged, err = gc.removeUntaggedManifests(repo, index, referenced)
   190  			if err != nil {
   191  				return err
   192  			}
   193  		}
   194  
   195  		/* if we gced any manifest then loop again and gc manifests with
   196  		a subject pointing to the last ones which were gced. */
   197  		stop = !gcedReferrer && !gcedUntagged
   198  	}
   199  
   200  	return nil
   201  }
   202  
   203  /*
   204  garbageCollectIndexReferrers will gc all referrers with a missing subject recursively
   205  
   206  rootIndex is indexJson, need to pass it down to garbageCollectReferrer()
   207  rootIndex is the place we look for referrers.
   208  */
   209  func (gc GarbageCollect) removeIndexReferrers(repo string, rootIndex *ispec.Index, index ispec.Index,
   210  ) (bool, error) {
   211  	var count int
   212  
   213  	var err error
   214  
   215  	for _, desc := range index.Manifests {
   216  		switch desc.MediaType {
   217  		case ispec.MediaTypeImageIndex:
   218  			indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log)
   219  			if err != nil {
   220  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()).
   221  					Msg("failed to read multiarch(index) image")
   222  
   223  				return false, err
   224  			}
   225  
   226  			gced, err := gc.removeReferrer(repo, rootIndex, desc, indexImage.Subject, indexImage.ArtifactType)
   227  			if err != nil {
   228  				return false, err
   229  			}
   230  
   231  			/* if we gc index then no need to continue searching for referrers inside it.
   232  			they will be gced when the next garbage collect is executed(if they are older than retentionDelay),
   233  			 because manifests part of indexes will still be referenced in index.json */
   234  			if gced {
   235  				return true, nil
   236  			}
   237  
   238  			gced, err = gc.removeIndexReferrers(repo, rootIndex, indexImage)
   239  			if err != nil {
   240  				return false, err
   241  			}
   242  
   243  			if gced {
   244  				count++
   245  			}
   246  		case ispec.MediaTypeImageManifest:
   247  			image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log)
   248  			if err != nil {
   249  				gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).Str("digest", desc.Digest.String()).
   250  					Msg("failed to read manifest image")
   251  
   252  				return false, err
   253  			}
   254  
   255  			artifactType := zcommon.GetManifestArtifactType(image)
   256  
   257  			gced, err := gc.removeReferrer(repo, rootIndex, desc, image.Subject, artifactType)
   258  			if err != nil {
   259  				return false, err
   260  			}
   261  
   262  			if gced {
   263  				count++
   264  			}
   265  		}
   266  	}
   267  
   268  	return count > 0, err
   269  }
   270  
   271  func (gc GarbageCollect) removeReferrer(repo string, index *ispec.Index, manifestDesc ispec.Descriptor,
   272  	subject *ispec.Descriptor, artifactType string,
   273  ) (bool, error) {
   274  	var gced bool
   275  
   276  	var err error
   277  
   278  	if subject != nil {
   279  		// try to find subject in index.json
   280  		referenced := isManifestReferencedInIndex(index, subject.Digest)
   281  
   282  		var signatureType string
   283  		// check if its notation or cosign signature
   284  		if artifactType == zcommon.ArtifactTypeNotation {
   285  			signatureType = storage.NotationType
   286  		} else if artifactType == zcommon.ArtifactTypeCosign {
   287  			signatureType = storage.CosignType
   288  		}
   289  
   290  		if !referenced {
   291  			gced, err = gc.gcManifest(repo, index, manifestDesc, signatureType, subject.Digest, gc.opts.ImageRetention.Delay)
   292  			if err != nil {
   293  				return false, err
   294  			}
   295  
   296  			if gced {
   297  				gc.log.Info().Str("module", "gc").
   298  					Str("repository", repo).
   299  					Str("reference", manifestDesc.Digest.String()).
   300  					Str("subject", subject.Digest.String()).
   301  					Str("decision", "delete").
   302  					Str("reason", "deleteReferrers").Msg("removed manifest without reference")
   303  
   304  				if gc.auditLog != nil {
   305  					gc.auditLog.Info().Str("module", "gc").
   306  						Str("repository", repo).
   307  						Str("reference", manifestDesc.Digest.String()).
   308  						Str("subject", subject.Digest.String()).
   309  						Str("decision", "delete").
   310  						Str("reason", "deleteReferrers").Msg("removed manifest without reference")
   311  				}
   312  			}
   313  		}
   314  	}
   315  
   316  	// cosign
   317  	tag, ok := getDescriptorTag(manifestDesc)
   318  	if ok {
   319  		if isCosignTag(tag) {
   320  			subjectDigest := getSubjectFromCosignTag(tag)
   321  			referenced := isManifestReferencedInIndex(index, subjectDigest)
   322  
   323  			if !referenced {
   324  				gced, err = gc.gcManifest(repo, index, manifestDesc, storage.CosignType, subjectDigest, gc.opts.Delay)
   325  				if err != nil {
   326  					return false, err
   327  				}
   328  
   329  				if gced {
   330  					gc.log.Info().Str("module", "gc").
   331  						Bool("dry-run", gc.opts.ImageRetention.DryRun).
   332  						Str("repository", repo).
   333  						Str("reference", tag).
   334  						Str("subject", subjectDigest.String()).
   335  						Str("decision", "delete").
   336  						Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference")
   337  
   338  					if gc.auditLog != nil {
   339  						gc.auditLog.Info().Str("module", "gc").
   340  							Bool("dry-run", gc.opts.ImageRetention.DryRun).
   341  							Str("repository", repo).
   342  							Str("reference", tag).
   343  							Str("subject", subjectDigest.String()).
   344  							Str("decision", "delete").
   345  							Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference")
   346  					}
   347  				}
   348  			}
   349  		}
   350  	}
   351  
   352  	return gced, nil
   353  }
   354  
   355  func (gc GarbageCollect) removeTagsPerRetentionPolicy(ctx context.Context, repo string, index *ispec.Index) error {
   356  	if !gc.policyMgr.HasTagRetention(repo) {
   357  		return nil
   358  	}
   359  
   360  	repoMeta, err := gc.metaDB.GetRepoMeta(ctx, repo)
   361  	if err != nil {
   362  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   363  			Msg("failed to get repoMeta")
   364  
   365  		return err
   366  	}
   367  
   368  	retainTags := gc.policyMgr.GetRetainedTags(ctx, repoMeta, *index)
   369  
   370  	// remove
   371  	for _, desc := range index.Manifests {
   372  		if zcommon.IsContextDone(ctx) {
   373  			return ctx.Err()
   374  		}
   375  
   376  		// check tag
   377  		tag, ok := getDescriptorTag(desc)
   378  		if ok && !zcommon.Contains(retainTags, tag) {
   379  			// remove tags which should not be retained
   380  			_, err := gc.removeManifest(repo, index, desc, tag, "", "")
   381  			if err != nil && !errors.Is(err, zerr.ErrManifestNotFound) {
   382  				return err
   383  			}
   384  		}
   385  	}
   386  
   387  	return nil
   388  }
   389  
   390  // gcManifest removes a manifest entry from an index and syncs metaDB accordingly if the blob is older than gc.Delay.
   391  func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec.Descriptor,
   392  	signatureType string, subjectDigest godigest.Digest, delay time.Duration,
   393  ) (bool, error) {
   394  	var gced bool
   395  
   396  	canGC, err := isBlobOlderThan(gc.imgStore, repo, desc.Digest, delay, gc.log)
   397  	if err != nil {
   398  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()).
   399  			Str("delay", delay.String()).Msg("failed to check if blob is older than delay")
   400  
   401  		return false, err
   402  	}
   403  
   404  	if canGC {
   405  		if gced, err = gc.removeManifest(repo, index, desc, desc.Digest.String(), signatureType, subjectDigest); err != nil {
   406  			return false, err
   407  		}
   408  	}
   409  
   410  	return gced, nil
   411  }
   412  
   413  // removeManifest removes a manifest entry from an index and syncs metaDB accordingly.
   414  func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index,
   415  	desc ispec.Descriptor, reference string, signatureType string, subjectDigest godigest.Digest,
   416  ) (bool, error) {
   417  	_, err := common.RemoveManifestDescByReference(index, reference, true)
   418  	if err != nil {
   419  		if errors.Is(err, zerr.ErrManifestConflict) {
   420  			return false, nil
   421  		}
   422  
   423  		return false, err
   424  	}
   425  
   426  	if gc.opts.ImageRetention.DryRun {
   427  		return true, nil
   428  	}
   429  
   430  	// sync metaDB
   431  	if gc.metaDB != nil {
   432  		if signatureType != "" {
   433  			err = gc.metaDB.DeleteSignature(repo, subjectDigest, mTypes.SignatureMetadata{
   434  				SignatureDigest: desc.Digest.String(),
   435  				SignatureType:   signatureType,
   436  			})
   437  			if err != nil {
   438  				gc.log.Error().Err(err).Str("module", "gc").Str("component", "metadb").
   439  					Msg("failed to remove signature in metaDB")
   440  
   441  				return false, err
   442  			}
   443  		} else {
   444  			err := gc.metaDB.RemoveRepoReference(repo, reference, desc.Digest)
   445  			if err != nil {
   446  				gc.log.Error().Err(err).Str("module", "gc").Str("component", "metadb").
   447  					Msg("failed to remove repo reference in metaDB")
   448  
   449  				return false, err
   450  			}
   451  		}
   452  	}
   453  
   454  	return true, nil
   455  }
   456  
   457  func (gc GarbageCollect) removeUntaggedManifests(repo string, index *ispec.Index,
   458  	referenced map[godigest.Digest]bool,
   459  ) (bool, error) {
   460  	var gced bool
   461  
   462  	var err error
   463  
   464  	gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests without tags")
   465  
   466  	for _, desc := range index.Manifests {
   467  		// skip manifests referenced in image indexes
   468  		if _, referenced := referenced[desc.Digest]; referenced {
   469  			continue
   470  		}
   471  
   472  		// remove untagged images
   473  		if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex {
   474  			_, ok := getDescriptorTag(desc)
   475  			if !ok {
   476  				gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.ImageRetention.Delay)
   477  				if err != nil {
   478  					return false, err
   479  				}
   480  
   481  				if gced {
   482  					gc.log.Info().Str("module", "gc").
   483  						Bool("dry-run", gc.opts.ImageRetention.DryRun).
   484  						Str("repository", repo).
   485  						Str("reference", desc.Digest.String()).
   486  						Str("decision", "delete").
   487  						Str("reason", "deleteUntagged").Msg("removed untagged manifest")
   488  
   489  					if gc.auditLog != nil {
   490  						gc.auditLog.Info().Str("module", "gc").
   491  							Bool("dry-run", gc.opts.ImageRetention.DryRun).
   492  							Str("repository", repo).
   493  							Str("reference", desc.Digest.String()).
   494  							Str("decision", "delete").
   495  							Str("reason", "deleteUntagged").Msg("removed untagged manifest")
   496  					}
   497  				}
   498  			}
   499  		}
   500  	}
   501  
   502  	return gced, nil
   503  }
   504  
   505  // Adds both referenced manifests and referrers from an index.
   506  func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, repo string,
   507  	referenced map[godigest.Digest]bool,
   508  ) error {
   509  	for _, desc := range index.Manifests {
   510  		switch desc.MediaType {
   511  		case ispec.MediaTypeImageIndex:
   512  			indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log)
   513  			if err != nil {
   514  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   515  					Str("digest", desc.Digest.String()).Msg("failed to read multiarch(index) image")
   516  
   517  				return err
   518  			}
   519  
   520  			if indexImage.Subject != nil {
   521  				referenced[desc.Digest] = true
   522  			}
   523  
   524  			for _, indexDesc := range indexImage.Manifests {
   525  				referenced[indexDesc.Digest] = true
   526  			}
   527  
   528  			if err := gc.identifyManifestsReferencedInIndex(indexImage, repo, referenced); err != nil {
   529  				return err
   530  			}
   531  		case ispec.MediaTypeImageManifest:
   532  			image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log)
   533  			if err != nil {
   534  				gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).
   535  					Str("digest", desc.Digest.String()).Msg("failed to read manifest image")
   536  
   537  				return err
   538  			}
   539  
   540  			if image.Subject != nil {
   541  				referenced[desc.Digest] = true
   542  			}
   543  		}
   544  	}
   545  
   546  	return nil
   547  }
   548  
   549  // removeUnreferencedBlobs gc all blobs which are not referenced by any manifest found in repo's index.json.
   550  func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duration, log zlog.Logger,
   551  ) error {
   552  	gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("cleaning orphan blobs")
   553  
   554  	refBlobs := map[string]bool{}
   555  
   556  	index, err := common.GetIndex(gc.imgStore, repo, gc.log)
   557  	if err != nil {
   558  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("failed to read index.json in repo")
   559  
   560  		return err
   561  	}
   562  
   563  	err = gc.addIndexBlobsToReferences(repo, index, refBlobs)
   564  	if err != nil {
   565  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("failed to get referenced blobs in repo")
   566  
   567  		return err
   568  	}
   569  
   570  	allBlobs, err := gc.imgStore.GetAllBlobs(repo)
   571  	if err != nil {
   572  		// /blobs/sha256/ may be empty in the case of s3, no need to return err, we want to skip
   573  		if errors.As(err, &driver.PathNotFoundError{}) {
   574  			return nil
   575  		}
   576  
   577  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("failed to get all blobs")
   578  
   579  		return err
   580  	}
   581  
   582  	gcBlobs := make([]godigest.Digest, 0)
   583  
   584  	for _, blob := range allBlobs {
   585  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob)
   586  		if err = digest.Validate(); err != nil {
   587  			log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob).
   588  				Msg("failed to parse digest")
   589  
   590  			return err
   591  		}
   592  
   593  		if _, ok := refBlobs[digest.String()]; !ok {
   594  			canGC, err := isBlobOlderThan(gc.imgStore, repo, digest, delay, log)
   595  			if err != nil {
   596  				log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob).
   597  					Msg("failed to determine GC delay")
   598  
   599  				return err
   600  			}
   601  
   602  			if canGC {
   603  				gcBlobs = append(gcBlobs, digest)
   604  			}
   605  		}
   606  	}
   607  
   608  	// if we removed all blobs from repo
   609  	removeRepo := len(gcBlobs) > 0 && len(gcBlobs) == len(allBlobs)
   610  
   611  	reaped, err := gc.imgStore.CleanupRepo(repo, gcBlobs, removeRepo)
   612  	if err != nil {
   613  		return err
   614  	}
   615  
   616  	log.Info().Str("module", "gc").Str("repository", repo).Int("count", reaped).
   617  		Msg("garbage collected blobs")
   618  
   619  	return nil
   620  }
   621  
   622  // used by removeUnreferencedBlobs()
   623  // addIndexBlobsToReferences adds referenced blobs found in referenced manifests (index.json) in refblobs map.
   624  func (gc GarbageCollect) addIndexBlobsToReferences(repo string, index ispec.Index, refBlobs map[string]bool,
   625  ) error {
   626  	for _, desc := range index.Manifests {
   627  		switch desc.MediaType {
   628  		case ispec.MediaTypeImageIndex:
   629  			if err := gc.addImageIndexBlobsToReferences(repo, desc.Digest, refBlobs); err != nil {
   630  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   631  					Str("digest", desc.Digest.String()).Msg("failed to read blobs in multiarch(index) image")
   632  
   633  				return err
   634  			}
   635  		case ispec.MediaTypeImageManifest:
   636  			if err := gc.addImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil {
   637  				gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   638  					Str("digest", desc.Digest.String()).Msg("failed to read blobs in image manifest")
   639  
   640  				return err
   641  			}
   642  		}
   643  	}
   644  
   645  	return nil
   646  }
   647  
   648  func (gc GarbageCollect) addImageIndexBlobsToReferences(repo string, mdigest godigest.Digest, refBlobs map[string]bool,
   649  ) error {
   650  	index, err := common.GetImageIndex(gc.imgStore, repo, mdigest, gc.log)
   651  	if err != nil {
   652  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", mdigest.String()).
   653  			Msg("failed to read manifest image")
   654  
   655  		return err
   656  	}
   657  
   658  	refBlobs[mdigest.String()] = true
   659  
   660  	// if there is a Subject, it may not exist yet and that is ok
   661  	if index.Subject != nil {
   662  		refBlobs[index.Subject.Digest.String()] = true
   663  	}
   664  
   665  	for _, manifest := range index.Manifests {
   666  		refBlobs[manifest.Digest.String()] = true
   667  	}
   668  
   669  	return nil
   670  }
   671  
   672  func (gc GarbageCollect) addImageManifestBlobsToReferences(repo string, mdigest godigest.Digest,
   673  	refBlobs map[string]bool,
   674  ) error {
   675  	manifestContent, err := common.GetImageManifest(gc.imgStore, repo, mdigest, gc.log)
   676  	if err != nil {
   677  		gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).
   678  			Str("digest", mdigest.String()).Msg("failed to read manifest image")
   679  
   680  		return err
   681  	}
   682  
   683  	refBlobs[mdigest.String()] = true
   684  	refBlobs[manifestContent.Config.Digest.String()] = true
   685  
   686  	// if there is a Subject, it may not exist yet and that is ok
   687  	if manifestContent.Subject != nil {
   688  		refBlobs[manifestContent.Subject.Digest.String()] = true
   689  	}
   690  
   691  	for _, layer := range manifestContent.Layers {
   692  		refBlobs[layer.Digest.String()] = true
   693  	}
   694  
   695  	return nil
   696  }
   697  
   698  func isManifestReferencedInIndex(index *ispec.Index, digest godigest.Digest) bool {
   699  	for _, manifest := range index.Manifests {
   700  		if manifest.Digest == digest {
   701  			return true
   702  		}
   703  	}
   704  
   705  	return false
   706  }
   707  
   708  func isBlobOlderThan(imgStore types.ImageStore, repo string,
   709  	digest godigest.Digest, delay time.Duration, log zlog.Logger,
   710  ) (bool, error) {
   711  	_, _, modtime, err := imgStore.StatBlob(repo, digest)
   712  	if err != nil {
   713  		log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", digest.String()).
   714  			Msg("failed to stat blob")
   715  
   716  		return false, err
   717  	}
   718  
   719  	if modtime.Add(delay).After(time.Now()) {
   720  		return false, nil
   721  	}
   722  
   723  	return true, nil
   724  }
   725  
   726  func getSubjectFromCosignTag(tag string) godigest.Digest {
   727  	alg := strings.Split(tag, "-")[0]
   728  	encoded := strings.Split(strings.Split(tag, "-")[1], ".sig")[0]
   729  
   730  	return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded)
   731  }
   732  
   733  func getDescriptorTag(desc ispec.Descriptor) (string, bool) {
   734  	tag, ok := desc.Annotations[ispec.AnnotationRefName]
   735  
   736  	return tag, ok
   737  }
   738  
   739  // this function will check if tag is a cosign tag (signature or sbom).
   740  func isCosignTag(tag string) bool {
   741  	if strings.HasPrefix(tag, "sha256-") &&
   742  		(strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) {
   743  		return true
   744  	}
   745  
   746  	return false
   747  }
   748  
   749  /*
   750  	GCTaskGenerator takes all repositories found in the storage.imagestore
   751  
   752  and it will execute garbage collection for each repository by creating a task
   753  for each repository and pushing it to the task scheduler.
   754  */
   755  type GCTaskGenerator struct {
   756  	imgStore types.ImageStore
   757  	gc       GarbageCollect
   758  	lastRepo string
   759  	nextRun  time.Time
   760  	done     bool
   761  	rand     *rand.Rand
   762  }
   763  
   764  func (gen *GCTaskGenerator) getRandomDelay() int {
   765  	maxDelay := 30
   766  
   767  	return gen.rand.Intn(maxDelay)
   768  }
   769  
   770  func (gen *GCTaskGenerator) Name() string {
   771  	return "GCTaskGenerator"
   772  }
   773  
   774  func (gen *GCTaskGenerator) Next() (scheduler.Task, error) {
   775  	if gen.lastRepo == "" && gen.nextRun.IsZero() {
   776  		gen.rand = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) //nolint: gosec
   777  	}
   778  
   779  	delay := gen.getRandomDelay()
   780  
   781  	gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second)
   782  
   783  	repo, err := gen.imgStore.GetNextRepository(gen.lastRepo)
   784  	if err != nil {
   785  		return nil, err
   786  	}
   787  
   788  	if repo == "" {
   789  		gen.done = true
   790  
   791  		return nil, nil
   792  	}
   793  
   794  	gen.lastRepo = repo
   795  
   796  	return NewGCTask(gen.imgStore, gen.gc, repo), nil
   797  }
   798  
   799  func (gen *GCTaskGenerator) IsDone() bool {
   800  	return gen.done
   801  }
   802  
   803  func (gen *GCTaskGenerator) IsReady() bool {
   804  	return time.Now().After(gen.nextRun)
   805  }
   806  
   807  func (gen *GCTaskGenerator) Reset() {
   808  	gen.lastRepo = ""
   809  	gen.done = false
   810  	gen.nextRun = time.Time{}
   811  }
   812  
   813  type gcTask struct {
   814  	imgStore types.ImageStore
   815  	gc       GarbageCollect
   816  	repo     string
   817  }
   818  
   819  func NewGCTask(imgStore types.ImageStore, gc GarbageCollect, repo string,
   820  ) *gcTask {
   821  	return &gcTask{imgStore, gc, repo}
   822  }
   823  
   824  func (gct *gcTask) DoWork(ctx context.Context) error {
   825  	// run task
   826  	return gct.gc.CleanRepo(ctx, gct.repo) //nolint: contextcheck
   827  }
   828  
   829  func (gct *gcTask) String() string {
   830  	return fmt.Sprintf("{Name: %s, repo: %s}",
   831  		gct.Name(), gct.repo)
   832  }
   833  
   834  func (gct *gcTask) Name() string {
   835  	return "GCTask"
   836  }