zotregistry.io/zot@v1.4.4-0.20231124084042-02a8ed785457/pkg/extensions/sync/sync.go (about)

     1  //go:build sync
     2  // +build sync
     3  
     4  package sync
     5  
     6  import (
     7  	"context"
     8  
     9  	"github.com/containers/common/pkg/retry"
    10  	"github.com/containers/image/v5/types"
    11  	"github.com/opencontainers/go-digest"
    12  
    13  	"zotregistry.io/zot/pkg/log"
    14  	"zotregistry.io/zot/pkg/scheduler"
    15  )
    16  
    17  // below types are used by containers/image to copy images
    18  // types.ImageReference - describes a registry/repo:tag
    19  // types.SystemContext - describes a registry/oci layout config
    20  
    21  // Sync general functionalities, one service per registry config.
    22  type Service interface {
    23  	// Get next repo from remote /v2/_catalog, will return empty string when there is no repo left.
    24  	GetNextRepo(lastRepo string) (string, error) // used by task scheduler
    25  	// Sync a repo with all of its tags and references (signatures, artifacts, sboms) into ImageStore.
    26  	SyncRepo(ctx context.Context, repo string) error // used by periodically sync
    27  	// Sync an image (repo:tag || repo:digest) into ImageStore.
    28  	SyncImage(ctx context.Context, repo, reference string) error // used by sync on demand
    29  	// Sync a single reference for an image.
    30  	SyncReference(ctx context.Context, repo string, subjectDigestStr string,
    31  		referenceType string) error // used by sync on demand
    32  	// Remove all internal catalog entries.
    33  	ResetCatalog() // used by scheduler to empty out the catalog after a sync periodically roundtrip finishes
    34  	// Sync supports multiple urls per registry, before a sync repo/image/ref 'ping' each url.
    35  	SetNextAvailableURL() error // used by all sync methods
    36  	// Returns retry options from registry config.
    37  	GetRetryOptions() *retry.Options // used by sync on demand to retry in background
    38  }
    39  
    40  // Local and remote registries must implement this interface.
    41  type Registry interface {
    42  	// Get temporary ImageReference, is used by functions in containers/image package
    43  	GetImageReference(repo string, tag string) (types.ImageReference, error)
    44  	// Get local oci layout context, is used by functions in containers/image package
    45  	GetContext() *types.SystemContext
    46  }
    47  
    48  /*
    49  Temporary oci layout, sync first pulls an image to this oci layout (using oci:// transport)
    50  then moves them into ImageStore.
    51  */
    52  type OciLayoutStorage interface {
    53  	Registry
    54  }
    55  
    56  // Remote registry.
    57  type Remote interface {
    58  	Registry
    59  	// Get a list of repos (catalog)
    60  	GetRepositories(ctx context.Context) ([]string, error)
    61  	// Get a list of tags given a repo
    62  	GetRepoTags(repo string) ([]string, error)
    63  	// Get manifest content, mediaType, digest given an ImageReference
    64  	GetManifestContent(imageReference types.ImageReference) ([]byte, string, digest.Digest, error)
    65  }
    66  
    67  // Local registry.
    68  type Local interface {
    69  	Registry
    70  	// Check if an image is already synced
    71  	CanSkipImage(repo, tag string, imageDigest digest.Digest) (bool, error)
    72  	// CommitImage moves a synced repo/ref from temporary oci layout to ImageStore
    73  	CommitImage(imageReference types.ImageReference, repo, tag string) error
    74  }
    75  
    76  type TaskGenerator struct {
    77  	Service  Service
    78  	lastRepo string
    79  	done     bool
    80  	log      log.Logger
    81  }
    82  
    83  func NewTaskGenerator(service Service, log log.Logger) *TaskGenerator {
    84  	return &TaskGenerator{
    85  		Service:  service,
    86  		done:     false,
    87  		lastRepo: "",
    88  		log:      log,
    89  	}
    90  }
    91  
    92  func (gen *TaskGenerator) Next() (scheduler.Task, error) {
    93  	if err := gen.Service.SetNextAvailableURL(); err != nil {
    94  		return nil, err
    95  	}
    96  
    97  	repo, err := gen.Service.GetNextRepo(gen.lastRepo)
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  
   102  	if repo == "" {
   103  		gen.log.Info().Msg("sync: finished syncing all repos")
   104  		gen.done = true
   105  
   106  		return nil, nil
   107  	}
   108  
   109  	gen.lastRepo = repo
   110  
   111  	return newSyncRepoTask(gen.lastRepo, gen.Service), nil
   112  }
   113  
   114  func (gen *TaskGenerator) IsDone() bool {
   115  	return gen.done
   116  }
   117  
   118  func (gen *TaskGenerator) IsReady() bool {
   119  	return true
   120  }
   121  
   122  func (gen *TaskGenerator) Reset() {
   123  	gen.lastRepo = ""
   124  	gen.Service.ResetCatalog()
   125  	gen.done = false
   126  }
   127  
   128  type syncRepoTask struct {
   129  	repo    string
   130  	service Service
   131  }
   132  
   133  func newSyncRepoTask(repo string, service Service) *syncRepoTask {
   134  	return &syncRepoTask{repo, service}
   135  }
   136  
   137  func (srt *syncRepoTask) DoWork(ctx context.Context) error {
   138  	return srt.service.SyncRepo(ctx, srt.repo)
   139  }