k8s.io/registry.k8s.io@v0.3.1/cmd/geranos/walkimages.go (about)

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"fmt"
    21  	"net/http"
    22  
    23  	"golang.org/x/sync/errgroup"
    24  
    25  	"k8s.io/klog/v2"
    26  
    27  	"github.com/google/go-containerregistry/pkg/name"
    28  	v1 "github.com/google/go-containerregistry/pkg/v1"
    29  	"github.com/google/go-containerregistry/pkg/v1/google"
    30  	"github.com/google/go-containerregistry/pkg/v1/partial"
    31  	"github.com/google/go-containerregistry/pkg/v1/remote"
    32  	"github.com/google/go-containerregistry/pkg/v1/types"
    33  )
    34  
    35  // WalkImageLAyersFunc is used to visit an image
    36  type WalkImageLayersFunc func(ref name.Reference, layers []v1.Layer) error
    37  
    38  // Unfortunately this is only doable on GCP currently.
    39  //
    40  // TODO: To support other registries in the meantime, we could require a list of
    41  // image names as an input and plumb that through, then list tags and get something
    42  // close to this. The _catalog endpoint + tag listing could also work in some cases.
    43  //
    44  // However, even then, this is more complete because it lists all manifests, not just tags.
    45  // It's also simpler and more efficient.
    46  //
    47  // See: https://github.com/opencontainers/distribution-spec/issues/222
    48  func WalkImageLayersGCP(transport http.RoundTripper, repo name.Repository, walkImageLayers WalkImageLayersFunc, skipImage func(string) bool) error {
    49  	g := new(errgroup.Group)
    50  	// TODO: This is really just an approximation to avoid exceeding typical socket limits
    51  	// See also quota limits:
    52  	// https://cloud.google.com/artifact-registry/quotas
    53  	g.SetLimit(1000)
    54  	g.Go(func() error {
    55  		return google.Walk(repo, func(r name.Repository, tags *google.Tags, err error) error {
    56  			if err != nil {
    57  				return err
    58  			}
    59  			for digest, metadata := range tags.Manifests {
    60  				digest := digest
    61  				// google.Walk already walks the child manifests
    62  				if metadata.MediaType == string(types.DockerManifestList) || metadata.MediaType == string(types.OCIImageIndex) {
    63  					continue
    64  				}
    65  				ref, err := name.ParseReference(fmt.Sprintf("%s@%s", r, digest))
    66  				if err != nil {
    67  					return err
    68  				}
    69  				g.Go(func() error {
    70  					if skipImage(digest) {
    71  						klog.V(4).Infof("Skipping already-uploaded: %s", ref)
    72  						return nil
    73  					}
    74  					return walkManifestLayers(transport, ref, walkImageLayers)
    75  				})
    76  			}
    77  			return nil
    78  		}, google.WithTransport(transport))
    79  	})
    80  	return g.Wait()
    81  }
    82  
    83  func walkManifestLayers(transport http.RoundTripper, ref name.Reference, walkImageLayers WalkImageLayersFunc) error {
    84  	desc, err := remote.Get(ref, remote.WithTransport(transport))
    85  	if err != nil {
    86  		return err
    87  	}
    88  
    89  	// google.Walk already resolves these to individual manifests
    90  	if desc.MediaType.IsIndex() {
    91  		klog.Warningf("Skipping Index: %s", ref.String())
    92  		return nil
    93  	}
    94  
    95  	// Specially handle schema 1
    96  	// https://github.com/google/go-containerregistry/issues/377
    97  	if desc.MediaType == types.DockerManifestSchema1 || desc.MediaType == types.DockerManifestSchema1Signed {
    98  		layers, err := layersForV1(transport, ref, desc)
    99  		if err != nil {
   100  			return err
   101  		}
   102  		return walkImageLayers(ref, layers)
   103  	}
   104  
   105  	// we don't expect anything other than index, or image ...
   106  	if !desc.MediaType.IsImage() {
   107  		klog.Warningf("Un-handled type: %s for %s", desc.MediaType, ref.String())
   108  		return nil
   109  	}
   110  
   111  	// Handle normal images
   112  	image, err := desc.Image()
   113  	if err != nil {
   114  		return err
   115  	}
   116  	layers, err := imageToLayers(image)
   117  	if err != nil {
   118  		return err
   119  	}
   120  	return walkImageLayers(ref, layers)
   121  }
   122  
   123  func imageToLayers(image v1.Image) ([]v1.Layer, error) {
   124  	layers, err := image.Layers()
   125  	if err != nil {
   126  		return nil, err
   127  	}
   128  	configLayer, err := partial.ConfigLayer(image)
   129  	if err != nil {
   130  		return nil, err
   131  	}
   132  	return append(layers, configLayer), nil
   133  }