k8s.io/registry.k8s.io@v0.3.1/cmd/geranos/main.go (about)

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"flag"
    21  	"os"
    22  
    23  	"github.com/google/go-containerregistry/pkg/crane"
    24  	"github.com/google/go-containerregistry/pkg/name"
    25  	v1 "github.com/google/go-containerregistry/pkg/v1"
    26  
    27  	"k8s.io/klog/v2"
    28  )
    29  
    30  func main() {
    31  	Main()
    32  }
    33  
    34  // Main is the application entrypoint, which injects globals to Run
    35  func Main() {
    36  	klog.InitFlags(flag.CommandLine)
    37  	flag.Parse()
    38  	if err := Run(os.Args); err != nil {
    39  		klog.Fatal(err)
    40  	}
    41  }
    42  
    43  // Run implements the actual application logic, accepting global inputs
    44  func Run(_ []string) error {
    45  	// one of the backing registries for registry.k8s.io
    46  	// TODO: make configurable later
    47  	const sourceRegistry = "us-central1-docker.pkg.dev/k8s-artifacts-prod/images"
    48  
    49  	// TODO: make configurable later
    50  	const s3Bucket = "prod-registry-k8s-io-us-east-2"
    51  
    52  	// 80*60s = 4800 RPM, below our current 5000 RPM per-user limit on the registry
    53  	// Even with the host node making other registry API calls
    54  	registryRateLimit := NewRateLimitRoundTripper(80)
    55  
    56  	repo, err := name.NewRepository(sourceRegistry)
    57  	if err != nil {
    58  		return err
    59  	}
    60  	s3Uploader, err := newS3Uploader(os.Getenv("REALLY_UPLOAD") == "")
    61  	if err != nil {
    62  		return err
    63  	}
    64  
    65  	// copy layers from all images in the repo
    66  	// TODO: print some progress logs at lower frequency instead of logging each image
    67  	// We will punt this temporarily, as we're about to refactor how this works anyhow
    68  	// to avoid fetching manifests for images we've already uploaded
    69  	err = WalkImageLayersGCP(registryRateLimit, repo,
    70  		func(ref name.Reference, layers []v1.Layer) error {
    71  			klog.Infof("Processing image: %s", ref.String())
    72  			return s3Uploader.UploadImage(s3Bucket, ref, layers, crane.WithTransport(registryRateLimit))
    73  		},
    74  		func(imageHash string) bool {
    75  			s, _ := s3Uploader.ImageAlreadyUploaded(s3Bucket, imageHash)
    76  			return s
    77  		})
    78  	if err == nil {
    79  		klog.Info("Done!")
    80  	}
    81  	return err
    82  }