k8s.io/registry.k8s.io@v0.3.1/cmd/archeio/internal/app/buckets.go (about)

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package app
    18  
    19  import (
    20  	"net/http"
    21  	"sync"
    22  	"time"
    23  
    24  	"k8s.io/klog/v2"
    25  )
    26  
    27  // awsRegionToHostURL returns the base S3 bucket URL for an OCI layer blob given the AWS region
    28  //
    29  // blobs in the buckets should be stored at /containers/images/sha256:$hash
    30  func awsRegionToHostURL(region, defaultURL string) string {
    31  	switch region {
    32  	// each of these has the region in which we have a bucket listed first
    33  	// and then additional regions we're mapping to that bucket
    34  	// based roughly on physical adjacency (and therefore _presumed_ latency)
    35  	//
    36  	// if you add a bucket, add a case for the region it is in, and consider
    37  	// shifting other regions that do not have their own bucket
    38  
    39  	// US East (N. Virginia)
    40  	case "us-east-1", "sa-east-1":
    41  		return "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com"
    42  	// US East (Ohio)
    43  	case "us-east-2", "ca-central-1":
    44  		return "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com"
    45  	// US West (N. California)
    46  	case "us-west-1":
    47  		return "https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com"
    48  	// US West (Oregon)
    49  	case "us-west-2", "ca-west-1":
    50  		return "https://prod-registry-k8s-io-us-west-2.s3.dualstack.us-west-2.amazonaws.com"
    51  	// Asia Pacific (Mumbai)
    52  	case "ap-south-1", "ap-south-2", "me-south-1", "me-central-1":
    53  		return "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com"
    54  	// Asia Pacific (Tokyo)
    55  	case "ap-northeast-1", "ap-northeast-2", "ap-northeast-3":
    56  		return "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com"
    57  	// Asia Pacific (Singapore)
    58  	case "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-east-1", "cn-northwest-1", "cn-north-1":
    59  		return "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com"
    60  	// Europe (Frankfurt)
    61  	case "eu-central-1", "eu-central-2", "eu-south-1", "eu-south-2", "il-central-1":
    62  		return "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com"
    63  	// Europe (Ireland)
    64  	case "eu-west-1", "af-south-1", "eu-west-2", "eu-west-3", "eu-north-1":
    65  		return "https://prod-registry-k8s-io-eu-west-1.s3.dualstack.eu-west-1.amazonaws.com"
    66  	default:
    67  		return defaultURL
    68  	}
    69  }
    70  
    71  // blobChecker are used to check if a blob exists, possibly with caching
    72  type blobChecker interface {
    73  	// BlobExists should check that blobURL exists
    74  	// bucket and layerHash may be used for caching purposes
    75  	BlobExists(blobURL string) bool
    76  }
    77  
    78  // cachedBlobChecker just performs an HTTP HEAD check against the blob
    79  //
    80  // TODO: potentially replace with a caching implementation
    81  // should be plenty fast for now, HTTP HEAD on s3 is cheap
    82  type cachedBlobChecker struct {
    83  	blobCache
    84  }
    85  
    86  func newCachedBlobChecker() *cachedBlobChecker {
    87  	return &cachedBlobChecker{}
    88  }
    89  
    90  type blobCache struct {
    91  	m sync.Map
    92  }
    93  
    94  func (b *blobCache) Get(blobURL string) bool {
    95  	_, exists := b.m.Load(blobURL)
    96  	return exists
    97  }
    98  
    99  func (b *blobCache) Put(blobURL string) {
   100  	b.m.Store(blobURL, struct{}{})
   101  }
   102  
   103  func (c *cachedBlobChecker) BlobExists(blobURL string) bool {
   104  	if c.blobCache.Get(blobURL) {
   105  		klog.V(3).InfoS("blob existence cache hit", "url", blobURL)
   106  		return true
   107  	}
   108  	klog.V(3).InfoS("blob existence cache miss", "url", blobURL)
   109  	// NOTE: this client will still share http.DefaultTransport
   110  	// We do not wish to share the rest of the client state currently
   111  	client := &http.Client{
   112  		// ensure sensible timeouts
   113  		Timeout: time.Second * 5,
   114  	}
   115  	r, err := client.Head(blobURL)
   116  	// fallback to assuming blob is unavailable on errors
   117  	if err != nil {
   118  		return false
   119  	}
   120  	r.Body.Close()
   121  	// if the blob exists it HEAD should return 200 OK
   122  	// this is true for S3 and for OCI registries
   123  	if r.StatusCode == http.StatusOK {
   124  		c.blobCache.Put(blobURL)
   125  		return true
   126  	}
   127  	return false
   128  }