k8s.io/apiserver@v0.31.1/pkg/server/storage/storage_factory.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storage
    18  
    19  import (
    20  	"crypto/tls"
    21  	"crypto/x509"
    22  	"os"
    23  	"strings"
    24  
    25  	"k8s.io/apimachinery/pkg/runtime"
    26  	"k8s.io/apimachinery/pkg/runtime/schema"
    27  	"k8s.io/apimachinery/pkg/util/sets"
    28  	"k8s.io/apiserver/pkg/storage/storagebackend"
    29  	"k8s.io/klog/v2"
    30  )
    31  
    32  // Backend describes the storage servers, the information here should be enough
    33  // for health validations.
    34  type Backend struct {
    35  	// the url of storage backend like: https://etcd.domain:2379
    36  	Server string
    37  	// the required tls config
    38  	TLSConfig *tls.Config
    39  }
    40  
    41  // StorageFactory is the interface to locate the storage for a given GroupResource
    42  type StorageFactory interface {
    43  	// New finds the storage destination for the given group and resource. It will
    44  	// return an error if the group has no storage destination configured.
    45  	NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
    46  
    47  	// ResourcePrefix returns the overridden resource prefix for the GroupResource
    48  	// This allows for cohabitation of resources with different native types and provides
    49  	// centralized control over the shape of etcd directories
    50  	ResourcePrefix(groupResource schema.GroupResource) string
    51  
    52  	// Configs gets configurations for all of registered storage destinations.
    53  	Configs() []storagebackend.Config
    54  
    55  	// Backends gets all backends for all registered storage destinations.
    56  	// Used for getting all instances for health validations.
    57  	// Deprecated: Use Configs instead
    58  	Backends() []Backend
    59  }
    60  
    61  // DefaultStorageFactory takes a GroupResource and returns back its storage interface.  This result includes:
    62  // 1. Merged etcd config, including: auth, server locations, prefixes
    63  // 2. Resource encodings for storage: group,version,kind to store as
    64  // 3. Cohabitating default: some resources like hpa are exposed through multiple APIs.  They must agree on 1 and 2
    65  type DefaultStorageFactory struct {
    66  	// StorageConfig describes how to create a storage backend in general.
    67  	// Its authentication information will be used for every storage.Interface returned.
    68  	StorageConfig storagebackend.Config
    69  
    70  	Overrides map[schema.GroupResource]groupResourceOverrides
    71  
    72  	DefaultResourcePrefixes map[schema.GroupResource]string
    73  
    74  	// DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used.
    75  	DefaultMediaType string
    76  
    77  	// DefaultSerializer is used to create encoders and decoders for the storage.Interface.
    78  	DefaultSerializer runtime.StorageSerializer
    79  
    80  	// ResourceEncodingConfig describes how to encode a particular GroupVersionResource
    81  	ResourceEncodingConfig ResourceEncodingConfig
    82  
    83  	// APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API
    84  	// This is discrete from resource enablement because those are separate concerns.  How this source is configured
    85  	// is left to the caller.
    86  	APIResourceConfigSource APIResourceConfigSource
    87  
    88  	// newStorageCodecFn exists to be overwritten for unit testing.
    89  	newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error)
    90  }
    91  
    92  type groupResourceOverrides struct {
    93  	// etcdLocation contains the list of "special" locations that are used for particular GroupResources
    94  	// These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource
    95  	etcdLocation []string
    96  	// etcdPrefix is the base location for a GroupResource.
    97  	etcdPrefix string
    98  	// etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location
    99  	// If empty, the default mapping is used.  If the default mapping doesn't contain an entry, it will use
   100  	// the ToLowered name of the resource, not including the group.
   101  	etcdResourcePrefix string
   102  	// mediaType is the desired serializer to choose. If empty, the default is chosen.
   103  	mediaType string
   104  	// serializer contains the list of "special" serializers for a GroupResource.  Resource=* means for the entire group
   105  	serializer runtime.StorageSerializer
   106  	// cohabitatingResources keeps track of which resources must be stored together.  This happens when we have multiple ways
   107  	// of exposing one set of concepts.  autoscaling.HPA and extensions.HPA as a for instance
   108  	// The order of the slice matters!  It is the priority order of lookup for finding a storage location
   109  	cohabitatingResources []schema.GroupResource
   110  	// encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized.
   111  	encoderDecoratorFn func(runtime.Encoder) runtime.Encoder
   112  	// decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of
   113  	// returned decoders will be priority for attempt to decode.
   114  	decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
   115  }
   116  
   117  // Apply overrides the provided config and options if the override has a value in that position
   118  func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) {
   119  	if len(o.etcdLocation) > 0 {
   120  		config.Transport.ServerList = o.etcdLocation
   121  	}
   122  	if len(o.etcdPrefix) > 0 {
   123  		config.Prefix = o.etcdPrefix
   124  	}
   125  
   126  	if len(o.mediaType) > 0 {
   127  		options.StorageMediaType = o.mediaType
   128  	}
   129  	if o.serializer != nil {
   130  		options.StorageSerializer = o.serializer
   131  	}
   132  	if o.encoderDecoratorFn != nil {
   133  		options.EncoderDecoratorFn = o.encoderDecoratorFn
   134  	}
   135  	if o.decoderDecoratorFn != nil {
   136  		options.DecoderDecoratorFn = o.decoderDecoratorFn
   137  	}
   138  }
   139  
   140  var _ StorageFactory = &DefaultStorageFactory{}
   141  
   142  const AllResources = "*"
   143  
   144  func NewDefaultStorageFactory(
   145  	config storagebackend.Config,
   146  	defaultMediaType string,
   147  	defaultSerializer runtime.StorageSerializer,
   148  	resourceEncodingConfig ResourceEncodingConfig,
   149  	resourceConfig APIResourceConfigSource,
   150  	specialDefaultResourcePrefixes map[schema.GroupResource]string,
   151  ) *DefaultStorageFactory {
   152  	if len(defaultMediaType) == 0 {
   153  		defaultMediaType = runtime.ContentTypeJSON
   154  	}
   155  	return &DefaultStorageFactory{
   156  		StorageConfig:           config,
   157  		Overrides:               map[schema.GroupResource]groupResourceOverrides{},
   158  		DefaultMediaType:        defaultMediaType,
   159  		DefaultSerializer:       defaultSerializer,
   160  		ResourceEncodingConfig:  resourceEncodingConfig,
   161  		APIResourceConfigSource: resourceConfig,
   162  		DefaultResourcePrefixes: specialDefaultResourcePrefixes,
   163  
   164  		newStorageCodecFn: NewStorageCodec,
   165  	}
   166  }
   167  
   168  func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) {
   169  	overrides := s.Overrides[groupResource]
   170  	overrides.etcdLocation = location
   171  	s.Overrides[groupResource] = overrides
   172  }
   173  
   174  func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) {
   175  	overrides := s.Overrides[groupResource]
   176  	overrides.etcdPrefix = prefix
   177  	s.Overrides[groupResource] = overrides
   178  }
   179  
   180  // SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir.  You'll end up in `etcdPrefix/resourceEtcdPrefix`.
   181  func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) {
   182  	overrides := s.Overrides[groupResource]
   183  	overrides.etcdResourcePrefix = prefix
   184  	s.Overrides[groupResource] = overrides
   185  }
   186  
   187  func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) {
   188  	overrides := s.Overrides[groupResource]
   189  	overrides.mediaType = mediaType
   190  	overrides.serializer = serializer
   191  	s.Overrides[groupResource] = overrides
   192  }
   193  
   194  // AddCohabitatingResources links resources together the order of the slice matters!  its the priority order of lookup for finding a storage location
   195  func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) {
   196  	for _, groupResource := range groupResources {
   197  		overrides := s.Overrides[groupResource]
   198  		overrides.cohabitatingResources = groupResources
   199  		s.Overrides[groupResource] = overrides
   200  	}
   201  }
   202  
   203  func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) {
   204  	for _, groupResource := range groupResources {
   205  		overrides := s.Overrides[groupResource]
   206  		overrides.encoderDecoratorFn = encoderDecoratorFn
   207  		overrides.decoderDecoratorFn = decoderDecoratorFn
   208  		s.Overrides[groupResource] = overrides
   209  	}
   210  }
   211  
   212  func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource {
   213  	return schema.GroupResource{Group: resource.Group, Resource: AllResources}
   214  }
   215  
   216  func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource {
   217  	for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources {
   218  		// TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters
   219  		if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) {
   220  			return potentialStorageResource
   221  		}
   222  	}
   223  
   224  	return groupResource
   225  }
   226  
   227  // New finds the storage destination for the given group and resource. It will
   228  // return an error if the group has no storage destination configured.
   229  func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
   230  	chosenStorageResource := s.getStorageGroupResource(groupResource)
   231  
   232  	// operate on copy
   233  	storageConfig := s.StorageConfig
   234  	codecConfig := StorageCodecConfig{
   235  		StorageMediaType:  s.DefaultMediaType,
   236  		StorageSerializer: s.DefaultSerializer,
   237  	}
   238  
   239  	if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok {
   240  		override.Apply(&storageConfig, &codecConfig)
   241  	}
   242  	if override, ok := s.Overrides[chosenStorageResource]; ok {
   243  		override.Apply(&storageConfig, &codecConfig)
   244  	}
   245  
   246  	var err error
   247  	if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
   248  		codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example)
   249  		if err != nil {
   250  			return nil, err
   251  		}
   252  	} else {
   253  		codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
   254  		if err != nil {
   255  			return nil, err
   256  		}
   257  	}
   258  
   259  	codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
   260  	if err != nil {
   261  		return nil, err
   262  	}
   263  
   264  	codecConfig.Config = storageConfig
   265  
   266  	storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)
   267  	if err != nil {
   268  		return nil, err
   269  	}
   270  	klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config)
   271  
   272  	return storageConfig.ForResource(groupResource), nil
   273  }
   274  
   275  // Configs implements StorageFactory.
   276  func (s *DefaultStorageFactory) Configs() []storagebackend.Config {
   277  	return configs(s.StorageConfig, s.Overrides)
   278  }
   279  
   280  // Configs gets configurations for all of registered storage destinations.
   281  func Configs(storageConfig storagebackend.Config) []storagebackend.Config {
   282  	return configs(storageConfig, nil)
   283  }
   284  
   285  // Returns all storage configurations including those for group resource overrides
   286  func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config {
   287  	configs := []storagebackend.Config{storageConfig}
   288  
   289  	for _, override := range grOverrides {
   290  		if len(override.etcdLocation) == 0 {
   291  			continue
   292  		}
   293  		// copy
   294  		newConfig := storageConfig
   295  		override.Apply(&newConfig, &StorageCodecConfig{})
   296  		newConfig.Transport.ServerList = override.etcdLocation
   297  		configs = append(configs, newConfig)
   298  	}
   299  	return configs
   300  }
   301  
   302  // Backends implements StorageFactory.
   303  func (s *DefaultStorageFactory) Backends() []Backend {
   304  	return backends(s.StorageConfig, s.Overrides)
   305  }
   306  
   307  // Backends returns all backends for all registered storage destinations.
   308  // Used for getting all instances for health validations.
   309  // Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber.
   310  func Backends(storageConfig storagebackend.Config) []Backend {
   311  	return backends(storageConfig, nil)
   312  }
   313  
   314  func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend {
   315  	servers := sets.NewString(storageConfig.Transport.ServerList...)
   316  
   317  	for _, overrides := range grOverrides {
   318  		servers.Insert(overrides.etcdLocation...)
   319  	}
   320  
   321  	tlsConfig := &tls.Config{
   322  		InsecureSkipVerify: true,
   323  	}
   324  	if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 {
   325  		cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile)
   326  		if err != nil {
   327  			klog.Errorf("failed to load key pair while getting backends: %s", err)
   328  		} else {
   329  			tlsConfig.Certificates = []tls.Certificate{cert}
   330  		}
   331  	}
   332  	if len(storageConfig.Transport.TrustedCAFile) > 0 {
   333  		if caCert, err := os.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil {
   334  			klog.Errorf("failed to read ca file while getting backends: %s", err)
   335  		} else {
   336  			caPool := x509.NewCertPool()
   337  			caPool.AppendCertsFromPEM(caCert)
   338  			tlsConfig.RootCAs = caPool
   339  			tlsConfig.InsecureSkipVerify = false
   340  		}
   341  	}
   342  
   343  	backends := []Backend{}
   344  	for server := range servers {
   345  		backends = append(backends, Backend{
   346  			Server: server,
   347  			// We can't share TLSConfig across different backends to avoid races.
   348  			// For more details see: https://pr.k8s.io/59338
   349  			TLSConfig: tlsConfig.Clone(),
   350  		})
   351  	}
   352  	return backends
   353  }
   354  
   355  func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string {
   356  	chosenStorageResource := s.getStorageGroupResource(groupResource)
   357  	groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)]
   358  	exactResourceOverride := s.Overrides[chosenStorageResource]
   359  
   360  	etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource]
   361  	if len(groupOverride.etcdResourcePrefix) > 0 {
   362  		etcdResourcePrefix = groupOverride.etcdResourcePrefix
   363  	}
   364  	if len(exactResourceOverride.etcdResourcePrefix) > 0 {
   365  		etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix
   366  	}
   367  	if len(etcdResourcePrefix) == 0 {
   368  		etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource)
   369  	}
   370  
   371  	return etcdResourcePrefix
   372  }