k8s.io/apiserver@v0.31.1/pkg/server/storage/storage_factory.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package storage 18 19 import ( 20 "crypto/tls" 21 "crypto/x509" 22 "os" 23 "strings" 24 25 "k8s.io/apimachinery/pkg/runtime" 26 "k8s.io/apimachinery/pkg/runtime/schema" 27 "k8s.io/apimachinery/pkg/util/sets" 28 "k8s.io/apiserver/pkg/storage/storagebackend" 29 "k8s.io/klog/v2" 30 ) 31 32 // Backend describes the storage servers, the information here should be enough 33 // for health validations. 34 type Backend struct { 35 // the url of storage backend like: https://etcd.domain:2379 36 Server string 37 // the required tls config 38 TLSConfig *tls.Config 39 } 40 41 // StorageFactory is the interface to locate the storage for a given GroupResource 42 type StorageFactory interface { 43 // New finds the storage destination for the given group and resource. It will 44 // return an error if the group has no storage destination configured. 45 NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) 46 47 // ResourcePrefix returns the overridden resource prefix for the GroupResource 48 // This allows for cohabitation of resources with different native types and provides 49 // centralized control over the shape of etcd directories 50 ResourcePrefix(groupResource schema.GroupResource) string 51 52 // Configs gets configurations for all of registered storage destinations. 53 Configs() []storagebackend.Config 54 55 // Backends gets all backends for all registered storage destinations. 56 // Used for getting all instances for health validations. 57 // Deprecated: Use Configs instead 58 Backends() []Backend 59 } 60 61 // DefaultStorageFactory takes a GroupResource and returns back its storage interface. This result includes: 62 // 1. Merged etcd config, including: auth, server locations, prefixes 63 // 2. Resource encodings for storage: group,version,kind to store as 64 // 3. Cohabitating default: some resources like hpa are exposed through multiple APIs. They must agree on 1 and 2 65 type DefaultStorageFactory struct { 66 // StorageConfig describes how to create a storage backend in general. 67 // Its authentication information will be used for every storage.Interface returned. 68 StorageConfig storagebackend.Config 69 70 Overrides map[schema.GroupResource]groupResourceOverrides 71 72 DefaultResourcePrefixes map[schema.GroupResource]string 73 74 // DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used. 75 DefaultMediaType string 76 77 // DefaultSerializer is used to create encoders and decoders for the storage.Interface. 78 DefaultSerializer runtime.StorageSerializer 79 80 // ResourceEncodingConfig describes how to encode a particular GroupVersionResource 81 ResourceEncodingConfig ResourceEncodingConfig 82 83 // APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API 84 // This is discrete from resource enablement because those are separate concerns. How this source is configured 85 // is left to the caller. 86 APIResourceConfigSource APIResourceConfigSource 87 88 // newStorageCodecFn exists to be overwritten for unit testing. 89 newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error) 90 } 91 92 type groupResourceOverrides struct { 93 // etcdLocation contains the list of "special" locations that are used for particular GroupResources 94 // These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource 95 etcdLocation []string 96 // etcdPrefix is the base location for a GroupResource. 97 etcdPrefix string 98 // etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location 99 // If empty, the default mapping is used. If the default mapping doesn't contain an entry, it will use 100 // the ToLowered name of the resource, not including the group. 101 etcdResourcePrefix string 102 // mediaType is the desired serializer to choose. If empty, the default is chosen. 103 mediaType string 104 // serializer contains the list of "special" serializers for a GroupResource. Resource=* means for the entire group 105 serializer runtime.StorageSerializer 106 // cohabitatingResources keeps track of which resources must be stored together. This happens when we have multiple ways 107 // of exposing one set of concepts. autoscaling.HPA and extensions.HPA as a for instance 108 // The order of the slice matters! It is the priority order of lookup for finding a storage location 109 cohabitatingResources []schema.GroupResource 110 // encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized. 111 encoderDecoratorFn func(runtime.Encoder) runtime.Encoder 112 // decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of 113 // returned decoders will be priority for attempt to decode. 114 decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder 115 } 116 117 // Apply overrides the provided config and options if the override has a value in that position 118 func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) { 119 if len(o.etcdLocation) > 0 { 120 config.Transport.ServerList = o.etcdLocation 121 } 122 if len(o.etcdPrefix) > 0 { 123 config.Prefix = o.etcdPrefix 124 } 125 126 if len(o.mediaType) > 0 { 127 options.StorageMediaType = o.mediaType 128 } 129 if o.serializer != nil { 130 options.StorageSerializer = o.serializer 131 } 132 if o.encoderDecoratorFn != nil { 133 options.EncoderDecoratorFn = o.encoderDecoratorFn 134 } 135 if o.decoderDecoratorFn != nil { 136 options.DecoderDecoratorFn = o.decoderDecoratorFn 137 } 138 } 139 140 var _ StorageFactory = &DefaultStorageFactory{} 141 142 const AllResources = "*" 143 144 func NewDefaultStorageFactory( 145 config storagebackend.Config, 146 defaultMediaType string, 147 defaultSerializer runtime.StorageSerializer, 148 resourceEncodingConfig ResourceEncodingConfig, 149 resourceConfig APIResourceConfigSource, 150 specialDefaultResourcePrefixes map[schema.GroupResource]string, 151 ) *DefaultStorageFactory { 152 if len(defaultMediaType) == 0 { 153 defaultMediaType = runtime.ContentTypeJSON 154 } 155 return &DefaultStorageFactory{ 156 StorageConfig: config, 157 Overrides: map[schema.GroupResource]groupResourceOverrides{}, 158 DefaultMediaType: defaultMediaType, 159 DefaultSerializer: defaultSerializer, 160 ResourceEncodingConfig: resourceEncodingConfig, 161 APIResourceConfigSource: resourceConfig, 162 DefaultResourcePrefixes: specialDefaultResourcePrefixes, 163 164 newStorageCodecFn: NewStorageCodec, 165 } 166 } 167 168 func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) { 169 overrides := s.Overrides[groupResource] 170 overrides.etcdLocation = location 171 s.Overrides[groupResource] = overrides 172 } 173 174 func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) { 175 overrides := s.Overrides[groupResource] 176 overrides.etcdPrefix = prefix 177 s.Overrides[groupResource] = overrides 178 } 179 180 // SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir. You'll end up in `etcdPrefix/resourceEtcdPrefix`. 181 func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) { 182 overrides := s.Overrides[groupResource] 183 overrides.etcdResourcePrefix = prefix 184 s.Overrides[groupResource] = overrides 185 } 186 187 func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) { 188 overrides := s.Overrides[groupResource] 189 overrides.mediaType = mediaType 190 overrides.serializer = serializer 191 s.Overrides[groupResource] = overrides 192 } 193 194 // AddCohabitatingResources links resources together the order of the slice matters! its the priority order of lookup for finding a storage location 195 func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) { 196 for _, groupResource := range groupResources { 197 overrides := s.Overrides[groupResource] 198 overrides.cohabitatingResources = groupResources 199 s.Overrides[groupResource] = overrides 200 } 201 } 202 203 func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) { 204 for _, groupResource := range groupResources { 205 overrides := s.Overrides[groupResource] 206 overrides.encoderDecoratorFn = encoderDecoratorFn 207 overrides.decoderDecoratorFn = decoderDecoratorFn 208 s.Overrides[groupResource] = overrides 209 } 210 } 211 212 func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource { 213 return schema.GroupResource{Group: resource.Group, Resource: AllResources} 214 } 215 216 func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource { 217 for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources { 218 // TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters 219 if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) { 220 return potentialStorageResource 221 } 222 } 223 224 return groupResource 225 } 226 227 // New finds the storage destination for the given group and resource. It will 228 // return an error if the group has no storage destination configured. 229 func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { 230 chosenStorageResource := s.getStorageGroupResource(groupResource) 231 232 // operate on copy 233 storageConfig := s.StorageConfig 234 codecConfig := StorageCodecConfig{ 235 StorageMediaType: s.DefaultMediaType, 236 StorageSerializer: s.DefaultSerializer, 237 } 238 239 if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok { 240 override.Apply(&storageConfig, &codecConfig) 241 } 242 if override, ok := s.Overrides[chosenStorageResource]; ok { 243 override.Apply(&storageConfig, &codecConfig) 244 } 245 246 var err error 247 if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok { 248 codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example) 249 if err != nil { 250 return nil, err 251 } 252 } else { 253 codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) 254 if err != nil { 255 return nil, err 256 } 257 } 258 259 codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource) 260 if err != nil { 261 return nil, err 262 } 263 264 codecConfig.Config = storageConfig 265 266 storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) 267 if err != nil { 268 return nil, err 269 } 270 klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config) 271 272 return storageConfig.ForResource(groupResource), nil 273 } 274 275 // Configs implements StorageFactory. 276 func (s *DefaultStorageFactory) Configs() []storagebackend.Config { 277 return configs(s.StorageConfig, s.Overrides) 278 } 279 280 // Configs gets configurations for all of registered storage destinations. 281 func Configs(storageConfig storagebackend.Config) []storagebackend.Config { 282 return configs(storageConfig, nil) 283 } 284 285 // Returns all storage configurations including those for group resource overrides 286 func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config { 287 configs := []storagebackend.Config{storageConfig} 288 289 for _, override := range grOverrides { 290 if len(override.etcdLocation) == 0 { 291 continue 292 } 293 // copy 294 newConfig := storageConfig 295 override.Apply(&newConfig, &StorageCodecConfig{}) 296 newConfig.Transport.ServerList = override.etcdLocation 297 configs = append(configs, newConfig) 298 } 299 return configs 300 } 301 302 // Backends implements StorageFactory. 303 func (s *DefaultStorageFactory) Backends() []Backend { 304 return backends(s.StorageConfig, s.Overrides) 305 } 306 307 // Backends returns all backends for all registered storage destinations. 308 // Used for getting all instances for health validations. 309 // Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber. 310 func Backends(storageConfig storagebackend.Config) []Backend { 311 return backends(storageConfig, nil) 312 } 313 314 func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend { 315 servers := sets.NewString(storageConfig.Transport.ServerList...) 316 317 for _, overrides := range grOverrides { 318 servers.Insert(overrides.etcdLocation...) 319 } 320 321 tlsConfig := &tls.Config{ 322 InsecureSkipVerify: true, 323 } 324 if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 { 325 cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile) 326 if err != nil { 327 klog.Errorf("failed to load key pair while getting backends: %s", err) 328 } else { 329 tlsConfig.Certificates = []tls.Certificate{cert} 330 } 331 } 332 if len(storageConfig.Transport.TrustedCAFile) > 0 { 333 if caCert, err := os.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil { 334 klog.Errorf("failed to read ca file while getting backends: %s", err) 335 } else { 336 caPool := x509.NewCertPool() 337 caPool.AppendCertsFromPEM(caCert) 338 tlsConfig.RootCAs = caPool 339 tlsConfig.InsecureSkipVerify = false 340 } 341 } 342 343 backends := []Backend{} 344 for server := range servers { 345 backends = append(backends, Backend{ 346 Server: server, 347 // We can't share TLSConfig across different backends to avoid races. 348 // For more details see: https://pr.k8s.io/59338 349 TLSConfig: tlsConfig.Clone(), 350 }) 351 } 352 return backends 353 } 354 355 func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string { 356 chosenStorageResource := s.getStorageGroupResource(groupResource) 357 groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)] 358 exactResourceOverride := s.Overrides[chosenStorageResource] 359 360 etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource] 361 if len(groupOverride.etcdResourcePrefix) > 0 { 362 etcdResourcePrefix = groupOverride.etcdResourcePrefix 363 } 364 if len(exactResourceOverride.etcdResourcePrefix) > 0 { 365 etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix 366 } 367 if len(etcdResourcePrefix) == 0 { 368 etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource) 369 } 370 371 return etcdResourcePrefix 372 }