k8s.io/apiserver@v0.31.1/pkg/server/options/etcd.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package options 18 19 import ( 20 "context" 21 "fmt" 22 "net/http" 23 "strconv" 24 "strings" 25 "time" 26 27 "github.com/spf13/pflag" 28 29 "k8s.io/apimachinery/pkg/runtime" 30 "k8s.io/apimachinery/pkg/runtime/schema" 31 "k8s.io/apimachinery/pkg/util/sets" 32 "k8s.io/apimachinery/pkg/util/wait" 33 "k8s.io/apiserver/pkg/registry/generic" 34 genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" 35 "k8s.io/apiserver/pkg/server" 36 "k8s.io/apiserver/pkg/server/healthz" 37 "k8s.io/apiserver/pkg/server/options/encryptionconfig" 38 encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller" 39 serverstorage "k8s.io/apiserver/pkg/server/storage" 40 "k8s.io/apiserver/pkg/storage/etcd3/metrics" 41 "k8s.io/apiserver/pkg/storage/storagebackend" 42 storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory" 43 storagevalue "k8s.io/apiserver/pkg/storage/value" 44 "k8s.io/klog/v2" 45 ) 46 47 type EtcdOptions struct { 48 StorageConfig storagebackend.Config 49 EncryptionProviderConfigFilepath string 50 EncryptionProviderConfigAutomaticReload bool 51 52 EtcdServersOverrides []string 53 54 // To enable protobuf as storage format, it is enough 55 // to set it to "application/vnd.kubernetes.protobuf". 56 DefaultStorageMediaType string 57 DeleteCollectionWorkers int 58 EnableGarbageCollection bool 59 60 // Set EnableWatchCache to false to disable all watch caches 61 EnableWatchCache bool 62 // Set DefaultWatchCacheSize to zero to disable watch caches for those resources that have no explicit cache size set 63 DefaultWatchCacheSize int 64 // WatchCacheSizes represents override to a given resource 65 WatchCacheSizes []string 66 67 // SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints. 68 // This allows multiple invocations of the Apply methods without duplication of said endpoints. 69 SkipHealthEndpoints bool 70 } 71 72 var storageTypes = sets.NewString( 73 storagebackend.StorageTypeETCD3, 74 ) 75 76 func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions { 77 options := &EtcdOptions{ 78 StorageConfig: *backendConfig, 79 DefaultStorageMediaType: "application/json", 80 DeleteCollectionWorkers: 1, 81 EnableGarbageCollection: true, 82 EnableWatchCache: true, 83 DefaultWatchCacheSize: 100, 84 } 85 options.StorageConfig.CountMetricPollPeriod = time.Minute 86 return options 87 } 88 89 var storageMediaTypes = sets.New( 90 runtime.ContentTypeJSON, 91 runtime.ContentTypeYAML, 92 runtime.ContentTypeProtobuf, 93 ) 94 95 func (s *EtcdOptions) Validate() []error { 96 if s == nil { 97 return nil 98 } 99 100 allErrors := []error{} 101 if len(s.StorageConfig.Transport.ServerList) == 0 { 102 allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified")) 103 } 104 105 if s.StorageConfig.Type != storagebackend.StorageTypeUnset && !storageTypes.Has(s.StorageConfig.Type) { 106 allErrors = append(allErrors, fmt.Errorf("--storage-backend invalid, allowed values: %s. If not specified, it will default to 'etcd3'", strings.Join(storageTypes.List(), ", "))) 107 } 108 109 for _, override := range s.EtcdServersOverrides { 110 tokens := strings.Split(override, "#") 111 if len(tokens) != 2 { 112 allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated")) 113 continue 114 } 115 116 apiresource := strings.Split(tokens[0], "/") 117 if len(apiresource) != 2 { 118 allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated")) 119 continue 120 } 121 122 } 123 124 if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload { 125 allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config")) 126 } 127 128 if s.DefaultStorageMediaType != "" && !storageMediaTypes.Has(s.DefaultStorageMediaType) { 129 allErrors = append(allErrors, fmt.Errorf("--storage-media-type %q invalid, allowed values: %s", s.DefaultStorageMediaType, strings.Join(sets.List(storageMediaTypes), ", "))) 130 } 131 132 return allErrors 133 } 134 135 // AddFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet 136 func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { 137 if s == nil { 138 return 139 } 140 141 fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+ 142 "Per-resource etcd servers overrides, comma separated. The individual override "+ 143 "format: group/resource#servers, where servers are URLs, semicolon separated. "+ 144 "Note that this applies only to resources compiled into this server binary. ") 145 146 fs.StringVar(&s.DefaultStorageMediaType, "storage-media-type", s.DefaultStorageMediaType, ""+ 147 "The media type to use to store objects in storage. "+ 148 "Some resources or storage backends may only support a specific media type and will ignore this setting. "+ 149 "Supported media types: [application/json, application/yaml, application/vnd.kubernetes.protobuf]") 150 fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers, 151 "Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.") 152 153 fs.BoolVar(&s.EnableGarbageCollection, "enable-garbage-collector", s.EnableGarbageCollection, ""+ 154 "Enables the generic garbage collector. MUST be synced with the corresponding flag "+ 155 "of the kube-controller-manager.") 156 157 fs.BoolVar(&s.EnableWatchCache, "watch-cache", s.EnableWatchCache, 158 "Enable watch caching in the apiserver") 159 160 fs.IntVar(&s.DefaultWatchCacheSize, "default-watch-cache-size", s.DefaultWatchCacheSize, 161 "Default watch cache size. If zero, watch cache will be disabled for resources that do not have a default watch size set.") 162 163 fs.MarkDeprecated("default-watch-cache-size", 164 "watch caches are sized automatically and this flag will be removed in a future version") 165 166 fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, ""+ 167 "Watch cache size settings for some resources (pods, nodes, etc.), comma separated. "+ 168 "The individual setting format: resource[.group]#size, where resource is lowercase plural (no version), "+ 169 "group is omitted for resources of apiVersion v1 (the legacy core API) and included for others, "+ 170 "and size is a number. This option is only meaningful for resources built into the apiserver, "+ 171 "not ones defined by CRDs or aggregated from external servers, and is only consulted if the "+ 172 "watch-cache is enabled. The only meaningful size setting to supply here is zero, which means to "+ 173 "disable watch caching for the associated resource; all non-zero values are equivalent and mean "+ 174 "to not disable watch caching for that resource") 175 176 fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type, 177 "The storage backend for persistence. Options: 'etcd3' (default).") 178 179 fs.StringSliceVar(&s.StorageConfig.Transport.ServerList, "etcd-servers", s.StorageConfig.Transport.ServerList, 180 "List of etcd servers to connect with (scheme://ip:port), comma separated.") 181 182 fs.StringVar(&s.StorageConfig.Prefix, "etcd-prefix", s.StorageConfig.Prefix, 183 "The prefix to prepend to all resource paths in etcd.") 184 185 fs.StringVar(&s.StorageConfig.Transport.KeyFile, "etcd-keyfile", s.StorageConfig.Transport.KeyFile, 186 "SSL key file used to secure etcd communication.") 187 188 fs.StringVar(&s.StorageConfig.Transport.CertFile, "etcd-certfile", s.StorageConfig.Transport.CertFile, 189 "SSL certification file used to secure etcd communication.") 190 191 fs.StringVar(&s.StorageConfig.Transport.TrustedCAFile, "etcd-cafile", s.StorageConfig.Transport.TrustedCAFile, 192 "SSL Certificate Authority file used to secure etcd communication.") 193 194 fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath, 195 "The file containing configuration for encryption providers to be used for storing secrets in etcd") 196 197 fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload, 198 "Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+ 199 "Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.") 200 201 fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval, 202 "The interval of compaction requests. If 0, the compaction request from apiserver is disabled.") 203 204 fs.DurationVar(&s.StorageConfig.CountMetricPollPeriod, "etcd-count-metric-poll-period", s.StorageConfig.CountMetricPollPeriod, ""+ 205 "Frequency of polling etcd for number of resources per type. 0 disables the metric collection.") 206 207 fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval, 208 "The interval of requests to poll etcd and update metric. 0 disables the metric collection") 209 210 fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout, 211 "The timeout to use when checking etcd health.") 212 213 fs.DurationVar(&s.StorageConfig.ReadycheckTimeout, "etcd-readycheck-timeout", s.StorageConfig.ReadycheckTimeout, 214 "The timeout to use when checking etcd readiness") 215 216 fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, 217 "The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.") 218 } 219 220 // ApplyTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions). 221 func (s *EtcdOptions) ApplyTo(c *server.Config) error { 222 if s == nil { 223 return nil 224 } 225 226 storageConfigCopy := s.StorageConfig 227 if storageConfigCopy.StorageObjectCountTracker == nil { 228 storageConfigCopy.StorageObjectCountTracker = c.StorageObjectCountTracker 229 } 230 231 return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: storageConfigCopy}, c) 232 } 233 234 // ApplyWithStorageFactoryTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions). 235 func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error { 236 if s == nil { 237 return nil 238 } 239 240 if !s.SkipHealthEndpoints { 241 if err := s.addEtcdHealthEndpoint(c); err != nil { 242 return err 243 } 244 } 245 246 // setup encryption 247 if err := s.maybeApplyResourceTransformers(c); err != nil { 248 return err 249 } 250 251 metrics.SetStorageMonitorGetter(monitorGetter(factory)) 252 253 c.RESTOptionsGetter = s.CreateRESTOptionsGetter(factory, c.ResourceTransformers) 254 return nil 255 } 256 257 func monitorGetter(factory serverstorage.StorageFactory) func() (monitors []metrics.Monitor, err error) { 258 return func() (monitors []metrics.Monitor, err error) { 259 defer func() { 260 if err != nil { 261 for _, m := range monitors { 262 m.Close() 263 } 264 } 265 }() 266 267 var m metrics.Monitor 268 for _, cfg := range factory.Configs() { 269 m, err = storagefactory.CreateMonitor(cfg) 270 if err != nil { 271 return nil, err 272 } 273 monitors = append(monitors, m) 274 } 275 return monitors, nil 276 } 277 } 278 279 func (s *EtcdOptions) CreateRESTOptionsGetter(factory serverstorage.StorageFactory, resourceTransformers storagevalue.ResourceTransformers) generic.RESTOptionsGetter { 280 if resourceTransformers != nil { 281 factory = &transformerStorageFactory{ 282 delegate: factory, 283 resourceTransformers: resourceTransformers, 284 } 285 } 286 return &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory} 287 } 288 289 func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err error) { 290 if c.ResourceTransformers != nil { 291 return nil 292 } 293 if len(s.EncryptionProviderConfigFilepath) == 0 { 294 return nil 295 } 296 297 ctxServer := wait.ContextForChannel(c.DrainedNotify()) 298 ctxTransformers, closeTransformers := context.WithCancel(ctxServer) 299 defer func() { 300 // in case of error, we want to close partially initialized (if any) transformers 301 if err != nil { 302 closeTransformers() 303 } 304 }() 305 306 encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID) 307 if err != nil { 308 return err 309 } 310 311 if s.EncryptionProviderConfigAutomaticReload { 312 // with reload=true we will always have 1 health check 313 if len(encryptionConfiguration.HealthChecks) != 1 { 314 return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled") 315 } 316 317 // Here the dynamic transformers take ownership of the transformers and their cancellation. 318 dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod) 319 320 // add post start hook to start hot reload controller 321 // adding this hook here will ensure that it gets configured exactly once 322 err = c.AddPostStartHook( 323 "start-encryption-provider-config-automatic-reload", 324 func(_ server.PostStartHookContext) error { 325 dynamicEncryptionConfigController := encryptionconfigcontroller.NewDynamicEncryptionConfiguration( 326 "encryption-provider-config-automatic-reload-controller", 327 s.EncryptionProviderConfigFilepath, 328 dynamicTransformers, 329 encryptionConfiguration.EncryptionFileContentHash, 330 c.APIServerID, 331 ) 332 333 go dynamicEncryptionConfigController.Run(ctxServer) 334 335 return nil 336 }, 337 ) 338 if err != nil { 339 return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err) 340 } 341 342 c.ResourceTransformers = dynamicTransformers 343 if !s.SkipHealthEndpoints { 344 addHealthChecksWithoutLivez(c, dynamicTransformers) 345 } 346 } else { 347 c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers) 348 if !s.SkipHealthEndpoints { 349 addHealthChecksWithoutLivez(c, encryptionConfiguration.HealthChecks...) 350 } 351 } 352 353 return nil 354 } 355 356 func addHealthChecksWithoutLivez(c *server.Config, healthChecks ...healthz.HealthChecker) { 357 c.HealthzChecks = append(c.HealthzChecks, healthChecks...) 358 c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...) 359 } 360 361 func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error { 362 healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify()) 363 if err != nil { 364 return err 365 } 366 c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error { 367 return healthCheck() 368 })) 369 370 readyCheck, err := storagefactory.CreateReadyCheck(s.StorageConfig, c.DrainedNotify()) 371 if err != nil { 372 return err 373 } 374 c.AddReadyzChecks(healthz.NamedCheck("etcd-readiness", func(r *http.Request) error { 375 return readyCheck() 376 })) 377 378 return nil 379 } 380 381 type StorageFactoryRestOptionsFactory struct { 382 Options EtcdOptions 383 StorageFactory serverstorage.StorageFactory 384 } 385 386 func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) { 387 storageConfig, err := f.StorageFactory.NewConfig(resource, example) 388 if err != nil { 389 return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error()) 390 } 391 392 ret := generic.RESTOptions{ 393 StorageConfig: storageConfig, 394 Decorator: generic.UndecoratedStorage, 395 DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers, 396 EnableGarbageCollection: f.Options.EnableGarbageCollection, 397 ResourcePrefix: f.StorageFactory.ResourcePrefix(resource), 398 CountMetricPollPeriod: f.Options.StorageConfig.CountMetricPollPeriod, 399 StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker, 400 } 401 402 if ret.StorageObjectCountTracker == nil { 403 ret.StorageObjectCountTracker = storageConfig.StorageObjectCountTracker 404 } 405 406 if f.Options.EnableWatchCache { 407 sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes) 408 if err != nil { 409 return generic.RESTOptions{}, err 410 } 411 size, ok := sizes[resource] 412 if ok && size > 0 { 413 klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource) 414 } 415 if ok && size <= 0 { 416 klog.V(3).InfoS("Not using watch cache", "resource", resource) 417 ret.Decorator = generic.UndecoratedStorage 418 } else { 419 klog.V(3).InfoS("Using watch cache", "resource", resource) 420 ret.Decorator = genericregistry.StorageWithCacher() 421 } 422 } 423 424 return ret, nil 425 } 426 427 // ParseWatchCacheSizes turns a list of cache size values into a map of group resources 428 // to requested sizes. 429 func ParseWatchCacheSizes(cacheSizes []string) (map[schema.GroupResource]int, error) { 430 watchCacheSizes := make(map[schema.GroupResource]int) 431 for _, c := range cacheSizes { 432 tokens := strings.Split(c, "#") 433 if len(tokens) != 2 { 434 return nil, fmt.Errorf("invalid value of watch cache size: %s", c) 435 } 436 437 size, err := strconv.Atoi(tokens[1]) 438 if err != nil { 439 return nil, fmt.Errorf("invalid size of watch cache size: %s", c) 440 } 441 if size < 0 { 442 return nil, fmt.Errorf("watch cache size cannot be negative: %s", c) 443 } 444 watchCacheSizes[schema.ParseGroupResource(tokens[0])] = size 445 } 446 return watchCacheSizes, nil 447 } 448 449 // WriteWatchCacheSizes turns a map of cache size values into a list of string specifications. 450 func WriteWatchCacheSizes(watchCacheSizes map[schema.GroupResource]int) ([]string, error) { 451 var cacheSizes []string 452 453 for resource, size := range watchCacheSizes { 454 if size < 0 { 455 return nil, fmt.Errorf("watch cache size cannot be negative for resource %s", resource) 456 } 457 cacheSizes = append(cacheSizes, fmt.Sprintf("%s#%d", resource.String(), size)) 458 } 459 return cacheSizes, nil 460 } 461 462 var _ serverstorage.StorageFactory = &SimpleStorageFactory{} 463 464 // SimpleStorageFactory provides a StorageFactory implementation that should be used when different 465 // resources essentially share the same storage config (as defined by the given storagebackend.Config). 466 // It assumes the resources are stored at a path that is purely based on the schema.GroupResource. 467 // Users that need flexibility and per resource overrides should use DefaultStorageFactory instead. 468 type SimpleStorageFactory struct { 469 StorageConfig storagebackend.Config 470 } 471 472 func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { 473 return s.StorageConfig.ForResource(resource), nil 474 } 475 476 func (s *SimpleStorageFactory) ResourcePrefix(resource schema.GroupResource) string { 477 return resource.Group + "/" + resource.Resource 478 } 479 480 func (s *SimpleStorageFactory) Configs() []storagebackend.Config { 481 return serverstorage.Configs(s.StorageConfig) 482 } 483 484 func (s *SimpleStorageFactory) Backends() []serverstorage.Backend { 485 // nothing should ever call this method but we still provide a functional implementation 486 return serverstorage.Backends(s.StorageConfig) 487 } 488 489 var _ serverstorage.StorageFactory = &transformerStorageFactory{} 490 491 type transformerStorageFactory struct { 492 delegate serverstorage.StorageFactory 493 resourceTransformers storagevalue.ResourceTransformers 494 } 495 496 func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) { 497 config, err := t.delegate.NewConfig(resource, example) 498 if err != nil { 499 return nil, err 500 } 501 502 configCopy := *config 503 resourceConfig := configCopy.Config 504 resourceConfig.Transformer = t.resourceTransformers.TransformerForResource(resource) 505 configCopy.Config = resourceConfig 506 507 return &configCopy, nil 508 } 509 510 func (t *transformerStorageFactory) ResourcePrefix(resource schema.GroupResource) string { 511 return t.delegate.ResourcePrefix(resource) 512 } 513 514 func (t *transformerStorageFactory) Configs() []storagebackend.Config { 515 return t.delegate.Configs() 516 } 517 518 func (t *transformerStorageFactory) Backends() []serverstorage.Backend { 519 return t.delegate.Backends() 520 }