k8s.io/apiserver@v0.31.1/pkg/util/version/registry.go (about) 1 /* 2 Copyright 2024 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package version 18 19 import ( 20 "fmt" 21 "sort" 22 "strings" 23 "sync" 24 25 "github.com/spf13/pflag" 26 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 27 "k8s.io/apimachinery/pkg/util/version" 28 cliflag "k8s.io/component-base/cli/flag" 29 "k8s.io/component-base/featuregate" 30 "k8s.io/klog/v2" 31 ) 32 33 // DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access. 34 // Example usage: 35 // // register the component effective version and feature gate first 36 // _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) 37 // wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") 38 // wardleFeatureGate := featuregate.NewFeatureGate() 39 // utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) 40 // 41 // cmd := &cobra.Command{ 42 // ... 43 // // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE 44 // PersistentPreRunE: func(*cobra.Command, []string) error { 45 // if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { 46 // return err 47 // } 48 // ... 49 // }, 50 // RunE: func(c *cobra.Command, args []string) error { 51 // // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere 52 // }, 53 // } 54 // 55 // flags := cmd.Flags() 56 // // add flags 57 // utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) 58 var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() 59 60 const ( 61 DefaultKubeComponent = "kube" 62 63 klogLevel = 2 64 ) 65 66 type VersionMapping func(from *version.Version) *version.Version 67 68 // ComponentGlobals stores the global variables for a component for easy access. 69 type ComponentGlobals struct { 70 effectiveVersion MutableEffectiveVersion 71 featureGate featuregate.MutableVersionedFeatureGate 72 73 // emulationVersionMapping contains the mapping from the emulation version of this component 74 // to the emulation version of another component. 75 emulationVersionMapping map[string]VersionMapping 76 // dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component. 77 // If true, the emulation version cannot be set from the flag, or version mapping from another component. 78 dependentEmulationVersion bool 79 // minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component 80 // to the min compatibility version of another component. 81 minCompatibilityVersionMapping map[string]VersionMapping 82 // dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component 83 // If true, the min compatibility version cannot be set from the flag, or version mapping from another component. 84 dependentMinCompatibilityVersion bool 85 } 86 87 type ComponentGlobalsRegistry interface { 88 // EffectiveVersionFor returns the EffectiveVersion registered under the component. 89 // Returns nil if the component is not registered. 90 EffectiveVersionFor(component string) EffectiveVersion 91 // FeatureGateFor returns the FeatureGate registered under the component. 92 // Returns nil if the component is not registered. 93 FeatureGateFor(component string) featuregate.FeatureGate 94 // Register registers the EffectiveVersion and FeatureGate for a component. 95 // returns error if the component is already registered. 96 Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error 97 // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. 98 // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. 99 ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) 100 // AddFlags adds flags of "--emulated-version" and "--feature-gates" 101 AddFlags(fs *pflag.FlagSet) 102 // Set sets the flags for all global variables for all components registered. 103 Set() error 104 // SetFallback calls Set() if it has never been called. 105 SetFallback() error 106 // Validate calls the Validate() function for all the global variables for all components registered. 107 Validate() []error 108 // Reset removes all stored ComponentGlobals, configurations, and version mappings. 109 Reset() 110 // SetEmulationVersionMapping sets the mapping from the emulation version of one component 111 // to the emulation version of another component. 112 // Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent, 113 // and cannot be set from cmd flags anymore. 114 // For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed. 115 SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error 116 } 117 118 type componentGlobalsRegistry struct { 119 componentGlobals map[string]*ComponentGlobals 120 mutex sync.RWMutex 121 // list of component name to emulation version set from the flag. 122 emulationVersionConfig []string 123 // map of component name to the list of feature gates set from the flag. 124 featureGatesConfig map[string][]string 125 // set stores if the Set() function for the registry is already called. 126 set bool 127 } 128 129 func NewComponentGlobalsRegistry() *componentGlobalsRegistry { 130 return &componentGlobalsRegistry{ 131 componentGlobals: make(map[string]*ComponentGlobals), 132 emulationVersionConfig: nil, 133 featureGatesConfig: nil, 134 } 135 } 136 137 func (r *componentGlobalsRegistry) Reset() { 138 r.mutex.Lock() 139 defer r.mutex.Unlock() 140 r.componentGlobals = make(map[string]*ComponentGlobals) 141 r.emulationVersionConfig = nil 142 r.featureGatesConfig = nil 143 r.set = false 144 } 145 146 func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { 147 r.mutex.RLock() 148 defer r.mutex.RUnlock() 149 globals, ok := r.componentGlobals[component] 150 if !ok { 151 return nil 152 } 153 return globals.effectiveVersion 154 } 155 156 func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate { 157 r.mutex.RLock() 158 defer r.mutex.RUnlock() 159 globals, ok := r.componentGlobals[component] 160 if !ok { 161 return nil 162 } 163 return globals.featureGate 164 } 165 166 func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { 167 if _, ok := r.componentGlobals[component]; ok { 168 return fmt.Errorf("component globals of %s already registered", component) 169 } 170 if featureGate != nil { 171 if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil { 172 return err 173 } 174 } 175 c := ComponentGlobals{ 176 effectiveVersion: effectiveVersion, 177 featureGate: featureGate, 178 emulationVersionMapping: make(map[string]VersionMapping), 179 minCompatibilityVersionMapping: make(map[string]VersionMapping), 180 } 181 r.componentGlobals[component] = &c 182 return nil 183 } 184 185 func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { 186 if effectiveVersion == nil { 187 return fmt.Errorf("cannot register nil effectiveVersion") 188 } 189 r.mutex.Lock() 190 defer r.mutex.Unlock() 191 return r.unsafeRegister(component, effectiveVersion, featureGate) 192 } 193 194 func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { 195 r.mutex.Lock() 196 defer r.mutex.Unlock() 197 globals, ok := r.componentGlobals[component] 198 if ok { 199 return globals.effectiveVersion, globals.featureGate 200 } 201 utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate)) 202 return effectiveVersion, featureGate 203 } 204 205 func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string { 206 var known []string 207 for component, globals := range r.componentGlobals { 208 if globals.featureGate == nil { 209 continue 210 } 211 for _, f := range globals.featureGate.KnownFeatures() { 212 known = append(known, component+":"+f) 213 } 214 } 215 sort.Strings(known) 216 return known 217 } 218 219 func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string { 220 var vs []string 221 for component, globals := range r.componentGlobals { 222 binaryVer := globals.effectiveVersion.BinaryVersion() 223 if isEmulation { 224 if globals.dependentEmulationVersion { 225 continue 226 } 227 // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} 228 // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 229 vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, 230 binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) 231 } else { 232 if globals.dependentMinCompatibilityVersion { 233 continue 234 } 235 // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} 236 vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, 237 binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) 238 } 239 } 240 sort.Strings(vs) 241 return vs 242 } 243 244 func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { 245 if r == nil { 246 return 247 } 248 r.mutex.Lock() 249 defer r.mutex.Unlock() 250 for _, globals := range r.componentGlobals { 251 if globals.featureGate != nil { 252 globals.featureGate.Close() 253 } 254 } 255 if r.emulationVersionConfig != nil || r.featureGatesConfig != nil { 256 klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags") 257 } 258 r.emulationVersionConfig = []string{} 259 r.featureGatesConfig = make(map[string][]string) 260 261 fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+ 262 "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ 263 "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ 264 "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+ 265 "If the component is not specified, defaults to \"kube\"") 266 267 fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ 268 "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ 269 "Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n")) 270 } 271 272 type componentVersion struct { 273 component string 274 ver *version.Version 275 } 276 277 // getFullEmulationVersionConfig expands the given version config with version registered version mapping, 278 // and returns the map of component to Version. 279 func (r *componentGlobalsRegistry) getFullEmulationVersionConfig( 280 versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) { 281 result := map[string]*version.Version{} 282 setQueue := []componentVersion{} 283 for comp, ver := range versionConfigMap { 284 if _, ok := r.componentGlobals[comp]; !ok { 285 return result, fmt.Errorf("component not registered: %s", comp) 286 } 287 klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String()) 288 setQueue = append(setQueue, componentVersion{comp, ver}) 289 } 290 for len(setQueue) > 0 { 291 cv := setQueue[0] 292 if _, visited := result[cv.component]; visited { 293 return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component) 294 } 295 setQueue = setQueue[1:] 296 result[cv.component] = cv.ver 297 for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping { 298 toVer := f(cv.ver) 299 if toVer == nil { 300 return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp) 301 } 302 klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String()) 303 setQueue = append(setQueue, componentVersion{toComp, toVer}) 304 } 305 } 306 return result, nil 307 } 308 309 func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { 310 m := map[string]*version.Version{} 311 for _, compVer := range versionConfig { 312 // default to "kube" of component is not specified 313 k := "kube" 314 v := compVer 315 if strings.Contains(compVer, "=") { 316 arr := strings.SplitN(compVer, "=", 2) 317 if len(arr) != 2 { 318 return m, fmt.Errorf("malformed pair, expect string=string") 319 } 320 k = strings.TrimSpace(arr[0]) 321 v = strings.TrimSpace(arr[1]) 322 } 323 ver, err := version.Parse(v) 324 if err != nil { 325 return m, err 326 } 327 if ver.Patch() != 0 { 328 return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String()) 329 } 330 if existingVer, ok := m[k]; ok { 331 return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String()) 332 } 333 m[k] = ver 334 } 335 return m, nil 336 } 337 338 func (r *componentGlobalsRegistry) SetFallback() error { 339 r.mutex.Lock() 340 set := r.set 341 r.mutex.Unlock() 342 if set { 343 return nil 344 } 345 klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" + 346 " right after parsing flags to avoid using feature gates before their final values are set by the flags.") 347 return r.Set() 348 } 349 350 func (r *componentGlobalsRegistry) Set() error { 351 r.mutex.Lock() 352 defer r.mutex.Unlock() 353 r.set = true 354 emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) 355 if err != nil { 356 return err 357 } 358 for comp := range emulationVersionConfigMap { 359 if _, ok := r.componentGlobals[comp]; !ok { 360 return fmt.Errorf("component not registered: %s", comp) 361 } 362 // only components without any dependencies can be set from the flag. 363 if r.componentGlobals[comp].dependentEmulationVersion { 364 return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp) 365 } 366 } 367 if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil { 368 return err 369 } else { 370 for comp, ver := range emulationVersions { 371 r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) 372 } 373 } 374 // Set feature gate emulation version before setting feature gate flag values. 375 for comp, globals := range r.componentGlobals { 376 if globals.featureGate == nil { 377 continue 378 } 379 klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String()) 380 if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { 381 return err 382 } 383 } 384 for comp, fg := range r.featureGatesConfig { 385 if comp == "" { 386 if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok { 387 return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use") 388 } 389 comp = DefaultKubeComponent 390 } 391 if _, ok := r.componentGlobals[comp]; !ok { 392 return fmt.Errorf("component not registered: %s", comp) 393 } 394 featureGate := r.componentGlobals[comp].featureGate 395 if featureGate == nil { 396 return fmt.Errorf("component featureGate not registered: %s", comp) 397 } 398 flagVal := strings.Join(fg, ",") 399 klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal) 400 if err := featureGate.Set(flagVal); err != nil { 401 return err 402 } 403 } 404 return nil 405 } 406 407 func (r *componentGlobalsRegistry) Validate() []error { 408 var errs []error 409 r.mutex.Lock() 410 defer r.mutex.Unlock() 411 for _, globals := range r.componentGlobals { 412 errs = append(errs, globals.effectiveVersion.Validate()...) 413 if globals.featureGate != nil { 414 errs = append(errs, globals.featureGate.Validate()...) 415 } 416 } 417 return errs 418 } 419 420 func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error { 421 if f == nil { 422 return nil 423 } 424 klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent) 425 r.mutex.Lock() 426 defer r.mutex.Unlock() 427 if _, ok := r.componentGlobals[fromComponent]; !ok { 428 return fmt.Errorf("component not registered: %s", fromComponent) 429 } 430 if _, ok := r.componentGlobals[toComponent]; !ok { 431 return fmt.Errorf("component not registered: %s", toComponent) 432 } 433 // check multiple dependency 434 if r.componentGlobals[toComponent].dependentEmulationVersion { 435 return fmt.Errorf("mapping of %s already exists from another component", toComponent) 436 } 437 r.componentGlobals[toComponent].dependentEmulationVersion = true 438 439 versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping 440 if _, ok := versionMapping[toComponent]; ok { 441 return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent) 442 } 443 versionMapping[toComponent] = f 444 klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent) 445 defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion() 446 emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion}) 447 if err != nil { 448 return err 449 } 450 for comp, ver := range emulationVersions { 451 r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) 452 } 453 return nil 454 }