k8s.io/apiserver@v0.31.1/pkg/util/version/registry.go (about)

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package version
    18  
    19  import (
    20  	"fmt"
    21  	"sort"
    22  	"strings"
    23  	"sync"
    24  
    25  	"github.com/spf13/pflag"
    26  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    27  	"k8s.io/apimachinery/pkg/util/version"
    28  	cliflag "k8s.io/component-base/cli/flag"
    29  	"k8s.io/component-base/featuregate"
    30  	"k8s.io/klog/v2"
    31  )
    32  
    33  // DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
    34  // Example usage:
    35  // // register the component effective version and feature gate first
    36  // _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
    37  // wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
    38  // wardleFeatureGate := featuregate.NewFeatureGate()
    39  // utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
    40  //
    41  //	cmd := &cobra.Command{
    42  //	 ...
    43  //		// call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE
    44  //		PersistentPreRunE: func(*cobra.Command, []string) error {
    45  //			if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
    46  //				return err
    47  //			}
    48  //	 ...
    49  //		},
    50  //		RunE: func(c *cobra.Command, args []string) error {
    51  //			// call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere
    52  //		},
    53  //	}
    54  //
    55  // flags := cmd.Flags()
    56  // // add flags
    57  // utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
    58  var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
    59  
    60  const (
    61  	DefaultKubeComponent = "kube"
    62  
    63  	klogLevel = 2
    64  )
    65  
    66  type VersionMapping func(from *version.Version) *version.Version
    67  
    68  // ComponentGlobals stores the global variables for a component for easy access.
    69  type ComponentGlobals struct {
    70  	effectiveVersion MutableEffectiveVersion
    71  	featureGate      featuregate.MutableVersionedFeatureGate
    72  
    73  	// emulationVersionMapping contains the mapping from the emulation version of this component
    74  	// to the emulation version of another component.
    75  	emulationVersionMapping map[string]VersionMapping
    76  	// dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component.
    77  	// If true, the emulation version cannot be set from the flag, or version mapping from another component.
    78  	dependentEmulationVersion bool
    79  	// minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component
    80  	// to the min compatibility version of another component.
    81  	minCompatibilityVersionMapping map[string]VersionMapping
    82  	// dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component
    83  	// If true, the min compatibility version cannot be set from the flag, or version mapping from another component.
    84  	dependentMinCompatibilityVersion bool
    85  }
    86  
    87  type ComponentGlobalsRegistry interface {
    88  	// EffectiveVersionFor returns the EffectiveVersion registered under the component.
    89  	// Returns nil if the component is not registered.
    90  	EffectiveVersionFor(component string) EffectiveVersion
    91  	// FeatureGateFor returns the FeatureGate registered under the component.
    92  	// Returns nil if the component is not registered.
    93  	FeatureGateFor(component string) featuregate.FeatureGate
    94  	// Register registers the EffectiveVersion and FeatureGate for a component.
    95  	// returns error if the component is already registered.
    96  	Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error
    97  	// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
    98  	// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
    99  	ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
   100  	// AddFlags adds flags of "--emulated-version" and "--feature-gates"
   101  	AddFlags(fs *pflag.FlagSet)
   102  	// Set sets the flags for all global variables for all components registered.
   103  	Set() error
   104  	// SetFallback calls Set() if it has never been called.
   105  	SetFallback() error
   106  	// Validate calls the Validate() function for all the global variables for all components registered.
   107  	Validate() []error
   108  	// Reset removes all stored ComponentGlobals, configurations, and version mappings.
   109  	Reset()
   110  	// SetEmulationVersionMapping sets the mapping from the emulation version of one component
   111  	// to the emulation version of another component.
   112  	// Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent,
   113  	// and cannot be set from cmd flags anymore.
   114  	// For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed.
   115  	SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error
   116  }
   117  
   118  type componentGlobalsRegistry struct {
   119  	componentGlobals map[string]*ComponentGlobals
   120  	mutex            sync.RWMutex
   121  	// list of component name to emulation version set from the flag.
   122  	emulationVersionConfig []string
   123  	// map of component name to the list of feature gates set from the flag.
   124  	featureGatesConfig map[string][]string
   125  	// set stores if the Set() function for the registry is already called.
   126  	set bool
   127  }
   128  
   129  func NewComponentGlobalsRegistry() *componentGlobalsRegistry {
   130  	return &componentGlobalsRegistry{
   131  		componentGlobals:       make(map[string]*ComponentGlobals),
   132  		emulationVersionConfig: nil,
   133  		featureGatesConfig:     nil,
   134  	}
   135  }
   136  
   137  func (r *componentGlobalsRegistry) Reset() {
   138  	r.mutex.Lock()
   139  	defer r.mutex.Unlock()
   140  	r.componentGlobals = make(map[string]*ComponentGlobals)
   141  	r.emulationVersionConfig = nil
   142  	r.featureGatesConfig = nil
   143  	r.set = false
   144  }
   145  
   146  func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
   147  	r.mutex.RLock()
   148  	defer r.mutex.RUnlock()
   149  	globals, ok := r.componentGlobals[component]
   150  	if !ok {
   151  		return nil
   152  	}
   153  	return globals.effectiveVersion
   154  }
   155  
   156  func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate {
   157  	r.mutex.RLock()
   158  	defer r.mutex.RUnlock()
   159  	globals, ok := r.componentGlobals[component]
   160  	if !ok {
   161  		return nil
   162  	}
   163  	return globals.featureGate
   164  }
   165  
   166  func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
   167  	if _, ok := r.componentGlobals[component]; ok {
   168  		return fmt.Errorf("component globals of %s already registered", component)
   169  	}
   170  	if featureGate != nil {
   171  		if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil {
   172  			return err
   173  		}
   174  	}
   175  	c := ComponentGlobals{
   176  		effectiveVersion:               effectiveVersion,
   177  		featureGate:                    featureGate,
   178  		emulationVersionMapping:        make(map[string]VersionMapping),
   179  		minCompatibilityVersionMapping: make(map[string]VersionMapping),
   180  	}
   181  	r.componentGlobals[component] = &c
   182  	return nil
   183  }
   184  
   185  func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
   186  	if effectiveVersion == nil {
   187  		return fmt.Errorf("cannot register nil effectiveVersion")
   188  	}
   189  	r.mutex.Lock()
   190  	defer r.mutex.Unlock()
   191  	return r.unsafeRegister(component, effectiveVersion, featureGate)
   192  }
   193  
   194  func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) {
   195  	r.mutex.Lock()
   196  	defer r.mutex.Unlock()
   197  	globals, ok := r.componentGlobals[component]
   198  	if ok {
   199  		return globals.effectiveVersion, globals.featureGate
   200  	}
   201  	utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate))
   202  	return effectiveVersion, featureGate
   203  }
   204  
   205  func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string {
   206  	var known []string
   207  	for component, globals := range r.componentGlobals {
   208  		if globals.featureGate == nil {
   209  			continue
   210  		}
   211  		for _, f := range globals.featureGate.KnownFeatures() {
   212  			known = append(known, component+":"+f)
   213  		}
   214  	}
   215  	sort.Strings(known)
   216  	return known
   217  }
   218  
   219  func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string {
   220  	var vs []string
   221  	for component, globals := range r.componentGlobals {
   222  		binaryVer := globals.effectiveVersion.BinaryVersion()
   223  		if isEmulation {
   224  			if globals.dependentEmulationVersion {
   225  				continue
   226  			}
   227  			// emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor}
   228  			// TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32
   229  			vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
   230  				binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String()))
   231  		} else {
   232  			if globals.dependentMinCompatibilityVersion {
   233  				continue
   234  			}
   235  			// min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor}
   236  			vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
   237  				binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String()))
   238  		}
   239  	}
   240  	sort.Strings(vs)
   241  	return vs
   242  }
   243  
   244  func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) {
   245  	if r == nil {
   246  		return
   247  	}
   248  	r.mutex.Lock()
   249  	defer r.mutex.Unlock()
   250  	for _, globals := range r.componentGlobals {
   251  		if globals.featureGate != nil {
   252  			globals.featureGate.Close()
   253  		}
   254  	}
   255  	if r.emulationVersionConfig != nil || r.featureGatesConfig != nil {
   256  		klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags")
   257  	}
   258  	r.emulationVersionConfig = []string{}
   259  	r.featureGatesConfig = make(map[string][]string)
   260  
   261  	fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+
   262  		"The versions different components emulate their capabilities (APIs, features, ...) of.\n"+
   263  		"If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+
   264  		"Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+
   265  		"If the component is not specified, defaults to \"kube\"")
   266  
   267  	fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+
   268  		"If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+
   269  		"Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n"))
   270  }
   271  
   272  type componentVersion struct {
   273  	component string
   274  	ver       *version.Version
   275  }
   276  
   277  // getFullEmulationVersionConfig expands the given version config with version registered version mapping,
   278  // and returns the map of component to Version.
   279  func (r *componentGlobalsRegistry) getFullEmulationVersionConfig(
   280  	versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) {
   281  	result := map[string]*version.Version{}
   282  	setQueue := []componentVersion{}
   283  	for comp, ver := range versionConfigMap {
   284  		if _, ok := r.componentGlobals[comp]; !ok {
   285  			return result, fmt.Errorf("component not registered: %s", comp)
   286  		}
   287  		klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String())
   288  		setQueue = append(setQueue, componentVersion{comp, ver})
   289  	}
   290  	for len(setQueue) > 0 {
   291  		cv := setQueue[0]
   292  		if _, visited := result[cv.component]; visited {
   293  			return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component)
   294  		}
   295  		setQueue = setQueue[1:]
   296  		result[cv.component] = cv.ver
   297  		for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping {
   298  			toVer := f(cv.ver)
   299  			if toVer == nil {
   300  				return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp)
   301  			}
   302  			klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String())
   303  			setQueue = append(setQueue, componentVersion{toComp, toVer})
   304  		}
   305  	}
   306  	return result, nil
   307  }
   308  
   309  func toVersionMap(versionConfig []string) (map[string]*version.Version, error) {
   310  	m := map[string]*version.Version{}
   311  	for _, compVer := range versionConfig {
   312  		// default to "kube" of component is not specified
   313  		k := "kube"
   314  		v := compVer
   315  		if strings.Contains(compVer, "=") {
   316  			arr := strings.SplitN(compVer, "=", 2)
   317  			if len(arr) != 2 {
   318  				return m, fmt.Errorf("malformed pair, expect string=string")
   319  			}
   320  			k = strings.TrimSpace(arr[0])
   321  			v = strings.TrimSpace(arr[1])
   322  		}
   323  		ver, err := version.Parse(v)
   324  		if err != nil {
   325  			return m, err
   326  		}
   327  		if ver.Patch() != 0 {
   328  			return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String())
   329  		}
   330  		if existingVer, ok := m[k]; ok {
   331  			return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String())
   332  		}
   333  		m[k] = ver
   334  	}
   335  	return m, nil
   336  }
   337  
   338  func (r *componentGlobalsRegistry) SetFallback() error {
   339  	r.mutex.Lock()
   340  	set := r.set
   341  	r.mutex.Unlock()
   342  	if set {
   343  		return nil
   344  	}
   345  	klog.Warning("setting componentGlobalsRegistry in SetFallback. We recommend calling componentGlobalsRegistry.Set()" +
   346  		" right after parsing flags to avoid using feature gates before their final values are set by the flags.")
   347  	return r.Set()
   348  }
   349  
   350  func (r *componentGlobalsRegistry) Set() error {
   351  	r.mutex.Lock()
   352  	defer r.mutex.Unlock()
   353  	r.set = true
   354  	emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig)
   355  	if err != nil {
   356  		return err
   357  	}
   358  	for comp := range emulationVersionConfigMap {
   359  		if _, ok := r.componentGlobals[comp]; !ok {
   360  			return fmt.Errorf("component not registered: %s", comp)
   361  		}
   362  		// only components without any dependencies can be set from the flag.
   363  		if r.componentGlobals[comp].dependentEmulationVersion {
   364  			return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp)
   365  		}
   366  	}
   367  	if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil {
   368  		return err
   369  	} else {
   370  		for comp, ver := range emulationVersions {
   371  			r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver)
   372  		}
   373  	}
   374  	// Set feature gate emulation version before setting feature gate flag values.
   375  	for comp, globals := range r.componentGlobals {
   376  		if globals.featureGate == nil {
   377  			continue
   378  		}
   379  		klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String())
   380  		if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil {
   381  			return err
   382  		}
   383  	}
   384  	for comp, fg := range r.featureGatesConfig {
   385  		if comp == "" {
   386  			if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok {
   387  				return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use")
   388  			}
   389  			comp = DefaultKubeComponent
   390  		}
   391  		if _, ok := r.componentGlobals[comp]; !ok {
   392  			return fmt.Errorf("component not registered: %s", comp)
   393  		}
   394  		featureGate := r.componentGlobals[comp].featureGate
   395  		if featureGate == nil {
   396  			return fmt.Errorf("component featureGate not registered: %s", comp)
   397  		}
   398  		flagVal := strings.Join(fg, ",")
   399  		klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal)
   400  		if err := featureGate.Set(flagVal); err != nil {
   401  			return err
   402  		}
   403  	}
   404  	return nil
   405  }
   406  
   407  func (r *componentGlobalsRegistry) Validate() []error {
   408  	var errs []error
   409  	r.mutex.Lock()
   410  	defer r.mutex.Unlock()
   411  	for _, globals := range r.componentGlobals {
   412  		errs = append(errs, globals.effectiveVersion.Validate()...)
   413  		if globals.featureGate != nil {
   414  			errs = append(errs, globals.featureGate.Validate()...)
   415  		}
   416  	}
   417  	return errs
   418  }
   419  
   420  func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error {
   421  	if f == nil {
   422  		return nil
   423  	}
   424  	klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent)
   425  	r.mutex.Lock()
   426  	defer r.mutex.Unlock()
   427  	if _, ok := r.componentGlobals[fromComponent]; !ok {
   428  		return fmt.Errorf("component not registered: %s", fromComponent)
   429  	}
   430  	if _, ok := r.componentGlobals[toComponent]; !ok {
   431  		return fmt.Errorf("component not registered: %s", toComponent)
   432  	}
   433  	// check multiple dependency
   434  	if r.componentGlobals[toComponent].dependentEmulationVersion {
   435  		return fmt.Errorf("mapping of %s already exists from another component", toComponent)
   436  	}
   437  	r.componentGlobals[toComponent].dependentEmulationVersion = true
   438  
   439  	versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping
   440  	if _, ok := versionMapping[toComponent]; ok {
   441  		return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent)
   442  	}
   443  	versionMapping[toComponent] = f
   444  	klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent)
   445  	defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion()
   446  	emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion})
   447  	if err != nil {
   448  		return err
   449  	}
   450  	for comp, ver := range emulationVersions {
   451  		r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver)
   452  	}
   453  	return nil
   454  }