k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/resourcequota/resource_quota_monitor.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package resourcequota
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"sync"
    23  	"time"
    24  
    25  	"k8s.io/klog/v2"
    26  
    27  	"k8s.io/apimachinery/pkg/api/meta"
    28  	"k8s.io/apimachinery/pkg/runtime/schema"
    29  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    30  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    31  	"k8s.io/apimachinery/pkg/util/wait"
    32  	quota "k8s.io/apiserver/pkg/quota/v1"
    33  	"k8s.io/apiserver/pkg/quota/v1/generic"
    34  	"k8s.io/client-go/tools/cache"
    35  	"k8s.io/client-go/util/workqueue"
    36  	"k8s.io/controller-manager/pkg/informerfactory"
    37  	"k8s.io/kubernetes/pkg/controller"
    38  )
    39  
    40  type eventType int
    41  
    42  func (e eventType) String() string {
    43  	switch e {
    44  	case addEvent:
    45  		return "add"
    46  	case updateEvent:
    47  		return "update"
    48  	case deleteEvent:
    49  		return "delete"
    50  	default:
    51  		return fmt.Sprintf("unknown(%d)", int(e))
    52  	}
    53  }
    54  
    55  const (
    56  	addEvent eventType = iota
    57  	updateEvent
    58  	deleteEvent
    59  )
    60  
    61  type event struct {
    62  	eventType eventType
    63  	obj       interface{}
    64  	oldObj    interface{}
    65  	gvr       schema.GroupVersionResource
    66  }
    67  
    68  // QuotaMonitor contains all necessary information to track quotas and trigger replenishments
    69  type QuotaMonitor struct {
    70  	// each monitor list/watches a resource and determines if we should replenish quota
    71  	monitors    monitors
    72  	monitorLock sync.RWMutex
    73  	// informersStarted is closed after all the controllers have been initialized and are running.
    74  	// After that it is safe to start them here, before that it is not.
    75  	informersStarted <-chan struct{}
    76  
    77  	// stopCh drives shutdown. When a reception from it unblocks, monitors will shut down.
    78  	// This channel is also protected by monitorLock.
    79  	stopCh <-chan struct{}
    80  
    81  	// running tracks whether Run() has been called.
    82  	// it is protected by monitorLock.
    83  	running bool
    84  
    85  	// monitors are the producer of the resourceChanges queue
    86  	resourceChanges workqueue.TypedRateLimitingInterface[*event]
    87  
    88  	// interfaces with informers
    89  	informerFactory informerfactory.InformerFactory
    90  
    91  	// list of resources to ignore
    92  	ignoredResources map[schema.GroupResource]struct{}
    93  
    94  	// The period that should be used to re-sync the monitored resource
    95  	resyncPeriod controller.ResyncPeriodFunc
    96  
    97  	// callback to alert that a change may require quota recalculation
    98  	replenishmentFunc ReplenishmentFunc
    99  
   100  	// maintains list of evaluators
   101  	registry quota.Registry
   102  
   103  	updateFilter UpdateFilter
   104  }
   105  
   106  // NewMonitor creates a new instance of a QuotaMonitor
   107  func NewMonitor(informersStarted <-chan struct{}, informerFactory informerfactory.InformerFactory, ignoredResources map[schema.GroupResource]struct{}, resyncPeriod controller.ResyncPeriodFunc, replenishmentFunc ReplenishmentFunc, registry quota.Registry, updateFilter UpdateFilter) *QuotaMonitor {
   108  	return &QuotaMonitor{
   109  		informersStarted: informersStarted,
   110  		informerFactory:  informerFactory,
   111  		ignoredResources: ignoredResources,
   112  		resourceChanges: workqueue.NewTypedRateLimitingQueueWithConfig(
   113  			workqueue.DefaultTypedControllerRateLimiter[*event](),
   114  			workqueue.TypedRateLimitingQueueConfig[*event]{Name: "resource_quota_controller_resource_changes"},
   115  		),
   116  		resyncPeriod:      resyncPeriod,
   117  		replenishmentFunc: replenishmentFunc,
   118  		registry:          registry,
   119  		updateFilter:      updateFilter,
   120  	}
   121  }
   122  
   123  // monitor runs a Controller with a local stop channel.
   124  type monitor struct {
   125  	controller cache.Controller
   126  
   127  	// stopCh stops Controller. If stopCh is nil, the monitor is considered to be
   128  	// not yet started.
   129  	stopCh chan struct{}
   130  }
   131  
   132  // Run is intended to be called in a goroutine. Multiple calls of this is an
   133  // error.
   134  func (m *monitor) Run() {
   135  	m.controller.Run(m.stopCh)
   136  }
   137  
   138  type monitors map[schema.GroupVersionResource]*monitor
   139  
   140  // UpdateFilter is a function that returns true if the update event should be added to the resourceChanges queue.
   141  type UpdateFilter func(resource schema.GroupVersionResource, oldObj, newObj interface{}) bool
   142  
   143  func (qm *QuotaMonitor) controllerFor(ctx context.Context, resource schema.GroupVersionResource) (cache.Controller, error) {
   144  	logger := klog.FromContext(ctx)
   145  
   146  	handlers := cache.ResourceEventHandlerFuncs{
   147  		UpdateFunc: func(oldObj, newObj interface{}) {
   148  			if qm.updateFilter != nil && qm.updateFilter(resource, oldObj, newObj) {
   149  				event := &event{
   150  					eventType: updateEvent,
   151  					obj:       newObj,
   152  					oldObj:    oldObj,
   153  					gvr:       resource,
   154  				}
   155  				qm.resourceChanges.Add(event)
   156  			}
   157  		},
   158  		DeleteFunc: func(obj interface{}) {
   159  			// delta fifo may wrap the object in a cache.DeletedFinalStateUnknown, unwrap it
   160  			if deletedFinalStateUnknown, ok := obj.(cache.DeletedFinalStateUnknown); ok {
   161  				obj = deletedFinalStateUnknown.Obj
   162  			}
   163  			event := &event{
   164  				eventType: deleteEvent,
   165  				obj:       obj,
   166  				gvr:       resource,
   167  			}
   168  			qm.resourceChanges.Add(event)
   169  		},
   170  	}
   171  	shared, err := qm.informerFactory.ForResource(resource)
   172  	if err == nil {
   173  		logger.V(4).Info("QuotaMonitor using a shared informer", "resource", resource.String())
   174  		shared.Informer().AddEventHandlerWithResyncPeriod(handlers, qm.resyncPeriod())
   175  		return shared.Informer().GetController(), nil
   176  	}
   177  	logger.V(4).Error(err, "QuotaMonitor unable to use a shared informer", "resource", resource.String())
   178  
   179  	// TODO: if we can share storage with garbage collector, it may make sense to support other resources
   180  	// until that time, aggregated api servers will have to run their own controller to reconcile their own quota.
   181  	return nil, fmt.Errorf("unable to monitor quota for resource %q", resource.String())
   182  }
   183  
   184  // SyncMonitors rebuilds the monitor set according to the supplied resources,
   185  // creating or deleting monitors as necessary. It will return any error
   186  // encountered, but will make an attempt to create a monitor for each resource
   187  // instead of immediately exiting on an error. It may be called before or after
   188  // Run. Monitors are NOT started as part of the sync. To ensure all existing
   189  // monitors are started, call StartMonitors.
   190  func (qm *QuotaMonitor) SyncMonitors(ctx context.Context, resources map[schema.GroupVersionResource]struct{}) error {
   191  	logger := klog.FromContext(ctx)
   192  
   193  	qm.monitorLock.Lock()
   194  	defer qm.monitorLock.Unlock()
   195  
   196  	toRemove := qm.monitors
   197  	if toRemove == nil {
   198  		toRemove = monitors{}
   199  	}
   200  	current := monitors{}
   201  	var errs []error
   202  	kept := 0
   203  	added := 0
   204  	for resource := range resources {
   205  		if _, ok := qm.ignoredResources[resource.GroupResource()]; ok {
   206  			continue
   207  		}
   208  		if m, ok := toRemove[resource]; ok {
   209  			current[resource] = m
   210  			delete(toRemove, resource)
   211  			kept++
   212  			continue
   213  		}
   214  		c, err := qm.controllerFor(ctx, resource)
   215  		if err != nil {
   216  			errs = append(errs, fmt.Errorf("couldn't start monitor for resource %q: %v", resource, err))
   217  			continue
   218  		}
   219  
   220  		// check if we need to create an evaluator for this resource (if none previously registered)
   221  		evaluator := qm.registry.Get(resource.GroupResource())
   222  		if evaluator == nil {
   223  			listerFunc := generic.ListerFuncForResourceFunc(qm.informerFactory.ForResource)
   224  			listResourceFunc := generic.ListResourceUsingListerFunc(listerFunc, resource)
   225  			evaluator = generic.NewObjectCountEvaluator(resource.GroupResource(), listResourceFunc, "")
   226  			qm.registry.Add(evaluator)
   227  			logger.Info("QuotaMonitor created object count evaluator", "resource", resource.GroupResource())
   228  		}
   229  
   230  		// track the monitor
   231  		current[resource] = &monitor{controller: c}
   232  		added++
   233  	}
   234  	qm.monitors = current
   235  
   236  	for _, monitor := range toRemove {
   237  		if monitor.stopCh != nil {
   238  			close(monitor.stopCh)
   239  		}
   240  	}
   241  
   242  	logger.V(4).Info("quota synced monitors", "added", added, "kept", kept, "removed", len(toRemove))
   243  	// NewAggregate returns nil if errs is 0-length
   244  	return utilerrors.NewAggregate(errs)
   245  }
   246  
   247  // StartMonitors ensures the current set of monitors are running. Any newly
   248  // started monitors will also cause shared informers to be started.
   249  //
   250  // If called before Run, StartMonitors does nothing (as there is no stop channel
   251  // to support monitor/informer execution).
   252  func (qm *QuotaMonitor) StartMonitors(ctx context.Context) {
   253  	qm.monitorLock.Lock()
   254  	defer qm.monitorLock.Unlock()
   255  
   256  	if !qm.running {
   257  		return
   258  	}
   259  
   260  	// we're waiting until after the informer start that happens once all the controllers are initialized.  This ensures
   261  	// that they don't get unexpected events on their work queues.
   262  	<-qm.informersStarted
   263  
   264  	monitors := qm.monitors
   265  	started := 0
   266  	for _, monitor := range monitors {
   267  		if monitor.stopCh == nil {
   268  			monitor.stopCh = make(chan struct{})
   269  			qm.informerFactory.Start(qm.stopCh)
   270  			go monitor.Run()
   271  			started++
   272  		}
   273  	}
   274  	klog.FromContext(ctx).V(4).Info("QuotaMonitor finished starting monitors", "new", started, "total", len(monitors))
   275  }
   276  
   277  // IsSynced returns true if any monitors exist AND all those monitors'
   278  // controllers HasSynced functions return true. This means IsSynced could return
   279  // true at one time, and then later return false if all monitors were
   280  // reconstructed.
   281  func (qm *QuotaMonitor) IsSynced(ctx context.Context) bool {
   282  	logger := klog.FromContext(ctx)
   283  
   284  	qm.monitorLock.RLock()
   285  	defer qm.monitorLock.RUnlock()
   286  
   287  	if len(qm.monitors) == 0 {
   288  		logger.V(4).Info("quota monitor not synced: no monitors")
   289  		return false
   290  	}
   291  
   292  	for resource, monitor := range qm.monitors {
   293  		if !monitor.controller.HasSynced() {
   294  			logger.V(4).Info("quota monitor not synced", "resource", resource)
   295  			return false
   296  		}
   297  	}
   298  	return true
   299  }
   300  
   301  // Run sets the stop channel and starts monitor execution until stopCh is
   302  // closed. Any running monitors will be stopped before Run returns.
   303  func (qm *QuotaMonitor) Run(ctx context.Context) {
   304  	defer utilruntime.HandleCrash()
   305  
   306  	logger := klog.FromContext(ctx)
   307  
   308  	logger.Info("QuotaMonitor running")
   309  	defer logger.Info("QuotaMonitor stopping")
   310  
   311  	// Set up the stop channel.
   312  	qm.monitorLock.Lock()
   313  	qm.stopCh = ctx.Done()
   314  	qm.running = true
   315  	qm.monitorLock.Unlock()
   316  
   317  	// Start monitors and begin change processing until the stop channel is
   318  	// closed.
   319  	qm.StartMonitors(ctx)
   320  
   321  	// The following workers are hanging forever until the queue is
   322  	// shutted down, so we need to shut it down in a separate goroutine.
   323  	go func() {
   324  		defer utilruntime.HandleCrash()
   325  		defer qm.resourceChanges.ShutDown()
   326  
   327  		<-ctx.Done()
   328  	}()
   329  	wait.UntilWithContext(ctx, qm.runProcessResourceChanges, 1*time.Second)
   330  
   331  	// Stop any running monitors.
   332  	qm.monitorLock.Lock()
   333  	defer qm.monitorLock.Unlock()
   334  	monitors := qm.monitors
   335  	stopped := 0
   336  	for _, monitor := range monitors {
   337  		if monitor.stopCh != nil {
   338  			stopped++
   339  			close(monitor.stopCh)
   340  		}
   341  	}
   342  	logger.Info("QuotaMonitor stopped monitors", "stopped", stopped, "total", len(monitors))
   343  }
   344  
   345  func (qm *QuotaMonitor) runProcessResourceChanges(ctx context.Context) {
   346  	for qm.processResourceChanges(ctx) {
   347  	}
   348  }
   349  
   350  // Dequeueing an event from resourceChanges to process
   351  func (qm *QuotaMonitor) processResourceChanges(ctx context.Context) bool {
   352  	item, quit := qm.resourceChanges.Get()
   353  	if quit {
   354  		return false
   355  	}
   356  	defer qm.resourceChanges.Done(item)
   357  	event := item
   358  	obj := event.obj
   359  	accessor, err := meta.Accessor(obj)
   360  	if err != nil {
   361  		utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err))
   362  		return true
   363  	}
   364  	klog.FromContext(ctx).V(4).Info("QuotaMonitor process object",
   365  		"resource", event.gvr.String(),
   366  		"namespace", accessor.GetNamespace(),
   367  		"name", accessor.GetName(),
   368  		"uid", string(accessor.GetUID()),
   369  		"eventType", event.eventType,
   370  	)
   371  	qm.replenishmentFunc(ctx, event.gvr.GroupResource(), accessor.GetNamespace())
   372  	return true
   373  }