k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/resourcequota/resource_quota_monitor.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package resourcequota 18 19 import ( 20 "context" 21 "fmt" 22 "sync" 23 "time" 24 25 "k8s.io/klog/v2" 26 27 "k8s.io/apimachinery/pkg/api/meta" 28 "k8s.io/apimachinery/pkg/runtime/schema" 29 utilerrors "k8s.io/apimachinery/pkg/util/errors" 30 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 31 "k8s.io/apimachinery/pkg/util/wait" 32 quota "k8s.io/apiserver/pkg/quota/v1" 33 "k8s.io/apiserver/pkg/quota/v1/generic" 34 "k8s.io/client-go/tools/cache" 35 "k8s.io/client-go/util/workqueue" 36 "k8s.io/controller-manager/pkg/informerfactory" 37 "k8s.io/kubernetes/pkg/controller" 38 ) 39 40 type eventType int 41 42 func (e eventType) String() string { 43 switch e { 44 case addEvent: 45 return "add" 46 case updateEvent: 47 return "update" 48 case deleteEvent: 49 return "delete" 50 default: 51 return fmt.Sprintf("unknown(%d)", int(e)) 52 } 53 } 54 55 const ( 56 addEvent eventType = iota 57 updateEvent 58 deleteEvent 59 ) 60 61 type event struct { 62 eventType eventType 63 obj interface{} 64 oldObj interface{} 65 gvr schema.GroupVersionResource 66 } 67 68 // QuotaMonitor contains all necessary information to track quotas and trigger replenishments 69 type QuotaMonitor struct { 70 // each monitor list/watches a resource and determines if we should replenish quota 71 monitors monitors 72 monitorLock sync.RWMutex 73 // informersStarted is closed after all the controllers have been initialized and are running. 74 // After that it is safe to start them here, before that it is not. 75 informersStarted <-chan struct{} 76 77 // stopCh drives shutdown. When a reception from it unblocks, monitors will shut down. 78 // This channel is also protected by monitorLock. 79 stopCh <-chan struct{} 80 81 // running tracks whether Run() has been called. 82 // it is protected by monitorLock. 83 running bool 84 85 // monitors are the producer of the resourceChanges queue 86 resourceChanges workqueue.TypedRateLimitingInterface[*event] 87 88 // interfaces with informers 89 informerFactory informerfactory.InformerFactory 90 91 // list of resources to ignore 92 ignoredResources map[schema.GroupResource]struct{} 93 94 // The period that should be used to re-sync the monitored resource 95 resyncPeriod controller.ResyncPeriodFunc 96 97 // callback to alert that a change may require quota recalculation 98 replenishmentFunc ReplenishmentFunc 99 100 // maintains list of evaluators 101 registry quota.Registry 102 103 updateFilter UpdateFilter 104 } 105 106 // NewMonitor creates a new instance of a QuotaMonitor 107 func NewMonitor(informersStarted <-chan struct{}, informerFactory informerfactory.InformerFactory, ignoredResources map[schema.GroupResource]struct{}, resyncPeriod controller.ResyncPeriodFunc, replenishmentFunc ReplenishmentFunc, registry quota.Registry, updateFilter UpdateFilter) *QuotaMonitor { 108 return &QuotaMonitor{ 109 informersStarted: informersStarted, 110 informerFactory: informerFactory, 111 ignoredResources: ignoredResources, 112 resourceChanges: workqueue.NewTypedRateLimitingQueueWithConfig( 113 workqueue.DefaultTypedControllerRateLimiter[*event](), 114 workqueue.TypedRateLimitingQueueConfig[*event]{Name: "resource_quota_controller_resource_changes"}, 115 ), 116 resyncPeriod: resyncPeriod, 117 replenishmentFunc: replenishmentFunc, 118 registry: registry, 119 updateFilter: updateFilter, 120 } 121 } 122 123 // monitor runs a Controller with a local stop channel. 124 type monitor struct { 125 controller cache.Controller 126 127 // stopCh stops Controller. If stopCh is nil, the monitor is considered to be 128 // not yet started. 129 stopCh chan struct{} 130 } 131 132 // Run is intended to be called in a goroutine. Multiple calls of this is an 133 // error. 134 func (m *monitor) Run() { 135 m.controller.Run(m.stopCh) 136 } 137 138 type monitors map[schema.GroupVersionResource]*monitor 139 140 // UpdateFilter is a function that returns true if the update event should be added to the resourceChanges queue. 141 type UpdateFilter func(resource schema.GroupVersionResource, oldObj, newObj interface{}) bool 142 143 func (qm *QuotaMonitor) controllerFor(ctx context.Context, resource schema.GroupVersionResource) (cache.Controller, error) { 144 logger := klog.FromContext(ctx) 145 146 handlers := cache.ResourceEventHandlerFuncs{ 147 UpdateFunc: func(oldObj, newObj interface{}) { 148 if qm.updateFilter != nil && qm.updateFilter(resource, oldObj, newObj) { 149 event := &event{ 150 eventType: updateEvent, 151 obj: newObj, 152 oldObj: oldObj, 153 gvr: resource, 154 } 155 qm.resourceChanges.Add(event) 156 } 157 }, 158 DeleteFunc: func(obj interface{}) { 159 // delta fifo may wrap the object in a cache.DeletedFinalStateUnknown, unwrap it 160 if deletedFinalStateUnknown, ok := obj.(cache.DeletedFinalStateUnknown); ok { 161 obj = deletedFinalStateUnknown.Obj 162 } 163 event := &event{ 164 eventType: deleteEvent, 165 obj: obj, 166 gvr: resource, 167 } 168 qm.resourceChanges.Add(event) 169 }, 170 } 171 shared, err := qm.informerFactory.ForResource(resource) 172 if err == nil { 173 logger.V(4).Info("QuotaMonitor using a shared informer", "resource", resource.String()) 174 shared.Informer().AddEventHandlerWithResyncPeriod(handlers, qm.resyncPeriod()) 175 return shared.Informer().GetController(), nil 176 } 177 logger.V(4).Error(err, "QuotaMonitor unable to use a shared informer", "resource", resource.String()) 178 179 // TODO: if we can share storage with garbage collector, it may make sense to support other resources 180 // until that time, aggregated api servers will have to run their own controller to reconcile their own quota. 181 return nil, fmt.Errorf("unable to monitor quota for resource %q", resource.String()) 182 } 183 184 // SyncMonitors rebuilds the monitor set according to the supplied resources, 185 // creating or deleting monitors as necessary. It will return any error 186 // encountered, but will make an attempt to create a monitor for each resource 187 // instead of immediately exiting on an error. It may be called before or after 188 // Run. Monitors are NOT started as part of the sync. To ensure all existing 189 // monitors are started, call StartMonitors. 190 func (qm *QuotaMonitor) SyncMonitors(ctx context.Context, resources map[schema.GroupVersionResource]struct{}) error { 191 logger := klog.FromContext(ctx) 192 193 qm.monitorLock.Lock() 194 defer qm.monitorLock.Unlock() 195 196 toRemove := qm.monitors 197 if toRemove == nil { 198 toRemove = monitors{} 199 } 200 current := monitors{} 201 var errs []error 202 kept := 0 203 added := 0 204 for resource := range resources { 205 if _, ok := qm.ignoredResources[resource.GroupResource()]; ok { 206 continue 207 } 208 if m, ok := toRemove[resource]; ok { 209 current[resource] = m 210 delete(toRemove, resource) 211 kept++ 212 continue 213 } 214 c, err := qm.controllerFor(ctx, resource) 215 if err != nil { 216 errs = append(errs, fmt.Errorf("couldn't start monitor for resource %q: %v", resource, err)) 217 continue 218 } 219 220 // check if we need to create an evaluator for this resource (if none previously registered) 221 evaluator := qm.registry.Get(resource.GroupResource()) 222 if evaluator == nil { 223 listerFunc := generic.ListerFuncForResourceFunc(qm.informerFactory.ForResource) 224 listResourceFunc := generic.ListResourceUsingListerFunc(listerFunc, resource) 225 evaluator = generic.NewObjectCountEvaluator(resource.GroupResource(), listResourceFunc, "") 226 qm.registry.Add(evaluator) 227 logger.Info("QuotaMonitor created object count evaluator", "resource", resource.GroupResource()) 228 } 229 230 // track the monitor 231 current[resource] = &monitor{controller: c} 232 added++ 233 } 234 qm.monitors = current 235 236 for _, monitor := range toRemove { 237 if monitor.stopCh != nil { 238 close(monitor.stopCh) 239 } 240 } 241 242 logger.V(4).Info("quota synced monitors", "added", added, "kept", kept, "removed", len(toRemove)) 243 // NewAggregate returns nil if errs is 0-length 244 return utilerrors.NewAggregate(errs) 245 } 246 247 // StartMonitors ensures the current set of monitors are running. Any newly 248 // started monitors will also cause shared informers to be started. 249 // 250 // If called before Run, StartMonitors does nothing (as there is no stop channel 251 // to support monitor/informer execution). 252 func (qm *QuotaMonitor) StartMonitors(ctx context.Context) { 253 qm.monitorLock.Lock() 254 defer qm.monitorLock.Unlock() 255 256 if !qm.running { 257 return 258 } 259 260 // we're waiting until after the informer start that happens once all the controllers are initialized. This ensures 261 // that they don't get unexpected events on their work queues. 262 <-qm.informersStarted 263 264 monitors := qm.monitors 265 started := 0 266 for _, monitor := range monitors { 267 if monitor.stopCh == nil { 268 monitor.stopCh = make(chan struct{}) 269 qm.informerFactory.Start(qm.stopCh) 270 go monitor.Run() 271 started++ 272 } 273 } 274 klog.FromContext(ctx).V(4).Info("QuotaMonitor finished starting monitors", "new", started, "total", len(monitors)) 275 } 276 277 // IsSynced returns true if any monitors exist AND all those monitors' 278 // controllers HasSynced functions return true. This means IsSynced could return 279 // true at one time, and then later return false if all monitors were 280 // reconstructed. 281 func (qm *QuotaMonitor) IsSynced(ctx context.Context) bool { 282 logger := klog.FromContext(ctx) 283 284 qm.monitorLock.RLock() 285 defer qm.monitorLock.RUnlock() 286 287 if len(qm.monitors) == 0 { 288 logger.V(4).Info("quota monitor not synced: no monitors") 289 return false 290 } 291 292 for resource, monitor := range qm.monitors { 293 if !monitor.controller.HasSynced() { 294 logger.V(4).Info("quota monitor not synced", "resource", resource) 295 return false 296 } 297 } 298 return true 299 } 300 301 // Run sets the stop channel and starts monitor execution until stopCh is 302 // closed. Any running monitors will be stopped before Run returns. 303 func (qm *QuotaMonitor) Run(ctx context.Context) { 304 defer utilruntime.HandleCrash() 305 306 logger := klog.FromContext(ctx) 307 308 logger.Info("QuotaMonitor running") 309 defer logger.Info("QuotaMonitor stopping") 310 311 // Set up the stop channel. 312 qm.monitorLock.Lock() 313 qm.stopCh = ctx.Done() 314 qm.running = true 315 qm.monitorLock.Unlock() 316 317 // Start monitors and begin change processing until the stop channel is 318 // closed. 319 qm.StartMonitors(ctx) 320 321 // The following workers are hanging forever until the queue is 322 // shutted down, so we need to shut it down in a separate goroutine. 323 go func() { 324 defer utilruntime.HandleCrash() 325 defer qm.resourceChanges.ShutDown() 326 327 <-ctx.Done() 328 }() 329 wait.UntilWithContext(ctx, qm.runProcessResourceChanges, 1*time.Second) 330 331 // Stop any running monitors. 332 qm.monitorLock.Lock() 333 defer qm.monitorLock.Unlock() 334 monitors := qm.monitors 335 stopped := 0 336 for _, monitor := range monitors { 337 if monitor.stopCh != nil { 338 stopped++ 339 close(monitor.stopCh) 340 } 341 } 342 logger.Info("QuotaMonitor stopped monitors", "stopped", stopped, "total", len(monitors)) 343 } 344 345 func (qm *QuotaMonitor) runProcessResourceChanges(ctx context.Context) { 346 for qm.processResourceChanges(ctx) { 347 } 348 } 349 350 // Dequeueing an event from resourceChanges to process 351 func (qm *QuotaMonitor) processResourceChanges(ctx context.Context) bool { 352 item, quit := qm.resourceChanges.Get() 353 if quit { 354 return false 355 } 356 defer qm.resourceChanges.Done(item) 357 event := item 358 obj := event.obj 359 accessor, err := meta.Accessor(obj) 360 if err != nil { 361 utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err)) 362 return true 363 } 364 klog.FromContext(ctx).V(4).Info("QuotaMonitor process object", 365 "resource", event.gvr.String(), 366 "namespace", accessor.GetNamespace(), 367 "name", accessor.GetName(), 368 "uid", string(accessor.GetUID()), 369 "eventType", event.eventType, 370 ) 371 qm.replenishmentFunc(ctx, event.gvr.GroupResource(), accessor.GetNamespace()) 372 return true 373 }