k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/ttl/ttl_controller.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // The TTLController sets ttl annotations on nodes, based on cluster size. 18 // The annotations are consumed by Kubelets as suggestions for how long 19 // it can cache objects (e.g. secrets or config maps) before refetching 20 // from apiserver again. 21 // 22 // TODO: This is a temporary workaround for the Kubelet not being able to 23 // send "watch secrets attached to pods from my node" request. Once 24 // sending such request will be possible, we will modify Kubelet to 25 // use it and get rid of this controller completely. 26 27 package ttl 28 29 import ( 30 "context" 31 "fmt" 32 "math" 33 "strconv" 34 "sync" 35 "time" 36 37 v1 "k8s.io/api/core/v1" 38 apierrors "k8s.io/apimachinery/pkg/api/errors" 39 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 "k8s.io/apimachinery/pkg/types" 41 "k8s.io/apimachinery/pkg/util/json" 42 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 43 "k8s.io/apimachinery/pkg/util/strategicpatch" 44 "k8s.io/apimachinery/pkg/util/wait" 45 informers "k8s.io/client-go/informers/core/v1" 46 clientset "k8s.io/client-go/kubernetes" 47 listers "k8s.io/client-go/listers/core/v1" 48 "k8s.io/client-go/tools/cache" 49 "k8s.io/client-go/util/workqueue" 50 "k8s.io/kubernetes/pkg/controller" 51 52 "k8s.io/klog/v2" 53 ) 54 55 // Controller sets ttl annotations on nodes, based on cluster size. 56 type Controller struct { 57 kubeClient clientset.Interface 58 59 // nodeStore is a local cache of nodes. 60 nodeStore listers.NodeLister 61 62 // Nodes that need to be synced. 63 queue workqueue.TypedRateLimitingInterface[string] 64 65 // Returns true if all underlying informers are synced. 66 hasSynced func() bool 67 68 lock sync.RWMutex 69 70 // Number of nodes in the cluster. 71 nodeCount int 72 73 // Desired TTL for all nodes in the cluster. 74 desiredTTLSeconds int 75 76 // In which interval of cluster size we currently are. 77 boundaryStep int 78 } 79 80 // NewTTLController creates a new TTLController 81 func NewTTLController(ctx context.Context, nodeInformer informers.NodeInformer, kubeClient clientset.Interface) *Controller { 82 ttlc := &Controller{ 83 kubeClient: kubeClient, 84 queue: workqueue.NewTypedRateLimitingQueueWithConfig( 85 workqueue.DefaultTypedControllerRateLimiter[string](), 86 workqueue.TypedRateLimitingQueueConfig[string]{Name: "ttlcontroller"}, 87 ), 88 } 89 logger := klog.FromContext(ctx) 90 nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 91 AddFunc: func(obj interface{}) { 92 ttlc.addNode(logger, obj) 93 }, 94 UpdateFunc: func(old, newObj interface{}) { 95 ttlc.updateNode(logger, old, newObj) 96 }, 97 DeleteFunc: ttlc.deleteNode, 98 }) 99 100 ttlc.nodeStore = listers.NewNodeLister(nodeInformer.Informer().GetIndexer()) 101 ttlc.hasSynced = nodeInformer.Informer().HasSynced 102 103 return ttlc 104 } 105 106 type ttlBoundary struct { 107 sizeMin int 108 sizeMax int 109 ttlSeconds int 110 } 111 112 var ( 113 ttlBoundaries = []ttlBoundary{ 114 {sizeMin: 0, sizeMax: 100, ttlSeconds: 0}, 115 {sizeMin: 90, sizeMax: 500, ttlSeconds: 15}, 116 {sizeMin: 450, sizeMax: 1000, ttlSeconds: 30}, 117 {sizeMin: 900, sizeMax: 2000, ttlSeconds: 60}, 118 {sizeMin: 1800, sizeMax: math.MaxInt32, ttlSeconds: 300}, 119 } 120 ) 121 122 // Run begins watching and syncing. 123 func (ttlc *Controller) Run(ctx context.Context, workers int) { 124 defer utilruntime.HandleCrash() 125 defer ttlc.queue.ShutDown() 126 logger := klog.FromContext(ctx) 127 logger.Info("Starting TTL controller") 128 defer logger.Info("Shutting down TTL controller") 129 130 if !cache.WaitForNamedCacheSync("TTL", ctx.Done(), ttlc.hasSynced) { 131 return 132 } 133 134 for i := 0; i < workers; i++ { 135 go wait.UntilWithContext(ctx, ttlc.worker, time.Second) 136 } 137 138 <-ctx.Done() 139 } 140 141 func (ttlc *Controller) addNode(logger klog.Logger, obj interface{}) { 142 node, ok := obj.(*v1.Node) 143 if !ok { 144 utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj)) 145 return 146 } 147 148 func() { 149 ttlc.lock.Lock() 150 defer ttlc.lock.Unlock() 151 ttlc.nodeCount++ 152 if ttlc.nodeCount > ttlBoundaries[ttlc.boundaryStep].sizeMax { 153 ttlc.boundaryStep++ 154 ttlc.desiredTTLSeconds = ttlBoundaries[ttlc.boundaryStep].ttlSeconds 155 } 156 }() 157 ttlc.enqueueNode(logger, node) 158 } 159 160 func (ttlc *Controller) updateNode(logger klog.Logger, _, newObj interface{}) { 161 node, ok := newObj.(*v1.Node) 162 if !ok { 163 utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", newObj)) 164 return 165 } 166 // Processing all updates of nodes guarantees that we will update 167 // the ttl annotation, when cluster size changes. 168 // We are relying on the fact that Kubelet is updating node status 169 // every 10s (or generally every X seconds), which means that whenever 170 // required, its ttl annotation should be updated within that period. 171 ttlc.enqueueNode(logger, node) 172 } 173 174 func (ttlc *Controller) deleteNode(obj interface{}) { 175 _, ok := obj.(*v1.Node) 176 if !ok { 177 tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 178 if !ok { 179 utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj)) 180 return 181 } 182 _, ok = tombstone.Obj.(*v1.Node) 183 if !ok { 184 utilruntime.HandleError(fmt.Errorf("unexpected object types: %v", obj)) 185 return 186 } 187 } 188 189 func() { 190 ttlc.lock.Lock() 191 defer ttlc.lock.Unlock() 192 ttlc.nodeCount-- 193 if ttlc.nodeCount < ttlBoundaries[ttlc.boundaryStep].sizeMin { 194 ttlc.boundaryStep-- 195 ttlc.desiredTTLSeconds = ttlBoundaries[ttlc.boundaryStep].ttlSeconds 196 } 197 }() 198 // We are not processing the node, as it no longer exists. 199 } 200 201 func (ttlc *Controller) enqueueNode(logger klog.Logger, node *v1.Node) { 202 key, err := controller.KeyFunc(node) 203 if err != nil { 204 logger.Error(nil, "Couldn't get key for object", "object", klog.KObj(node)) 205 return 206 } 207 ttlc.queue.Add(key) 208 } 209 210 func (ttlc *Controller) worker(ctx context.Context) { 211 for ttlc.processItem(ctx) { 212 } 213 } 214 215 func (ttlc *Controller) processItem(ctx context.Context) bool { 216 key, quit := ttlc.queue.Get() 217 if quit { 218 return false 219 } 220 defer ttlc.queue.Done(key) 221 222 err := ttlc.updateNodeIfNeeded(ctx, key) 223 if err == nil { 224 ttlc.queue.Forget(key) 225 return true 226 } 227 228 ttlc.queue.AddRateLimited(key) 229 utilruntime.HandleError(err) 230 return true 231 } 232 233 func (ttlc *Controller) getDesiredTTLSeconds() int { 234 ttlc.lock.RLock() 235 defer ttlc.lock.RUnlock() 236 return ttlc.desiredTTLSeconds 237 } 238 239 func getIntFromAnnotation(ctx context.Context, node *v1.Node, annotationKey string) (int, bool) { 240 if node.Annotations == nil { 241 return 0, false 242 } 243 annotationValue, ok := node.Annotations[annotationKey] 244 if !ok { 245 return 0, false 246 } 247 intValue, err := strconv.Atoi(annotationValue) 248 if err != nil { 249 logger := klog.FromContext(ctx) 250 logger.Info("Could not convert the value with annotation key for the node", "annotationValue", 251 annotationValue, "annotationKey", annotationKey, "node", klog.KObj(node)) 252 return 0, false 253 } 254 return intValue, true 255 } 256 257 func setIntAnnotation(node *v1.Node, annotationKey string, value int) { 258 if node.Annotations == nil { 259 node.Annotations = make(map[string]string) 260 } 261 node.Annotations[annotationKey] = strconv.Itoa(value) 262 } 263 264 func (ttlc *Controller) patchNodeWithAnnotation(ctx context.Context, node *v1.Node, annotationKey string, value int) error { 265 oldData, err := json.Marshal(node) 266 if err != nil { 267 return err 268 } 269 setIntAnnotation(node, annotationKey, value) 270 newData, err := json.Marshal(node) 271 if err != nil { 272 return err 273 } 274 patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{}) 275 if err != nil { 276 return err 277 } 278 _, err = ttlc.kubeClient.CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) 279 logger := klog.FromContext(ctx) 280 if err != nil { 281 logger.V(2).Info("Failed to change ttl annotation for node", "node", klog.KObj(node), "err", err) 282 return err 283 } 284 logger.V(2).Info("Changed ttl annotation", "node", klog.KObj(node), "TTL", time.Duration(value)*time.Second) 285 return nil 286 } 287 288 func (ttlc *Controller) updateNodeIfNeeded(ctx context.Context, key string) error { 289 node, err := ttlc.nodeStore.Get(key) 290 if err != nil { 291 if apierrors.IsNotFound(err) { 292 return nil 293 } 294 return err 295 } 296 297 desiredTTL := ttlc.getDesiredTTLSeconds() 298 currentTTL, ok := getIntFromAnnotation(ctx, node, v1.ObjectTTLAnnotationKey) 299 if ok && currentTTL == desiredTTL { 300 return nil 301 } 302 303 return ttlc.patchNodeWithAnnotation(ctx, node.DeepCopy(), v1.ObjectTTLAnnotationKey, desiredTTL) 304 }