k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/ttl/ttl_controller.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // The TTLController sets ttl annotations on nodes, based on cluster size.
    18  // The annotations are consumed by Kubelets as suggestions for how long
    19  // it can cache objects (e.g. secrets or config maps) before refetching
    20  // from apiserver again.
    21  //
    22  // TODO: This is a temporary workaround for the Kubelet not being able to
    23  // send "watch secrets attached to pods from my node" request. Once
    24  // sending such request will be possible, we will modify Kubelet to
    25  // use it and get rid of this controller completely.
    26  
    27  package ttl
    28  
    29  import (
    30  	"context"
    31  	"fmt"
    32  	"math"
    33  	"strconv"
    34  	"sync"
    35  	"time"
    36  
    37  	v1 "k8s.io/api/core/v1"
    38  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    39  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    40  	"k8s.io/apimachinery/pkg/types"
    41  	"k8s.io/apimachinery/pkg/util/json"
    42  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    43  	"k8s.io/apimachinery/pkg/util/strategicpatch"
    44  	"k8s.io/apimachinery/pkg/util/wait"
    45  	informers "k8s.io/client-go/informers/core/v1"
    46  	clientset "k8s.io/client-go/kubernetes"
    47  	listers "k8s.io/client-go/listers/core/v1"
    48  	"k8s.io/client-go/tools/cache"
    49  	"k8s.io/client-go/util/workqueue"
    50  	"k8s.io/kubernetes/pkg/controller"
    51  
    52  	"k8s.io/klog/v2"
    53  )
    54  
    55  // Controller sets ttl annotations on nodes, based on cluster size.
    56  type Controller struct {
    57  	kubeClient clientset.Interface
    58  
    59  	// nodeStore is a local cache of nodes.
    60  	nodeStore listers.NodeLister
    61  
    62  	// Nodes that need to be synced.
    63  	queue workqueue.TypedRateLimitingInterface[string]
    64  
    65  	// Returns true if all underlying informers are synced.
    66  	hasSynced func() bool
    67  
    68  	lock sync.RWMutex
    69  
    70  	// Number of nodes in the cluster.
    71  	nodeCount int
    72  
    73  	// Desired TTL for all nodes in the cluster.
    74  	desiredTTLSeconds int
    75  
    76  	// In which interval of cluster size we currently are.
    77  	boundaryStep int
    78  }
    79  
    80  // NewTTLController creates a new TTLController
    81  func NewTTLController(ctx context.Context, nodeInformer informers.NodeInformer, kubeClient clientset.Interface) *Controller {
    82  	ttlc := &Controller{
    83  		kubeClient: kubeClient,
    84  		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
    85  			workqueue.DefaultTypedControllerRateLimiter[string](),
    86  			workqueue.TypedRateLimitingQueueConfig[string]{Name: "ttlcontroller"},
    87  		),
    88  	}
    89  	logger := klog.FromContext(ctx)
    90  	nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
    91  		AddFunc: func(obj interface{}) {
    92  			ttlc.addNode(logger, obj)
    93  		},
    94  		UpdateFunc: func(old, newObj interface{}) {
    95  			ttlc.updateNode(logger, old, newObj)
    96  		},
    97  		DeleteFunc: ttlc.deleteNode,
    98  	})
    99  
   100  	ttlc.nodeStore = listers.NewNodeLister(nodeInformer.Informer().GetIndexer())
   101  	ttlc.hasSynced = nodeInformer.Informer().HasSynced
   102  
   103  	return ttlc
   104  }
   105  
   106  type ttlBoundary struct {
   107  	sizeMin    int
   108  	sizeMax    int
   109  	ttlSeconds int
   110  }
   111  
   112  var (
   113  	ttlBoundaries = []ttlBoundary{
   114  		{sizeMin: 0, sizeMax: 100, ttlSeconds: 0},
   115  		{sizeMin: 90, sizeMax: 500, ttlSeconds: 15},
   116  		{sizeMin: 450, sizeMax: 1000, ttlSeconds: 30},
   117  		{sizeMin: 900, sizeMax: 2000, ttlSeconds: 60},
   118  		{sizeMin: 1800, sizeMax: math.MaxInt32, ttlSeconds: 300},
   119  	}
   120  )
   121  
   122  // Run begins watching and syncing.
   123  func (ttlc *Controller) Run(ctx context.Context, workers int) {
   124  	defer utilruntime.HandleCrash()
   125  	defer ttlc.queue.ShutDown()
   126  	logger := klog.FromContext(ctx)
   127  	logger.Info("Starting TTL controller")
   128  	defer logger.Info("Shutting down TTL controller")
   129  
   130  	if !cache.WaitForNamedCacheSync("TTL", ctx.Done(), ttlc.hasSynced) {
   131  		return
   132  	}
   133  
   134  	for i := 0; i < workers; i++ {
   135  		go wait.UntilWithContext(ctx, ttlc.worker, time.Second)
   136  	}
   137  
   138  	<-ctx.Done()
   139  }
   140  
   141  func (ttlc *Controller) addNode(logger klog.Logger, obj interface{}) {
   142  	node, ok := obj.(*v1.Node)
   143  	if !ok {
   144  		utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj))
   145  		return
   146  	}
   147  
   148  	func() {
   149  		ttlc.lock.Lock()
   150  		defer ttlc.lock.Unlock()
   151  		ttlc.nodeCount++
   152  		if ttlc.nodeCount > ttlBoundaries[ttlc.boundaryStep].sizeMax {
   153  			ttlc.boundaryStep++
   154  			ttlc.desiredTTLSeconds = ttlBoundaries[ttlc.boundaryStep].ttlSeconds
   155  		}
   156  	}()
   157  	ttlc.enqueueNode(logger, node)
   158  }
   159  
   160  func (ttlc *Controller) updateNode(logger klog.Logger, _, newObj interface{}) {
   161  	node, ok := newObj.(*v1.Node)
   162  	if !ok {
   163  		utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", newObj))
   164  		return
   165  	}
   166  	// Processing all updates of nodes guarantees that we will update
   167  	// the ttl annotation, when cluster size changes.
   168  	// We are relying on the fact that Kubelet is updating node status
   169  	// every 10s (or generally every X seconds), which means that whenever
   170  	// required, its ttl annotation should be updated within that period.
   171  	ttlc.enqueueNode(logger, node)
   172  }
   173  
   174  func (ttlc *Controller) deleteNode(obj interface{}) {
   175  	_, ok := obj.(*v1.Node)
   176  	if !ok {
   177  		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
   178  		if !ok {
   179  			utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj))
   180  			return
   181  		}
   182  		_, ok = tombstone.Obj.(*v1.Node)
   183  		if !ok {
   184  			utilruntime.HandleError(fmt.Errorf("unexpected object types: %v", obj))
   185  			return
   186  		}
   187  	}
   188  
   189  	func() {
   190  		ttlc.lock.Lock()
   191  		defer ttlc.lock.Unlock()
   192  		ttlc.nodeCount--
   193  		if ttlc.nodeCount < ttlBoundaries[ttlc.boundaryStep].sizeMin {
   194  			ttlc.boundaryStep--
   195  			ttlc.desiredTTLSeconds = ttlBoundaries[ttlc.boundaryStep].ttlSeconds
   196  		}
   197  	}()
   198  	// We are not processing the node, as it no longer exists.
   199  }
   200  
   201  func (ttlc *Controller) enqueueNode(logger klog.Logger, node *v1.Node) {
   202  	key, err := controller.KeyFunc(node)
   203  	if err != nil {
   204  		logger.Error(nil, "Couldn't get key for object", "object", klog.KObj(node))
   205  		return
   206  	}
   207  	ttlc.queue.Add(key)
   208  }
   209  
   210  func (ttlc *Controller) worker(ctx context.Context) {
   211  	for ttlc.processItem(ctx) {
   212  	}
   213  }
   214  
   215  func (ttlc *Controller) processItem(ctx context.Context) bool {
   216  	key, quit := ttlc.queue.Get()
   217  	if quit {
   218  		return false
   219  	}
   220  	defer ttlc.queue.Done(key)
   221  
   222  	err := ttlc.updateNodeIfNeeded(ctx, key)
   223  	if err == nil {
   224  		ttlc.queue.Forget(key)
   225  		return true
   226  	}
   227  
   228  	ttlc.queue.AddRateLimited(key)
   229  	utilruntime.HandleError(err)
   230  	return true
   231  }
   232  
   233  func (ttlc *Controller) getDesiredTTLSeconds() int {
   234  	ttlc.lock.RLock()
   235  	defer ttlc.lock.RUnlock()
   236  	return ttlc.desiredTTLSeconds
   237  }
   238  
   239  func getIntFromAnnotation(ctx context.Context, node *v1.Node, annotationKey string) (int, bool) {
   240  	if node.Annotations == nil {
   241  		return 0, false
   242  	}
   243  	annotationValue, ok := node.Annotations[annotationKey]
   244  	if !ok {
   245  		return 0, false
   246  	}
   247  	intValue, err := strconv.Atoi(annotationValue)
   248  	if err != nil {
   249  		logger := klog.FromContext(ctx)
   250  		logger.Info("Could not convert the value with annotation key for the node", "annotationValue",
   251  			annotationValue, "annotationKey", annotationKey, "node", klog.KObj(node))
   252  		return 0, false
   253  	}
   254  	return intValue, true
   255  }
   256  
   257  func setIntAnnotation(node *v1.Node, annotationKey string, value int) {
   258  	if node.Annotations == nil {
   259  		node.Annotations = make(map[string]string)
   260  	}
   261  	node.Annotations[annotationKey] = strconv.Itoa(value)
   262  }
   263  
   264  func (ttlc *Controller) patchNodeWithAnnotation(ctx context.Context, node *v1.Node, annotationKey string, value int) error {
   265  	oldData, err := json.Marshal(node)
   266  	if err != nil {
   267  		return err
   268  	}
   269  	setIntAnnotation(node, annotationKey, value)
   270  	newData, err := json.Marshal(node)
   271  	if err != nil {
   272  		return err
   273  	}
   274  	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
   275  	if err != nil {
   276  		return err
   277  	}
   278  	_, err = ttlc.kubeClient.CoreV1().Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
   279  	logger := klog.FromContext(ctx)
   280  	if err != nil {
   281  		logger.V(2).Info("Failed to change ttl annotation for node", "node", klog.KObj(node), "err", err)
   282  		return err
   283  	}
   284  	logger.V(2).Info("Changed ttl annotation", "node", klog.KObj(node), "TTL", time.Duration(value)*time.Second)
   285  	return nil
   286  }
   287  
   288  func (ttlc *Controller) updateNodeIfNeeded(ctx context.Context, key string) error {
   289  	node, err := ttlc.nodeStore.Get(key)
   290  	if err != nil {
   291  		if apierrors.IsNotFound(err) {
   292  			return nil
   293  		}
   294  		return err
   295  	}
   296  
   297  	desiredTTL := ttlc.getDesiredTTLSeconds()
   298  	currentTTL, ok := getIntFromAnnotation(ctx, node, v1.ObjectTTLAnnotationKey)
   299  	if ok && currentTTL == desiredTTL {
   300  		return nil
   301  	}
   302  
   303  	return ttlc.patchNodeWithAnnotation(ctx, node.DeepCopy(), v1.ObjectTTLAnnotationKey, desiredTTL)
   304  }