k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/storageversiongc/gc_controller.go (about)

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storageversiongc
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1"
    25  	coordinationv1 "k8s.io/api/coordination/v1"
    26  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    29  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    30  	"k8s.io/apimachinery/pkg/util/wait"
    31  	"k8s.io/apiserver/pkg/storageversion"
    32  	apiserverinternalinformers "k8s.io/client-go/informers/apiserverinternal/v1alpha1"
    33  	coordinformers "k8s.io/client-go/informers/coordination/v1"
    34  	"k8s.io/client-go/kubernetes"
    35  	coordlisters "k8s.io/client-go/listers/coordination/v1"
    36  	"k8s.io/client-go/tools/cache"
    37  	"k8s.io/client-go/util/workqueue"
    38  	"k8s.io/kubernetes/pkg/controlplane"
    39  
    40  	"k8s.io/klog/v2"
    41  )
    42  
    43  // Controller watches kube-apiserver leases and storageversions, and delete stale
    44  // storage version entries and objects.
    45  type Controller struct {
    46  	kubeclientset kubernetes.Interface
    47  
    48  	leaseLister  coordlisters.LeaseLister
    49  	leasesSynced cache.InformerSynced
    50  
    51  	storageVersionSynced cache.InformerSynced
    52  
    53  	leaseQueue          workqueue.TypedRateLimitingInterface[string]
    54  	storageVersionQueue workqueue.TypedRateLimitingInterface[string]
    55  }
    56  
    57  // NewStorageVersionGC creates a new Controller.
    58  func NewStorageVersionGC(ctx context.Context, clientset kubernetes.Interface, leaseInformer coordinformers.LeaseInformer, storageVersionInformer apiserverinternalinformers.StorageVersionInformer) *Controller {
    59  	c := &Controller{
    60  		kubeclientset:        clientset,
    61  		leaseLister:          leaseInformer.Lister(),
    62  		leasesSynced:         leaseInformer.Informer().HasSynced,
    63  		storageVersionSynced: storageVersionInformer.Informer().HasSynced,
    64  		leaseQueue: workqueue.NewTypedRateLimitingQueueWithConfig(
    65  			workqueue.DefaultTypedControllerRateLimiter[string](),
    66  			workqueue.TypedRateLimitingQueueConfig[string]{Name: "storage_version_garbage_collector_leases"},
    67  		),
    68  		storageVersionQueue: workqueue.NewTypedRateLimitingQueueWithConfig(
    69  			workqueue.DefaultTypedControllerRateLimiter[string](),
    70  			workqueue.TypedRateLimitingQueueConfig[string]{Name: "storage_version_garbage_collector_storageversions"},
    71  		),
    72  	}
    73  	logger := klog.FromContext(ctx)
    74  	leaseInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
    75  		DeleteFunc: func(obj interface{}) {
    76  			c.onDeleteLease(logger, obj)
    77  		},
    78  	})
    79  	// use the default resync period from the informer
    80  	storageVersionInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
    81  		AddFunc: func(obj interface{}) {
    82  			c.onAddStorageVersion(logger, obj)
    83  		},
    84  		UpdateFunc: func(old, newObj interface{}) {
    85  			c.onUpdateStorageVersion(logger, old, newObj)
    86  		},
    87  	})
    88  
    89  	return c
    90  }
    91  
    92  // Run starts one worker.
    93  func (c *Controller) Run(ctx context.Context) {
    94  	logger := klog.FromContext(ctx)
    95  	defer utilruntime.HandleCrash()
    96  	defer c.leaseQueue.ShutDown()
    97  	defer c.storageVersionQueue.ShutDown()
    98  	defer logger.Info("Shutting down storage version garbage collector")
    99  
   100  	logger.Info("Starting storage version garbage collector")
   101  
   102  	if !cache.WaitForCacheSync(ctx.Done(), c.leasesSynced, c.storageVersionSynced) {
   103  		utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
   104  		return
   105  	}
   106  
   107  	// Identity lease deletion and storageversion update don't happen too often. Start one
   108  	// worker for each of them.
   109  	// runLeaseWorker handles legit identity lease deletion, while runStorageVersionWorker
   110  	// handles storageversion creation/update with non-existing id. The latter should rarely
   111  	// happen. It's okay for the two workers to conflict on update.
   112  	go wait.UntilWithContext(ctx, c.runLeaseWorker, time.Second)
   113  	go wait.UntilWithContext(ctx, c.runStorageVersionWorker, time.Second)
   114  
   115  	<-ctx.Done()
   116  }
   117  
   118  func (c *Controller) runLeaseWorker(ctx context.Context) {
   119  	for c.processNextLease(ctx) {
   120  	}
   121  }
   122  
   123  func (c *Controller) processNextLease(ctx context.Context) bool {
   124  	key, quit := c.leaseQueue.Get()
   125  	if quit {
   126  		return false
   127  	}
   128  	defer c.leaseQueue.Done(key)
   129  
   130  	err := c.processDeletedLease(ctx, key)
   131  	if err == nil {
   132  		c.leaseQueue.Forget(key)
   133  		return true
   134  	}
   135  
   136  	utilruntime.HandleError(fmt.Errorf("lease %v failed with: %v", key, err))
   137  	c.leaseQueue.AddRateLimited(key)
   138  	return true
   139  }
   140  
   141  func (c *Controller) runStorageVersionWorker(ctx context.Context) {
   142  	for c.processNextStorageVersion(ctx) {
   143  	}
   144  }
   145  
   146  func (c *Controller) processNextStorageVersion(ctx context.Context) bool {
   147  	key, quit := c.storageVersionQueue.Get()
   148  	if quit {
   149  		return false
   150  	}
   151  	defer c.storageVersionQueue.Done(key)
   152  
   153  	err := c.syncStorageVersion(ctx, key)
   154  	if err == nil {
   155  		c.storageVersionQueue.Forget(key)
   156  		return true
   157  	}
   158  
   159  	utilruntime.HandleError(fmt.Errorf("storage version %v failed with: %v", key, err))
   160  	c.storageVersionQueue.AddRateLimited(key)
   161  	return true
   162  }
   163  
   164  func (c *Controller) processDeletedLease(ctx context.Context, name string) error {
   165  	_, err := c.kubeclientset.CoordinationV1().Leases(metav1.NamespaceSystem).Get(ctx, name, metav1.GetOptions{})
   166  	// the lease isn't deleted, nothing we need to do here
   167  	if err == nil {
   168  		return nil
   169  	}
   170  	if !apierrors.IsNotFound(err) {
   171  		return err
   172  	}
   173  	// the frequency of this call won't be too high because we only trigger on identity lease deletions
   174  	storageVersionList, err := c.kubeclientset.InternalV1alpha1().StorageVersions().List(ctx, metav1.ListOptions{})
   175  	if err != nil {
   176  		return err
   177  	}
   178  
   179  	var errors []error
   180  	for _, sv := range storageVersionList.Items {
   181  		var serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion
   182  		hasStaleRecord := false
   183  		for _, ssv := range sv.Status.StorageVersions {
   184  			if ssv.APIServerID == name {
   185  				hasStaleRecord = true
   186  				continue
   187  			}
   188  			serverStorageVersions = append(serverStorageVersions, ssv)
   189  		}
   190  		if !hasStaleRecord {
   191  			continue
   192  		}
   193  		if err := c.updateOrDeleteStorageVersion(ctx, &sv, serverStorageVersions); err != nil {
   194  			errors = append(errors, err)
   195  		}
   196  	}
   197  
   198  	return utilerrors.NewAggregate(errors)
   199  }
   200  
   201  func (c *Controller) syncStorageVersion(ctx context.Context, name string) error {
   202  	sv, err := c.kubeclientset.InternalV1alpha1().StorageVersions().Get(ctx, name, metav1.GetOptions{})
   203  	if apierrors.IsNotFound(err) {
   204  		// The problematic storage version that was added/updated recently is gone.
   205  		// Nothing we need to do here.
   206  		return nil
   207  	}
   208  	if err != nil {
   209  		return err
   210  	}
   211  
   212  	hasInvalidID := false
   213  	var serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion
   214  	for _, v := range sv.Status.StorageVersions {
   215  		lease, err := c.kubeclientset.CoordinationV1().Leases(metav1.NamespaceSystem).Get(ctx, v.APIServerID, metav1.GetOptions{})
   216  		if err != nil || lease == nil || lease.Labels == nil ||
   217  			lease.Labels[controlplane.IdentityLeaseComponentLabelKey] != controlplane.KubeAPIServer {
   218  			// We cannot find a corresponding identity lease from apiserver as well.
   219  			// We need to clean up this storage version.
   220  			hasInvalidID = true
   221  			continue
   222  		}
   223  		serverStorageVersions = append(serverStorageVersions, v)
   224  	}
   225  	if !hasInvalidID {
   226  		return nil
   227  	}
   228  	return c.updateOrDeleteStorageVersion(ctx, sv, serverStorageVersions)
   229  }
   230  
   231  func (c *Controller) onAddStorageVersion(logger klog.Logger, obj interface{}) {
   232  	castObj := obj.(*apiserverinternalv1alpha1.StorageVersion)
   233  	c.enqueueStorageVersion(logger, castObj)
   234  }
   235  
   236  func (c *Controller) onUpdateStorageVersion(logger klog.Logger, oldObj, newObj interface{}) {
   237  	castNewObj := newObj.(*apiserverinternalv1alpha1.StorageVersion)
   238  	c.enqueueStorageVersion(logger, castNewObj)
   239  }
   240  
   241  // enqueueStorageVersion enqueues the storage version if it has entry for invalid apiserver
   242  func (c *Controller) enqueueStorageVersion(logger klog.Logger, obj *apiserverinternalv1alpha1.StorageVersion) {
   243  	for _, sv := range obj.Status.StorageVersions {
   244  		lease, err := c.leaseLister.Leases(metav1.NamespaceSystem).Get(sv.APIServerID)
   245  		if err != nil || lease == nil || lease.Labels == nil ||
   246  			lease.Labels[controlplane.IdentityLeaseComponentLabelKey] != controlplane.KubeAPIServer {
   247  			// we cannot find a corresponding identity lease in cache, enqueue the storageversion
   248  			logger.V(4).Info("Observed storage version with invalid apiserver entry", "objName", obj.Name)
   249  			c.storageVersionQueue.Add(obj.Name)
   250  			return
   251  		}
   252  	}
   253  }
   254  
   255  func (c *Controller) onDeleteLease(logger klog.Logger, obj interface{}) {
   256  	castObj, ok := obj.(*coordinationv1.Lease)
   257  	if !ok {
   258  		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
   259  		if !ok {
   260  			utilruntime.HandleError(fmt.Errorf("couldn't get object from tombstone %#v", obj))
   261  			return
   262  		}
   263  		castObj, ok = tombstone.Obj.(*coordinationv1.Lease)
   264  		if !ok {
   265  			utilruntime.HandleError(fmt.Errorf("tombstone contained object that is not a Lease %#v", obj))
   266  			return
   267  		}
   268  	}
   269  
   270  	if castObj.Namespace == metav1.NamespaceSystem &&
   271  		castObj.Labels != nil &&
   272  		castObj.Labels[controlplane.IdentityLeaseComponentLabelKey] == controlplane.KubeAPIServer {
   273  		logger.V(4).Info("Observed lease deleted", "castObjName", castObj.Name)
   274  		c.enqueueLease(castObj)
   275  	}
   276  }
   277  
   278  func (c *Controller) enqueueLease(obj *coordinationv1.Lease) {
   279  	c.leaseQueue.Add(obj.Name)
   280  }
   281  
   282  func (c *Controller) updateOrDeleteStorageVersion(ctx context.Context, sv *apiserverinternalv1alpha1.StorageVersion, serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion) error {
   283  	if len(serverStorageVersions) == 0 {
   284  		return c.kubeclientset.InternalV1alpha1().StorageVersions().Delete(
   285  			ctx, sv.Name, metav1.DeleteOptions{})
   286  	}
   287  	sv.Status.StorageVersions = serverStorageVersions
   288  	storageversion.SetCommonEncodingVersion(sv)
   289  	_, err := c.kubeclientset.InternalV1alpha1().StorageVersions().UpdateStatus(
   290  		ctx, sv, metav1.UpdateOptions{})
   291  	return err
   292  }