k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/storageversiongc/gc_controller.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package storageversiongc 18 19 import ( 20 "context" 21 "fmt" 22 "time" 23 24 apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1" 25 coordinationv1 "k8s.io/api/coordination/v1" 26 apierrors "k8s.io/apimachinery/pkg/api/errors" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 utilerrors "k8s.io/apimachinery/pkg/util/errors" 29 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 30 "k8s.io/apimachinery/pkg/util/wait" 31 "k8s.io/apiserver/pkg/storageversion" 32 apiserverinternalinformers "k8s.io/client-go/informers/apiserverinternal/v1alpha1" 33 coordinformers "k8s.io/client-go/informers/coordination/v1" 34 "k8s.io/client-go/kubernetes" 35 coordlisters "k8s.io/client-go/listers/coordination/v1" 36 "k8s.io/client-go/tools/cache" 37 "k8s.io/client-go/util/workqueue" 38 "k8s.io/kubernetes/pkg/controlplane" 39 40 "k8s.io/klog/v2" 41 ) 42 43 // Controller watches kube-apiserver leases and storageversions, and delete stale 44 // storage version entries and objects. 45 type Controller struct { 46 kubeclientset kubernetes.Interface 47 48 leaseLister coordlisters.LeaseLister 49 leasesSynced cache.InformerSynced 50 51 storageVersionSynced cache.InformerSynced 52 53 leaseQueue workqueue.TypedRateLimitingInterface[string] 54 storageVersionQueue workqueue.TypedRateLimitingInterface[string] 55 } 56 57 // NewStorageVersionGC creates a new Controller. 58 func NewStorageVersionGC(ctx context.Context, clientset kubernetes.Interface, leaseInformer coordinformers.LeaseInformer, storageVersionInformer apiserverinternalinformers.StorageVersionInformer) *Controller { 59 c := &Controller{ 60 kubeclientset: clientset, 61 leaseLister: leaseInformer.Lister(), 62 leasesSynced: leaseInformer.Informer().HasSynced, 63 storageVersionSynced: storageVersionInformer.Informer().HasSynced, 64 leaseQueue: workqueue.NewTypedRateLimitingQueueWithConfig( 65 workqueue.DefaultTypedControllerRateLimiter[string](), 66 workqueue.TypedRateLimitingQueueConfig[string]{Name: "storage_version_garbage_collector_leases"}, 67 ), 68 storageVersionQueue: workqueue.NewTypedRateLimitingQueueWithConfig( 69 workqueue.DefaultTypedControllerRateLimiter[string](), 70 workqueue.TypedRateLimitingQueueConfig[string]{Name: "storage_version_garbage_collector_storageversions"}, 71 ), 72 } 73 logger := klog.FromContext(ctx) 74 leaseInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 75 DeleteFunc: func(obj interface{}) { 76 c.onDeleteLease(logger, obj) 77 }, 78 }) 79 // use the default resync period from the informer 80 storageVersionInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 81 AddFunc: func(obj interface{}) { 82 c.onAddStorageVersion(logger, obj) 83 }, 84 UpdateFunc: func(old, newObj interface{}) { 85 c.onUpdateStorageVersion(logger, old, newObj) 86 }, 87 }) 88 89 return c 90 } 91 92 // Run starts one worker. 93 func (c *Controller) Run(ctx context.Context) { 94 logger := klog.FromContext(ctx) 95 defer utilruntime.HandleCrash() 96 defer c.leaseQueue.ShutDown() 97 defer c.storageVersionQueue.ShutDown() 98 defer logger.Info("Shutting down storage version garbage collector") 99 100 logger.Info("Starting storage version garbage collector") 101 102 if !cache.WaitForCacheSync(ctx.Done(), c.leasesSynced, c.storageVersionSynced) { 103 utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) 104 return 105 } 106 107 // Identity lease deletion and storageversion update don't happen too often. Start one 108 // worker for each of them. 109 // runLeaseWorker handles legit identity lease deletion, while runStorageVersionWorker 110 // handles storageversion creation/update with non-existing id. The latter should rarely 111 // happen. It's okay for the two workers to conflict on update. 112 go wait.UntilWithContext(ctx, c.runLeaseWorker, time.Second) 113 go wait.UntilWithContext(ctx, c.runStorageVersionWorker, time.Second) 114 115 <-ctx.Done() 116 } 117 118 func (c *Controller) runLeaseWorker(ctx context.Context) { 119 for c.processNextLease(ctx) { 120 } 121 } 122 123 func (c *Controller) processNextLease(ctx context.Context) bool { 124 key, quit := c.leaseQueue.Get() 125 if quit { 126 return false 127 } 128 defer c.leaseQueue.Done(key) 129 130 err := c.processDeletedLease(ctx, key) 131 if err == nil { 132 c.leaseQueue.Forget(key) 133 return true 134 } 135 136 utilruntime.HandleError(fmt.Errorf("lease %v failed with: %v", key, err)) 137 c.leaseQueue.AddRateLimited(key) 138 return true 139 } 140 141 func (c *Controller) runStorageVersionWorker(ctx context.Context) { 142 for c.processNextStorageVersion(ctx) { 143 } 144 } 145 146 func (c *Controller) processNextStorageVersion(ctx context.Context) bool { 147 key, quit := c.storageVersionQueue.Get() 148 if quit { 149 return false 150 } 151 defer c.storageVersionQueue.Done(key) 152 153 err := c.syncStorageVersion(ctx, key) 154 if err == nil { 155 c.storageVersionQueue.Forget(key) 156 return true 157 } 158 159 utilruntime.HandleError(fmt.Errorf("storage version %v failed with: %v", key, err)) 160 c.storageVersionQueue.AddRateLimited(key) 161 return true 162 } 163 164 func (c *Controller) processDeletedLease(ctx context.Context, name string) error { 165 _, err := c.kubeclientset.CoordinationV1().Leases(metav1.NamespaceSystem).Get(ctx, name, metav1.GetOptions{}) 166 // the lease isn't deleted, nothing we need to do here 167 if err == nil { 168 return nil 169 } 170 if !apierrors.IsNotFound(err) { 171 return err 172 } 173 // the frequency of this call won't be too high because we only trigger on identity lease deletions 174 storageVersionList, err := c.kubeclientset.InternalV1alpha1().StorageVersions().List(ctx, metav1.ListOptions{}) 175 if err != nil { 176 return err 177 } 178 179 var errors []error 180 for _, sv := range storageVersionList.Items { 181 var serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion 182 hasStaleRecord := false 183 for _, ssv := range sv.Status.StorageVersions { 184 if ssv.APIServerID == name { 185 hasStaleRecord = true 186 continue 187 } 188 serverStorageVersions = append(serverStorageVersions, ssv) 189 } 190 if !hasStaleRecord { 191 continue 192 } 193 if err := c.updateOrDeleteStorageVersion(ctx, &sv, serverStorageVersions); err != nil { 194 errors = append(errors, err) 195 } 196 } 197 198 return utilerrors.NewAggregate(errors) 199 } 200 201 func (c *Controller) syncStorageVersion(ctx context.Context, name string) error { 202 sv, err := c.kubeclientset.InternalV1alpha1().StorageVersions().Get(ctx, name, metav1.GetOptions{}) 203 if apierrors.IsNotFound(err) { 204 // The problematic storage version that was added/updated recently is gone. 205 // Nothing we need to do here. 206 return nil 207 } 208 if err != nil { 209 return err 210 } 211 212 hasInvalidID := false 213 var serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion 214 for _, v := range sv.Status.StorageVersions { 215 lease, err := c.kubeclientset.CoordinationV1().Leases(metav1.NamespaceSystem).Get(ctx, v.APIServerID, metav1.GetOptions{}) 216 if err != nil || lease == nil || lease.Labels == nil || 217 lease.Labels[controlplane.IdentityLeaseComponentLabelKey] != controlplane.KubeAPIServer { 218 // We cannot find a corresponding identity lease from apiserver as well. 219 // We need to clean up this storage version. 220 hasInvalidID = true 221 continue 222 } 223 serverStorageVersions = append(serverStorageVersions, v) 224 } 225 if !hasInvalidID { 226 return nil 227 } 228 return c.updateOrDeleteStorageVersion(ctx, sv, serverStorageVersions) 229 } 230 231 func (c *Controller) onAddStorageVersion(logger klog.Logger, obj interface{}) { 232 castObj := obj.(*apiserverinternalv1alpha1.StorageVersion) 233 c.enqueueStorageVersion(logger, castObj) 234 } 235 236 func (c *Controller) onUpdateStorageVersion(logger klog.Logger, oldObj, newObj interface{}) { 237 castNewObj := newObj.(*apiserverinternalv1alpha1.StorageVersion) 238 c.enqueueStorageVersion(logger, castNewObj) 239 } 240 241 // enqueueStorageVersion enqueues the storage version if it has entry for invalid apiserver 242 func (c *Controller) enqueueStorageVersion(logger klog.Logger, obj *apiserverinternalv1alpha1.StorageVersion) { 243 for _, sv := range obj.Status.StorageVersions { 244 lease, err := c.leaseLister.Leases(metav1.NamespaceSystem).Get(sv.APIServerID) 245 if err != nil || lease == nil || lease.Labels == nil || 246 lease.Labels[controlplane.IdentityLeaseComponentLabelKey] != controlplane.KubeAPIServer { 247 // we cannot find a corresponding identity lease in cache, enqueue the storageversion 248 logger.V(4).Info("Observed storage version with invalid apiserver entry", "objName", obj.Name) 249 c.storageVersionQueue.Add(obj.Name) 250 return 251 } 252 } 253 } 254 255 func (c *Controller) onDeleteLease(logger klog.Logger, obj interface{}) { 256 castObj, ok := obj.(*coordinationv1.Lease) 257 if !ok { 258 tombstone, ok := obj.(cache.DeletedFinalStateUnknown) 259 if !ok { 260 utilruntime.HandleError(fmt.Errorf("couldn't get object from tombstone %#v", obj)) 261 return 262 } 263 castObj, ok = tombstone.Obj.(*coordinationv1.Lease) 264 if !ok { 265 utilruntime.HandleError(fmt.Errorf("tombstone contained object that is not a Lease %#v", obj)) 266 return 267 } 268 } 269 270 if castObj.Namespace == metav1.NamespaceSystem && 271 castObj.Labels != nil && 272 castObj.Labels[controlplane.IdentityLeaseComponentLabelKey] == controlplane.KubeAPIServer { 273 logger.V(4).Info("Observed lease deleted", "castObjName", castObj.Name) 274 c.enqueueLease(castObj) 275 } 276 } 277 278 func (c *Controller) enqueueLease(obj *coordinationv1.Lease) { 279 c.leaseQueue.Add(obj.Name) 280 } 281 282 func (c *Controller) updateOrDeleteStorageVersion(ctx context.Context, sv *apiserverinternalv1alpha1.StorageVersion, serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion) error { 283 if len(serverStorageVersions) == 0 { 284 return c.kubeclientset.InternalV1alpha1().StorageVersions().Delete( 285 ctx, sv.Name, metav1.DeleteOptions{}) 286 } 287 sv.Status.StorageVersions = serverStorageVersions 288 storageversion.SetCommonEncodingVersion(sv) 289 _, err := c.kubeclientset.InternalV1alpha1().StorageVersions().UpdateStatus( 290 ctx, sv, metav1.UpdateOptions{}) 291 return err 292 }