k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/storageversionmigrator/storageversionmigrator.go (about) 1 /* 2 Copyright 2024 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package storageversionmigrator 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "time" 24 25 "k8s.io/klog/v2" 26 27 "k8s.io/apimachinery/pkg/api/meta" 28 "k8s.io/apimachinery/pkg/types" 29 "k8s.io/apimachinery/pkg/util/wait" 30 "k8s.io/client-go/dynamic" 31 "k8s.io/client-go/kubernetes" 32 "k8s.io/client-go/tools/cache" 33 "k8s.io/client-go/util/workqueue" 34 "k8s.io/kubernetes/pkg/controller" 35 "k8s.io/kubernetes/pkg/controller/garbagecollector" 36 37 svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1" 38 apierrors "k8s.io/apimachinery/pkg/api/errors" 39 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 41 svminformers "k8s.io/client-go/informers/storagemigration/v1alpha1" 42 svmlisters "k8s.io/client-go/listers/storagemigration/v1alpha1" 43 ) 44 45 const ( 46 workers = 5 47 migrationSuccessStatusReason = "StorageVersionMigrationSucceeded" 48 migrationRunningStatusReason = "StorageVersionMigrationInProgress" 49 migrationFailedStatusReason = "StorageVersionMigrationFailed" 50 ) 51 52 type SVMController struct { 53 controllerName string 54 kubeClient kubernetes.Interface 55 dynamicClient *dynamic.DynamicClient 56 svmListers svmlisters.StorageVersionMigrationLister 57 svmSynced cache.InformerSynced 58 queue workqueue.TypedRateLimitingInterface[string] 59 restMapper meta.RESTMapper 60 dependencyGraphBuilder *garbagecollector.GraphBuilder 61 } 62 63 func NewSVMController( 64 ctx context.Context, 65 kubeClient kubernetes.Interface, 66 dynamicClient *dynamic.DynamicClient, 67 svmInformer svminformers.StorageVersionMigrationInformer, 68 controllerName string, 69 mapper meta.ResettableRESTMapper, 70 dependencyGraphBuilder *garbagecollector.GraphBuilder, 71 ) *SVMController { 72 logger := klog.FromContext(ctx) 73 74 svmController := &SVMController{ 75 kubeClient: kubeClient, 76 dynamicClient: dynamicClient, 77 controllerName: controllerName, 78 svmListers: svmInformer.Lister(), 79 svmSynced: svmInformer.Informer().HasSynced, 80 restMapper: mapper, 81 dependencyGraphBuilder: dependencyGraphBuilder, 82 queue: workqueue.NewTypedRateLimitingQueueWithConfig( 83 workqueue.DefaultTypedControllerRateLimiter[string](), 84 workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}, 85 ), 86 } 87 88 _, _ = svmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 89 AddFunc: func(obj interface{}) { 90 svmController.addSVM(logger, obj) 91 }, 92 UpdateFunc: func(oldObj, newObj interface{}) { 93 svmController.updateSVM(logger, oldObj, newObj) 94 }, 95 }) 96 97 return svmController 98 } 99 100 func (svmc *SVMController) Name() string { 101 return svmc.controllerName 102 } 103 104 func (svmc *SVMController) addSVM(logger klog.Logger, obj interface{}) { 105 svm := obj.(*svmv1alpha1.StorageVersionMigration) 106 logger.V(4).Info("Adding", "svm", klog.KObj(svm)) 107 svmc.enqueue(svm) 108 } 109 110 func (svmc *SVMController) updateSVM(logger klog.Logger, oldObj, newObj interface{}) { 111 oldSVM := oldObj.(*svmv1alpha1.StorageVersionMigration) 112 newSVM := newObj.(*svmv1alpha1.StorageVersionMigration) 113 logger.V(4).Info("Updating", "svm", klog.KObj(oldSVM)) 114 svmc.enqueue(newSVM) 115 } 116 117 func (svmc *SVMController) enqueue(svm *svmv1alpha1.StorageVersionMigration) { 118 key, err := controller.KeyFunc(svm) 119 if err != nil { 120 utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %w", svm, err)) 121 return 122 } 123 124 svmc.queue.Add(key) 125 } 126 127 func (svmc *SVMController) Run(ctx context.Context) { 128 defer utilruntime.HandleCrash() 129 defer svmc.queue.ShutDown() 130 131 logger := klog.FromContext(ctx) 132 logger.Info("Starting", "controller", svmc.controllerName) 133 defer logger.Info("Shutting down", "controller", svmc.controllerName) 134 135 if !cache.WaitForNamedCacheSync(svmc.controllerName, ctx.Done(), svmc.svmSynced) { 136 return 137 } 138 139 for i := 0; i < workers; i++ { 140 go wait.UntilWithContext(ctx, svmc.worker, time.Second) 141 } 142 143 <-ctx.Done() 144 } 145 146 func (svmc *SVMController) worker(ctx context.Context) { 147 for svmc.processNext(ctx) { 148 } 149 } 150 151 func (svmc *SVMController) processNext(ctx context.Context) bool { 152 key, quit := svmc.queue.Get() 153 if quit { 154 return false 155 } 156 defer svmc.queue.Done(key) 157 158 err := svmc.sync(ctx, key) 159 if err == nil { 160 svmc.queue.Forget(key) 161 return true 162 } 163 164 klog.FromContext(ctx).V(2).Info("Error syncing SVM resource, retrying", "svm", key, "err", err) 165 svmc.queue.AddRateLimited(key) 166 167 return true 168 } 169 170 func (svmc *SVMController) sync(ctx context.Context, key string) error { 171 logger := klog.FromContext(ctx) 172 startTime := time.Now() 173 174 if svmc.dependencyGraphBuilder == nil { 175 logger.V(4).Info("dependency graph builder is not set. we will skip migration") 176 return nil 177 } 178 179 // SVM is a cluster scoped resource so we don't care about the namespace 180 _, name, err := cache.SplitMetaNamespaceKey(key) 181 if err != nil { 182 return err 183 } 184 185 svm, err := svmc.svmListers.Get(name) 186 if apierrors.IsNotFound(err) { 187 // no work to do, don't fail and requeue 188 return nil 189 } 190 if err != nil { 191 return err 192 } 193 // working with a copy to avoid race condition between this and resource version controller 194 toBeProcessedSVM := svm.DeepCopy() 195 196 if IsConditionTrue(toBeProcessedSVM, svmv1alpha1.MigrationSucceeded) || IsConditionTrue(toBeProcessedSVM, svmv1alpha1.MigrationFailed) { 197 logger.V(4).Info("Migration has already succeeded or failed previously, skipping", "svm", name) 198 return nil 199 } 200 201 if len(toBeProcessedSVM.Status.ResourceVersion) == 0 { 202 logger.V(4).Info("The latest resource version is empty. We will attempt to migrate once the resource version is available.") 203 return nil 204 } 205 gvr := getGVRFromResource(toBeProcessedSVM) 206 207 resourceMonitor, err := svmc.dependencyGraphBuilder.GetMonitor(ctx, gvr) 208 if resourceMonitor != nil { 209 if err != nil { 210 // non nil monitor indicates that error is due to resource not being synced 211 return fmt.Errorf("dependency graph is not synced, requeuing to attempt again") 212 } 213 } else { 214 // we can't migrate a resource that doesn't exist in the GC 215 _, err = svmc.kubeClient.StoragemigrationV1alpha1(). 216 StorageVersionMigrations(). 217 UpdateStatus( 218 ctx, 219 setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationFailed, migrationFailedStatusReason), 220 metav1.UpdateOptions{}, 221 ) 222 if err != nil { 223 return err 224 } 225 logger.V(4).Error(fmt.Errorf("error migrating the resource"), "resource does not exist in GC", "gvr", gvr.String()) 226 227 return nil 228 } 229 230 gcListResourceVersion, err := convertResourceVersionToInt(resourceMonitor.Controller.LastSyncResourceVersion()) 231 if err != nil { 232 return err 233 } 234 listResourceVersion, err := convertResourceVersionToInt(toBeProcessedSVM.Status.ResourceVersion) 235 if err != nil { 236 return err 237 } 238 239 if gcListResourceVersion < listResourceVersion { 240 return fmt.Errorf("GC cache is not up to date, requeuing to attempt again. gcListResourceVersion: %d, listResourceVersion: %d", gcListResourceVersion, listResourceVersion) 241 } 242 243 toBeProcessedSVM, err = svmc.kubeClient.StoragemigrationV1alpha1(). 244 StorageVersionMigrations(). 245 UpdateStatus( 246 ctx, 247 setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationRunning, migrationRunningStatusReason), 248 metav1.UpdateOptions{}, 249 ) 250 if err != nil { 251 return err 252 } 253 254 gvk, err := svmc.restMapper.KindFor(gvr) 255 if err != nil { 256 return err 257 } 258 typeMeta := metav1.TypeMeta{} 259 typeMeta.APIVersion, typeMeta.Kind = gvk.ToAPIVersionAndKind() 260 data, err := json.Marshal(typeMeta) 261 if err != nil { 262 return err 263 } 264 265 // ToDo: implement a mechanism to resume migration from the last migrated resource in case of a failure 266 // process storage migration 267 for _, gvrKey := range resourceMonitor.Store.ListKeys() { 268 namespace, name, err := cache.SplitMetaNamespaceKey(gvrKey) 269 if err != nil { 270 return err 271 } 272 273 _, err = svmc.dynamicClient.Resource(gvr). 274 Namespace(namespace). 275 Patch(ctx, 276 name, 277 types.ApplyPatchType, 278 data, 279 metav1.PatchOptions{ 280 FieldManager: svmc.controllerName, 281 }, 282 ) 283 if err != nil { 284 // in case of NotFound or Conflict, we can stop processing migration for that resource 285 if apierrors.IsNotFound(err) || apierrors.IsConflict(err) { 286 continue 287 } 288 289 _, err = svmc.kubeClient.StoragemigrationV1alpha1(). 290 StorageVersionMigrations(). 291 UpdateStatus( 292 ctx, 293 setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationFailed, migrationFailedStatusReason), 294 metav1.UpdateOptions{}, 295 ) 296 if err != nil { 297 return err 298 } 299 logger.V(4).Error(err, "Failed to migrate the resource", "name", gvrKey, "gvr", gvr.String(), "reason", apierrors.ReasonForError(err)) 300 301 return nil 302 // Todo: add retry for scenarios where API server returns rate limiting error 303 } 304 logger.V(4).Info("Successfully migrated the resource", "name", gvrKey, "gvr", gvr.String()) 305 } 306 307 _, err = svmc.kubeClient.StoragemigrationV1alpha1(). 308 StorageVersionMigrations(). 309 UpdateStatus( 310 ctx, 311 setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationSucceeded, migrationSuccessStatusReason), 312 metav1.UpdateOptions{}, 313 ) 314 if err != nil { 315 return err 316 } 317 318 logger.V(4).Info("Finished syncing svm resource", "key", key, "gvr", gvr.String(), "elapsed", time.Since(startTime)) 319 return nil 320 }