k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/controller/storageversionmigrator/storageversionmigrator.go (about)

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storageversionmigrator
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"time"
    24  
    25  	"k8s.io/klog/v2"
    26  
    27  	"k8s.io/apimachinery/pkg/api/meta"
    28  	"k8s.io/apimachinery/pkg/types"
    29  	"k8s.io/apimachinery/pkg/util/wait"
    30  	"k8s.io/client-go/dynamic"
    31  	"k8s.io/client-go/kubernetes"
    32  	"k8s.io/client-go/tools/cache"
    33  	"k8s.io/client-go/util/workqueue"
    34  	"k8s.io/kubernetes/pkg/controller"
    35  	"k8s.io/kubernetes/pkg/controller/garbagecollector"
    36  
    37  	svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1"
    38  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    39  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    40  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    41  	svminformers "k8s.io/client-go/informers/storagemigration/v1alpha1"
    42  	svmlisters "k8s.io/client-go/listers/storagemigration/v1alpha1"
    43  )
    44  
    45  const (
    46  	workers                      = 5
    47  	migrationSuccessStatusReason = "StorageVersionMigrationSucceeded"
    48  	migrationRunningStatusReason = "StorageVersionMigrationInProgress"
    49  	migrationFailedStatusReason  = "StorageVersionMigrationFailed"
    50  )
    51  
    52  type SVMController struct {
    53  	controllerName         string
    54  	kubeClient             kubernetes.Interface
    55  	dynamicClient          *dynamic.DynamicClient
    56  	svmListers             svmlisters.StorageVersionMigrationLister
    57  	svmSynced              cache.InformerSynced
    58  	queue                  workqueue.TypedRateLimitingInterface[string]
    59  	restMapper             meta.RESTMapper
    60  	dependencyGraphBuilder *garbagecollector.GraphBuilder
    61  }
    62  
    63  func NewSVMController(
    64  	ctx context.Context,
    65  	kubeClient kubernetes.Interface,
    66  	dynamicClient *dynamic.DynamicClient,
    67  	svmInformer svminformers.StorageVersionMigrationInformer,
    68  	controllerName string,
    69  	mapper meta.ResettableRESTMapper,
    70  	dependencyGraphBuilder *garbagecollector.GraphBuilder,
    71  ) *SVMController {
    72  	logger := klog.FromContext(ctx)
    73  
    74  	svmController := &SVMController{
    75  		kubeClient:             kubeClient,
    76  		dynamicClient:          dynamicClient,
    77  		controllerName:         controllerName,
    78  		svmListers:             svmInformer.Lister(),
    79  		svmSynced:              svmInformer.Informer().HasSynced,
    80  		restMapper:             mapper,
    81  		dependencyGraphBuilder: dependencyGraphBuilder,
    82  		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
    83  			workqueue.DefaultTypedControllerRateLimiter[string](),
    84  			workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName},
    85  		),
    86  	}
    87  
    88  	_, _ = svmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
    89  		AddFunc: func(obj interface{}) {
    90  			svmController.addSVM(logger, obj)
    91  		},
    92  		UpdateFunc: func(oldObj, newObj interface{}) {
    93  			svmController.updateSVM(logger, oldObj, newObj)
    94  		},
    95  	})
    96  
    97  	return svmController
    98  }
    99  
   100  func (svmc *SVMController) Name() string {
   101  	return svmc.controllerName
   102  }
   103  
   104  func (svmc *SVMController) addSVM(logger klog.Logger, obj interface{}) {
   105  	svm := obj.(*svmv1alpha1.StorageVersionMigration)
   106  	logger.V(4).Info("Adding", "svm", klog.KObj(svm))
   107  	svmc.enqueue(svm)
   108  }
   109  
   110  func (svmc *SVMController) updateSVM(logger klog.Logger, oldObj, newObj interface{}) {
   111  	oldSVM := oldObj.(*svmv1alpha1.StorageVersionMigration)
   112  	newSVM := newObj.(*svmv1alpha1.StorageVersionMigration)
   113  	logger.V(4).Info("Updating", "svm", klog.KObj(oldSVM))
   114  	svmc.enqueue(newSVM)
   115  }
   116  
   117  func (svmc *SVMController) enqueue(svm *svmv1alpha1.StorageVersionMigration) {
   118  	key, err := controller.KeyFunc(svm)
   119  	if err != nil {
   120  		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %w", svm, err))
   121  		return
   122  	}
   123  
   124  	svmc.queue.Add(key)
   125  }
   126  
   127  func (svmc *SVMController) Run(ctx context.Context) {
   128  	defer utilruntime.HandleCrash()
   129  	defer svmc.queue.ShutDown()
   130  
   131  	logger := klog.FromContext(ctx)
   132  	logger.Info("Starting", "controller", svmc.controllerName)
   133  	defer logger.Info("Shutting down", "controller", svmc.controllerName)
   134  
   135  	if !cache.WaitForNamedCacheSync(svmc.controllerName, ctx.Done(), svmc.svmSynced) {
   136  		return
   137  	}
   138  
   139  	for i := 0; i < workers; i++ {
   140  		go wait.UntilWithContext(ctx, svmc.worker, time.Second)
   141  	}
   142  
   143  	<-ctx.Done()
   144  }
   145  
   146  func (svmc *SVMController) worker(ctx context.Context) {
   147  	for svmc.processNext(ctx) {
   148  	}
   149  }
   150  
   151  func (svmc *SVMController) processNext(ctx context.Context) bool {
   152  	key, quit := svmc.queue.Get()
   153  	if quit {
   154  		return false
   155  	}
   156  	defer svmc.queue.Done(key)
   157  
   158  	err := svmc.sync(ctx, key)
   159  	if err == nil {
   160  		svmc.queue.Forget(key)
   161  		return true
   162  	}
   163  
   164  	klog.FromContext(ctx).V(2).Info("Error syncing SVM resource, retrying", "svm", key, "err", err)
   165  	svmc.queue.AddRateLimited(key)
   166  
   167  	return true
   168  }
   169  
   170  func (svmc *SVMController) sync(ctx context.Context, key string) error {
   171  	logger := klog.FromContext(ctx)
   172  	startTime := time.Now()
   173  
   174  	if svmc.dependencyGraphBuilder == nil {
   175  		logger.V(4).Info("dependency graph builder is not set. we will skip migration")
   176  		return nil
   177  	}
   178  
   179  	// SVM is a cluster scoped resource so we don't care about the namespace
   180  	_, name, err := cache.SplitMetaNamespaceKey(key)
   181  	if err != nil {
   182  		return err
   183  	}
   184  
   185  	svm, err := svmc.svmListers.Get(name)
   186  	if apierrors.IsNotFound(err) {
   187  		// no work to do, don't fail and requeue
   188  		return nil
   189  	}
   190  	if err != nil {
   191  		return err
   192  	}
   193  	// working with a copy to avoid race condition between this and resource version controller
   194  	toBeProcessedSVM := svm.DeepCopy()
   195  
   196  	if IsConditionTrue(toBeProcessedSVM, svmv1alpha1.MigrationSucceeded) || IsConditionTrue(toBeProcessedSVM, svmv1alpha1.MigrationFailed) {
   197  		logger.V(4).Info("Migration has already succeeded or failed previously, skipping", "svm", name)
   198  		return nil
   199  	}
   200  
   201  	if len(toBeProcessedSVM.Status.ResourceVersion) == 0 {
   202  		logger.V(4).Info("The latest resource version is empty. We will attempt to migrate once the resource version is available.")
   203  		return nil
   204  	}
   205  	gvr := getGVRFromResource(toBeProcessedSVM)
   206  
   207  	resourceMonitor, err := svmc.dependencyGraphBuilder.GetMonitor(ctx, gvr)
   208  	if resourceMonitor != nil {
   209  		if err != nil {
   210  			// non nil monitor indicates that error is due to resource not being synced
   211  			return fmt.Errorf("dependency graph is not synced, requeuing to attempt again")
   212  		}
   213  	} else {
   214  		// we can't migrate a resource that doesn't exist in the GC
   215  		_, err = svmc.kubeClient.StoragemigrationV1alpha1().
   216  			StorageVersionMigrations().
   217  			UpdateStatus(
   218  				ctx,
   219  				setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationFailed, migrationFailedStatusReason),
   220  				metav1.UpdateOptions{},
   221  			)
   222  		if err != nil {
   223  			return err
   224  		}
   225  		logger.V(4).Error(fmt.Errorf("error migrating the resource"), "resource does not exist in GC", "gvr", gvr.String())
   226  
   227  		return nil
   228  	}
   229  
   230  	gcListResourceVersion, err := convertResourceVersionToInt(resourceMonitor.Controller.LastSyncResourceVersion())
   231  	if err != nil {
   232  		return err
   233  	}
   234  	listResourceVersion, err := convertResourceVersionToInt(toBeProcessedSVM.Status.ResourceVersion)
   235  	if err != nil {
   236  		return err
   237  	}
   238  
   239  	if gcListResourceVersion < listResourceVersion {
   240  		return fmt.Errorf("GC cache is not up to date, requeuing to attempt again. gcListResourceVersion: %d, listResourceVersion: %d", gcListResourceVersion, listResourceVersion)
   241  	}
   242  
   243  	toBeProcessedSVM, err = svmc.kubeClient.StoragemigrationV1alpha1().
   244  		StorageVersionMigrations().
   245  		UpdateStatus(
   246  			ctx,
   247  			setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationRunning, migrationRunningStatusReason),
   248  			metav1.UpdateOptions{},
   249  		)
   250  	if err != nil {
   251  		return err
   252  	}
   253  
   254  	gvk, err := svmc.restMapper.KindFor(gvr)
   255  	if err != nil {
   256  		return err
   257  	}
   258  	typeMeta := metav1.TypeMeta{}
   259  	typeMeta.APIVersion, typeMeta.Kind = gvk.ToAPIVersionAndKind()
   260  	data, err := json.Marshal(typeMeta)
   261  	if err != nil {
   262  		return err
   263  	}
   264  
   265  	// ToDo: implement a mechanism to resume migration from the last migrated resource in case of a failure
   266  	// process storage migration
   267  	for _, gvrKey := range resourceMonitor.Store.ListKeys() {
   268  		namespace, name, err := cache.SplitMetaNamespaceKey(gvrKey)
   269  		if err != nil {
   270  			return err
   271  		}
   272  
   273  		_, err = svmc.dynamicClient.Resource(gvr).
   274  			Namespace(namespace).
   275  			Patch(ctx,
   276  				name,
   277  				types.ApplyPatchType,
   278  				data,
   279  				metav1.PatchOptions{
   280  					FieldManager: svmc.controllerName,
   281  				},
   282  			)
   283  		if err != nil {
   284  			// in case of NotFound or Conflict, we can stop processing migration for that resource
   285  			if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
   286  				continue
   287  			}
   288  
   289  			_, err = svmc.kubeClient.StoragemigrationV1alpha1().
   290  				StorageVersionMigrations().
   291  				UpdateStatus(
   292  					ctx,
   293  					setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationFailed, migrationFailedStatusReason),
   294  					metav1.UpdateOptions{},
   295  				)
   296  			if err != nil {
   297  				return err
   298  			}
   299  			logger.V(4).Error(err, "Failed to migrate the resource", "name", gvrKey, "gvr", gvr.String(), "reason", apierrors.ReasonForError(err))
   300  
   301  			return nil
   302  			// Todo: add retry for scenarios where API server returns rate limiting error
   303  		}
   304  		logger.V(4).Info("Successfully migrated the resource", "name", gvrKey, "gvr", gvr.String())
   305  	}
   306  
   307  	_, err = svmc.kubeClient.StoragemigrationV1alpha1().
   308  		StorageVersionMigrations().
   309  		UpdateStatus(
   310  			ctx,
   311  			setStatusConditions(toBeProcessedSVM, svmv1alpha1.MigrationSucceeded, migrationSuccessStatusReason),
   312  			metav1.UpdateOptions{},
   313  		)
   314  	if err != nil {
   315  		return err
   316  	}
   317  
   318  	logger.V(4).Info("Finished syncing svm resource", "key", key, "gvr", gvr.String(), "elapsed", time.Since(startTime))
   319  	return nil
   320  }