sigs.k8s.io/cluster-api-provider-azure@v1.17.0/controllers/azureasomanagedcontrolplane_controller.go (about)

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controllers
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  
    24  	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
    25  	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
    26  	corev1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    29  	infrav1alpha "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha1"
    30  	"sigs.k8s.io/cluster-api-provider-azure/pkg/mutators"
    31  	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
    32  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    33  	"sigs.k8s.io/cluster-api/controllers/external"
    34  	"sigs.k8s.io/cluster-api/util"
    35  	"sigs.k8s.io/cluster-api/util/annotations"
    36  	"sigs.k8s.io/cluster-api/util/patch"
    37  	"sigs.k8s.io/cluster-api/util/predicates"
    38  	"sigs.k8s.io/cluster-api/util/secret"
    39  	ctrl "sigs.k8s.io/controller-runtime"
    40  	"sigs.k8s.io/controller-runtime/pkg/builder"
    41  	"sigs.k8s.io/controller-runtime/pkg/client"
    42  	"sigs.k8s.io/controller-runtime/pkg/controller"
    43  	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
    44  	"sigs.k8s.io/controller-runtime/pkg/handler"
    45  	"sigs.k8s.io/controller-runtime/pkg/reconcile"
    46  )
    47  
    48  var errInvalidClusterKind = errors.New("AzureASOManagedControlPlane cannot be used without AzureASOManagedCluster")
    49  
    50  // AzureASOManagedControlPlaneReconciler reconciles a AzureASOManagedControlPlane object.
    51  type AzureASOManagedControlPlaneReconciler struct {
    52  	client.Client
    53  	WatchFilterValue string
    54  
    55  	newResourceReconciler func(*infrav1alpha.AzureASOManagedControlPlane, []*unstructured.Unstructured) resourceReconciler
    56  }
    57  
    58  // SetupWithManager sets up the controller with the Manager.
    59  func (r *AzureASOManagedControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
    60  	_, log, done := tele.StartSpanWithLogger(ctx,
    61  		"controllers.AzureASOManagedControlPlaneReconciler.SetupWithManager",
    62  		tele.KVP("controller", infrav1alpha.AzureASOManagedControlPlaneKind),
    63  	)
    64  	defer done()
    65  
    66  	c, err := ctrl.NewControllerManagedBy(mgr).
    67  		WithOptions(options).
    68  		For(&infrav1alpha.AzureASOManagedControlPlane{}).
    69  		WithEventFilter(predicates.ResourceHasFilterLabel(log, r.WatchFilterValue)).
    70  		Watches(&clusterv1.Cluster{},
    71  			handler.EnqueueRequestsFromMapFunc(clusterToAzureASOManagedControlPlane),
    72  			builder.WithPredicates(
    73  				predicates.ResourceHasFilterLabel(log, r.WatchFilterValue),
    74  				ClusterPauseChangeAndInfrastructureReady(log),
    75  			),
    76  		).
    77  		// User errors that CAPZ passes through agentPoolProfiles on create must be fixed in the
    78  		// AzureASOManagedMachinePool, so trigger a reconciliation to consume those fixes.
    79  		Watches(
    80  			&infrav1alpha.AzureASOManagedMachinePool{},
    81  			handler.EnqueueRequestsFromMapFunc(r.azureASOManagedMachinePoolToAzureASOManagedControlPlane),
    82  		).
    83  		Owns(&corev1.Secret{}).
    84  		Build(r)
    85  	if err != nil {
    86  		return err
    87  	}
    88  
    89  	externalTracker := &external.ObjectTracker{
    90  		Cache:      mgr.GetCache(),
    91  		Controller: c,
    92  	}
    93  
    94  	r.newResourceReconciler = func(asoManagedCluster *infrav1alpha.AzureASOManagedControlPlane, resources []*unstructured.Unstructured) resourceReconciler {
    95  		return &ResourceReconciler{
    96  			Client:    r.Client,
    97  			resources: resources,
    98  			owner:     asoManagedCluster,
    99  			watcher:   externalTracker,
   100  		}
   101  	}
   102  
   103  	return nil
   104  }
   105  
   106  func clusterToAzureASOManagedControlPlane(_ context.Context, o client.Object) []ctrl.Request {
   107  	controlPlaneRef := o.(*clusterv1.Cluster).Spec.ControlPlaneRef
   108  	if controlPlaneRef != nil &&
   109  		controlPlaneRef.APIVersion == infrav1alpha.GroupVersion.Identifier() &&
   110  		controlPlaneRef.Kind == infrav1alpha.AzureASOManagedControlPlaneKind {
   111  		return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: controlPlaneRef.Namespace, Name: controlPlaneRef.Name}}}
   112  	}
   113  	return nil
   114  }
   115  
   116  func (r *AzureASOManagedControlPlaneReconciler) azureASOManagedMachinePoolToAzureASOManagedControlPlane(ctx context.Context, o client.Object) []ctrl.Request {
   117  	asoManagedMachinePool := o.(*infrav1alpha.AzureASOManagedMachinePool)
   118  	clusterName := asoManagedMachinePool.Labels[clusterv1.ClusterNameLabel]
   119  	if clusterName == "" {
   120  		return nil
   121  	}
   122  	cluster, err := util.GetClusterByName(ctx, r.Client, asoManagedMachinePool.Namespace, clusterName)
   123  	if client.IgnoreNotFound(err) != nil || cluster == nil {
   124  		return nil
   125  	}
   126  	return clusterToAzureASOManagedControlPlane(ctx, cluster)
   127  }
   128  
   129  //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes,verbs=get;list;watch;create;update;patch;delete
   130  //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes/status,verbs=get;update;patch
   131  //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes/finalizers,verbs=update
   132  
   133  // Reconcile reconciles an AzureASOManagedControlPlane.
   134  func (r *AzureASOManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, resultErr error) {
   135  	ctx, _, done := tele.StartSpanWithLogger(ctx,
   136  		"controllers.AzureASOManagedControlPlaneReconciler.Reconcile",
   137  		tele.KVP("namespace", req.Namespace),
   138  		tele.KVP("name", req.Name),
   139  		tele.KVP("kind", infrav1alpha.AzureASOManagedControlPlaneKind),
   140  	)
   141  	defer done()
   142  
   143  	asoManagedControlPlane := &infrav1alpha.AzureASOManagedControlPlane{}
   144  	err := r.Get(ctx, req.NamespacedName, asoManagedControlPlane)
   145  	if err != nil {
   146  		return ctrl.Result{}, client.IgnoreNotFound(err)
   147  	}
   148  
   149  	patchHelper, err := patch.NewHelper(asoManagedControlPlane, r.Client)
   150  	if err != nil {
   151  		return ctrl.Result{}, fmt.Errorf("failed to create patch helper: %w", err)
   152  	}
   153  	defer func() {
   154  		err := patchHelper.Patch(ctx, asoManagedControlPlane)
   155  		if err != nil && resultErr == nil {
   156  			resultErr = err
   157  			result = ctrl.Result{}
   158  		}
   159  	}()
   160  
   161  	asoManagedControlPlane.Status.Ready = false
   162  	asoManagedControlPlane.Status.Initialized = false
   163  
   164  	cluster, err := util.GetOwnerCluster(ctx, r.Client, asoManagedControlPlane.ObjectMeta)
   165  	if err != nil {
   166  		return ctrl.Result{}, err
   167  	}
   168  
   169  	if cluster != nil && cluster.Spec.Paused ||
   170  		annotations.HasPaused(asoManagedControlPlane) {
   171  		return r.reconcilePaused(ctx, asoManagedControlPlane)
   172  	}
   173  
   174  	if !asoManagedControlPlane.GetDeletionTimestamp().IsZero() {
   175  		return r.reconcileDelete(ctx, asoManagedControlPlane)
   176  	}
   177  
   178  	return r.reconcileNormal(ctx, asoManagedControlPlane, cluster)
   179  }
   180  
   181  func (r *AzureASOManagedControlPlaneReconciler) reconcileNormal(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane, cluster *clusterv1.Cluster) (ctrl.Result, error) {
   182  	ctx, log, done := tele.StartSpanWithLogger(ctx,
   183  		"controllers.AzureASOManagedControlPlaneReconciler.reconcileNormal",
   184  	)
   185  	defer done()
   186  	log.V(4).Info("reconciling normally")
   187  
   188  	if cluster == nil {
   189  		log.V(4).Info("Cluster Controller has not yet set OwnerRef")
   190  		return ctrl.Result{}, nil
   191  	}
   192  	if cluster.Spec.InfrastructureRef == nil ||
   193  		cluster.Spec.InfrastructureRef.APIVersion != infrav1alpha.GroupVersion.Identifier() ||
   194  		cluster.Spec.InfrastructureRef.Kind != infrav1alpha.AzureASOManagedClusterKind {
   195  		return ctrl.Result{}, reconcile.TerminalError(errInvalidClusterKind)
   196  	}
   197  
   198  	needsPatch := controllerutil.AddFinalizer(asoManagedControlPlane, infrav1alpha.AzureASOManagedControlPlaneFinalizer)
   199  	needsPatch = AddBlockMoveAnnotation(asoManagedControlPlane) || needsPatch
   200  	if needsPatch {
   201  		return ctrl.Result{Requeue: true}, nil
   202  	}
   203  
   204  	resources, err := mutators.ApplyMutators(ctx, asoManagedControlPlane.Spec.Resources, mutators.SetManagedClusterDefaults(r.Client, asoManagedControlPlane, cluster))
   205  	if err != nil {
   206  		return ctrl.Result{}, err
   207  	}
   208  
   209  	var managedClusterName string
   210  	for _, resource := range resources {
   211  		if resource.GroupVersionKind().Group == asocontainerservicev1.GroupVersion.Group &&
   212  			resource.GroupVersionKind().Kind == "ManagedCluster" {
   213  			managedClusterName = resource.GetName()
   214  			break
   215  		}
   216  	}
   217  	if managedClusterName == "" {
   218  		return ctrl.Result{}, reconcile.TerminalError(mutators.ErrNoManagedClusterDefined)
   219  	}
   220  
   221  	resourceReconciler := r.newResourceReconciler(asoManagedControlPlane, resources)
   222  	err = resourceReconciler.Reconcile(ctx)
   223  	if err != nil {
   224  		return ctrl.Result{}, fmt.Errorf("failed to reconcile resources: %w", err)
   225  	}
   226  	for _, status := range asoManagedControlPlane.Status.Resources {
   227  		if !status.Ready {
   228  			return ctrl.Result{}, nil
   229  		}
   230  	}
   231  
   232  	managedCluster := &asocontainerservicev1.ManagedCluster{}
   233  	err = r.Get(ctx, client.ObjectKey{Namespace: asoManagedControlPlane.Namespace, Name: managedClusterName}, managedCluster)
   234  	if err != nil {
   235  		return ctrl.Result{}, fmt.Errorf("error getting ManagedCluster: %w", err)
   236  	}
   237  
   238  	asoManagedControlPlane.Status.ControlPlaneEndpoint = getControlPlaneEndpoint(managedCluster)
   239  	if managedCluster.Status.CurrentKubernetesVersion != nil {
   240  		asoManagedControlPlane.Status.Version = "v" + *managedCluster.Status.CurrentKubernetesVersion
   241  	}
   242  
   243  	err = r.reconcileKubeconfig(ctx, asoManagedControlPlane, cluster, managedCluster)
   244  	if err != nil {
   245  		return ctrl.Result{}, fmt.Errorf("failed to reconcile kubeconfig: %w", err)
   246  	}
   247  
   248  	asoManagedControlPlane.Status.Ready = !asoManagedControlPlane.Status.ControlPlaneEndpoint.IsZero()
   249  	// The AKS API doesn't allow us to distinguish between CAPI's definitions of "initialized" and "ready" so
   250  	// we treat them equivalently.
   251  	asoManagedControlPlane.Status.Initialized = asoManagedControlPlane.Status.Ready
   252  
   253  	return ctrl.Result{}, nil
   254  }
   255  
   256  func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane, cluster *clusterv1.Cluster, managedCluster *asocontainerservicev1.ManagedCluster) error {
   257  	ctx, _, done := tele.StartSpanWithLogger(ctx,
   258  		"controllers.AzureASOManagedControlPlaneReconciler.reconcileKubeconfig",
   259  	)
   260  	defer done()
   261  
   262  	var secretRef *genruntime.SecretDestination
   263  	if managedCluster.Spec.OperatorSpec != nil &&
   264  		managedCluster.Spec.OperatorSpec.Secrets != nil {
   265  		secretRef = managedCluster.Spec.OperatorSpec.Secrets.UserCredentials
   266  		if managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials != nil {
   267  			secretRef = managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials
   268  		}
   269  	}
   270  	if secretRef == nil {
   271  		return reconcile.TerminalError(fmt.Errorf("ManagedCluster must define at least one of spec.operatorSpec.secrets.{userCredentials,adminCredentials}"))
   272  	}
   273  	asoKubeconfig := &corev1.Secret{}
   274  	err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretRef.Name}, asoKubeconfig)
   275  	if err != nil {
   276  		return fmt.Errorf("failed to fetch secret created by ASO: %w", err)
   277  	}
   278  
   279  	expectedSecret := &corev1.Secret{
   280  		TypeMeta: metav1.TypeMeta{
   281  			APIVersion: corev1.SchemeGroupVersion.Identifier(),
   282  			Kind:       "Secret",
   283  		},
   284  		ObjectMeta: metav1.ObjectMeta{
   285  			Name:      secret.Name(cluster.Name, secret.Kubeconfig),
   286  			Namespace: cluster.Namespace,
   287  			OwnerReferences: []metav1.OwnerReference{
   288  				*metav1.NewControllerRef(asoManagedControlPlane, infrav1alpha.GroupVersion.WithKind(infrav1alpha.AzureASOManagedControlPlaneKind)),
   289  			},
   290  			Labels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name},
   291  		},
   292  		Data: map[string][]byte{
   293  			secret.KubeconfigDataName: asoKubeconfig.Data[secretRef.Key],
   294  		},
   295  	}
   296  
   297  	return r.Patch(ctx, expectedSecret, client.Apply, client.FieldOwner("capz-manager"), client.ForceOwnership)
   298  }
   299  
   300  //nolint:unparam // an empty ctrl.Result is always returned here, leaving it as-is to avoid churn in refactoring later if that changes.
   301  func (r *AzureASOManagedControlPlaneReconciler) reconcilePaused(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane) (ctrl.Result, error) {
   302  	ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureASOManagedControlPlaneReconciler.reconcilePaused")
   303  	defer done()
   304  	log.V(4).Info("reconciling pause")
   305  
   306  	resources, err := mutators.ToUnstructured(ctx, asoManagedControlPlane.Spec.Resources)
   307  	if err != nil {
   308  		return ctrl.Result{}, err
   309  	}
   310  	resourceReconciler := r.newResourceReconciler(asoManagedControlPlane, resources)
   311  	err = resourceReconciler.Pause(ctx)
   312  	if err != nil {
   313  		return ctrl.Result{}, fmt.Errorf("failed to pause resources: %w", err)
   314  	}
   315  
   316  	RemoveBlockMoveAnnotation(asoManagedControlPlane)
   317  
   318  	return ctrl.Result{}, nil
   319  }
   320  
   321  //nolint:unparam // an empty ctrl.Result is always returned here, leaving it as-is to avoid churn in refactoring later if that changes.
   322  func (r *AzureASOManagedControlPlaneReconciler) reconcileDelete(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane) (ctrl.Result, error) {
   323  	ctx, log, done := tele.StartSpanWithLogger(ctx,
   324  		"controllers.AzureASOManagedControlPlaneReconciler.reconcileDelete",
   325  	)
   326  	defer done()
   327  	log.V(4).Info("reconciling delete")
   328  
   329  	resources, err := mutators.ToUnstructured(ctx, asoManagedControlPlane.Spec.Resources)
   330  	if err != nil {
   331  		return ctrl.Result{}, err
   332  	}
   333  	resourceReconciler := r.newResourceReconciler(asoManagedControlPlane, resources)
   334  	err = resourceReconciler.Delete(ctx)
   335  	if err != nil {
   336  		return ctrl.Result{}, fmt.Errorf("failed to reconcile resources: %w", err)
   337  	}
   338  	if len(asoManagedControlPlane.Status.Resources) > 0 {
   339  		return ctrl.Result{}, nil
   340  	}
   341  
   342  	controllerutil.RemoveFinalizer(asoManagedControlPlane, infrav1alpha.AzureASOManagedControlPlaneFinalizer)
   343  	return ctrl.Result{}, nil
   344  }
   345  
   346  func getControlPlaneEndpoint(managedCluster *asocontainerservicev1.ManagedCluster) clusterv1.APIEndpoint {
   347  	if managedCluster.Status.PrivateFQDN != nil {
   348  		return clusterv1.APIEndpoint{
   349  			Host: *managedCluster.Status.PrivateFQDN,
   350  			Port: 443,
   351  		}
   352  	}
   353  	if managedCluster.Status.Fqdn != nil {
   354  		return clusterv1.APIEndpoint{
   355  			Host: *managedCluster.Status.Fqdn,
   356  			Port: 443,
   357  		}
   358  	}
   359  	return clusterv1.APIEndpoint{}
   360  }