sigs.k8s.io/cluster-api-provider-azure@v1.17.0/controllers/azureasomanagedcontrolplane_controller.go (about) 1 /* 2 Copyright 2024 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package controllers 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 24 asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" 25 "github.com/Azure/azure-service-operator/v2/pkg/genruntime" 26 corev1 "k8s.io/api/core/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 29 infrav1alpha "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha1" 30 "sigs.k8s.io/cluster-api-provider-azure/pkg/mutators" 31 "sigs.k8s.io/cluster-api-provider-azure/util/tele" 32 clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 33 "sigs.k8s.io/cluster-api/controllers/external" 34 "sigs.k8s.io/cluster-api/util" 35 "sigs.k8s.io/cluster-api/util/annotations" 36 "sigs.k8s.io/cluster-api/util/patch" 37 "sigs.k8s.io/cluster-api/util/predicates" 38 "sigs.k8s.io/cluster-api/util/secret" 39 ctrl "sigs.k8s.io/controller-runtime" 40 "sigs.k8s.io/controller-runtime/pkg/builder" 41 "sigs.k8s.io/controller-runtime/pkg/client" 42 "sigs.k8s.io/controller-runtime/pkg/controller" 43 "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 44 "sigs.k8s.io/controller-runtime/pkg/handler" 45 "sigs.k8s.io/controller-runtime/pkg/reconcile" 46 ) 47 48 var errInvalidClusterKind = errors.New("AzureASOManagedControlPlane cannot be used without AzureASOManagedCluster") 49 50 // AzureASOManagedControlPlaneReconciler reconciles a AzureASOManagedControlPlane object. 51 type AzureASOManagedControlPlaneReconciler struct { 52 client.Client 53 WatchFilterValue string 54 55 newResourceReconciler func(*infrav1alpha.AzureASOManagedControlPlane, []*unstructured.Unstructured) resourceReconciler 56 } 57 58 // SetupWithManager sets up the controller with the Manager. 59 func (r *AzureASOManagedControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { 60 _, log, done := tele.StartSpanWithLogger(ctx, 61 "controllers.AzureASOManagedControlPlaneReconciler.SetupWithManager", 62 tele.KVP("controller", infrav1alpha.AzureASOManagedControlPlaneKind), 63 ) 64 defer done() 65 66 c, err := ctrl.NewControllerManagedBy(mgr). 67 WithOptions(options). 68 For(&infrav1alpha.AzureASOManagedControlPlane{}). 69 WithEventFilter(predicates.ResourceHasFilterLabel(log, r.WatchFilterValue)). 70 Watches(&clusterv1.Cluster{}, 71 handler.EnqueueRequestsFromMapFunc(clusterToAzureASOManagedControlPlane), 72 builder.WithPredicates( 73 predicates.ResourceHasFilterLabel(log, r.WatchFilterValue), 74 ClusterPauseChangeAndInfrastructureReady(log), 75 ), 76 ). 77 // User errors that CAPZ passes through agentPoolProfiles on create must be fixed in the 78 // AzureASOManagedMachinePool, so trigger a reconciliation to consume those fixes. 79 Watches( 80 &infrav1alpha.AzureASOManagedMachinePool{}, 81 handler.EnqueueRequestsFromMapFunc(r.azureASOManagedMachinePoolToAzureASOManagedControlPlane), 82 ). 83 Owns(&corev1.Secret{}). 84 Build(r) 85 if err != nil { 86 return err 87 } 88 89 externalTracker := &external.ObjectTracker{ 90 Cache: mgr.GetCache(), 91 Controller: c, 92 } 93 94 r.newResourceReconciler = func(asoManagedCluster *infrav1alpha.AzureASOManagedControlPlane, resources []*unstructured.Unstructured) resourceReconciler { 95 return &ResourceReconciler{ 96 Client: r.Client, 97 resources: resources, 98 owner: asoManagedCluster, 99 watcher: externalTracker, 100 } 101 } 102 103 return nil 104 } 105 106 func clusterToAzureASOManagedControlPlane(_ context.Context, o client.Object) []ctrl.Request { 107 controlPlaneRef := o.(*clusterv1.Cluster).Spec.ControlPlaneRef 108 if controlPlaneRef != nil && 109 controlPlaneRef.APIVersion == infrav1alpha.GroupVersion.Identifier() && 110 controlPlaneRef.Kind == infrav1alpha.AzureASOManagedControlPlaneKind { 111 return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: controlPlaneRef.Namespace, Name: controlPlaneRef.Name}}} 112 } 113 return nil 114 } 115 116 func (r *AzureASOManagedControlPlaneReconciler) azureASOManagedMachinePoolToAzureASOManagedControlPlane(ctx context.Context, o client.Object) []ctrl.Request { 117 asoManagedMachinePool := o.(*infrav1alpha.AzureASOManagedMachinePool) 118 clusterName := asoManagedMachinePool.Labels[clusterv1.ClusterNameLabel] 119 if clusterName == "" { 120 return nil 121 } 122 cluster, err := util.GetClusterByName(ctx, r.Client, asoManagedMachinePool.Namespace, clusterName) 123 if client.IgnoreNotFound(err) != nil || cluster == nil { 124 return nil 125 } 126 return clusterToAzureASOManagedControlPlane(ctx, cluster) 127 } 128 129 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes,verbs=get;list;watch;create;update;patch;delete 130 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes/status,verbs=get;update;patch 131 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes/finalizers,verbs=update 132 133 // Reconcile reconciles an AzureASOManagedControlPlane. 134 func (r *AzureASOManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, resultErr error) { 135 ctx, _, done := tele.StartSpanWithLogger(ctx, 136 "controllers.AzureASOManagedControlPlaneReconciler.Reconcile", 137 tele.KVP("namespace", req.Namespace), 138 tele.KVP("name", req.Name), 139 tele.KVP("kind", infrav1alpha.AzureASOManagedControlPlaneKind), 140 ) 141 defer done() 142 143 asoManagedControlPlane := &infrav1alpha.AzureASOManagedControlPlane{} 144 err := r.Get(ctx, req.NamespacedName, asoManagedControlPlane) 145 if err != nil { 146 return ctrl.Result{}, client.IgnoreNotFound(err) 147 } 148 149 patchHelper, err := patch.NewHelper(asoManagedControlPlane, r.Client) 150 if err != nil { 151 return ctrl.Result{}, fmt.Errorf("failed to create patch helper: %w", err) 152 } 153 defer func() { 154 err := patchHelper.Patch(ctx, asoManagedControlPlane) 155 if err != nil && resultErr == nil { 156 resultErr = err 157 result = ctrl.Result{} 158 } 159 }() 160 161 asoManagedControlPlane.Status.Ready = false 162 asoManagedControlPlane.Status.Initialized = false 163 164 cluster, err := util.GetOwnerCluster(ctx, r.Client, asoManagedControlPlane.ObjectMeta) 165 if err != nil { 166 return ctrl.Result{}, err 167 } 168 169 if cluster != nil && cluster.Spec.Paused || 170 annotations.HasPaused(asoManagedControlPlane) { 171 return r.reconcilePaused(ctx, asoManagedControlPlane) 172 } 173 174 if !asoManagedControlPlane.GetDeletionTimestamp().IsZero() { 175 return r.reconcileDelete(ctx, asoManagedControlPlane) 176 } 177 178 return r.reconcileNormal(ctx, asoManagedControlPlane, cluster) 179 } 180 181 func (r *AzureASOManagedControlPlaneReconciler) reconcileNormal(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane, cluster *clusterv1.Cluster) (ctrl.Result, error) { 182 ctx, log, done := tele.StartSpanWithLogger(ctx, 183 "controllers.AzureASOManagedControlPlaneReconciler.reconcileNormal", 184 ) 185 defer done() 186 log.V(4).Info("reconciling normally") 187 188 if cluster == nil { 189 log.V(4).Info("Cluster Controller has not yet set OwnerRef") 190 return ctrl.Result{}, nil 191 } 192 if cluster.Spec.InfrastructureRef == nil || 193 cluster.Spec.InfrastructureRef.APIVersion != infrav1alpha.GroupVersion.Identifier() || 194 cluster.Spec.InfrastructureRef.Kind != infrav1alpha.AzureASOManagedClusterKind { 195 return ctrl.Result{}, reconcile.TerminalError(errInvalidClusterKind) 196 } 197 198 needsPatch := controllerutil.AddFinalizer(asoManagedControlPlane, infrav1alpha.AzureASOManagedControlPlaneFinalizer) 199 needsPatch = AddBlockMoveAnnotation(asoManagedControlPlane) || needsPatch 200 if needsPatch { 201 return ctrl.Result{Requeue: true}, nil 202 } 203 204 resources, err := mutators.ApplyMutators(ctx, asoManagedControlPlane.Spec.Resources, mutators.SetManagedClusterDefaults(r.Client, asoManagedControlPlane, cluster)) 205 if err != nil { 206 return ctrl.Result{}, err 207 } 208 209 var managedClusterName string 210 for _, resource := range resources { 211 if resource.GroupVersionKind().Group == asocontainerservicev1.GroupVersion.Group && 212 resource.GroupVersionKind().Kind == "ManagedCluster" { 213 managedClusterName = resource.GetName() 214 break 215 } 216 } 217 if managedClusterName == "" { 218 return ctrl.Result{}, reconcile.TerminalError(mutators.ErrNoManagedClusterDefined) 219 } 220 221 resourceReconciler := r.newResourceReconciler(asoManagedControlPlane, resources) 222 err = resourceReconciler.Reconcile(ctx) 223 if err != nil { 224 return ctrl.Result{}, fmt.Errorf("failed to reconcile resources: %w", err) 225 } 226 for _, status := range asoManagedControlPlane.Status.Resources { 227 if !status.Ready { 228 return ctrl.Result{}, nil 229 } 230 } 231 232 managedCluster := &asocontainerservicev1.ManagedCluster{} 233 err = r.Get(ctx, client.ObjectKey{Namespace: asoManagedControlPlane.Namespace, Name: managedClusterName}, managedCluster) 234 if err != nil { 235 return ctrl.Result{}, fmt.Errorf("error getting ManagedCluster: %w", err) 236 } 237 238 asoManagedControlPlane.Status.ControlPlaneEndpoint = getControlPlaneEndpoint(managedCluster) 239 if managedCluster.Status.CurrentKubernetesVersion != nil { 240 asoManagedControlPlane.Status.Version = "v" + *managedCluster.Status.CurrentKubernetesVersion 241 } 242 243 err = r.reconcileKubeconfig(ctx, asoManagedControlPlane, cluster, managedCluster) 244 if err != nil { 245 return ctrl.Result{}, fmt.Errorf("failed to reconcile kubeconfig: %w", err) 246 } 247 248 asoManagedControlPlane.Status.Ready = !asoManagedControlPlane.Status.ControlPlaneEndpoint.IsZero() 249 // The AKS API doesn't allow us to distinguish between CAPI's definitions of "initialized" and "ready" so 250 // we treat them equivalently. 251 asoManagedControlPlane.Status.Initialized = asoManagedControlPlane.Status.Ready 252 253 return ctrl.Result{}, nil 254 } 255 256 func (r *AzureASOManagedControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane, cluster *clusterv1.Cluster, managedCluster *asocontainerservicev1.ManagedCluster) error { 257 ctx, _, done := tele.StartSpanWithLogger(ctx, 258 "controllers.AzureASOManagedControlPlaneReconciler.reconcileKubeconfig", 259 ) 260 defer done() 261 262 var secretRef *genruntime.SecretDestination 263 if managedCluster.Spec.OperatorSpec != nil && 264 managedCluster.Spec.OperatorSpec.Secrets != nil { 265 secretRef = managedCluster.Spec.OperatorSpec.Secrets.UserCredentials 266 if managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials != nil { 267 secretRef = managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials 268 } 269 } 270 if secretRef == nil { 271 return reconcile.TerminalError(fmt.Errorf("ManagedCluster must define at least one of spec.operatorSpec.secrets.{userCredentials,adminCredentials}")) 272 } 273 asoKubeconfig := &corev1.Secret{} 274 err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretRef.Name}, asoKubeconfig) 275 if err != nil { 276 return fmt.Errorf("failed to fetch secret created by ASO: %w", err) 277 } 278 279 expectedSecret := &corev1.Secret{ 280 TypeMeta: metav1.TypeMeta{ 281 APIVersion: corev1.SchemeGroupVersion.Identifier(), 282 Kind: "Secret", 283 }, 284 ObjectMeta: metav1.ObjectMeta{ 285 Name: secret.Name(cluster.Name, secret.Kubeconfig), 286 Namespace: cluster.Namespace, 287 OwnerReferences: []metav1.OwnerReference{ 288 *metav1.NewControllerRef(asoManagedControlPlane, infrav1alpha.GroupVersion.WithKind(infrav1alpha.AzureASOManagedControlPlaneKind)), 289 }, 290 Labels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name}, 291 }, 292 Data: map[string][]byte{ 293 secret.KubeconfigDataName: asoKubeconfig.Data[secretRef.Key], 294 }, 295 } 296 297 return r.Patch(ctx, expectedSecret, client.Apply, client.FieldOwner("capz-manager"), client.ForceOwnership) 298 } 299 300 //nolint:unparam // an empty ctrl.Result is always returned here, leaving it as-is to avoid churn in refactoring later if that changes. 301 func (r *AzureASOManagedControlPlaneReconciler) reconcilePaused(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane) (ctrl.Result, error) { 302 ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureASOManagedControlPlaneReconciler.reconcilePaused") 303 defer done() 304 log.V(4).Info("reconciling pause") 305 306 resources, err := mutators.ToUnstructured(ctx, asoManagedControlPlane.Spec.Resources) 307 if err != nil { 308 return ctrl.Result{}, err 309 } 310 resourceReconciler := r.newResourceReconciler(asoManagedControlPlane, resources) 311 err = resourceReconciler.Pause(ctx) 312 if err != nil { 313 return ctrl.Result{}, fmt.Errorf("failed to pause resources: %w", err) 314 } 315 316 RemoveBlockMoveAnnotation(asoManagedControlPlane) 317 318 return ctrl.Result{}, nil 319 } 320 321 //nolint:unparam // an empty ctrl.Result is always returned here, leaving it as-is to avoid churn in refactoring later if that changes. 322 func (r *AzureASOManagedControlPlaneReconciler) reconcileDelete(ctx context.Context, asoManagedControlPlane *infrav1alpha.AzureASOManagedControlPlane) (ctrl.Result, error) { 323 ctx, log, done := tele.StartSpanWithLogger(ctx, 324 "controllers.AzureASOManagedControlPlaneReconciler.reconcileDelete", 325 ) 326 defer done() 327 log.V(4).Info("reconciling delete") 328 329 resources, err := mutators.ToUnstructured(ctx, asoManagedControlPlane.Spec.Resources) 330 if err != nil { 331 return ctrl.Result{}, err 332 } 333 resourceReconciler := r.newResourceReconciler(asoManagedControlPlane, resources) 334 err = resourceReconciler.Delete(ctx) 335 if err != nil { 336 return ctrl.Result{}, fmt.Errorf("failed to reconcile resources: %w", err) 337 } 338 if len(asoManagedControlPlane.Status.Resources) > 0 { 339 return ctrl.Result{}, nil 340 } 341 342 controllerutil.RemoveFinalizer(asoManagedControlPlane, infrav1alpha.AzureASOManagedControlPlaneFinalizer) 343 return ctrl.Result{}, nil 344 } 345 346 func getControlPlaneEndpoint(managedCluster *asocontainerservicev1.ManagedCluster) clusterv1.APIEndpoint { 347 if managedCluster.Status.PrivateFQDN != nil { 348 return clusterv1.APIEndpoint{ 349 Host: *managedCluster.Status.PrivateFQDN, 350 Port: 443, 351 } 352 } 353 if managedCluster.Status.Fqdn != nil { 354 return clusterv1.APIEndpoint{ 355 Host: *managedCluster.Status.Fqdn, 356 Port: 443, 357 } 358 } 359 return clusterv1.APIEndpoint{} 360 }