sigs.k8s.io/cluster-api-provider-azure@v1.14.3/azure/scope/managedcontrolplane.go (about)

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package scope
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"strings"
    24  	"time"
    25  
    26  	asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
    27  	asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501"
    28  	asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101"
    29  	asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
    30  	asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
    31  	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
    32  	"github.com/pkg/errors"
    33  	"golang.org/x/mod/semver"
    34  	"gopkg.in/yaml.v3"
    35  	corev1 "k8s.io/api/core/v1"
    36  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    39  	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
    40  	"k8s.io/utils/ptr"
    41  	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
    42  	"sigs.k8s.io/cluster-api-provider-azure/azure"
    43  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions"
    44  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers"
    45  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/groups"
    46  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters"
    47  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
    48  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
    49  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualnetworks"
    50  	"sigs.k8s.io/cluster-api-provider-azure/util/futures"
    51  	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
    52  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    53  	"sigs.k8s.io/cluster-api/controllers/remote"
    54  	"sigs.k8s.io/cluster-api/util/conditions"
    55  	"sigs.k8s.io/cluster-api/util/patch"
    56  	"sigs.k8s.io/cluster-api/util/secret"
    57  	"sigs.k8s.io/controller-runtime/pkg/client"
    58  	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
    59  )
    60  
    61  const (
    62  	resourceHealthWarningInitialGracePeriod = 1 * time.Hour
    63  	// managedControlPlaneScopeName is the sourceName, or more specifically the UserAgent, of client used to store the Cluster Info configmap.
    64  	managedControlPlaneScopeName = "azuremanagedcontrolplane-scope"
    65  )
    66  
    67  // ManagedControlPlaneScopeParams defines the input parameters used to create a new managed
    68  // control plane.
    69  type ManagedControlPlaneScopeParams struct {
    70  	AzureClients
    71  	Client              client.Client
    72  	Cluster             *clusterv1.Cluster
    73  	ControlPlane        *infrav1.AzureManagedControlPlane
    74  	ManagedMachinePools []ManagedMachinePool
    75  	Cache               *ManagedControlPlaneCache
    76  	Timeouts            azure.AsyncReconciler
    77  }
    78  
    79  // NewManagedControlPlaneScope creates a new Scope from the supplied parameters.
    80  // This is meant to be called for each reconcile iteration.
    81  func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlaneScopeParams) (*ManagedControlPlaneScope, error) {
    82  	ctx, _, done := tele.StartSpanWithLogger(ctx, "scope.NewManagedControlPlaneScope")
    83  	defer done()
    84  
    85  	if params.Cluster == nil {
    86  		return nil, errors.New("failed to generate new scope from nil Cluster")
    87  	}
    88  
    89  	if params.ControlPlane == nil {
    90  		return nil, errors.New("failed to generate new scope from nil ControlPlane")
    91  	}
    92  
    93  	credentialsProvider, err := NewManagedControlPlaneCredentialsProvider(ctx, params.Client, params.ControlPlane)
    94  	if err != nil {
    95  		return nil, errors.Wrap(err, "failed to init credentials provider")
    96  	}
    97  
    98  	if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment, credentialsProvider); err != nil {
    99  		return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity")
   100  	}
   101  
   102  	if params.Cache == nil {
   103  		params.Cache = &ManagedControlPlaneCache{}
   104  	}
   105  
   106  	helper, err := patch.NewHelper(params.ControlPlane, params.Client)
   107  	if err != nil {
   108  		return nil, errors.Wrap(err, "failed to init patch helper")
   109  	}
   110  
   111  	return &ManagedControlPlaneScope{
   112  		Client:              params.Client,
   113  		AzureClients:        params.AzureClients,
   114  		Cluster:             params.Cluster,
   115  		ControlPlane:        params.ControlPlane,
   116  		ManagedMachinePools: params.ManagedMachinePools,
   117  		PatchHelper:         helper,
   118  		cache:               params.Cache,
   119  		AsyncReconciler:     params.Timeouts,
   120  	}, nil
   121  }
   122  
   123  // ManagedControlPlaneScope defines the basic context for an actuator to operate upon.
   124  type ManagedControlPlaneScope struct {
   125  	Client              client.Client
   126  	PatchHelper         *patch.Helper
   127  	adminKubeConfigData []byte
   128  	userKubeConfigData  []byte
   129  	cache               *ManagedControlPlaneCache
   130  
   131  	AzureClients
   132  	Cluster             *clusterv1.Cluster
   133  	ControlPlane        *infrav1.AzureManagedControlPlane
   134  	ManagedMachinePools []ManagedMachinePool
   135  	azure.AsyncReconciler
   136  }
   137  
   138  // ManagedControlPlaneCache stores ManagedControlPlane data locally so we don't have to hit the API multiple times within the same reconcile loop.
   139  type ManagedControlPlaneCache struct {
   140  	isVnetManaged *bool
   141  }
   142  
   143  // GetClient returns the controller-runtime client.
   144  func (s *ManagedControlPlaneScope) GetClient() client.Client {
   145  	return s.Client
   146  }
   147  
   148  // ASOOwner implements aso.Scope.
   149  func (s *ManagedControlPlaneScope) ASOOwner() client.Object {
   150  	return s.ControlPlane
   151  }
   152  
   153  // GetDeletionTimestamp returns the deletion timestamp of the cluster.
   154  func (s *ManagedControlPlaneScope) GetDeletionTimestamp() *metav1.Time {
   155  	return s.Cluster.DeletionTimestamp
   156  }
   157  
   158  // ResourceGroup returns the managed control plane's resource group.
   159  func (s *ManagedControlPlaneScope) ResourceGroup() string {
   160  	if s.ControlPlane == nil {
   161  		return ""
   162  	}
   163  	return s.ControlPlane.Spec.ResourceGroupName
   164  }
   165  
   166  // NodeResourceGroup returns the managed control plane's node resource group.
   167  func (s *ManagedControlPlaneScope) NodeResourceGroup() string {
   168  	if s.ControlPlane == nil {
   169  		return ""
   170  	}
   171  	return s.ControlPlane.Spec.NodeResourceGroupName
   172  }
   173  
   174  // ClusterName returns the managed control plane's name.
   175  func (s *ManagedControlPlaneScope) ClusterName() string {
   176  	return s.Cluster.Name
   177  }
   178  
   179  // Location returns the managed control plane's Azure location, or an empty string.
   180  func (s *ManagedControlPlaneScope) Location() string {
   181  	if s.ControlPlane == nil {
   182  		return ""
   183  	}
   184  	return s.ControlPlane.Spec.Location
   185  }
   186  
   187  // ExtendedLocation has not been implemented for AzureManagedControlPlane.
   188  func (s *ManagedControlPlaneScope) ExtendedLocation() *infrav1.ExtendedLocationSpec {
   189  	return nil
   190  }
   191  
   192  // ExtendedLocationName has not been implemented for AzureManagedControlPlane.
   193  func (s *ManagedControlPlaneScope) ExtendedLocationName() string {
   194  	return ""
   195  }
   196  
   197  // ExtendedLocationType has not been implemented for AzureManagedControlPlane.
   198  func (s *ManagedControlPlaneScope) ExtendedLocationType() string {
   199  	return ""
   200  }
   201  
   202  // AvailabilitySetEnabled is always false for a managed control plane.
   203  func (s *ManagedControlPlaneScope) AvailabilitySetEnabled() bool {
   204  	return false // not applicable for a managed control plane
   205  }
   206  
   207  // AdditionalTags returns AdditionalTags from the ControlPlane spec.
   208  func (s *ManagedControlPlaneScope) AdditionalTags() infrav1.Tags {
   209  	tags := make(infrav1.Tags)
   210  	if s.ControlPlane.Spec.AdditionalTags != nil {
   211  		tags = s.ControlPlane.Spec.AdditionalTags.DeepCopy()
   212  	}
   213  	return tags
   214  }
   215  
   216  // AzureFleetMembership returns the cluster AzureFleetMembership.
   217  func (s *ManagedControlPlaneScope) AzureFleetMembership() *infrav1.FleetsMember {
   218  	return s.ControlPlane.Spec.FleetsMember
   219  }
   220  
   221  // SubscriptionID returns the Azure client Subscription ID.
   222  func (s *ManagedControlPlaneScope) SubscriptionID() string {
   223  	return s.AzureClients.SubscriptionID()
   224  }
   225  
   226  // BaseURI returns the Azure ResourceManagerEndpoint.
   227  func (s *ManagedControlPlaneScope) BaseURI() string {
   228  	return s.AzureClients.ResourceManagerEndpoint
   229  }
   230  
   231  // PatchObject persists the cluster configuration and status.
   232  func (s *ManagedControlPlaneScope) PatchObject(ctx context.Context) error {
   233  	ctx, _, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.PatchObject")
   234  	defer done()
   235  
   236  	conditions.SetSummary(s.ControlPlane)
   237  
   238  	return s.PatchHelper.Patch(
   239  		ctx,
   240  		s.ControlPlane,
   241  		patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{
   242  			clusterv1.ReadyCondition,
   243  			infrav1.ResourceGroupReadyCondition,
   244  			infrav1.VNetReadyCondition,
   245  			infrav1.SubnetsReadyCondition,
   246  			infrav1.ManagedClusterRunningCondition,
   247  			infrav1.AgentPoolsReadyCondition,
   248  			infrav1.AzureResourceAvailableCondition,
   249  		}})
   250  }
   251  
   252  // Close closes the current scope persisting the cluster configuration and status.
   253  func (s *ManagedControlPlaneScope) Close(ctx context.Context) error {
   254  	ctx, _, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.Close")
   255  	defer done()
   256  
   257  	return s.PatchObject(ctx)
   258  }
   259  
   260  // Vnet returns the cluster Vnet.
   261  func (s *ManagedControlPlaneScope) Vnet() *infrav1.VnetSpec {
   262  	return &infrav1.VnetSpec{
   263  		ResourceGroup: s.ControlPlane.Spec.VirtualNetwork.ResourceGroup,
   264  		Name:          s.ControlPlane.Spec.VirtualNetwork.Name,
   265  		VnetClassSpec: infrav1.VnetClassSpec{
   266  			CIDRBlocks: []string{s.ControlPlane.Spec.VirtualNetwork.CIDRBlock},
   267  		},
   268  	}
   269  }
   270  
   271  // GroupSpecs returns the resource group spec.
   272  func (s *ManagedControlPlaneScope) GroupSpecs() []azure.ASOResourceSpecGetter[*asoresourcesv1.ResourceGroup] {
   273  	specs := []azure.ASOResourceSpecGetter[*asoresourcesv1.ResourceGroup]{
   274  		&groups.GroupSpec{
   275  			Name:           s.ResourceGroup(),
   276  			AzureName:      s.ResourceGroup(),
   277  			Location:       s.Location(),
   278  			ClusterName:    s.ClusterName(),
   279  			AdditionalTags: s.AdditionalTags(),
   280  		},
   281  	}
   282  	if s.Vnet().ResourceGroup != "" && s.Vnet().ResourceGroup != s.ResourceGroup() {
   283  		specs = append(specs, &groups.GroupSpec{
   284  			Name:           azure.GetNormalizedKubernetesName(s.Vnet().ResourceGroup),
   285  			AzureName:      s.Vnet().ResourceGroup,
   286  			Location:       s.Location(),
   287  			ClusterName:    s.ClusterName(),
   288  			AdditionalTags: s.AdditionalTags(),
   289  		})
   290  	}
   291  	return specs
   292  }
   293  
   294  // VNetSpec returns the virtual network spec.
   295  func (s *ManagedControlPlaneScope) VNetSpec() azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetwork] {
   296  	return &virtualnetworks.VNetSpec{
   297  		ResourceGroup:  s.Vnet().ResourceGroup,
   298  		Name:           s.Vnet().Name,
   299  		CIDRs:          s.Vnet().CIDRBlocks,
   300  		Location:       s.Location(),
   301  		ClusterName:    s.ClusterName(),
   302  		AdditionalTags: s.AdditionalTags(),
   303  	}
   304  }
   305  
   306  // AzureFleetsMemberSpec returns the fleet spec.
   307  func (s *ManagedControlPlaneScope) AzureFleetsMemberSpec() []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember] {
   308  	if s.AzureFleetMembership() == nil {
   309  		return nil
   310  	}
   311  	return []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember]{&fleetsmembers.AzureFleetsMemberSpec{
   312  		Name:                 s.AzureFleetMembership().Name,
   313  		ClusterName:          s.ClusterName(),
   314  		ClusterResourceGroup: s.ResourceGroup(),
   315  		Group:                s.AzureFleetMembership().Group,
   316  		SubscriptionID:       s.SubscriptionID(),
   317  		ManagerName:          s.AzureFleetMembership().ManagerName,
   318  		ManagerResourceGroup: s.AzureFleetMembership().ManagerResourceGroup,
   319  	}}
   320  }
   321  
   322  // ControlPlaneRouteTable returns the cluster controlplane routetable.
   323  func (s *ManagedControlPlaneScope) ControlPlaneRouteTable() infrav1.RouteTable {
   324  	return infrav1.RouteTable{}
   325  }
   326  
   327  // NodeRouteTable returns the cluster node routetable.
   328  func (s *ManagedControlPlaneScope) NodeRouteTable() infrav1.RouteTable {
   329  	return infrav1.RouteTable{}
   330  }
   331  
   332  // NodeNatGateway returns the cluster node NAT gateway.
   333  func (s *ManagedControlPlaneScope) NodeNatGateway() infrav1.NatGateway {
   334  	return infrav1.NatGateway{}
   335  }
   336  
   337  // SubnetSpecs returns the subnets specs.
   338  func (s *ManagedControlPlaneScope) SubnetSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetworksSubnet] {
   339  	return []azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetworksSubnet]{
   340  		&subnets.SubnetSpec{
   341  			Name:              s.NodeSubnet().Name,
   342  			ResourceGroup:     s.ResourceGroup(),
   343  			SubscriptionID:    s.SubscriptionID(),
   344  			CIDRs:             s.NodeSubnet().CIDRBlocks,
   345  			VNetName:          s.Vnet().Name,
   346  			VNetResourceGroup: s.Vnet().ResourceGroup,
   347  			IsVNetManaged:     s.IsVnetManaged(),
   348  			ServiceEndpoints:  s.NodeSubnet().ServiceEndpoints,
   349  		},
   350  	}
   351  }
   352  
   353  // Subnets returns the subnets specs.
   354  func (s *ManagedControlPlaneScope) Subnets() infrav1.Subnets {
   355  	return infrav1.Subnets{}
   356  }
   357  
   358  // NodeSubnet returns the cluster node subnet.
   359  func (s *ManagedControlPlaneScope) NodeSubnet() infrav1.SubnetSpec {
   360  	return infrav1.SubnetSpec{
   361  		SubnetClassSpec: infrav1.SubnetClassSpec{
   362  			CIDRBlocks:       []string{s.ControlPlane.Spec.VirtualNetwork.Subnet.CIDRBlock},
   363  			Name:             s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   364  			ServiceEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.ServiceEndpoints,
   365  			PrivateEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints,
   366  		},
   367  	}
   368  }
   369  
   370  // SetSubnet sets the passed subnet spec into the scope.
   371  // This is not used when using a managed control plane.
   372  func (s *ManagedControlPlaneScope) SetSubnet(_ infrav1.SubnetSpec) {
   373  	// no-op
   374  }
   375  
   376  // UpdateSubnetCIDRs updates the subnet CIDRs for the subnet with the same name.
   377  // This is not used when using a managed control plane.
   378  func (s *ManagedControlPlaneScope) UpdateSubnetCIDRs(_ string, _ []string) {
   379  	// no-op
   380  }
   381  
   382  // UpdateSubnetID updates the subnet ID for the subnet with the same name.
   383  // This is not used when using a managed control plane.
   384  func (s *ManagedControlPlaneScope) UpdateSubnetID(_ string, _ string) {
   385  	// no-op
   386  }
   387  
   388  // ControlPlaneSubnet returns the cluster control plane subnet.
   389  func (s *ManagedControlPlaneScope) ControlPlaneSubnet() infrav1.SubnetSpec {
   390  	return infrav1.SubnetSpec{}
   391  }
   392  
   393  // NodeSubnets returns the subnets with the node role.
   394  func (s *ManagedControlPlaneScope) NodeSubnets() []infrav1.SubnetSpec {
   395  	return []infrav1.SubnetSpec{
   396  		{
   397  			SubnetClassSpec: infrav1.SubnetClassSpec{
   398  				CIDRBlocks:       []string{s.ControlPlane.Spec.VirtualNetwork.Subnet.CIDRBlock},
   399  				Name:             s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   400  				ServiceEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.ServiceEndpoints,
   401  				PrivateEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints,
   402  			},
   403  		},
   404  	}
   405  }
   406  
   407  // Subnet returns the subnet with the provided name.
   408  func (s *ManagedControlPlaneScope) Subnet(name string) infrav1.SubnetSpec {
   409  	subnet := infrav1.SubnetSpec{}
   410  	if name == s.ControlPlane.Spec.VirtualNetwork.Subnet.Name {
   411  		subnet.Name = s.ControlPlane.Spec.VirtualNetwork.Subnet.Name
   412  		subnet.CIDRBlocks = []string{s.ControlPlane.Spec.VirtualNetwork.Subnet.CIDRBlock}
   413  		subnet.ServiceEndpoints = s.ControlPlane.Spec.VirtualNetwork.Subnet.ServiceEndpoints
   414  		subnet.PrivateEndpoints = s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints
   415  	}
   416  
   417  	return subnet
   418  }
   419  
   420  // IsIPv6Enabled returns true if a cluster is ipv6 enabled.
   421  // Currently always false as managed control planes do not currently implement ipv6.
   422  func (s *ManagedControlPlaneScope) IsIPv6Enabled() bool {
   423  	return false
   424  }
   425  
   426  // IsVnetManaged returns true if the vnet is managed.
   427  func (s *ManagedControlPlaneScope) IsVnetManaged() bool {
   428  	if s.cache.isVnetManaged != nil {
   429  		return ptr.Deref(s.cache.isVnetManaged, false)
   430  	}
   431  	// TODO refactor `IsVnetManaged` so that it is able to use an upstream context
   432  	// see https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/2581
   433  	ctx := context.Background()
   434  	ctx, log, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.IsVnetManaged")
   435  	defer done()
   436  
   437  	vnet := s.VNetSpec().ResourceRef()
   438  	vnet.SetNamespace(s.ASOOwner().GetNamespace())
   439  	err := s.Client.Get(ctx, client.ObjectKeyFromObject(vnet), vnet)
   440  	if err != nil {
   441  		log.Error(err, "Unable to determine if ManagedControlPlaneScope VNET is managed by capz, assuming unmanaged", "AzureManagedCluster", s.ClusterName())
   442  		return false
   443  	}
   444  
   445  	isManaged := infrav1.Tags(vnet.Status.Tags).HasOwned(s.ClusterName())
   446  	s.cache.isVnetManaged = ptr.To(isManaged)
   447  	return isManaged
   448  }
   449  
   450  // APIServerLB returns the API Server LB spec.
   451  func (s *ManagedControlPlaneScope) APIServerLB() *infrav1.LoadBalancerSpec {
   452  	return nil // does not apply for AKS
   453  }
   454  
   455  // APIServerLBName returns the API Server LB name.
   456  func (s *ManagedControlPlaneScope) APIServerLBName() string {
   457  	return "" // does not apply for AKS
   458  }
   459  
   460  // APIServerLBPoolName returns the API Server LB backend pool name.
   461  func (s *ManagedControlPlaneScope) APIServerLBPoolName() string {
   462  	return "" // does not apply for AKS
   463  }
   464  
   465  // IsAPIServerPrivate returns true if the API Server LB is of type Internal.
   466  // Currently always false as managed control planes do not currently implement private clusters.
   467  func (s *ManagedControlPlaneScope) IsAPIServerPrivate() bool {
   468  	return false
   469  }
   470  
   471  // OutboundLBName returns the name of the outbound LB.
   472  // Note: for managed clusters, the outbound LB lifecycle is not managed.
   473  func (s *ManagedControlPlaneScope) OutboundLBName(_ string) string {
   474  	return "kubernetes"
   475  }
   476  
   477  // OutboundPoolName returns the outbound LB backend pool name.
   478  func (s *ManagedControlPlaneScope) OutboundPoolName(_ string) string {
   479  	return "aksOutboundBackendPool" // hard-coded in aks
   480  }
   481  
   482  // GetPrivateDNSZoneName returns the Private DNS Zone from the spec or generate it from cluster name.
   483  // Currently always empty as managed control planes do not currently implement private clusters.
   484  func (s *ManagedControlPlaneScope) GetPrivateDNSZoneName() string {
   485  	return ""
   486  }
   487  
   488  // CloudProviderConfigOverrides returns the cloud provider config overrides for the cluster.
   489  func (s *ManagedControlPlaneScope) CloudProviderConfigOverrides() *infrav1.CloudProviderConfigOverrides {
   490  	return nil
   491  }
   492  
   493  // FailureDomains returns the failure domains for the cluster.
   494  func (s *ManagedControlPlaneScope) FailureDomains() []*string {
   495  	return []*string{}
   496  }
   497  
   498  // AreLocalAccountsDisabled checks if local accounts are disabled for aad enabled managed clusters.
   499  func (s *ManagedControlPlaneScope) AreLocalAccountsDisabled() bool {
   500  	if s.IsAADEnabled() &&
   501  		s.ControlPlane.Spec.DisableLocalAccounts != nil &&
   502  		*s.ControlPlane.Spec.DisableLocalAccounts {
   503  		return true
   504  	}
   505  	return false
   506  }
   507  
   508  // IsAADEnabled checks if azure active directory is enabled for managed clusters.
   509  func (s *ManagedControlPlaneScope) IsAADEnabled() bool {
   510  	if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed {
   511  		return true
   512  	}
   513  	return false
   514  }
   515  
   516  // SetVersionStatus sets the k8s version in status.
   517  func (s *ManagedControlPlaneScope) SetVersionStatus(version string) {
   518  	s.ControlPlane.Status.Version = version
   519  }
   520  
   521  // SetAutoUpgradeVersionStatus sets the auto upgrade version in status.
   522  func (s *ManagedControlPlaneScope) SetAutoUpgradeVersionStatus(version string) {
   523  	s.ControlPlane.Status.AutoUpgradeVersion = version
   524  }
   525  
   526  // IsManagedVersionUpgrade checks if version is auto managed by AKS.
   527  func (s *ManagedControlPlaneScope) IsManagedVersionUpgrade() bool {
   528  	return isManagedVersionUpgrade(s.ControlPlane)
   529  }
   530  
   531  // IsPreviewEnabled checks if the preview feature is enabled.
   532  func (s *ManagedControlPlaneScope) IsPreviewEnabled() bool {
   533  	return ptr.Deref(s.ControlPlane.Spec.EnablePreviewFeatures, false)
   534  }
   535  
   536  func isManagedVersionUpgrade(managedControlPlane *infrav1.AzureManagedControlPlane) bool {
   537  	return managedControlPlane.Spec.AutoUpgradeProfile != nil &&
   538  		managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != nil &&
   539  		(*managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1.UpgradeChannelNone &&
   540  			*managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1.UpgradeChannelNodeImage)
   541  }
   542  
   543  // ManagedClusterSpec returns the managed cluster spec.
   544  func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ASOResourceSpecGetter[genruntime.MetaObject] {
   545  	managedClusterSpec := managedclusters.ManagedClusterSpec{
   546  		Name:              s.ControlPlane.Name,
   547  		ResourceGroup:     s.ControlPlane.Spec.ResourceGroupName,
   548  		NodeResourceGroup: s.ControlPlane.Spec.NodeResourceGroupName,
   549  		ClusterName:       s.ClusterName(),
   550  		Location:          s.ControlPlane.Spec.Location,
   551  		Tags:              s.ControlPlane.Spec.AdditionalTags,
   552  		Version:           strings.TrimPrefix(s.ControlPlane.Spec.Version, "v"),
   553  		DNSServiceIP:      s.ControlPlane.Spec.DNSServiceIP,
   554  		VnetSubnetID: azure.SubnetID(
   555  			s.ControlPlane.Spec.SubscriptionID,
   556  			s.Vnet().ResourceGroup,
   557  			s.ControlPlane.Spec.VirtualNetwork.Name,
   558  			s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   559  		),
   560  		GetAllAgentPools:            s.GetAllAgentPoolSpecs,
   561  		OutboundType:                s.ControlPlane.Spec.OutboundType,
   562  		Identity:                    s.ControlPlane.Spec.Identity,
   563  		KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity,
   564  		NetworkPluginMode:           s.ControlPlane.Spec.NetworkPluginMode,
   565  		DNSPrefix:                   s.ControlPlane.Spec.DNSPrefix,
   566  		Patches:                     s.ControlPlane.Spec.ASOManagedClusterPatches,
   567  		Preview:                     ptr.Deref(s.ControlPlane.Spec.EnablePreviewFeatures, false),
   568  	}
   569  
   570  	if s.ControlPlane.Spec.SSHPublicKey != nil {
   571  		managedClusterSpec.SSHPublicKey = *s.ControlPlane.Spec.SSHPublicKey
   572  	}
   573  	if s.ControlPlane.Spec.NetworkPlugin != nil {
   574  		managedClusterSpec.NetworkPlugin = *s.ControlPlane.Spec.NetworkPlugin
   575  	}
   576  	if s.ControlPlane.Spec.NetworkPolicy != nil {
   577  		managedClusterSpec.NetworkPolicy = *s.ControlPlane.Spec.NetworkPolicy
   578  	}
   579  	if s.ControlPlane.Spec.NetworkDataplane != nil {
   580  		managedClusterSpec.NetworkDataplane = s.ControlPlane.Spec.NetworkDataplane
   581  	}
   582  	if s.ControlPlane.Spec.LoadBalancerSKU != nil {
   583  		// CAPZ accepts Standard/Basic, Azure accepts standard/basic
   584  		managedClusterSpec.LoadBalancerSKU = strings.ToLower(*s.ControlPlane.Spec.LoadBalancerSKU)
   585  	}
   586  
   587  	if clusterNetwork := s.Cluster.Spec.ClusterNetwork; clusterNetwork != nil {
   588  		if clusterNetwork.Services != nil && len(clusterNetwork.Services.CIDRBlocks) == 1 {
   589  			managedClusterSpec.ServiceCIDR = clusterNetwork.Services.CIDRBlocks[0]
   590  		}
   591  		if clusterNetwork.Pods != nil && len(clusterNetwork.Pods.CIDRBlocks) == 1 {
   592  			managedClusterSpec.PodCIDR = clusterNetwork.Pods.CIDRBlocks[0]
   593  		}
   594  	}
   595  
   596  	if s.ControlPlane.Spec.AADProfile != nil {
   597  		managedClusterSpec.AADProfile = &managedclusters.AADProfile{
   598  			Managed:             s.ControlPlane.Spec.AADProfile.Managed,
   599  			EnableAzureRBAC:     s.ControlPlane.Spec.AADProfile.Managed,
   600  			AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs,
   601  		}
   602  		if s.ControlPlane.Spec.DisableLocalAccounts != nil {
   603  			managedClusterSpec.DisableLocalAccounts = s.ControlPlane.Spec.DisableLocalAccounts
   604  		}
   605  	}
   606  
   607  	if s.ControlPlane.Spec.AddonProfiles != nil {
   608  		for _, profile := range s.ControlPlane.Spec.AddonProfiles {
   609  			managedClusterSpec.AddonProfiles = append(managedClusterSpec.AddonProfiles, managedclusters.AddonProfile{
   610  				Name:    profile.Name,
   611  				Enabled: profile.Enabled,
   612  				Config:  profile.Config,
   613  			})
   614  		}
   615  	}
   616  
   617  	if s.ControlPlane.Spec.SKU != nil {
   618  		managedClusterSpec.SKU = &managedclusters.SKU{
   619  			Tier: string(s.ControlPlane.Spec.SKU.Tier),
   620  		}
   621  	}
   622  
   623  	if s.ControlPlane.Spec.LoadBalancerProfile != nil {
   624  		managedClusterSpec.LoadBalancerProfile = &managedclusters.LoadBalancerProfile{
   625  			ManagedOutboundIPs:     s.ControlPlane.Spec.LoadBalancerProfile.ManagedOutboundIPs,
   626  			OutboundIPPrefixes:     s.ControlPlane.Spec.LoadBalancerProfile.OutboundIPPrefixes,
   627  			OutboundIPs:            s.ControlPlane.Spec.LoadBalancerProfile.OutboundIPs,
   628  			AllocatedOutboundPorts: s.ControlPlane.Spec.LoadBalancerProfile.AllocatedOutboundPorts,
   629  			IdleTimeoutInMinutes:   s.ControlPlane.Spec.LoadBalancerProfile.IdleTimeoutInMinutes,
   630  		}
   631  	}
   632  
   633  	if s.ControlPlane.Spec.APIServerAccessProfile != nil {
   634  		managedClusterSpec.APIServerAccessProfile = &managedclusters.APIServerAccessProfile{
   635  			AuthorizedIPRanges:             s.ControlPlane.Spec.APIServerAccessProfile.AuthorizedIPRanges,
   636  			EnablePrivateCluster:           s.ControlPlane.Spec.APIServerAccessProfile.EnablePrivateCluster,
   637  			PrivateDNSZone:                 s.ControlPlane.Spec.APIServerAccessProfile.PrivateDNSZone,
   638  			EnablePrivateClusterPublicFQDN: s.ControlPlane.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN,
   639  		}
   640  	}
   641  
   642  	if s.ControlPlane.Spec.AutoScalerProfile != nil {
   643  		managedClusterSpec.AutoScalerProfile = &managedclusters.AutoScalerProfile{
   644  			BalanceSimilarNodeGroups:      (*string)(s.ControlPlane.Spec.AutoScalerProfile.BalanceSimilarNodeGroups),
   645  			Expander:                      (*string)(s.ControlPlane.Spec.AutoScalerProfile.Expander),
   646  			MaxEmptyBulkDelete:            s.ControlPlane.Spec.AutoScalerProfile.MaxEmptyBulkDelete,
   647  			MaxGracefulTerminationSec:     s.ControlPlane.Spec.AutoScalerProfile.MaxGracefulTerminationSec,
   648  			MaxNodeProvisionTime:          s.ControlPlane.Spec.AutoScalerProfile.MaxNodeProvisionTime,
   649  			MaxTotalUnreadyPercentage:     s.ControlPlane.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage,
   650  			NewPodScaleUpDelay:            s.ControlPlane.Spec.AutoScalerProfile.NewPodScaleUpDelay,
   651  			OkTotalUnreadyCount:           s.ControlPlane.Spec.AutoScalerProfile.OkTotalUnreadyCount,
   652  			ScanInterval:                  s.ControlPlane.Spec.AutoScalerProfile.ScanInterval,
   653  			ScaleDownDelayAfterAdd:        s.ControlPlane.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd,
   654  			ScaleDownDelayAfterDelete:     s.ControlPlane.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete,
   655  			ScaleDownDelayAfterFailure:    s.ControlPlane.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure,
   656  			ScaleDownUnneededTime:         s.ControlPlane.Spec.AutoScalerProfile.ScaleDownUnneededTime,
   657  			ScaleDownUnreadyTime:          s.ControlPlane.Spec.AutoScalerProfile.ScaleDownUnreadyTime,
   658  			ScaleDownUtilizationThreshold: s.ControlPlane.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold,
   659  			SkipNodesWithLocalStorage:     (*string)(s.ControlPlane.Spec.AutoScalerProfile.SkipNodesWithLocalStorage),
   660  			SkipNodesWithSystemPods:       (*string)(s.ControlPlane.Spec.AutoScalerProfile.SkipNodesWithSystemPods),
   661  		}
   662  	}
   663  
   664  	if s.ControlPlane.Spec.HTTPProxyConfig != nil {
   665  		managedClusterSpec.HTTPProxyConfig = &managedclusters.HTTPProxyConfig{
   666  			HTTPProxy:  s.ControlPlane.Spec.HTTPProxyConfig.HTTPProxy,
   667  			HTTPSProxy: s.ControlPlane.Spec.HTTPProxyConfig.HTTPSProxy,
   668  			NoProxy:    s.ControlPlane.Spec.HTTPProxyConfig.NoProxy,
   669  			TrustedCA:  s.ControlPlane.Spec.HTTPProxyConfig.TrustedCA,
   670  		}
   671  	}
   672  
   673  	if s.ControlPlane.Spec.OIDCIssuerProfile != nil {
   674  		managedClusterSpec.OIDCIssuerProfile = &managedclusters.OIDCIssuerProfile{
   675  			Enabled: s.ControlPlane.Spec.OIDCIssuerProfile.Enabled,
   676  		}
   677  	}
   678  
   679  	if s.ControlPlane.Spec.AutoUpgradeProfile != nil {
   680  		managedClusterSpec.AutoUpgradeProfile = &managedclusters.ManagedClusterAutoUpgradeProfile{}
   681  		if s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != nil {
   682  			managedClusterSpec.AutoUpgradeProfile.UpgradeChannel = s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel
   683  		}
   684  	}
   685  
   686  	if s.ControlPlane.Spec.SecurityProfile != nil {
   687  		managedClusterSpec.SecurityProfile = s.getManagedClusterSecurityProfile()
   688  	}
   689  
   690  	return &managedClusterSpec
   691  }
   692  
   693  // GetManagedClusterSecurityProfile gets the security profile for managed cluster.
   694  func (s *ManagedControlPlaneScope) getManagedClusterSecurityProfile() *managedclusters.ManagedClusterSecurityProfile {
   695  	securityProfile := &managedclusters.ManagedClusterSecurityProfile{}
   696  	if s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms != nil {
   697  		securityProfile.AzureKeyVaultKms = &managedclusters.AzureKeyVaultKms{
   698  			Enabled: ptr.To(s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.Enabled),
   699  			KeyID:   ptr.To(s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyID),
   700  		}
   701  		if s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess != nil {
   702  			securityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess = s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess
   703  		}
   704  		if s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID != nil {
   705  			securityProfile.AzureKeyVaultKms.KeyVaultResourceID = s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID
   706  		}
   707  	}
   708  
   709  	if s.ControlPlane.Spec.SecurityProfile.Defender != nil {
   710  		securityProfile.Defender = &managedclusters.ManagedClusterSecurityProfileDefender{
   711  			LogAnalyticsWorkspaceResourceID: ptr.To(s.ControlPlane.Spec.SecurityProfile.Defender.LogAnalyticsWorkspaceResourceID),
   712  			SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{
   713  				Enabled: ptr.To(s.ControlPlane.Spec.SecurityProfile.Defender.SecurityMonitoring.Enabled),
   714  			},
   715  		}
   716  	}
   717  
   718  	if s.ControlPlane.Spec.SecurityProfile.ImageCleaner != nil {
   719  		securityProfile.ImageCleaner = &managedclusters.ManagedClusterSecurityProfileImageCleaner{
   720  			Enabled:       ptr.To(s.ControlPlane.Spec.SecurityProfile.ImageCleaner.Enabled),
   721  			IntervalHours: s.ControlPlane.Spec.SecurityProfile.ImageCleaner.IntervalHours,
   722  		}
   723  	}
   724  
   725  	if s.ControlPlane.Spec.SecurityProfile.WorkloadIdentity != nil {
   726  		securityProfile.WorkloadIdentity = &managedclusters.ManagedClusterSecurityProfileWorkloadIdentity{
   727  			Enabled: ptr.To(s.ControlPlane.Spec.SecurityProfile.WorkloadIdentity.Enabled),
   728  		}
   729  	}
   730  
   731  	return securityProfile
   732  }
   733  
   734  // GetAllAgentPoolSpecs gets a slice of azure.AgentPoolSpec for the list of agent pools.
   735  func (s *ManagedControlPlaneScope) GetAllAgentPoolSpecs() ([]azure.ASOResourceSpecGetter[genruntime.MetaObject], error) {
   736  	var (
   737  		ammps           = make([]azure.ASOResourceSpecGetter[genruntime.MetaObject], 0, len(s.ManagedMachinePools))
   738  		foundSystemPool = false
   739  	)
   740  	for _, pool := range s.ManagedMachinePools {
   741  		// TODO: this should be in a webhook: https://github.com/kubernetes-sigs/cluster-api/issues/6040
   742  		if pool.MachinePool != nil && pool.MachinePool.Spec.Template.Spec.Version != nil {
   743  			version := *pool.MachinePool.Spec.Template.Spec.Version
   744  			if semver.Compare(version, s.ControlPlane.Spec.Version) > 0 {
   745  				return nil, errors.New("MachinePool version cannot be greater than the AzureManagedControlPlane version")
   746  			}
   747  		}
   748  
   749  		if pool.InfraMachinePool != nil && pool.InfraMachinePool.Spec.Mode == string(infrav1.NodePoolModeSystem) {
   750  			foundSystemPool = true
   751  		}
   752  
   753  		ammp := buildAgentPoolSpec(s.ControlPlane, pool.MachinePool, pool.InfraMachinePool)
   754  		ammps = append(ammps, ammp)
   755  	}
   756  
   757  	if !foundSystemPool {
   758  		return nil, errors.New("failed to fetch azuremanagedMachine pool with mode:System, require at least 1 system node pool")
   759  	}
   760  
   761  	return ammps, nil
   762  }
   763  
   764  // SetControlPlaneEndpoint sets a control plane endpoint.
   765  func (s *ManagedControlPlaneScope) SetControlPlaneEndpoint(endpoint clusterv1.APIEndpoint) {
   766  	s.ControlPlane.Spec.ControlPlaneEndpoint.Host = endpoint.Host
   767  	s.ControlPlane.Spec.ControlPlaneEndpoint.Port = endpoint.Port
   768  }
   769  
   770  // MakeEmptyKubeConfigSecret creates an empty secret object that is used for storing kubeconfig secret data.
   771  func (s *ManagedControlPlaneScope) MakeEmptyKubeConfigSecret() corev1.Secret {
   772  	return corev1.Secret{
   773  		ObjectMeta: metav1.ObjectMeta{
   774  			Name:      secret.Name(s.Cluster.Name, secret.Kubeconfig),
   775  			Namespace: s.Cluster.Namespace,
   776  			OwnerReferences: []metav1.OwnerReference{
   777  				*metav1.NewControllerRef(s.ControlPlane, infrav1.GroupVersion.WithKind(infrav1.AzureManagedControlPlaneKind)),
   778  			},
   779  			Labels: map[string]string{clusterv1.ClusterNameLabel: s.Cluster.Name},
   780  		},
   781  	}
   782  }
   783  
   784  // GetAdminKubeconfigData returns admin kubeconfig.
   785  func (s *ManagedControlPlaneScope) GetAdminKubeconfigData() []byte {
   786  	return s.adminKubeConfigData
   787  }
   788  
   789  // SetAdminKubeconfigData sets admin kubeconfig data.
   790  func (s *ManagedControlPlaneScope) SetAdminKubeconfigData(kubeConfigData []byte) {
   791  	s.adminKubeConfigData = kubeConfigData
   792  }
   793  
   794  // GetUserKubeconfigData returns user kubeconfig, required when using AAD with AKS cluster.
   795  func (s *ManagedControlPlaneScope) GetUserKubeconfigData() []byte {
   796  	return s.userKubeConfigData
   797  }
   798  
   799  // SetUserKubeconfigData sets userKubeconfig data.
   800  func (s *ManagedControlPlaneScope) SetUserKubeconfigData(kubeConfigData []byte) {
   801  	s.userKubeConfigData = kubeConfigData
   802  }
   803  
   804  // MakeClusterCA returns a cluster CA Secret for the managed control plane.
   805  func (s *ManagedControlPlaneScope) MakeClusterCA() *corev1.Secret {
   806  	return &corev1.Secret{
   807  		ObjectMeta: metav1.ObjectMeta{
   808  			Name:      secret.Name(s.Cluster.Name, secret.ClusterCA),
   809  			Namespace: s.Cluster.Namespace,
   810  			OwnerReferences: []metav1.OwnerReference{
   811  				*metav1.NewControllerRef(s.ControlPlane, infrav1.GroupVersion.WithKind(infrav1.AzureManagedControlPlaneKind)),
   812  			},
   813  		},
   814  	}
   815  }
   816  
   817  // StoreClusterInfo stores the discovery cluster-info configmap in the kube-public namespace on the AKS cluster so kubeadm can access it to join nodes.
   818  func (s *ManagedControlPlaneScope) StoreClusterInfo(ctx context.Context, caData []byte) error {
   819  	remoteclient, err := remote.NewClusterClient(ctx, managedControlPlaneScopeName, s.Client, types.NamespacedName{
   820  		Namespace: s.Cluster.Namespace,
   821  		Name:      s.Cluster.Name,
   822  	})
   823  	if err != nil {
   824  		return errors.Wrap(err, "failed to create remote cluster kubeclient")
   825  	}
   826  
   827  	discoveryFile := clientcmdapi.NewConfig()
   828  	discoveryFile.Clusters[""] = &clientcmdapi.Cluster{
   829  		CertificateAuthorityData: caData,
   830  		Server: fmt.Sprintf(
   831  			"%s:%d",
   832  			s.ControlPlane.Spec.ControlPlaneEndpoint.Host,
   833  			s.ControlPlane.Spec.ControlPlaneEndpoint.Port,
   834  		),
   835  	}
   836  
   837  	data, err := yaml.Marshal(&discoveryFile)
   838  	if err != nil {
   839  		return errors.Wrap(err, "failed to serialize cluster-info to yaml")
   840  	}
   841  
   842  	clusterInfo := &corev1.ConfigMap{
   843  		ObjectMeta: metav1.ObjectMeta{
   844  			Name:      bootstrapapi.ConfigMapClusterInfo,
   845  			Namespace: metav1.NamespacePublic,
   846  		},
   847  		Data: map[string]string{
   848  			bootstrapapi.KubeConfigKey: string(data),
   849  		},
   850  	}
   851  
   852  	if _, err := controllerutil.CreateOrUpdate(ctx, remoteclient, clusterInfo, func() error {
   853  		clusterInfo.Data[bootstrapapi.KubeConfigKey] = string(data)
   854  		return nil
   855  	}); err != nil {
   856  		return errors.Wrapf(err, "failed to reconcile certificate authority data secret for cluster")
   857  	}
   858  
   859  	return nil
   860  }
   861  
   862  // SetLongRunningOperationState will set the future on the AzureManagedControlPlane status to allow the resource to continue
   863  // in the next reconciliation.
   864  func (s *ManagedControlPlaneScope) SetLongRunningOperationState(future *infrav1.Future) {
   865  	futures.Set(s.ControlPlane, future)
   866  }
   867  
   868  // GetLongRunningOperationState will get the future on the AzureManagedControlPlane status.
   869  func (s *ManagedControlPlaneScope) GetLongRunningOperationState(name, service, futureType string) *infrav1.Future {
   870  	return futures.Get(s.ControlPlane, name, service, futureType)
   871  }
   872  
   873  // DeleteLongRunningOperationState will delete the future from the AzureManagedControlPlane status.
   874  func (s *ManagedControlPlaneScope) DeleteLongRunningOperationState(name, service, futureType string) {
   875  	futures.Delete(s.ControlPlane, name, service, futureType)
   876  }
   877  
   878  // UpdateDeleteStatus updates a condition on the AzureManagedControlPlane status after a DELETE operation.
   879  func (s *ManagedControlPlaneScope) UpdateDeleteStatus(condition clusterv1.ConditionType, service string, err error) {
   880  	switch {
   881  	case err == nil:
   882  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.DeletedReason, clusterv1.ConditionSeverityInfo, "%s successfully deleted", service)
   883  	case azure.IsOperationNotDoneError(err):
   884  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.DeletingReason, clusterv1.ConditionSeverityInfo, "%s deleting", service)
   885  	default:
   886  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.DeletionFailedReason, clusterv1.ConditionSeverityError, "%s failed to delete. err: %s", service, err.Error())
   887  	}
   888  }
   889  
   890  // UpdatePutStatus updates a condition on the AzureManagedControlPlane status after a PUT operation.
   891  func (s *ManagedControlPlaneScope) UpdatePutStatus(condition clusterv1.ConditionType, service string, err error) {
   892  	switch {
   893  	case err == nil:
   894  		conditions.MarkTrue(s.ControlPlane, condition)
   895  	case azure.IsOperationNotDoneError(err):
   896  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.CreatingReason, clusterv1.ConditionSeverityInfo, "%s creating or updating", service)
   897  	default:
   898  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.FailedReason, clusterv1.ConditionSeverityError, "%s failed to create or update. err: %s", service, err.Error())
   899  	}
   900  }
   901  
   902  // UpdatePatchStatus updates a condition on the AzureManagedControlPlane status after a PATCH operation.
   903  func (s *ManagedControlPlaneScope) UpdatePatchStatus(condition clusterv1.ConditionType, service string, err error) {
   904  	switch {
   905  	case err == nil:
   906  		conditions.MarkTrue(s.ControlPlane, condition)
   907  	case azure.IsOperationNotDoneError(err):
   908  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.UpdatingReason, clusterv1.ConditionSeverityInfo, "%s updating", service)
   909  	default:
   910  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.FailedReason, clusterv1.ConditionSeverityError, "%s failed to update. err: %s", service, err.Error())
   911  	}
   912  }
   913  
   914  // AnnotationJSON returns a map[string]interface from a JSON annotation.
   915  func (s *ManagedControlPlaneScope) AnnotationJSON(annotation string) (map[string]interface{}, error) {
   916  	out := map[string]interface{}{}
   917  	jsonAnnotation := s.ControlPlane.GetAnnotations()[annotation]
   918  	if jsonAnnotation == "" {
   919  		return out, nil
   920  	}
   921  	err := json.Unmarshal([]byte(jsonAnnotation), &out)
   922  	if err != nil {
   923  		return out, err
   924  	}
   925  	return out, nil
   926  }
   927  
   928  // UpdateAnnotationJSON updates the `annotation` with
   929  // `content`. `content` in this case should be a `map[string]interface{}`
   930  // suitable for turning into JSON. This `content` map will be marshalled into a
   931  // JSON string before being set as the given `annotation`.
   932  func (s *ManagedControlPlaneScope) UpdateAnnotationJSON(annotation string, content map[string]interface{}) error {
   933  	b, err := json.Marshal(content)
   934  	if err != nil {
   935  		return err
   936  	}
   937  	s.SetAnnotation(annotation, string(b))
   938  	return nil
   939  }
   940  
   941  // SetAnnotation sets a key value annotation on the ControlPlane.
   942  func (s *ManagedControlPlaneScope) SetAnnotation(key, value string) {
   943  	if s.ControlPlane.Annotations == nil {
   944  		s.ControlPlane.Annotations = map[string]string{}
   945  	}
   946  	s.ControlPlane.Annotations[key] = value
   947  }
   948  
   949  // AvailabilityStatusResource refers to the AzureManagedControlPlane.
   950  func (s *ManagedControlPlaneScope) AvailabilityStatusResource() conditions.Setter {
   951  	return s.ControlPlane
   952  }
   953  
   954  // AvailabilityStatusResourceURI constructs the ID of the underlying AKS resource.
   955  func (s *ManagedControlPlaneScope) AvailabilityStatusResourceURI() string {
   956  	return azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name)
   957  }
   958  
   959  // AvailabilityStatusFilter ignores the health metrics connection error that
   960  // occurs on startup for every AKS cluster.
   961  func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Condition) *clusterv1.Condition {
   962  	if time.Since(s.ControlPlane.CreationTimestamp.Time) < resourceHealthWarningInitialGracePeriod &&
   963  		cond.Severity == clusterv1.ConditionSeverityWarning {
   964  		return conditions.TrueCondition(infrav1.AzureResourceAvailableCondition)
   965  	}
   966  	return cond
   967  }
   968  
   969  // PrivateEndpointSpecs returns the private endpoint specs.
   970  func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1api20220701.PrivateEndpoint] {
   971  	privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1api20220701.PrivateEndpoint], 0, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))
   972  
   973  	for _, privateEndpoint := range s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints {
   974  		privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
   975  			Name:                       privateEndpoint.Name,
   976  			ResourceGroup:              s.Vnet().ResourceGroup,
   977  			Location:                   privateEndpoint.Location,
   978  			CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
   979  			PrivateIPAddresses:         privateEndpoint.PrivateIPAddresses,
   980  			SubnetID: azure.SubnetID(
   981  				s.ControlPlane.Spec.SubscriptionID,
   982  				s.Vnet().ResourceGroup,
   983  				s.ControlPlane.Spec.VirtualNetwork.Name,
   984  				s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   985  			),
   986  			ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
   987  			ManualApproval:            privateEndpoint.ManualApproval,
   988  			ClusterName:               s.ClusterName(),
   989  			AdditionalTags:            s.AdditionalTags(),
   990  		}
   991  
   992  		for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
   993  			pl := privateendpoints.PrivateLinkServiceConnection{
   994  				PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
   995  				Name:                 privateLinkServiceConnection.Name,
   996  				RequestMessage:       privateLinkServiceConnection.RequestMessage,
   997  				GroupIDs:             privateLinkServiceConnection.GroupIDs,
   998  			}
   999  			privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
  1000  		}
  1001  		privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
  1002  	}
  1003  
  1004  	return privateEndpointSpecs
  1005  }
  1006  
  1007  // SetOIDCIssuerProfileStatus sets the status for the OIDC issuer profile config.
  1008  func (s *ManagedControlPlaneScope) SetOIDCIssuerProfileStatus(oidc *infrav1.OIDCIssuerProfileStatus) {
  1009  	s.ControlPlane.Status.OIDCIssuerProfile = oidc
  1010  }
  1011  
  1012  // AKSExtension returns the cluster AKS extensions.
  1013  func (s *ManagedControlPlaneScope) AKSExtension() []infrav1.AKSExtension {
  1014  	return s.ControlPlane.Spec.Extensions
  1015  }
  1016  
  1017  // AKSExtensionSpecs returns the AKS extension specs.
  1018  func (s *ManagedControlPlaneScope) AKSExtensionSpecs() []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] {
  1019  	if s.AKSExtension() == nil {
  1020  		return nil
  1021  	}
  1022  	extensionSpecs := make([]azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension], 0, len(s.ControlPlane.Spec.Extensions))
  1023  	for _, extension := range s.AKSExtension() {
  1024  		extensionSpec := &aksextensions.AKSExtensionSpec{
  1025  			Name:                    extension.Name,
  1026  			Namespace:               s.Cluster.Namespace,
  1027  			AutoUpgradeMinorVersion: extension.AutoUpgradeMinorVersion,
  1028  			ConfigurationSettings:   extension.ConfigurationSettings,
  1029  			ExtensionType:           extension.ExtensionType,
  1030  			ReleaseTrain:            extension.ReleaseTrain,
  1031  			Version:                 extension.Version,
  1032  			Owner:                   azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name),
  1033  			Plan:                    extension.Plan,
  1034  			AKSAssignedIdentityType: extension.AKSAssignedIdentityType,
  1035  			ExtensionIdentity:       extension.Identity,
  1036  		}
  1037  
  1038  		extensionSpecs = append(extensionSpecs, extensionSpec)
  1039  	}
  1040  
  1041  	return extensionSpecs
  1042  }