sigs.k8s.io/cluster-api-provider-azure@v1.17.0/azure/scope/managedcontrolplane.go (about)

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package scope
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"strings"
    24  	"time"
    25  
    26  	asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
    27  	asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501"
    28  	asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101"
    29  	asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
    30  	asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
    31  	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
    32  	"github.com/pkg/errors"
    33  	"golang.org/x/mod/semver"
    34  	"gopkg.in/yaml.v3"
    35  	corev1 "k8s.io/api/core/v1"
    36  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    39  	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
    40  	"k8s.io/utils/ptr"
    41  	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
    42  	"sigs.k8s.io/cluster-api-provider-azure/azure"
    43  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/aksextensions"
    44  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers"
    45  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/groups"
    46  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters"
    47  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
    48  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
    49  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualnetworks"
    50  	"sigs.k8s.io/cluster-api-provider-azure/util/futures"
    51  	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
    52  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    53  	"sigs.k8s.io/cluster-api/controllers/remote"
    54  	"sigs.k8s.io/cluster-api/util/conditions"
    55  	"sigs.k8s.io/cluster-api/util/patch"
    56  	"sigs.k8s.io/cluster-api/util/secret"
    57  	"sigs.k8s.io/controller-runtime/pkg/client"
    58  	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
    59  )
    60  
    61  const (
    62  	resourceHealthWarningInitialGracePeriod = 1 * time.Hour
    63  	// managedControlPlaneScopeName is the sourceName, or more specifically the UserAgent, of client used to store the Cluster Info configmap.
    64  	managedControlPlaneScopeName = "azuremanagedcontrolplane-scope"
    65  )
    66  
    67  // ManagedControlPlaneScopeParams defines the input parameters used to create a new managed
    68  // control plane.
    69  type ManagedControlPlaneScopeParams struct {
    70  	AzureClients
    71  	Client              client.Client
    72  	Cluster             *clusterv1.Cluster
    73  	ControlPlane        *infrav1.AzureManagedControlPlane
    74  	ManagedMachinePools []ManagedMachinePool
    75  	Cache               *ManagedControlPlaneCache
    76  	Timeouts            azure.AsyncReconciler
    77  }
    78  
    79  // NewManagedControlPlaneScope creates a new Scope from the supplied parameters.
    80  // This is meant to be called for each reconcile iteration.
    81  func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlaneScopeParams) (*ManagedControlPlaneScope, error) {
    82  	ctx, _, done := tele.StartSpanWithLogger(ctx, "scope.NewManagedControlPlaneScope")
    83  	defer done()
    84  
    85  	if params.Cluster == nil {
    86  		return nil, errors.New("failed to generate new scope from nil Cluster")
    87  	}
    88  
    89  	if params.ControlPlane == nil {
    90  		return nil, errors.New("failed to generate new scope from nil ControlPlane")
    91  	}
    92  
    93  	credentialsProvider, err := NewAzureCredentialsProvider(ctx, params.Client, params.ControlPlane.Spec.IdentityRef, params.ControlPlane.Namespace)
    94  	if err != nil {
    95  		return nil, errors.Wrap(err, "failed to init credentials provider")
    96  	}
    97  
    98  	if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment, credentialsProvider); err != nil {
    99  		return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity")
   100  	}
   101  
   102  	if params.Cache == nil {
   103  		params.Cache = &ManagedControlPlaneCache{}
   104  	}
   105  
   106  	helper, err := patch.NewHelper(params.ControlPlane, params.Client)
   107  	if err != nil {
   108  		return nil, errors.Wrap(err, "failed to init patch helper")
   109  	}
   110  
   111  	return &ManagedControlPlaneScope{
   112  		Client:              params.Client,
   113  		AzureClients:        params.AzureClients,
   114  		Cluster:             params.Cluster,
   115  		ControlPlane:        params.ControlPlane,
   116  		ManagedMachinePools: params.ManagedMachinePools,
   117  		PatchHelper:         helper,
   118  		cache:               params.Cache,
   119  		AsyncReconciler:     params.Timeouts,
   120  	}, nil
   121  }
   122  
   123  // ManagedControlPlaneScope defines the basic context for an actuator to operate upon.
   124  type ManagedControlPlaneScope struct {
   125  	Client              client.Client
   126  	PatchHelper         *patch.Helper
   127  	adminKubeConfigData []byte
   128  	userKubeConfigData  []byte
   129  	cache               *ManagedControlPlaneCache
   130  
   131  	AzureClients
   132  	Cluster             *clusterv1.Cluster
   133  	ControlPlane        *infrav1.AzureManagedControlPlane
   134  	ManagedMachinePools []ManagedMachinePool
   135  	azure.AsyncReconciler
   136  }
   137  
   138  // ManagedControlPlaneCache stores ManagedControlPlane data locally so we don't have to hit the API multiple times within the same reconcile loop.
   139  type ManagedControlPlaneCache struct {
   140  	isVnetManaged *bool
   141  }
   142  
   143  // GetClient returns the controller-runtime client.
   144  func (s *ManagedControlPlaneScope) GetClient() client.Client {
   145  	return s.Client
   146  }
   147  
   148  // ASOOwner implements aso.Scope.
   149  func (s *ManagedControlPlaneScope) ASOOwner() client.Object {
   150  	return s.ControlPlane
   151  }
   152  
   153  // GetDeletionTimestamp returns the deletion timestamp of the cluster.
   154  func (s *ManagedControlPlaneScope) GetDeletionTimestamp() *metav1.Time {
   155  	return s.Cluster.DeletionTimestamp
   156  }
   157  
   158  // ResourceGroup returns the managed control plane's resource group.
   159  func (s *ManagedControlPlaneScope) ResourceGroup() string {
   160  	if s.ControlPlane == nil {
   161  		return ""
   162  	}
   163  	return s.ControlPlane.Spec.ResourceGroupName
   164  }
   165  
   166  // NodeResourceGroup returns the managed control plane's node resource group.
   167  func (s *ManagedControlPlaneScope) NodeResourceGroup() string {
   168  	if s.ControlPlane == nil {
   169  		return ""
   170  	}
   171  	return s.ControlPlane.Spec.NodeResourceGroupName
   172  }
   173  
   174  // ClusterName returns the managed control plane's name.
   175  func (s *ManagedControlPlaneScope) ClusterName() string {
   176  	return s.Cluster.Name
   177  }
   178  
   179  // Location returns the managed control plane's Azure location, or an empty string.
   180  func (s *ManagedControlPlaneScope) Location() string {
   181  	if s.ControlPlane == nil {
   182  		return ""
   183  	}
   184  	return s.ControlPlane.Spec.Location
   185  }
   186  
   187  // ExtendedLocation has not been implemented for AzureManagedControlPlane.
   188  func (s *ManagedControlPlaneScope) ExtendedLocation() *infrav1.ExtendedLocationSpec {
   189  	return nil
   190  }
   191  
   192  // ExtendedLocationName has not been implemented for AzureManagedControlPlane.
   193  func (s *ManagedControlPlaneScope) ExtendedLocationName() string {
   194  	return ""
   195  }
   196  
   197  // ExtendedLocationType has not been implemented for AzureManagedControlPlane.
   198  func (s *ManagedControlPlaneScope) ExtendedLocationType() string {
   199  	return ""
   200  }
   201  
   202  // AvailabilitySetEnabled is always false for a managed control plane.
   203  func (s *ManagedControlPlaneScope) AvailabilitySetEnabled() bool {
   204  	return false // not applicable for a managed control plane
   205  }
   206  
   207  // AdditionalTags returns AdditionalTags from the ControlPlane spec.
   208  func (s *ManagedControlPlaneScope) AdditionalTags() infrav1.Tags {
   209  	tags := make(infrav1.Tags)
   210  	if s.ControlPlane.Spec.AdditionalTags != nil {
   211  		tags = s.ControlPlane.Spec.AdditionalTags.DeepCopy()
   212  	}
   213  	return tags
   214  }
   215  
   216  // AzureFleetMembership returns the cluster AzureFleetMembership.
   217  func (s *ManagedControlPlaneScope) AzureFleetMembership() *infrav1.FleetsMember {
   218  	return s.ControlPlane.Spec.FleetsMember
   219  }
   220  
   221  // SubscriptionID returns the Azure client Subscription ID.
   222  func (s *ManagedControlPlaneScope) SubscriptionID() string {
   223  	return s.AzureClients.SubscriptionID()
   224  }
   225  
   226  // BaseURI returns the Azure ResourceManagerEndpoint.
   227  func (s *ManagedControlPlaneScope) BaseURI() string {
   228  	return s.AzureClients.ResourceManagerEndpoint
   229  }
   230  
   231  // PatchObject persists the cluster configuration and status.
   232  func (s *ManagedControlPlaneScope) PatchObject(ctx context.Context) error {
   233  	ctx, _, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.PatchObject")
   234  	defer done()
   235  
   236  	conditions.SetSummary(s.ControlPlane)
   237  
   238  	return s.PatchHelper.Patch(
   239  		ctx,
   240  		s.ControlPlane,
   241  		patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{
   242  			clusterv1.ReadyCondition,
   243  			infrav1.ResourceGroupReadyCondition,
   244  			infrav1.VNetReadyCondition,
   245  			infrav1.SubnetsReadyCondition,
   246  			infrav1.ManagedClusterRunningCondition,
   247  			infrav1.AgentPoolsReadyCondition,
   248  			infrav1.AzureResourceAvailableCondition,
   249  		}})
   250  }
   251  
   252  // Close closes the current scope persisting the cluster configuration and status.
   253  func (s *ManagedControlPlaneScope) Close(ctx context.Context) error {
   254  	ctx, _, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.Close")
   255  	defer done()
   256  
   257  	return s.PatchObject(ctx)
   258  }
   259  
   260  // Vnet returns the cluster Vnet.
   261  func (s *ManagedControlPlaneScope) Vnet() *infrav1.VnetSpec {
   262  	return &infrav1.VnetSpec{
   263  		ResourceGroup: s.ControlPlane.Spec.VirtualNetwork.ResourceGroup,
   264  		Name:          s.ControlPlane.Spec.VirtualNetwork.Name,
   265  		VnetClassSpec: infrav1.VnetClassSpec{
   266  			CIDRBlocks: []string{s.ControlPlane.Spec.VirtualNetwork.CIDRBlock},
   267  		},
   268  	}
   269  }
   270  
   271  // GroupSpecs returns the resource group spec.
   272  func (s *ManagedControlPlaneScope) GroupSpecs() []azure.ASOResourceSpecGetter[*asoresourcesv1.ResourceGroup] {
   273  	specs := []azure.ASOResourceSpecGetter[*asoresourcesv1.ResourceGroup]{
   274  		&groups.GroupSpec{
   275  			Name:           s.ResourceGroup(),
   276  			AzureName:      s.ResourceGroup(),
   277  			Location:       s.Location(),
   278  			ClusterName:    s.ClusterName(),
   279  			AdditionalTags: s.AdditionalTags(),
   280  		},
   281  	}
   282  	if s.Vnet().ResourceGroup != "" && s.Vnet().ResourceGroup != s.ResourceGroup() {
   283  		specs = append(specs, &groups.GroupSpec{
   284  			Name:           azure.GetNormalizedKubernetesName(s.Vnet().ResourceGroup),
   285  			AzureName:      s.Vnet().ResourceGroup,
   286  			Location:       s.Location(),
   287  			ClusterName:    s.ClusterName(),
   288  			AdditionalTags: s.AdditionalTags(),
   289  		})
   290  	}
   291  	return specs
   292  }
   293  
   294  // VNetSpec returns the virtual network spec.
   295  func (s *ManagedControlPlaneScope) VNetSpec() azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetwork] {
   296  	return &virtualnetworks.VNetSpec{
   297  		ResourceGroup:  s.Vnet().ResourceGroup,
   298  		Name:           s.Vnet().Name,
   299  		CIDRs:          s.Vnet().CIDRBlocks,
   300  		Location:       s.Location(),
   301  		ClusterName:    s.ClusterName(),
   302  		AdditionalTags: s.AdditionalTags(),
   303  	}
   304  }
   305  
   306  // AzureFleetsMemberSpec returns the fleet spec.
   307  func (s *ManagedControlPlaneScope) AzureFleetsMemberSpec() []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember] {
   308  	if s.AzureFleetMembership() == nil {
   309  		return nil
   310  	}
   311  	return []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember]{&fleetsmembers.AzureFleetsMemberSpec{
   312  		Name:                 s.AzureFleetMembership().Name,
   313  		ClusterName:          s.ClusterName(),
   314  		ClusterResourceGroup: s.ResourceGroup(),
   315  		Group:                s.AzureFleetMembership().Group,
   316  		SubscriptionID:       s.SubscriptionID(),
   317  		ManagerName:          s.AzureFleetMembership().ManagerName,
   318  		ManagerResourceGroup: s.AzureFleetMembership().ManagerResourceGroup,
   319  	}}
   320  }
   321  
   322  // ControlPlaneRouteTable returns the cluster controlplane routetable.
   323  func (s *ManagedControlPlaneScope) ControlPlaneRouteTable() infrav1.RouteTable {
   324  	return infrav1.RouteTable{}
   325  }
   326  
   327  // NodeRouteTable returns the cluster node routetable.
   328  func (s *ManagedControlPlaneScope) NodeRouteTable() infrav1.RouteTable {
   329  	return infrav1.RouteTable{}
   330  }
   331  
   332  // NodeNatGateway returns the cluster node NAT gateway.
   333  func (s *ManagedControlPlaneScope) NodeNatGateway() infrav1.NatGateway {
   334  	return infrav1.NatGateway{}
   335  }
   336  
   337  // SubnetSpecs returns the subnets specs.
   338  func (s *ManagedControlPlaneScope) SubnetSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetworksSubnet] {
   339  	return []azure.ASOResourceSpecGetter[*asonetworkv1api20201101.VirtualNetworksSubnet]{
   340  		&subnets.SubnetSpec{
   341  			Name:              s.NodeSubnet().Name,
   342  			ResourceGroup:     s.ResourceGroup(),
   343  			SubscriptionID:    s.SubscriptionID(),
   344  			CIDRs:             s.NodeSubnet().CIDRBlocks,
   345  			VNetName:          s.Vnet().Name,
   346  			VNetResourceGroup: s.Vnet().ResourceGroup,
   347  			IsVNetManaged:     s.IsVnetManaged(),
   348  			ServiceEndpoints:  s.NodeSubnet().ServiceEndpoints,
   349  		},
   350  	}
   351  }
   352  
   353  // Subnets returns the subnets specs.
   354  func (s *ManagedControlPlaneScope) Subnets() infrav1.Subnets {
   355  	return infrav1.Subnets{}
   356  }
   357  
   358  // NodeSubnet returns the cluster node subnet.
   359  func (s *ManagedControlPlaneScope) NodeSubnet() infrav1.SubnetSpec {
   360  	return infrav1.SubnetSpec{
   361  		SubnetClassSpec: infrav1.SubnetClassSpec{
   362  			CIDRBlocks:       []string{s.ControlPlane.Spec.VirtualNetwork.Subnet.CIDRBlock},
   363  			Name:             s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   364  			ServiceEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.ServiceEndpoints,
   365  			PrivateEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints,
   366  		},
   367  	}
   368  }
   369  
   370  // SetSubnet sets the passed subnet spec into the scope.
   371  // This is not used when using a managed control plane.
   372  func (s *ManagedControlPlaneScope) SetSubnet(_ infrav1.SubnetSpec) {
   373  	// no-op
   374  }
   375  
   376  // UpdateSubnetCIDRs updates the subnet CIDRs for the subnet with the same name.
   377  // This is not used when using a managed control plane.
   378  func (s *ManagedControlPlaneScope) UpdateSubnetCIDRs(_ string, _ []string) {
   379  	// no-op
   380  }
   381  
   382  // UpdateSubnetID updates the subnet ID for the subnet with the same name.
   383  // This is not used when using a managed control plane.
   384  func (s *ManagedControlPlaneScope) UpdateSubnetID(_ string, _ string) {
   385  	// no-op
   386  }
   387  
   388  // ControlPlaneSubnet returns the cluster control plane subnet.
   389  func (s *ManagedControlPlaneScope) ControlPlaneSubnet() infrav1.SubnetSpec {
   390  	return infrav1.SubnetSpec{}
   391  }
   392  
   393  // NodeSubnets returns the subnets with the node role.
   394  func (s *ManagedControlPlaneScope) NodeSubnets() []infrav1.SubnetSpec {
   395  	return []infrav1.SubnetSpec{
   396  		{
   397  			SubnetClassSpec: infrav1.SubnetClassSpec{
   398  				CIDRBlocks:       []string{s.ControlPlane.Spec.VirtualNetwork.Subnet.CIDRBlock},
   399  				Name:             s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   400  				ServiceEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.ServiceEndpoints,
   401  				PrivateEndpoints: s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints,
   402  			},
   403  		},
   404  	}
   405  }
   406  
   407  // Subnet returns the subnet with the provided name.
   408  func (s *ManagedControlPlaneScope) Subnet(name string) infrav1.SubnetSpec {
   409  	subnet := infrav1.SubnetSpec{}
   410  	if name == s.ControlPlane.Spec.VirtualNetwork.Subnet.Name {
   411  		subnet.Name = s.ControlPlane.Spec.VirtualNetwork.Subnet.Name
   412  		subnet.CIDRBlocks = []string{s.ControlPlane.Spec.VirtualNetwork.Subnet.CIDRBlock}
   413  		subnet.ServiceEndpoints = s.ControlPlane.Spec.VirtualNetwork.Subnet.ServiceEndpoints
   414  		subnet.PrivateEndpoints = s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints
   415  	}
   416  
   417  	return subnet
   418  }
   419  
   420  // IsIPv6Enabled returns true if a cluster is ipv6 enabled.
   421  // Currently always false as managed control planes do not currently implement ipv6.
   422  func (s *ManagedControlPlaneScope) IsIPv6Enabled() bool {
   423  	return false
   424  }
   425  
   426  // IsVnetManaged returns true if the vnet is managed.
   427  func (s *ManagedControlPlaneScope) IsVnetManaged() bool {
   428  	if s.cache.isVnetManaged != nil {
   429  		return ptr.Deref(s.cache.isVnetManaged, false)
   430  	}
   431  	// TODO refactor `IsVnetManaged` so that it is able to use an upstream context
   432  	// see https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/2581
   433  	ctx := context.Background()
   434  	ctx, log, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.IsVnetManaged")
   435  	defer done()
   436  
   437  	vnet := s.VNetSpec().ResourceRef()
   438  	vnet.SetNamespace(s.ASOOwner().GetNamespace())
   439  	err := s.Client.Get(ctx, client.ObjectKeyFromObject(vnet), vnet)
   440  	if err != nil {
   441  		log.Error(err, "Unable to determine if ManagedControlPlaneScope VNET is managed by capz, assuming unmanaged", "AzureManagedCluster", s.ClusterName())
   442  		return false
   443  	}
   444  
   445  	isManaged := infrav1.Tags(vnet.Status.Tags).HasOwned(s.ClusterName())
   446  	s.cache.isVnetManaged = ptr.To(isManaged)
   447  	return isManaged
   448  }
   449  
   450  // APIServerLB returns the API Server LB spec.
   451  func (s *ManagedControlPlaneScope) APIServerLB() *infrav1.LoadBalancerSpec {
   452  	return nil // does not apply for AKS
   453  }
   454  
   455  // APIServerLBName returns the API Server LB name.
   456  func (s *ManagedControlPlaneScope) APIServerLBName() string {
   457  	return "" // does not apply for AKS
   458  }
   459  
   460  // APIServerLBPoolName returns the API Server LB backend pool name.
   461  func (s *ManagedControlPlaneScope) APIServerLBPoolName() string {
   462  	return "" // does not apply for AKS
   463  }
   464  
   465  // IsAPIServerPrivate returns true if the API Server LB is of type Internal.
   466  // Currently always false as managed control planes do not currently implement private clusters.
   467  func (s *ManagedControlPlaneScope) IsAPIServerPrivate() bool {
   468  	return false
   469  }
   470  
   471  // OutboundLBName returns the name of the outbound LB.
   472  // Note: for managed clusters, the outbound LB lifecycle is not managed.
   473  func (s *ManagedControlPlaneScope) OutboundLBName(_ string) string {
   474  	return "kubernetes"
   475  }
   476  
   477  // OutboundPoolName returns the outbound LB backend pool name.
   478  func (s *ManagedControlPlaneScope) OutboundPoolName(_ string) string {
   479  	return "aksOutboundBackendPool" // hard-coded in aks
   480  }
   481  
   482  // GetPrivateDNSZoneName returns the Private DNS Zone from the spec or generate it from cluster name.
   483  // Currently always empty as managed control planes do not currently implement private clusters.
   484  func (s *ManagedControlPlaneScope) GetPrivateDNSZoneName() string {
   485  	return ""
   486  }
   487  
   488  // CloudProviderConfigOverrides returns the cloud provider config overrides for the cluster.
   489  func (s *ManagedControlPlaneScope) CloudProviderConfigOverrides() *infrav1.CloudProviderConfigOverrides {
   490  	return nil
   491  }
   492  
   493  // FailureDomains returns the failure domains for the cluster.
   494  func (s *ManagedControlPlaneScope) FailureDomains() []*string {
   495  	return []*string{}
   496  }
   497  
   498  // AreLocalAccountsDisabled checks if local accounts are disabled for aad enabled managed clusters.
   499  func (s *ManagedControlPlaneScope) AreLocalAccountsDisabled() bool {
   500  	if s.IsAADEnabled() &&
   501  		s.ControlPlane.Spec.DisableLocalAccounts != nil &&
   502  		*s.ControlPlane.Spec.DisableLocalAccounts {
   503  		return true
   504  	}
   505  	return false
   506  }
   507  
   508  // IsAADEnabled checks if azure active directory is enabled for managed clusters.
   509  func (s *ManagedControlPlaneScope) IsAADEnabled() bool {
   510  	if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed {
   511  		return true
   512  	}
   513  	return false
   514  }
   515  
   516  // SetVersionStatus sets the k8s version in status.
   517  func (s *ManagedControlPlaneScope) SetVersionStatus(version string) {
   518  	s.ControlPlane.Status.Version = version
   519  }
   520  
   521  // SetAutoUpgradeVersionStatus sets the auto upgrade version in status.
   522  func (s *ManagedControlPlaneScope) SetAutoUpgradeVersionStatus(version string) {
   523  	s.ControlPlane.Status.AutoUpgradeVersion = version
   524  }
   525  
   526  // IsManagedVersionUpgrade checks if version is auto managed by AKS.
   527  func (s *ManagedControlPlaneScope) IsManagedVersionUpgrade() bool {
   528  	return isManagedVersionUpgrade(s.ControlPlane)
   529  }
   530  
   531  func isManagedVersionUpgrade(managedControlPlane *infrav1.AzureManagedControlPlane) bool {
   532  	return managedControlPlane.Spec.AutoUpgradeProfile != nil &&
   533  		managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != nil &&
   534  		(*managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1.UpgradeChannelNone &&
   535  			*managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1.UpgradeChannelNodeImage)
   536  }
   537  
   538  // ManagedClusterSpec returns the managed cluster spec.
   539  func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ASOResourceSpecGetter[genruntime.MetaObject] {
   540  	managedClusterSpec := managedclusters.ManagedClusterSpec{
   541  		Name:              s.ControlPlane.Name,
   542  		ResourceGroup:     s.ControlPlane.Spec.ResourceGroupName,
   543  		NodeResourceGroup: s.ControlPlane.Spec.NodeResourceGroupName,
   544  		ClusterName:       s.ClusterName(),
   545  		Location:          s.ControlPlane.Spec.Location,
   546  		Tags:              s.ControlPlane.Spec.AdditionalTags,
   547  		Version:           strings.TrimPrefix(s.ControlPlane.Spec.Version, "v"),
   548  		DNSServiceIP:      s.ControlPlane.Spec.DNSServiceIP,
   549  		VnetSubnetID: azure.SubnetID(
   550  			s.ControlPlane.Spec.SubscriptionID,
   551  			s.Vnet().ResourceGroup,
   552  			s.ControlPlane.Spec.VirtualNetwork.Name,
   553  			s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   554  		),
   555  		GetAllAgentPools:            s.GetAllAgentPoolSpecs,
   556  		OutboundType:                s.ControlPlane.Spec.OutboundType,
   557  		Identity:                    s.ControlPlane.Spec.Identity,
   558  		KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity,
   559  		NetworkPluginMode:           s.ControlPlane.Spec.NetworkPluginMode,
   560  		DNSPrefix:                   s.ControlPlane.Spec.DNSPrefix,
   561  		Patches:                     s.ControlPlane.Spec.ASOManagedClusterPatches,
   562  		Preview:                     ptr.Deref(s.ControlPlane.Spec.EnablePreviewFeatures, false),
   563  	}
   564  
   565  	if s.ControlPlane.Spec.SSHPublicKey != nil {
   566  		managedClusterSpec.SSHPublicKey = *s.ControlPlane.Spec.SSHPublicKey
   567  	}
   568  	if s.ControlPlane.Spec.NetworkPlugin != nil {
   569  		managedClusterSpec.NetworkPlugin = *s.ControlPlane.Spec.NetworkPlugin
   570  	}
   571  	if s.ControlPlane.Spec.NetworkPolicy != nil {
   572  		managedClusterSpec.NetworkPolicy = *s.ControlPlane.Spec.NetworkPolicy
   573  	}
   574  	if s.ControlPlane.Spec.NetworkDataplane != nil {
   575  		managedClusterSpec.NetworkDataplane = s.ControlPlane.Spec.NetworkDataplane
   576  	}
   577  	if s.ControlPlane.Spec.LoadBalancerSKU != nil {
   578  		// CAPZ accepts Standard/Basic, Azure accepts standard/basic
   579  		managedClusterSpec.LoadBalancerSKU = strings.ToLower(*s.ControlPlane.Spec.LoadBalancerSKU)
   580  	}
   581  
   582  	if clusterNetwork := s.Cluster.Spec.ClusterNetwork; clusterNetwork != nil {
   583  		if clusterNetwork.Services != nil && len(clusterNetwork.Services.CIDRBlocks) == 1 {
   584  			managedClusterSpec.ServiceCIDR = clusterNetwork.Services.CIDRBlocks[0]
   585  		}
   586  		if clusterNetwork.Pods != nil && len(clusterNetwork.Pods.CIDRBlocks) == 1 {
   587  			managedClusterSpec.PodCIDR = clusterNetwork.Pods.CIDRBlocks[0]
   588  		}
   589  	}
   590  
   591  	if s.ControlPlane.Spec.AADProfile != nil {
   592  		managedClusterSpec.AADProfile = &managedclusters.AADProfile{
   593  			Managed:             s.ControlPlane.Spec.AADProfile.Managed,
   594  			EnableAzureRBAC:     s.ControlPlane.Spec.AADProfile.Managed,
   595  			AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs,
   596  		}
   597  		if s.ControlPlane.Spec.DisableLocalAccounts != nil {
   598  			managedClusterSpec.DisableLocalAccounts = s.ControlPlane.Spec.DisableLocalAccounts
   599  		}
   600  	}
   601  
   602  	if s.ControlPlane.Spec.AddonProfiles != nil {
   603  		for _, profile := range s.ControlPlane.Spec.AddonProfiles {
   604  			managedClusterSpec.AddonProfiles = append(managedClusterSpec.AddonProfiles, managedclusters.AddonProfile{
   605  				Name:    profile.Name,
   606  				Enabled: profile.Enabled,
   607  				Config:  profile.Config,
   608  			})
   609  		}
   610  	}
   611  
   612  	if s.ControlPlane.Spec.SKU != nil {
   613  		managedClusterSpec.SKU = &managedclusters.SKU{
   614  			Tier: string(s.ControlPlane.Spec.SKU.Tier),
   615  		}
   616  	}
   617  
   618  	if s.ControlPlane.Spec.LoadBalancerProfile != nil {
   619  		managedClusterSpec.LoadBalancerProfile = &managedclusters.LoadBalancerProfile{
   620  			ManagedOutboundIPs:     s.ControlPlane.Spec.LoadBalancerProfile.ManagedOutboundIPs,
   621  			OutboundIPPrefixes:     s.ControlPlane.Spec.LoadBalancerProfile.OutboundIPPrefixes,
   622  			OutboundIPs:            s.ControlPlane.Spec.LoadBalancerProfile.OutboundIPs,
   623  			AllocatedOutboundPorts: s.ControlPlane.Spec.LoadBalancerProfile.AllocatedOutboundPorts,
   624  			IdleTimeoutInMinutes:   s.ControlPlane.Spec.LoadBalancerProfile.IdleTimeoutInMinutes,
   625  		}
   626  	}
   627  
   628  	if s.ControlPlane.Spec.APIServerAccessProfile != nil {
   629  		managedClusterSpec.APIServerAccessProfile = &managedclusters.APIServerAccessProfile{
   630  			AuthorizedIPRanges:             s.ControlPlane.Spec.APIServerAccessProfile.AuthorizedIPRanges,
   631  			EnablePrivateCluster:           s.ControlPlane.Spec.APIServerAccessProfile.EnablePrivateCluster,
   632  			PrivateDNSZone:                 s.ControlPlane.Spec.APIServerAccessProfile.PrivateDNSZone,
   633  			EnablePrivateClusterPublicFQDN: s.ControlPlane.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN,
   634  		}
   635  	}
   636  
   637  	if s.ControlPlane.Spec.AutoScalerProfile != nil {
   638  		managedClusterSpec.AutoScalerProfile = &managedclusters.AutoScalerProfile{
   639  			BalanceSimilarNodeGroups:      (*string)(s.ControlPlane.Spec.AutoScalerProfile.BalanceSimilarNodeGroups),
   640  			Expander:                      (*string)(s.ControlPlane.Spec.AutoScalerProfile.Expander),
   641  			MaxEmptyBulkDelete:            s.ControlPlane.Spec.AutoScalerProfile.MaxEmptyBulkDelete,
   642  			MaxGracefulTerminationSec:     s.ControlPlane.Spec.AutoScalerProfile.MaxGracefulTerminationSec,
   643  			MaxNodeProvisionTime:          s.ControlPlane.Spec.AutoScalerProfile.MaxNodeProvisionTime,
   644  			MaxTotalUnreadyPercentage:     s.ControlPlane.Spec.AutoScalerProfile.MaxTotalUnreadyPercentage,
   645  			NewPodScaleUpDelay:            s.ControlPlane.Spec.AutoScalerProfile.NewPodScaleUpDelay,
   646  			OkTotalUnreadyCount:           s.ControlPlane.Spec.AutoScalerProfile.OkTotalUnreadyCount,
   647  			ScanInterval:                  s.ControlPlane.Spec.AutoScalerProfile.ScanInterval,
   648  			ScaleDownDelayAfterAdd:        s.ControlPlane.Spec.AutoScalerProfile.ScaleDownDelayAfterAdd,
   649  			ScaleDownDelayAfterDelete:     s.ControlPlane.Spec.AutoScalerProfile.ScaleDownDelayAfterDelete,
   650  			ScaleDownDelayAfterFailure:    s.ControlPlane.Spec.AutoScalerProfile.ScaleDownDelayAfterFailure,
   651  			ScaleDownUnneededTime:         s.ControlPlane.Spec.AutoScalerProfile.ScaleDownUnneededTime,
   652  			ScaleDownUnreadyTime:          s.ControlPlane.Spec.AutoScalerProfile.ScaleDownUnreadyTime,
   653  			ScaleDownUtilizationThreshold: s.ControlPlane.Spec.AutoScalerProfile.ScaleDownUtilizationThreshold,
   654  			SkipNodesWithLocalStorage:     (*string)(s.ControlPlane.Spec.AutoScalerProfile.SkipNodesWithLocalStorage),
   655  			SkipNodesWithSystemPods:       (*string)(s.ControlPlane.Spec.AutoScalerProfile.SkipNodesWithSystemPods),
   656  		}
   657  	}
   658  
   659  	if s.ControlPlane.Spec.HTTPProxyConfig != nil {
   660  		managedClusterSpec.HTTPProxyConfig = &managedclusters.HTTPProxyConfig{
   661  			HTTPProxy:  s.ControlPlane.Spec.HTTPProxyConfig.HTTPProxy,
   662  			HTTPSProxy: s.ControlPlane.Spec.HTTPProxyConfig.HTTPSProxy,
   663  			NoProxy:    s.ControlPlane.Spec.HTTPProxyConfig.NoProxy,
   664  			TrustedCA:  s.ControlPlane.Spec.HTTPProxyConfig.TrustedCA,
   665  		}
   666  	}
   667  
   668  	if s.ControlPlane.Spec.OIDCIssuerProfile != nil {
   669  		managedClusterSpec.OIDCIssuerProfile = &managedclusters.OIDCIssuerProfile{
   670  			Enabled: s.ControlPlane.Spec.OIDCIssuerProfile.Enabled,
   671  		}
   672  	}
   673  
   674  	if s.ControlPlane.Spec.AutoUpgradeProfile != nil {
   675  		managedClusterSpec.AutoUpgradeProfile = &managedclusters.ManagedClusterAutoUpgradeProfile{}
   676  		if s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != nil {
   677  			managedClusterSpec.AutoUpgradeProfile.UpgradeChannel = s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel
   678  		}
   679  	}
   680  
   681  	if s.ControlPlane.Spec.SecurityProfile != nil {
   682  		managedClusterSpec.SecurityProfile = s.getManagedClusterSecurityProfile()
   683  	}
   684  
   685  	return &managedClusterSpec
   686  }
   687  
   688  // GetManagedClusterSecurityProfile gets the security profile for managed cluster.
   689  func (s *ManagedControlPlaneScope) getManagedClusterSecurityProfile() *managedclusters.ManagedClusterSecurityProfile {
   690  	securityProfile := &managedclusters.ManagedClusterSecurityProfile{}
   691  	if s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms != nil {
   692  		securityProfile.AzureKeyVaultKms = &managedclusters.AzureKeyVaultKms{
   693  			Enabled: ptr.To(s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.Enabled),
   694  			KeyID:   ptr.To(s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyID),
   695  		}
   696  		if s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess != nil {
   697  			securityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess = s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess
   698  		}
   699  		if s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID != nil {
   700  			securityProfile.AzureKeyVaultKms.KeyVaultResourceID = s.ControlPlane.Spec.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID
   701  		}
   702  	}
   703  
   704  	if s.ControlPlane.Spec.SecurityProfile.Defender != nil {
   705  		securityProfile.Defender = &managedclusters.ManagedClusterSecurityProfileDefender{
   706  			LogAnalyticsWorkspaceResourceID: ptr.To(s.ControlPlane.Spec.SecurityProfile.Defender.LogAnalyticsWorkspaceResourceID),
   707  			SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{
   708  				Enabled: ptr.To(s.ControlPlane.Spec.SecurityProfile.Defender.SecurityMonitoring.Enabled),
   709  			},
   710  		}
   711  	}
   712  
   713  	if s.ControlPlane.Spec.SecurityProfile.ImageCleaner != nil {
   714  		securityProfile.ImageCleaner = &managedclusters.ManagedClusterSecurityProfileImageCleaner{
   715  			Enabled:       ptr.To(s.ControlPlane.Spec.SecurityProfile.ImageCleaner.Enabled),
   716  			IntervalHours: s.ControlPlane.Spec.SecurityProfile.ImageCleaner.IntervalHours,
   717  		}
   718  	}
   719  
   720  	if s.ControlPlane.Spec.SecurityProfile.WorkloadIdentity != nil {
   721  		securityProfile.WorkloadIdentity = &managedclusters.ManagedClusterSecurityProfileWorkloadIdentity{
   722  			Enabled: ptr.To(s.ControlPlane.Spec.SecurityProfile.WorkloadIdentity.Enabled),
   723  		}
   724  	}
   725  
   726  	return securityProfile
   727  }
   728  
   729  // GetAllAgentPoolSpecs gets a slice of azure.AgentPoolSpec for the list of agent pools.
   730  func (s *ManagedControlPlaneScope) GetAllAgentPoolSpecs() ([]azure.ASOResourceSpecGetter[genruntime.MetaObject], error) {
   731  	var (
   732  		ammps           = make([]azure.ASOResourceSpecGetter[genruntime.MetaObject], 0, len(s.ManagedMachinePools))
   733  		foundSystemPool = false
   734  	)
   735  	for _, pool := range s.ManagedMachinePools {
   736  		// TODO: this should be in a webhook: https://github.com/kubernetes-sigs/cluster-api/issues/6040
   737  		if pool.MachinePool != nil && pool.MachinePool.Spec.Template.Spec.Version != nil {
   738  			version := *pool.MachinePool.Spec.Template.Spec.Version
   739  			if semver.Compare(version, s.ControlPlane.Spec.Version) > 0 {
   740  				return nil, errors.New("MachinePool version cannot be greater than the AzureManagedControlPlane version")
   741  			}
   742  		}
   743  
   744  		if pool.InfraMachinePool != nil && pool.InfraMachinePool.Spec.Mode == string(infrav1.NodePoolModeSystem) {
   745  			foundSystemPool = true
   746  		}
   747  
   748  		ammp := buildAgentPoolSpec(s.ControlPlane, pool.MachinePool, pool.InfraMachinePool)
   749  		ammps = append(ammps, ammp)
   750  	}
   751  
   752  	if !foundSystemPool {
   753  		return nil, errors.New("failed to fetch azuremanagedMachine pool with mode:System, require at least 1 system node pool")
   754  	}
   755  
   756  	return ammps, nil
   757  }
   758  
   759  // SetControlPlaneEndpoint sets a control plane endpoint.
   760  func (s *ManagedControlPlaneScope) SetControlPlaneEndpoint(endpoint clusterv1.APIEndpoint) {
   761  	s.ControlPlane.Spec.ControlPlaneEndpoint.Host = endpoint.Host
   762  	s.ControlPlane.Spec.ControlPlaneEndpoint.Port = endpoint.Port
   763  }
   764  
   765  // MakeEmptyKubeConfigSecret creates an empty secret object that is used for storing kubeconfig secret data.
   766  func (s *ManagedControlPlaneScope) MakeEmptyKubeConfigSecret() corev1.Secret {
   767  	return corev1.Secret{
   768  		ObjectMeta: metav1.ObjectMeta{
   769  			Name:      secret.Name(s.Cluster.Name, secret.Kubeconfig),
   770  			Namespace: s.Cluster.Namespace,
   771  			OwnerReferences: []metav1.OwnerReference{
   772  				*metav1.NewControllerRef(s.ControlPlane, infrav1.GroupVersion.WithKind(infrav1.AzureManagedControlPlaneKind)),
   773  			},
   774  			Labels: map[string]string{clusterv1.ClusterNameLabel: s.Cluster.Name},
   775  		},
   776  	}
   777  }
   778  
   779  // GetAdminKubeconfigData returns admin kubeconfig.
   780  func (s *ManagedControlPlaneScope) GetAdminKubeconfigData() []byte {
   781  	return s.adminKubeConfigData
   782  }
   783  
   784  // SetAdminKubeconfigData sets admin kubeconfig data.
   785  func (s *ManagedControlPlaneScope) SetAdminKubeconfigData(kubeConfigData []byte) {
   786  	s.adminKubeConfigData = kubeConfigData
   787  }
   788  
   789  // GetUserKubeconfigData returns user kubeconfig, required when using AAD with AKS cluster.
   790  func (s *ManagedControlPlaneScope) GetUserKubeconfigData() []byte {
   791  	return s.userKubeConfigData
   792  }
   793  
   794  // SetUserKubeconfigData sets userKubeconfig data.
   795  func (s *ManagedControlPlaneScope) SetUserKubeconfigData(kubeConfigData []byte) {
   796  	s.userKubeConfigData = kubeConfigData
   797  }
   798  
   799  // MakeClusterCA returns a cluster CA Secret for the managed control plane.
   800  func (s *ManagedControlPlaneScope) MakeClusterCA() *corev1.Secret {
   801  	return &corev1.Secret{
   802  		ObjectMeta: metav1.ObjectMeta{
   803  			Name:      secret.Name(s.Cluster.Name, secret.ClusterCA),
   804  			Namespace: s.Cluster.Namespace,
   805  			OwnerReferences: []metav1.OwnerReference{
   806  				*metav1.NewControllerRef(s.ControlPlane, infrav1.GroupVersion.WithKind(infrav1.AzureManagedControlPlaneKind)),
   807  			},
   808  		},
   809  	}
   810  }
   811  
   812  // StoreClusterInfo stores the discovery cluster-info configmap in the kube-public namespace on the AKS cluster so kubeadm can access it to join nodes.
   813  func (s *ManagedControlPlaneScope) StoreClusterInfo(ctx context.Context, caData []byte) error {
   814  	remoteclient, err := remote.NewClusterClient(ctx, managedControlPlaneScopeName, s.Client, types.NamespacedName{
   815  		Namespace: s.Cluster.Namespace,
   816  		Name:      s.Cluster.Name,
   817  	})
   818  	if err != nil {
   819  		return errors.Wrap(err, "failed to create remote cluster kubeclient")
   820  	}
   821  
   822  	discoveryFile := clientcmdapi.NewConfig()
   823  	discoveryFile.Clusters[""] = &clientcmdapi.Cluster{
   824  		CertificateAuthorityData: caData,
   825  		Server: fmt.Sprintf(
   826  			"%s:%d",
   827  			s.ControlPlane.Spec.ControlPlaneEndpoint.Host,
   828  			s.ControlPlane.Spec.ControlPlaneEndpoint.Port,
   829  		),
   830  	}
   831  
   832  	data, err := yaml.Marshal(&discoveryFile)
   833  	if err != nil {
   834  		return errors.Wrap(err, "failed to serialize cluster-info to yaml")
   835  	}
   836  
   837  	clusterInfo := &corev1.ConfigMap{
   838  		ObjectMeta: metav1.ObjectMeta{
   839  			Name:      bootstrapapi.ConfigMapClusterInfo,
   840  			Namespace: metav1.NamespacePublic,
   841  		},
   842  		Data: map[string]string{
   843  			bootstrapapi.KubeConfigKey: string(data),
   844  		},
   845  	}
   846  
   847  	if _, err := controllerutil.CreateOrUpdate(ctx, remoteclient, clusterInfo, func() error {
   848  		clusterInfo.Data[bootstrapapi.KubeConfigKey] = string(data)
   849  		return nil
   850  	}); err != nil {
   851  		return errors.Wrapf(err, "failed to reconcile certificate authority data secret for cluster")
   852  	}
   853  
   854  	return nil
   855  }
   856  
   857  // SetLongRunningOperationState will set the future on the AzureManagedControlPlane status to allow the resource to continue
   858  // in the next reconciliation.
   859  func (s *ManagedControlPlaneScope) SetLongRunningOperationState(future *infrav1.Future) {
   860  	futures.Set(s.ControlPlane, future)
   861  }
   862  
   863  // GetLongRunningOperationState will get the future on the AzureManagedControlPlane status.
   864  func (s *ManagedControlPlaneScope) GetLongRunningOperationState(name, service, futureType string) *infrav1.Future {
   865  	return futures.Get(s.ControlPlane, name, service, futureType)
   866  }
   867  
   868  // DeleteLongRunningOperationState will delete the future from the AzureManagedControlPlane status.
   869  func (s *ManagedControlPlaneScope) DeleteLongRunningOperationState(name, service, futureType string) {
   870  	futures.Delete(s.ControlPlane, name, service, futureType)
   871  }
   872  
   873  // UpdateDeleteStatus updates a condition on the AzureManagedControlPlane status after a DELETE operation.
   874  func (s *ManagedControlPlaneScope) UpdateDeleteStatus(condition clusterv1.ConditionType, service string, err error) {
   875  	switch {
   876  	case err == nil:
   877  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.DeletedReason, clusterv1.ConditionSeverityInfo, "%s successfully deleted", service)
   878  	case azure.IsOperationNotDoneError(err):
   879  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.DeletingReason, clusterv1.ConditionSeverityInfo, "%s deleting", service)
   880  	default:
   881  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.DeletionFailedReason, clusterv1.ConditionSeverityError, "%s failed to delete. err: %s", service, err.Error())
   882  	}
   883  }
   884  
   885  // UpdatePutStatus updates a condition on the AzureManagedControlPlane status after a PUT operation.
   886  func (s *ManagedControlPlaneScope) UpdatePutStatus(condition clusterv1.ConditionType, service string, err error) {
   887  	switch {
   888  	case err == nil:
   889  		conditions.MarkTrue(s.ControlPlane, condition)
   890  	case azure.IsOperationNotDoneError(err):
   891  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.CreatingReason, clusterv1.ConditionSeverityInfo, "%s creating or updating", service)
   892  	default:
   893  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.FailedReason, clusterv1.ConditionSeverityError, "%s failed to create or update. err: %s", service, err.Error())
   894  	}
   895  }
   896  
   897  // UpdatePatchStatus updates a condition on the AzureManagedControlPlane status after a PATCH operation.
   898  func (s *ManagedControlPlaneScope) UpdatePatchStatus(condition clusterv1.ConditionType, service string, err error) {
   899  	switch {
   900  	case err == nil:
   901  		conditions.MarkTrue(s.ControlPlane, condition)
   902  	case azure.IsOperationNotDoneError(err):
   903  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.UpdatingReason, clusterv1.ConditionSeverityInfo, "%s updating", service)
   904  	default:
   905  		conditions.MarkFalse(s.ControlPlane, condition, infrav1.FailedReason, clusterv1.ConditionSeverityError, "%s failed to update. err: %s", service, err.Error())
   906  	}
   907  }
   908  
   909  // AnnotationJSON returns a map[string]interface from a JSON annotation.
   910  func (s *ManagedControlPlaneScope) AnnotationJSON(annotation string) (map[string]interface{}, error) {
   911  	out := map[string]interface{}{}
   912  	jsonAnnotation := s.ControlPlane.GetAnnotations()[annotation]
   913  	if jsonAnnotation == "" {
   914  		return out, nil
   915  	}
   916  	err := json.Unmarshal([]byte(jsonAnnotation), &out)
   917  	if err != nil {
   918  		return out, err
   919  	}
   920  	return out, nil
   921  }
   922  
   923  // UpdateAnnotationJSON updates the `annotation` with
   924  // `content`. `content` in this case should be a `map[string]interface{}`
   925  // suitable for turning into JSON. This `content` map will be marshalled into a
   926  // JSON string before being set as the given `annotation`.
   927  func (s *ManagedControlPlaneScope) UpdateAnnotationJSON(annotation string, content map[string]interface{}) error {
   928  	b, err := json.Marshal(content)
   929  	if err != nil {
   930  		return err
   931  	}
   932  	s.SetAnnotation(annotation, string(b))
   933  	return nil
   934  }
   935  
   936  // SetAnnotation sets a key value annotation on the ControlPlane.
   937  func (s *ManagedControlPlaneScope) SetAnnotation(key, value string) {
   938  	if s.ControlPlane.Annotations == nil {
   939  		s.ControlPlane.Annotations = map[string]string{}
   940  	}
   941  	s.ControlPlane.Annotations[key] = value
   942  }
   943  
   944  // AvailabilityStatusResource refers to the AzureManagedControlPlane.
   945  func (s *ManagedControlPlaneScope) AvailabilityStatusResource() conditions.Setter {
   946  	return s.ControlPlane
   947  }
   948  
   949  // AvailabilityStatusResourceURI constructs the ID of the underlying AKS resource.
   950  func (s *ManagedControlPlaneScope) AvailabilityStatusResourceURI() string {
   951  	return azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name)
   952  }
   953  
   954  // AvailabilityStatusFilter ignores the health metrics connection error that
   955  // occurs on startup for every AKS cluster.
   956  func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Condition) *clusterv1.Condition {
   957  	if time.Since(s.ControlPlane.CreationTimestamp.Time) < resourceHealthWarningInitialGracePeriod &&
   958  		cond.Severity == clusterv1.ConditionSeverityWarning {
   959  		return conditions.TrueCondition(infrav1.AzureResourceAvailableCondition)
   960  	}
   961  	return cond
   962  }
   963  
   964  // PrivateEndpointSpecs returns the private endpoint specs.
   965  func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1api20220701.PrivateEndpoint] {
   966  	privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1api20220701.PrivateEndpoint], 0, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))
   967  
   968  	for _, privateEndpoint := range s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints {
   969  		privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
   970  			Name:                       privateEndpoint.Name,
   971  			ResourceGroup:              s.Vnet().ResourceGroup,
   972  			Location:                   privateEndpoint.Location,
   973  			CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
   974  			PrivateIPAddresses:         privateEndpoint.PrivateIPAddresses,
   975  			SubnetID: azure.SubnetID(
   976  				s.ControlPlane.Spec.SubscriptionID,
   977  				s.Vnet().ResourceGroup,
   978  				s.ControlPlane.Spec.VirtualNetwork.Name,
   979  				s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
   980  			),
   981  			ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
   982  			ManualApproval:            privateEndpoint.ManualApproval,
   983  			ClusterName:               s.ClusterName(),
   984  			AdditionalTags:            s.AdditionalTags(),
   985  		}
   986  
   987  		for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
   988  			pl := privateendpoints.PrivateLinkServiceConnection{
   989  				PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
   990  				Name:                 privateLinkServiceConnection.Name,
   991  				RequestMessage:       privateLinkServiceConnection.RequestMessage,
   992  				GroupIDs:             privateLinkServiceConnection.GroupIDs,
   993  			}
   994  			privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
   995  		}
   996  		privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
   997  	}
   998  
   999  	return privateEndpointSpecs
  1000  }
  1001  
  1002  // SetOIDCIssuerProfileStatus sets the status for the OIDC issuer profile config.
  1003  func (s *ManagedControlPlaneScope) SetOIDCIssuerProfileStatus(oidc *infrav1.OIDCIssuerProfileStatus) {
  1004  	s.ControlPlane.Status.OIDCIssuerProfile = oidc
  1005  }
  1006  
  1007  // AKSExtension returns the cluster AKS extensions.
  1008  func (s *ManagedControlPlaneScope) AKSExtension() []infrav1.AKSExtension {
  1009  	return s.ControlPlane.Spec.Extensions
  1010  }
  1011  
  1012  // AKSExtensionSpecs returns the AKS extension specs.
  1013  func (s *ManagedControlPlaneScope) AKSExtensionSpecs() []azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension] {
  1014  	if s.AKSExtension() == nil {
  1015  		return nil
  1016  	}
  1017  	extensionSpecs := make([]azure.ASOResourceSpecGetter[*asokubernetesconfigurationv1.Extension], 0, len(s.ControlPlane.Spec.Extensions))
  1018  	for _, extension := range s.AKSExtension() {
  1019  		extensionSpec := &aksextensions.AKSExtensionSpec{
  1020  			Name:                    extension.Name,
  1021  			Namespace:               s.Cluster.Namespace,
  1022  			AutoUpgradeMinorVersion: extension.AutoUpgradeMinorVersion,
  1023  			ConfigurationSettings:   extension.ConfigurationSettings,
  1024  			ExtensionType:           extension.ExtensionType,
  1025  			ReleaseTrain:            extension.ReleaseTrain,
  1026  			Version:                 extension.Version,
  1027  			Owner:                   azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name),
  1028  			Plan:                    extension.Plan,
  1029  			AKSAssignedIdentityType: extension.AKSAssignedIdentityType,
  1030  			ExtensionIdentity:       extension.Identity,
  1031  		}
  1032  
  1033  		extensionSpecs = append(extensionSpecs, extensionSpec)
  1034  	}
  1035  
  1036  	return extensionSpecs
  1037  }