sigs.k8s.io/cluster-api-provider-azure@v1.17.0/azure/services/managedclusters/spec.go (about)

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package managedclusters
    18  
    19  import (
    20  	"context"
    21  	"encoding/base64"
    22  	"fmt"
    23  	"net"
    24  
    25  	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
    26  	asocontainerservicev1hub "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001/storage"
    27  	asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231102preview"
    28  	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
    29  	"github.com/pkg/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/runtime"
    32  	"k8s.io/utils/ptr"
    33  	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
    34  	"sigs.k8s.io/cluster-api-provider-azure/azure"
    35  	"sigs.k8s.io/cluster-api-provider-azure/azure/converters"
    36  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools"
    37  	"sigs.k8s.io/cluster-api-provider-azure/azure/services/aso"
    38  	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
    39  	"sigs.k8s.io/cluster-api-provider-azure/util/versions"
    40  	"sigs.k8s.io/cluster-api/util/secret"
    41  	"sigs.k8s.io/controller-runtime/pkg/conversion"
    42  )
    43  
    44  // ManagedClusterSpec contains properties to create a managed cluster.
    45  type ManagedClusterSpec struct {
    46  	// Name is the name of this AKS Cluster.
    47  	Name string
    48  
    49  	// ResourceGroup is the name of the Azure resource group for this AKS Cluster.
    50  	ResourceGroup string
    51  
    52  	// NodeResourceGroup is the name of the Azure resource group containing IaaS VMs.
    53  	NodeResourceGroup string
    54  
    55  	// ClusterName is the name of the owning Cluster API Cluster resource.
    56  	ClusterName string
    57  
    58  	// VnetSubnetID is the Azure Resource ID for the subnet which should contain nodes.
    59  	VnetSubnetID string
    60  
    61  	// Location is a string matching one of the canonical Azure region names. Examples: "westus2", "eastus".
    62  	Location string
    63  
    64  	// Tags is a set of tags to add to this cluster.
    65  	Tags map[string]string
    66  
    67  	// Version defines the desired Kubernetes version.
    68  	Version string
    69  
    70  	// LoadBalancerSKU for the managed cluster. Possible values include: 'Standard', 'Basic'. Defaults to Standard.
    71  	LoadBalancerSKU string
    72  
    73  	// NetworkPlugin used for building Kubernetes network. Possible values include: 'azure', 'kubenet'. Defaults to azure.
    74  	NetworkPlugin string
    75  
    76  	// NetworkPluginMode is the mode the network plugin should use.
    77  	NetworkPluginMode *infrav1.NetworkPluginMode
    78  
    79  	// NetworkPolicy used for building Kubernetes network. Possible values include: 'azure', 'calico', 'cilium'.
    80  	NetworkPolicy string
    81  
    82  	// NetworkDataplane used for building Kubernetes network. Possible values include: 'azure', 'cilium'.
    83  	NetworkDataplane *infrav1.NetworkDataplaneType
    84  
    85  	// OutboundType used for building Kubernetes network. Possible values include: 'loadBalancer', 'managedNATGateway', 'userAssignedNATGateway', 'userDefinedRouting'.
    86  	OutboundType *infrav1.ManagedControlPlaneOutboundType
    87  
    88  	// SSHPublicKey is a string literal containing an ssh public key. Will autogenerate and discard if not provided.
    89  	SSHPublicKey string
    90  
    91  	// GetAllAgentPools is a function that returns the list of agent pool specifications in this cluster.
    92  	GetAllAgentPools func() ([]azure.ASOResourceSpecGetter[genruntime.MetaObject], error)
    93  
    94  	// PodCIDR is the CIDR block for IP addresses distributed to pods
    95  	PodCIDR string
    96  
    97  	// ServiceCIDR is the CIDR block for IP addresses distributed to services
    98  	ServiceCIDR string
    99  
   100  	// DNSServiceIP is an IP address assigned to the Kubernetes DNS service
   101  	DNSServiceIP *string
   102  
   103  	// AddonProfiles are the profiles of managed cluster add-on.
   104  	AddonProfiles []AddonProfile
   105  
   106  	// AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication.
   107  	AADProfile *AADProfile
   108  
   109  	// SKU is the SKU of the AKS to be provisioned.
   110  	SKU *SKU
   111  
   112  	// LoadBalancerProfile is the profile of the cluster load balancer.
   113  	LoadBalancerProfile *LoadBalancerProfile
   114  
   115  	// APIServerAccessProfile is the access profile for AKS API server.
   116  	APIServerAccessProfile *APIServerAccessProfile
   117  
   118  	// AutoScalerProfile is the parameters to be applied to the cluster-autoscaler when enabled.
   119  	AutoScalerProfile *AutoScalerProfile
   120  
   121  	// Identity is the AKS control plane Identity configuration
   122  	Identity *infrav1.Identity
   123  
   124  	// KubeletUserAssignedIdentity is the user-assigned identity for kubelet to authenticate to ACR.
   125  	KubeletUserAssignedIdentity string
   126  
   127  	// HTTPProxyConfig is the HTTP proxy configuration for the cluster.
   128  	HTTPProxyConfig *HTTPProxyConfig
   129  
   130  	// OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster.
   131  	OIDCIssuerProfile *OIDCIssuerProfile
   132  
   133  	// DNSPrefix allows the user to customize dns prefix.
   134  	DNSPrefix *string
   135  
   136  	// DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters.
   137  	DisableLocalAccounts *bool
   138  
   139  	// AutoUpgradeProfile defines auto upgrade configuration.
   140  	AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile
   141  
   142  	// SecurityProfile defines the security profile for the cluster.
   143  	SecurityProfile *ManagedClusterSecurityProfile
   144  
   145  	// Patches are extra patches to be applied to the ASO resource.
   146  	Patches []string
   147  
   148  	// Preview enables the preview API version.
   149  	Preview bool
   150  }
   151  
   152  // ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster.
   153  type ManagedClusterAutoUpgradeProfile struct {
   154  	// UpgradeChannel defines the channel for auto upgrade configuration.
   155  	UpgradeChannel *infrav1.UpgradeChannel
   156  }
   157  
   158  // HTTPProxyConfig is the HTTP proxy configuration for the cluster.
   159  type HTTPProxyConfig struct {
   160  	// HTTPProxy is the HTTP proxy server endpoint to use.
   161  	HTTPProxy *string `json:"httpProxy,omitempty"`
   162  
   163  	// HTTPSProxy is the HTTPS proxy server endpoint to use.
   164  	HTTPSProxy *string `json:"httpsProxy,omitempty"`
   165  
   166  	// NoProxy is the endpoints that should not go through proxy.
   167  	NoProxy []string `json:"noProxy,omitempty"`
   168  
   169  	// TrustedCA is the Alternative CA cert to use for connecting to proxy servers.
   170  	TrustedCA *string `json:"trustedCa,omitempty"`
   171  }
   172  
   173  // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication.
   174  type AADProfile struct {
   175  	// Managed defines whether to enable managed AAD.
   176  	Managed bool
   177  
   178  	// EnableAzureRBAC defines whether to enable Azure RBAC for Kubernetes authorization.
   179  	EnableAzureRBAC bool
   180  
   181  	// AdminGroupObjectIDs are the AAD group object IDs that will have admin role of the cluster.
   182  	AdminGroupObjectIDs []string
   183  }
   184  
   185  // AddonProfile is the profile of a managed cluster add-on.
   186  type AddonProfile struct {
   187  	Name    string
   188  	Config  map[string]string
   189  	Enabled bool
   190  }
   191  
   192  // SKU is an AKS SKU.
   193  type SKU struct {
   194  	// Tier is the tier of a managed cluster SKU.
   195  	Tier string
   196  }
   197  
   198  // LoadBalancerProfile is the profile of the cluster load balancer.
   199  type LoadBalancerProfile struct {
   200  	// Load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs.
   201  	// By default the AKS cluster automatically creates a public IP in the AKS-managed infrastructure resource group and assigns it to the load balancer outbound pool.
   202  	// Alternatively, you can assign your own custom public IP or public IP prefix at cluster creation time.
   203  	// See https://learn.microsoft.com/azure/aks/load-balancer-standard#provide-your-own-outbound-public-ips-or-prefixes
   204  
   205  	// ManagedOutboundIPs are the desired managed outbound IPs for the cluster load balancer.
   206  	ManagedOutboundIPs *int
   207  
   208  	// OutboundIPPrefixes are the desired outbound IP Prefix resources for the cluster load balancer.
   209  	OutboundIPPrefixes []string
   210  
   211  	// OutboundIPs are the desired outbound IP resources for the cluster load balancer.
   212  	OutboundIPs []string
   213  
   214  	// AllocatedOutboundPorts are the desired number of allocated SNAT ports per VM. Allowed values must be in the range of 0 to 64000 (inclusive). The default value is 0 which results in Azure dynamically allocating ports.
   215  	AllocatedOutboundPorts *int
   216  
   217  	// IdleTimeoutInMinutes  are the desired outbound flow idle timeout in minutes. Allowed values must be in the range of 4 to 120 (inclusive). The default value is 30 minutes.
   218  	IdleTimeoutInMinutes *int
   219  }
   220  
   221  // APIServerAccessProfile is the access profile for AKS API server.
   222  type APIServerAccessProfile struct {
   223  	// AuthorizedIPRanges are the authorized IP Ranges to kubernetes API server.
   224  	AuthorizedIPRanges []string
   225  	// EnablePrivateCluster defines hether to create the cluster as a private cluster or not.
   226  	EnablePrivateCluster *bool
   227  	// PrivateDNSZone is the private dns zone for private clusters.
   228  	PrivateDNSZone *string
   229  	// EnablePrivateClusterPublicFQDN defines whether to create additional public FQDN for private cluster or not.
   230  	EnablePrivateClusterPublicFQDN *bool
   231  }
   232  
   233  // AutoScalerProfile parameters to be applied to the cluster-autoscaler when enabled.
   234  type AutoScalerProfile struct {
   235  	// BalanceSimilarNodeGroups - Valid values are 'true' and 'false'
   236  	BalanceSimilarNodeGroups *string
   237  	// Expander - If not specified, the default is 'random'. See [expanders](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-expanders) for more information.
   238  	Expander *string
   239  	// MaxEmptyBulkDelete - The default is 10.
   240  	MaxEmptyBulkDelete *string
   241  	// MaxGracefulTerminationSec - The default is 600.
   242  	MaxGracefulTerminationSec *string
   243  	// MaxNodeProvisionTime - The default is '15m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported.
   244  	MaxNodeProvisionTime *string
   245  	// MaxTotalUnreadyPercentage - The default is 45. The maximum is 100 and the minimum is 0.
   246  	MaxTotalUnreadyPercentage *string
   247  	// NewPodScaleUpDelay - For scenarios like burst/batch scale where you don't want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they're a certain age. The default is '0s'. Values must be an integer followed by a unit ('s' for seconds, 'm' for minutes, 'h' for hours, etc).
   248  	NewPodScaleUpDelay *string
   249  	// OkTotalUnreadyCount - This must be an integer. The default is 3.
   250  	OkTotalUnreadyCount *string
   251  	// ScanInterval - The default is '10s'. Values must be an integer number of seconds.
   252  	ScanInterval *string
   253  	// ScaleDownDelayAfterAdd - The default is '10m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported.
   254  	ScaleDownDelayAfterAdd *string
   255  	// ScaleDownDelayAfterDelete - The default is the scan-interval. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported.
   256  	ScaleDownDelayAfterDelete *string
   257  	// ScaleDownDelayAfterFailure - The default is '3m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported.
   258  	ScaleDownDelayAfterFailure *string
   259  	// ScaleDownUnneededTime - The default is '10m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported.
   260  	ScaleDownUnneededTime *string
   261  	// ScaleDownUnreadyTime - The default is '20m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported.
   262  	ScaleDownUnreadyTime *string
   263  	// ScaleDownUtilizationThreshold - The default is '0.5'.
   264  	ScaleDownUtilizationThreshold *string
   265  	// SkipNodesWithLocalStorage - The default is true.
   266  	SkipNodesWithLocalStorage *string
   267  	// SkipNodesWithSystemPods - The default is true.
   268  	SkipNodesWithSystemPods *string
   269  }
   270  
   271  // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster.
   272  type OIDCIssuerProfile struct {
   273  	// Enabled is whether the OIDC issuer is enabled.
   274  	Enabled *bool
   275  }
   276  
   277  // ManagedClusterSecurityProfile defines the security profile for the cluster.
   278  type ManagedClusterSecurityProfile struct {
   279  	// AzureKeyVaultKms defines Azure Key Vault key management service settings for the security profile.
   280  	AzureKeyVaultKms *AzureKeyVaultKms
   281  
   282  	// Defender defines Microsoft Defender settings for the security profile.
   283  	Defender *ManagedClusterSecurityProfileDefender
   284  
   285  	// ImageCleaner settings for the security profile.
   286  	ImageCleaner *ManagedClusterSecurityProfileImageCleaner
   287  
   288  	// Workloadidentity enables Kubernetes applications to access Azure cloud resources securely with Azure AD.
   289  	WorkloadIdentity *ManagedClusterSecurityProfileWorkloadIdentity
   290  }
   291  
   292  // ManagedClusterSecurityProfileDefender defines Microsoft Defender settings for the security profile.
   293  type ManagedClusterSecurityProfileDefender struct {
   294  	// LogAnalyticsWorkspaceResourceID is the ID of the Log Analytics workspace that has to be associated with Microsoft Defender.
   295  	// When Microsoft Defender is enabled, this field is required and must be a valid workspace resource ID.
   296  	LogAnalyticsWorkspaceResourceID *string
   297  
   298  	// SecurityMonitoring profile defines the Microsoft Defender threat detection for Cloud settings for the security profile.
   299  	SecurityMonitoring *ManagedClusterSecurityProfileDefenderSecurityMonitoring
   300  }
   301  
   302  // ManagedClusterSecurityProfileDefenderSecurityMonitoring settings for the security profile threat detection.
   303  type ManagedClusterSecurityProfileDefenderSecurityMonitoring struct {
   304  	// Enabled enables Defender threat detection
   305  	Enabled *bool
   306  }
   307  
   308  // ManagedClusterSecurityProfileImageCleaner removes unused images from nodes, freeing up disk space and helping to reduce attack surface area.
   309  type ManagedClusterSecurityProfileImageCleaner struct {
   310  	// Enabled enables Image Cleaner on AKS cluster.
   311  	Enabled *bool
   312  
   313  	// Image Cleaner scanning interval in hours.
   314  	IntervalHours *int
   315  }
   316  
   317  // ManagedClusterSecurityProfileWorkloadIdentity defines Workload identity settings for the security profile.
   318  type ManagedClusterSecurityProfileWorkloadIdentity struct {
   319  	// Enabled enables workload identity.
   320  	Enabled *bool
   321  }
   322  
   323  // AzureKeyVaultKms Azure Key Vault key management service settings for the security profile.
   324  type AzureKeyVaultKms struct {
   325  	// Enabled enables Azure Key Vault key management service. The default is false.
   326  	Enabled *bool
   327  
   328  	// KeyID defines the Identifier of Azure Key Vault key.
   329  	// When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier.
   330  	KeyID *string
   331  
   332  	// KeyVaultNetworkAccess defines the network access of key vault.
   333  	// The possible values are Public and Private.
   334  	// Public means the key vault allows public access from all networks.
   335  	// Private means the key vault disables public access and enables private link. The default value is Public.
   336  	KeyVaultNetworkAccess *infrav1.KeyVaultNetworkAccessTypes
   337  
   338  	// KeyVaultResourceID is the Resource ID of key vault. When keyVaultNetworkAccess is Private, this field is required and must be a valid resource ID.
   339  	KeyVaultResourceID *string
   340  }
   341  
   342  // buildAutoScalerProfile builds the AutoScalerProfile for the ManagedClusterProperties.
   343  func buildAutoScalerProfile(autoScalerProfile *AutoScalerProfile) *asocontainerservicev1hub.ManagedClusterProperties_AutoScalerProfile {
   344  	if autoScalerProfile == nil {
   345  		return nil
   346  	}
   347  
   348  	mcAutoScalerProfile := &asocontainerservicev1hub.ManagedClusterProperties_AutoScalerProfile{
   349  		BalanceSimilarNodeGroups:      autoScalerProfile.BalanceSimilarNodeGroups,
   350  		MaxEmptyBulkDelete:            autoScalerProfile.MaxEmptyBulkDelete,
   351  		MaxGracefulTerminationSec:     autoScalerProfile.MaxGracefulTerminationSec,
   352  		MaxNodeProvisionTime:          autoScalerProfile.MaxNodeProvisionTime,
   353  		MaxTotalUnreadyPercentage:     autoScalerProfile.MaxTotalUnreadyPercentage,
   354  		NewPodScaleUpDelay:            autoScalerProfile.NewPodScaleUpDelay,
   355  		OkTotalUnreadyCount:           autoScalerProfile.OkTotalUnreadyCount,
   356  		ScanInterval:                  autoScalerProfile.ScanInterval,
   357  		ScaleDownDelayAfterAdd:        autoScalerProfile.ScaleDownDelayAfterAdd,
   358  		ScaleDownDelayAfterDelete:     autoScalerProfile.ScaleDownDelayAfterDelete,
   359  		ScaleDownDelayAfterFailure:    autoScalerProfile.ScaleDownDelayAfterFailure,
   360  		ScaleDownUnneededTime:         autoScalerProfile.ScaleDownUnneededTime,
   361  		ScaleDownUnreadyTime:          autoScalerProfile.ScaleDownUnreadyTime,
   362  		ScaleDownUtilizationThreshold: autoScalerProfile.ScaleDownUtilizationThreshold,
   363  		SkipNodesWithLocalStorage:     autoScalerProfile.SkipNodesWithLocalStorage,
   364  		SkipNodesWithSystemPods:       autoScalerProfile.SkipNodesWithSystemPods,
   365  	}
   366  	if autoScalerProfile.Expander != nil {
   367  		mcAutoScalerProfile.Expander = ptr.To(string(asocontainerservicev1.ManagedClusterProperties_AutoScalerProfile_Expander(*autoScalerProfile.Expander)))
   368  	}
   369  
   370  	return mcAutoScalerProfile
   371  }
   372  
   373  // getManagedClusterVersion gets the desired managed k8s version.
   374  // If autoupgrade channels is set to patch, stable or rapid, clusters can be upgraded to higher version by AKS.
   375  // If autoupgrade is triggered, existing kubernetes version will be higher than the user desired kubernetes version.
   376  // CAPZ should honour the upgrade and it should not downgrade to the lower desired version.
   377  func (s *ManagedClusterSpec) getManagedClusterVersion(existing *asocontainerservicev1hub.ManagedCluster) string {
   378  	if existing == nil || existing.Status.CurrentKubernetesVersion == nil {
   379  		return s.Version
   380  	}
   381  	return versions.GetHigherK8sVersion(s.Version, *existing.Status.CurrentKubernetesVersion)
   382  }
   383  
   384  // ResourceRef implements azure.ASOResourceSpecGetter.
   385  func (s *ManagedClusterSpec) ResourceRef() genruntime.MetaObject {
   386  	if s.Preview {
   387  		return &asocontainerservicev1preview.ManagedCluster{
   388  			ObjectMeta: metav1.ObjectMeta{
   389  				Name: azure.GetNormalizedKubernetesName(s.Name),
   390  			},
   391  		}
   392  	}
   393  	return &asocontainerservicev1.ManagedCluster{
   394  		ObjectMeta: metav1.ObjectMeta{
   395  			Name: azure.GetNormalizedKubernetesName(s.Name),
   396  		},
   397  	}
   398  }
   399  
   400  // Parameters returns the parameters for the managed clusters.
   401  //
   402  //nolint:gocyclo // Function requires a lot of nil checks that raise complexity.
   403  func (s *ManagedClusterSpec) Parameters(ctx context.Context, existingObj genruntime.MetaObject) (params genruntime.MetaObject, err error) {
   404  	ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.Service.Parameters")
   405  	defer done()
   406  
   407  	// If existing is preview, convert to stable then back to preview at the end of the function.
   408  	var existing *asocontainerservicev1hub.ManagedCluster
   409  	var existingStatus asocontainerservicev1preview.ManagedCluster_STATUS
   410  	if existingObj != nil {
   411  		hub := &asocontainerservicev1hub.ManagedCluster{}
   412  		if err := existingObj.(conversion.Convertible).ConvertTo(hub); err != nil {
   413  			return nil, err
   414  		}
   415  		existing = hub
   416  	}
   417  
   418  	managedCluster := existing
   419  	if managedCluster == nil {
   420  		managedCluster = &asocontainerservicev1hub.ManagedCluster{
   421  			Spec: asocontainerservicev1hub.ManagedCluster_Spec{
   422  				Tags: infrav1.Build(infrav1.BuildParams{
   423  					Lifecycle:   infrav1.ResourceLifecycleOwned,
   424  					ClusterName: s.ClusterName,
   425  					Name:        ptr.To(s.Name),
   426  					Role:        ptr.To(infrav1.CommonRole),
   427  					// Additional tags managed separately
   428  				}),
   429  			},
   430  		}
   431  	}
   432  
   433  	managedCluster.Spec.AzureName = s.Name
   434  	managedCluster.Spec.Owner = &genruntime.KnownResourceReference{
   435  		Name: azure.GetNormalizedKubernetesName(s.ResourceGroup),
   436  	}
   437  	managedCluster.Spec.Identity = &asocontainerservicev1hub.ManagedClusterIdentity{
   438  		Type: ptr.To(string(asocontainerservicev1.ManagedClusterIdentity_Type_SystemAssigned)),
   439  	}
   440  	managedCluster.Spec.Location = &s.Location
   441  	managedCluster.Spec.NodeResourceGroup = &s.NodeResourceGroup
   442  	managedCluster.Spec.EnableRBAC = ptr.To(true)
   443  	managedCluster.Spec.DnsPrefix = s.DNSPrefix
   444  
   445  	if kubernetesVersion := s.getManagedClusterVersion(existing); kubernetesVersion != "" {
   446  		managedCluster.Spec.KubernetesVersion = &kubernetesVersion
   447  	}
   448  
   449  	managedCluster.Spec.ServicePrincipalProfile = &asocontainerservicev1hub.ManagedClusterServicePrincipalProfile{
   450  		ClientId: ptr.To("msi"),
   451  	}
   452  	managedCluster.Spec.NetworkProfile = &asocontainerservicev1hub.ContainerServiceNetworkProfile{
   453  		NetworkPlugin:   azure.AliasOrNil[string](ptr.To(s.NetworkPlugin)),
   454  		LoadBalancerSku: azure.AliasOrNil[string](ptr.To(s.LoadBalancerSKU)),
   455  		NetworkPolicy:   azure.AliasOrNil[string](ptr.To(s.NetworkPolicy)),
   456  	}
   457  	if s.NetworkDataplane != nil {
   458  		managedCluster.Spec.NetworkProfile.NetworkDataplane = ptr.To(string(asocontainerservicev1.ContainerServiceNetworkProfile_NetworkDataplane(*s.NetworkDataplane)))
   459  	}
   460  	managedCluster.Spec.AutoScalerProfile = buildAutoScalerProfile(s.AutoScalerProfile)
   461  
   462  	var decodedSSHPublicKey []byte
   463  	if s.SSHPublicKey != "" {
   464  		decodedSSHPublicKey, err = base64.StdEncoding.DecodeString(s.SSHPublicKey)
   465  		if err != nil {
   466  			return nil, errors.Wrap(err, "failed to decode SSHPublicKey")
   467  		}
   468  	}
   469  
   470  	if decodedSSHPublicKey != nil {
   471  		managedCluster.Spec.LinuxProfile = &asocontainerservicev1hub.ContainerServiceLinuxProfile{
   472  			AdminUsername: ptr.To(azure.DefaultAKSUserName),
   473  			Ssh: &asocontainerservicev1hub.ContainerServiceSshConfiguration{
   474  				PublicKeys: []asocontainerservicev1hub.ContainerServiceSshPublicKey{
   475  					{
   476  						KeyData: ptr.To(string(decodedSSHPublicKey)),
   477  					},
   478  				},
   479  			},
   480  		}
   481  	}
   482  
   483  	if s.NetworkPluginMode != nil {
   484  		managedCluster.Spec.NetworkProfile.NetworkPluginMode = ptr.To(string(asocontainerservicev1.ContainerServiceNetworkProfile_NetworkPluginMode(*s.NetworkPluginMode)))
   485  	}
   486  
   487  	if s.PodCIDR != "" {
   488  		managedCluster.Spec.NetworkProfile.PodCidr = &s.PodCIDR
   489  	}
   490  
   491  	if s.ServiceCIDR != "" {
   492  		managedCluster.Spec.NetworkProfile.ServiceCidr = &s.ServiceCIDR
   493  		managedCluster.Spec.NetworkProfile.DnsServiceIP = s.DNSServiceIP
   494  		if s.DNSServiceIP == nil {
   495  			ip, _, err := net.ParseCIDR(s.ServiceCIDR)
   496  			if err != nil {
   497  				return nil, fmt.Errorf("failed to parse service cidr: %w", err)
   498  			}
   499  			// HACK: set the last octet of the IP to .10
   500  			// This ensures the dns IP is valid in the service cidr without forcing the user
   501  			// to specify it in both the Capi cluster and the Azure control plane.
   502  			// https://golang.org/src/net/ip.go#L48
   503  			ip[15] = byte(10)
   504  			dnsIP := ip.String()
   505  			managedCluster.Spec.NetworkProfile.DnsServiceIP = &dnsIP
   506  		}
   507  	}
   508  
   509  	// OperatorSpec defines how the Secrets generated by ASO should look for the AKS cluster kubeconfigs.
   510  	// There is no prescribed naming convention that must be followed.
   511  	managedCluster.Spec.OperatorSpec = &asocontainerservicev1hub.ManagedClusterOperatorSpec{
   512  		Secrets: &asocontainerservicev1hub.ManagedClusterOperatorSecrets{
   513  			AdminCredentials: &genruntime.SecretDestination{
   514  				Name: adminKubeconfigSecretName(s.ClusterName),
   515  				Key:  secret.KubeconfigDataName,
   516  			},
   517  		},
   518  	}
   519  
   520  	if s.AADProfile != nil {
   521  		managedCluster.Spec.AadProfile = &asocontainerservicev1hub.ManagedClusterAADProfile{
   522  			Managed:             &s.AADProfile.Managed,
   523  			EnableAzureRBAC:     &s.AADProfile.EnableAzureRBAC,
   524  			AdminGroupObjectIDs: s.AADProfile.AdminGroupObjectIDs,
   525  		}
   526  		if s.DisableLocalAccounts != nil {
   527  			managedCluster.Spec.DisableLocalAccounts = s.DisableLocalAccounts
   528  		}
   529  
   530  		if ptr.Deref(s.DisableLocalAccounts, false) {
   531  			// admin credentials cannot be fetched when local accounts are disabled
   532  			managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials = nil
   533  		}
   534  		if s.AADProfile.Managed {
   535  			managedCluster.Spec.OperatorSpec.Secrets.UserCredentials = &genruntime.SecretDestination{
   536  				Name: userKubeconfigSecretName(s.ClusterName),
   537  				Key:  secret.KubeconfigDataName,
   538  			}
   539  		}
   540  	}
   541  
   542  	for i := range s.AddonProfiles {
   543  		if managedCluster.Spec.AddonProfiles == nil {
   544  			managedCluster.Spec.AddonProfiles = map[string]asocontainerservicev1hub.ManagedClusterAddonProfile{}
   545  		}
   546  		item := s.AddonProfiles[i]
   547  		addonProfile := asocontainerservicev1hub.ManagedClusterAddonProfile{
   548  			Enabled: &item.Enabled,
   549  		}
   550  		if item.Config != nil {
   551  			addonProfile.Config = item.Config
   552  		}
   553  		managedCluster.Spec.AddonProfiles[item.Name] = addonProfile
   554  	}
   555  
   556  	if s.SKU != nil {
   557  		tierName := asocontainerservicev1.ManagedClusterSKU_Tier(s.SKU.Tier)
   558  		managedCluster.Spec.Sku = &asocontainerservicev1hub.ManagedClusterSKU{
   559  			Name: ptr.To(string(asocontainerservicev1.ManagedClusterSKU_Name("Base"))),
   560  			Tier: ptr.To(string(tierName)),
   561  		}
   562  	}
   563  
   564  	if s.LoadBalancerProfile != nil {
   565  		managedCluster.Spec.NetworkProfile.LoadBalancerProfile = s.GetLoadBalancerProfile()
   566  	}
   567  
   568  	if s.APIServerAccessProfile != nil {
   569  		managedCluster.Spec.ApiServerAccessProfile = &asocontainerservicev1hub.ManagedClusterAPIServerAccessProfile{
   570  			EnablePrivateCluster:           s.APIServerAccessProfile.EnablePrivateCluster,
   571  			PrivateDNSZone:                 s.APIServerAccessProfile.PrivateDNSZone,
   572  			EnablePrivateClusterPublicFQDN: s.APIServerAccessProfile.EnablePrivateClusterPublicFQDN,
   573  		}
   574  
   575  		if s.APIServerAccessProfile.AuthorizedIPRanges != nil {
   576  			managedCluster.Spec.ApiServerAccessProfile.AuthorizedIPRanges = s.APIServerAccessProfile.AuthorizedIPRanges
   577  		}
   578  	}
   579  
   580  	if s.OutboundType != nil {
   581  		managedCluster.Spec.NetworkProfile.OutboundType = ptr.To(string(asocontainerservicev1.ContainerServiceNetworkProfile_OutboundType(*s.OutboundType)))
   582  	}
   583  
   584  	if s.Identity != nil {
   585  		managedCluster.Spec.Identity, err = getIdentity(s.Identity)
   586  		if err != nil {
   587  			return nil, errors.Wrapf(err, "Identity is not valid: %s", err)
   588  		}
   589  	}
   590  
   591  	if s.KubeletUserAssignedIdentity != "" {
   592  		managedCluster.Spec.IdentityProfile = map[string]asocontainerservicev1hub.UserAssignedIdentity{
   593  			kubeletIdentityKey: {
   594  				ResourceReference: &genruntime.ResourceReference{
   595  					ARMID: s.KubeletUserAssignedIdentity,
   596  				},
   597  			},
   598  		}
   599  	}
   600  
   601  	if s.HTTPProxyConfig != nil {
   602  		managedCluster.Spec.HttpProxyConfig = &asocontainerservicev1hub.ManagedClusterHTTPProxyConfig{
   603  			HttpProxy:  s.HTTPProxyConfig.HTTPProxy,
   604  			HttpsProxy: s.HTTPProxyConfig.HTTPSProxy,
   605  			TrustedCa:  s.HTTPProxyConfig.TrustedCA,
   606  		}
   607  
   608  		if s.HTTPProxyConfig.NoProxy != nil {
   609  			managedCluster.Spec.HttpProxyConfig.NoProxy = s.HTTPProxyConfig.NoProxy
   610  		}
   611  	}
   612  
   613  	if s.OIDCIssuerProfile != nil {
   614  		managedCluster.Spec.OidcIssuerProfile = &asocontainerservicev1hub.ManagedClusterOIDCIssuerProfile{
   615  			Enabled: s.OIDCIssuerProfile.Enabled,
   616  		}
   617  		if ptr.Deref(s.OIDCIssuerProfile.Enabled, false) {
   618  			managedCluster.Spec.OperatorSpec.ConfigMaps = &asocontainerservicev1hub.ManagedClusterOperatorConfigMaps{
   619  				OIDCIssuerProfile: &genruntime.ConfigMapDestination{
   620  					Name: oidcIssuerURLConfigMapName(s.ClusterName),
   621  					Key:  oidcIssuerProfileURL,
   622  				},
   623  			}
   624  		}
   625  	}
   626  
   627  	if s.AutoUpgradeProfile != nil {
   628  		managedCluster.Spec.AutoUpgradeProfile = &asocontainerservicev1hub.ManagedClusterAutoUpgradeProfile{
   629  			UpgradeChannel: (*string)(s.AutoUpgradeProfile.UpgradeChannel),
   630  		}
   631  	}
   632  
   633  	if s.SecurityProfile != nil {
   634  		securityProfile := &asocontainerservicev1hub.ManagedClusterSecurityProfile{}
   635  		if s.SecurityProfile.AzureKeyVaultKms != nil {
   636  			securityProfile.AzureKeyVaultKms = &asocontainerservicev1hub.AzureKeyVaultKms{
   637  				Enabled: s.SecurityProfile.AzureKeyVaultKms.Enabled,
   638  				KeyId:   s.SecurityProfile.AzureKeyVaultKms.KeyID,
   639  			}
   640  			if s.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess != nil {
   641  				keyVaultNetworkAccess := string(*s.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess)
   642  				securityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess = ptr.To(string(asocontainerservicev1.AzureKeyVaultKms_KeyVaultNetworkAccess(keyVaultNetworkAccess)))
   643  			}
   644  			if s.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID != nil {
   645  				securityProfile.AzureKeyVaultKms.KeyVaultResourceReference = &genruntime.ResourceReference{
   646  					ARMID: *s.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID,
   647  				}
   648  			}
   649  		}
   650  		if s.SecurityProfile.Defender != nil {
   651  			securityProfile.Defender = &asocontainerservicev1hub.ManagedClusterSecurityProfileDefender{
   652  				LogAnalyticsWorkspaceResourceReference: &genruntime.ResourceReference{
   653  					ARMID: *s.SecurityProfile.Defender.LogAnalyticsWorkspaceResourceID,
   654  				},
   655  			}
   656  			if s.SecurityProfile.Defender.SecurityMonitoring != nil {
   657  				securityProfile.Defender.SecurityMonitoring = &asocontainerservicev1hub.ManagedClusterSecurityProfileDefenderSecurityMonitoring{
   658  					Enabled: s.SecurityProfile.Defender.SecurityMonitoring.Enabled,
   659  				}
   660  			}
   661  		}
   662  		if s.SecurityProfile.ImageCleaner != nil {
   663  			securityProfile.ImageCleaner = &asocontainerservicev1hub.ManagedClusterSecurityProfileImageCleaner{
   664  				Enabled:       s.SecurityProfile.ImageCleaner.Enabled,
   665  				IntervalHours: s.SecurityProfile.ImageCleaner.IntervalHours,
   666  			}
   667  		}
   668  		if s.SecurityProfile.WorkloadIdentity != nil {
   669  			securityProfile.WorkloadIdentity = &asocontainerservicev1hub.ManagedClusterSecurityProfileWorkloadIdentity{
   670  				Enabled: s.SecurityProfile.WorkloadIdentity.Enabled,
   671  			}
   672  		}
   673  		managedCluster.Spec.SecurityProfile = securityProfile
   674  	}
   675  
   676  	// Only include AgentPoolProfiles during initial cluster creation. Agent pools are managed solely by the
   677  	// AzureManagedMachinePool controller thereafter.
   678  	var prevAgentPoolProfiles []asocontainerservicev1hub.ManagedClusterAgentPoolProfile
   679  	managedCluster.Spec.AgentPoolProfiles = nil
   680  	if managedCluster.Status.AgentPoolProfiles == nil {
   681  		// Add all agent pools to cluster spec that will be submitted to the API
   682  		agentPoolSpecs, err := s.GetAllAgentPools()
   683  		if err != nil {
   684  			return nil, errors.Wrapf(err, "failed to get agent pool specs for managed cluster %s", s.Name)
   685  		}
   686  
   687  		scheme := runtime.NewScheme()
   688  		if err := asocontainerservicev1.AddToScheme(scheme); err != nil {
   689  			return nil, errors.Wrap(err, "error constructing scheme")
   690  		}
   691  		if err := asocontainerservicev1preview.AddToScheme(scheme); err != nil {
   692  			return nil, errors.Wrap(err, "error constructing scheme")
   693  		}
   694  		for _, agentPoolSpec := range agentPoolSpecs {
   695  			agentPool, err := aso.PatchedParameters(ctx, scheme, agentPoolSpec, nil)
   696  			if err != nil {
   697  				return nil, errors.Wrapf(err, "failed to get agent pool parameters for managed cluster %s", s.Name)
   698  			}
   699  			agentPoolSpecTyped := agentPoolSpec.(*agentpools.AgentPoolSpec)
   700  			hubAgentPool := &asocontainerservicev1hub.ManagedClustersAgentPool{}
   701  			if s.Preview {
   702  				agentPoolTyped := agentPool.(*asocontainerservicev1preview.ManagedClustersAgentPool)
   703  				if err := agentPoolTyped.ConvertTo(hubAgentPool); err != nil {
   704  					return nil, err
   705  				}
   706  			} else {
   707  				agentPoolTyped := agentPool.(*asocontainerservicev1.ManagedClustersAgentPool)
   708  				if err := agentPoolTyped.ConvertTo(hubAgentPool); err != nil {
   709  					return nil, err
   710  				}
   711  			}
   712  			hubAgentPool.Spec.AzureName = agentPoolSpecTyped.AzureName
   713  			profile := converters.AgentPoolToManagedClusterAgentPoolProfile(hubAgentPool)
   714  			prevAgentPoolProfiles = append(prevAgentPoolProfiles, profile)
   715  		}
   716  	}
   717  
   718  	managedCluster.Spec.AgentPoolProfiles = prevAgentPoolProfiles
   719  
   720  	if s.Preview {
   721  		prev := &asocontainerservicev1preview.ManagedCluster{}
   722  		if err := prev.ConvertFrom(managedCluster); err != nil {
   723  			return nil, err
   724  		}
   725  		if existing != nil {
   726  			prev.Status = existingStatus
   727  		}
   728  		return prev, nil
   729  	}
   730  
   731  	stable := &asocontainerservicev1.ManagedCluster{}
   732  	if err := stable.ConvertFrom(managedCluster); err != nil {
   733  		return nil, err
   734  	}
   735  
   736  	return stable, nil
   737  }
   738  
   739  // GetLoadBalancerProfile returns an asocontainerservicev1.ManagedClusterLoadBalancerProfile from the
   740  // information present in ManagedClusterSpec.LoadBalancerProfile.
   741  func (s *ManagedClusterSpec) GetLoadBalancerProfile() (loadBalancerProfile *asocontainerservicev1hub.ManagedClusterLoadBalancerProfile) {
   742  	loadBalancerProfile = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile{
   743  		AllocatedOutboundPorts: s.LoadBalancerProfile.AllocatedOutboundPorts,
   744  		IdleTimeoutInMinutes:   s.LoadBalancerProfile.IdleTimeoutInMinutes,
   745  	}
   746  	if s.LoadBalancerProfile.ManagedOutboundIPs != nil {
   747  		loadBalancerProfile.ManagedOutboundIPs = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile_ManagedOutboundIPs{Count: s.LoadBalancerProfile.ManagedOutboundIPs}
   748  	}
   749  	if len(s.LoadBalancerProfile.OutboundIPPrefixes) > 0 {
   750  		loadBalancerProfile.OutboundIPPrefixes = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile_OutboundIPPrefixes{
   751  			PublicIPPrefixes: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPPrefixes),
   752  		}
   753  	}
   754  	if len(s.LoadBalancerProfile.OutboundIPs) > 0 {
   755  		loadBalancerProfile.OutboundIPs = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile_OutboundIPs{
   756  			PublicIPs: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPs),
   757  		}
   758  	}
   759  	return
   760  }
   761  
   762  func convertToResourceReferences(resources []string) []asocontainerservicev1hub.ResourceReference {
   763  	resourceReferences := make([]asocontainerservicev1hub.ResourceReference, len(resources))
   764  	for i := range resources {
   765  		resourceReferences[i] = asocontainerservicev1hub.ResourceReference{
   766  			Reference: &genruntime.ResourceReference{
   767  				ARMID: resources[i],
   768  			},
   769  		}
   770  	}
   771  	return resourceReferences
   772  }
   773  
   774  func getIdentity(identity *infrav1.Identity) (managedClusterIdentity *asocontainerservicev1hub.ManagedClusterIdentity, err error) {
   775  	if identity.Type == "" {
   776  		return
   777  	}
   778  
   779  	managedClusterIdentity = &asocontainerservicev1hub.ManagedClusterIdentity{
   780  		Type: ptr.To(string(asocontainerservicev1.ManagedClusterIdentity_Type(identity.Type))),
   781  	}
   782  	if ptr.Deref(managedClusterIdentity.Type, "") == string(asocontainerservicev1.ManagedClusterIdentity_Type_UserAssigned) {
   783  		if identity.UserAssignedIdentityResourceID == "" {
   784  			err = errors.Errorf("Identity is set to \"UserAssigned\" but no UserAssignedIdentityResourceID is present")
   785  			return
   786  		}
   787  		managedClusterIdentity.UserAssignedIdentities = []asocontainerservicev1hub.UserAssignedIdentityDetails{
   788  			{
   789  				Reference: genruntime.ResourceReference{
   790  					ARMID: identity.UserAssignedIdentityResourceID,
   791  				},
   792  			},
   793  		}
   794  	}
   795  	return
   796  }
   797  
   798  func adminKubeconfigSecretName(clusterName string) string {
   799  	return secret.Name(clusterName+"-aso", secret.Kubeconfig)
   800  }
   801  
   802  func oidcIssuerURLConfigMapName(clusterName string) string {
   803  	return secret.Name(clusterName+"-aso", "oidc-issuer-profile")
   804  }
   805  
   806  func userKubeconfigSecretName(clusterName string) string {
   807  	return secret.Name(clusterName+"-user-aso", secret.Kubeconfig)
   808  }
   809  
   810  // WasManaged implements azure.ASOResourceSpecGetter.
   811  func (s *ManagedClusterSpec) WasManaged(resource genruntime.MetaObject) bool {
   812  	// CAPZ has never supported BYO managed clusters.
   813  	return true
   814  }
   815  
   816  var _ aso.TagsGetterSetter[genruntime.MetaObject] = (*ManagedClusterSpec)(nil)
   817  
   818  // GetAdditionalTags implements aso.TagsGetterSetter.
   819  func (s *ManagedClusterSpec) GetAdditionalTags() infrav1.Tags {
   820  	return s.Tags
   821  }
   822  
   823  // GetDesiredTags implements aso.TagsGetterSetter.
   824  func (s *ManagedClusterSpec) GetDesiredTags(resource genruntime.MetaObject) infrav1.Tags {
   825  	if s.Preview {
   826  		return resource.(*asocontainerservicev1preview.ManagedCluster).Spec.Tags
   827  	}
   828  	return resource.(*asocontainerservicev1.ManagedCluster).Spec.Tags
   829  }
   830  
   831  // SetTags implements aso.TagsGetterSetter.
   832  func (s *ManagedClusterSpec) SetTags(resource genruntime.MetaObject, tags infrav1.Tags) {
   833  	if s.Preview {
   834  		resource.(*asocontainerservicev1preview.ManagedCluster).Spec.Tags = tags
   835  		return
   836  	}
   837  	resource.(*asocontainerservicev1.ManagedCluster).Spec.Tags = tags
   838  }
   839  
   840  var _ aso.Patcher = (*ManagedClusterSpec)(nil)
   841  
   842  // ExtraPatches implements aso.Patcher.
   843  func (s *ManagedClusterSpec) ExtraPatches() []string {
   844  	return s.Patches
   845  }