sigs.k8s.io/cluster-api-provider-azure@v1.17.0/azure/services/managedclusters/spec.go (about) 1 /* 2 Copyright 2022 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package managedclusters 18 19 import ( 20 "context" 21 "encoding/base64" 22 "fmt" 23 "net" 24 25 asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" 26 asocontainerservicev1hub "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001/storage" 27 asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231102preview" 28 "github.com/Azure/azure-service-operator/v2/pkg/genruntime" 29 "github.com/pkg/errors" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/runtime" 32 "k8s.io/utils/ptr" 33 infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" 34 "sigs.k8s.io/cluster-api-provider-azure/azure" 35 "sigs.k8s.io/cluster-api-provider-azure/azure/converters" 36 "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" 37 "sigs.k8s.io/cluster-api-provider-azure/azure/services/aso" 38 "sigs.k8s.io/cluster-api-provider-azure/util/tele" 39 "sigs.k8s.io/cluster-api-provider-azure/util/versions" 40 "sigs.k8s.io/cluster-api/util/secret" 41 "sigs.k8s.io/controller-runtime/pkg/conversion" 42 ) 43 44 // ManagedClusterSpec contains properties to create a managed cluster. 45 type ManagedClusterSpec struct { 46 // Name is the name of this AKS Cluster. 47 Name string 48 49 // ResourceGroup is the name of the Azure resource group for this AKS Cluster. 50 ResourceGroup string 51 52 // NodeResourceGroup is the name of the Azure resource group containing IaaS VMs. 53 NodeResourceGroup string 54 55 // ClusterName is the name of the owning Cluster API Cluster resource. 56 ClusterName string 57 58 // VnetSubnetID is the Azure Resource ID for the subnet which should contain nodes. 59 VnetSubnetID string 60 61 // Location is a string matching one of the canonical Azure region names. Examples: "westus2", "eastus". 62 Location string 63 64 // Tags is a set of tags to add to this cluster. 65 Tags map[string]string 66 67 // Version defines the desired Kubernetes version. 68 Version string 69 70 // LoadBalancerSKU for the managed cluster. Possible values include: 'Standard', 'Basic'. Defaults to Standard. 71 LoadBalancerSKU string 72 73 // NetworkPlugin used for building Kubernetes network. Possible values include: 'azure', 'kubenet'. Defaults to azure. 74 NetworkPlugin string 75 76 // NetworkPluginMode is the mode the network plugin should use. 77 NetworkPluginMode *infrav1.NetworkPluginMode 78 79 // NetworkPolicy used for building Kubernetes network. Possible values include: 'azure', 'calico', 'cilium'. 80 NetworkPolicy string 81 82 // NetworkDataplane used for building Kubernetes network. Possible values include: 'azure', 'cilium'. 83 NetworkDataplane *infrav1.NetworkDataplaneType 84 85 // OutboundType used for building Kubernetes network. Possible values include: 'loadBalancer', 'managedNATGateway', 'userAssignedNATGateway', 'userDefinedRouting'. 86 OutboundType *infrav1.ManagedControlPlaneOutboundType 87 88 // SSHPublicKey is a string literal containing an ssh public key. Will autogenerate and discard if not provided. 89 SSHPublicKey string 90 91 // GetAllAgentPools is a function that returns the list of agent pool specifications in this cluster. 92 GetAllAgentPools func() ([]azure.ASOResourceSpecGetter[genruntime.MetaObject], error) 93 94 // PodCIDR is the CIDR block for IP addresses distributed to pods 95 PodCIDR string 96 97 // ServiceCIDR is the CIDR block for IP addresses distributed to services 98 ServiceCIDR string 99 100 // DNSServiceIP is an IP address assigned to the Kubernetes DNS service 101 DNSServiceIP *string 102 103 // AddonProfiles are the profiles of managed cluster add-on. 104 AddonProfiles []AddonProfile 105 106 // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. 107 AADProfile *AADProfile 108 109 // SKU is the SKU of the AKS to be provisioned. 110 SKU *SKU 111 112 // LoadBalancerProfile is the profile of the cluster load balancer. 113 LoadBalancerProfile *LoadBalancerProfile 114 115 // APIServerAccessProfile is the access profile for AKS API server. 116 APIServerAccessProfile *APIServerAccessProfile 117 118 // AutoScalerProfile is the parameters to be applied to the cluster-autoscaler when enabled. 119 AutoScalerProfile *AutoScalerProfile 120 121 // Identity is the AKS control plane Identity configuration 122 Identity *infrav1.Identity 123 124 // KubeletUserAssignedIdentity is the user-assigned identity for kubelet to authenticate to ACR. 125 KubeletUserAssignedIdentity string 126 127 // HTTPProxyConfig is the HTTP proxy configuration for the cluster. 128 HTTPProxyConfig *HTTPProxyConfig 129 130 // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. 131 OIDCIssuerProfile *OIDCIssuerProfile 132 133 // DNSPrefix allows the user to customize dns prefix. 134 DNSPrefix *string 135 136 // DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters. 137 DisableLocalAccounts *bool 138 139 // AutoUpgradeProfile defines auto upgrade configuration. 140 AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile 141 142 // SecurityProfile defines the security profile for the cluster. 143 SecurityProfile *ManagedClusterSecurityProfile 144 145 // Patches are extra patches to be applied to the ASO resource. 146 Patches []string 147 148 // Preview enables the preview API version. 149 Preview bool 150 } 151 152 // ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. 153 type ManagedClusterAutoUpgradeProfile struct { 154 // UpgradeChannel defines the channel for auto upgrade configuration. 155 UpgradeChannel *infrav1.UpgradeChannel 156 } 157 158 // HTTPProxyConfig is the HTTP proxy configuration for the cluster. 159 type HTTPProxyConfig struct { 160 // HTTPProxy is the HTTP proxy server endpoint to use. 161 HTTPProxy *string `json:"httpProxy,omitempty"` 162 163 // HTTPSProxy is the HTTPS proxy server endpoint to use. 164 HTTPSProxy *string `json:"httpsProxy,omitempty"` 165 166 // NoProxy is the endpoints that should not go through proxy. 167 NoProxy []string `json:"noProxy,omitempty"` 168 169 // TrustedCA is the Alternative CA cert to use for connecting to proxy servers. 170 TrustedCA *string `json:"trustedCa,omitempty"` 171 } 172 173 // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. 174 type AADProfile struct { 175 // Managed defines whether to enable managed AAD. 176 Managed bool 177 178 // EnableAzureRBAC defines whether to enable Azure RBAC for Kubernetes authorization. 179 EnableAzureRBAC bool 180 181 // AdminGroupObjectIDs are the AAD group object IDs that will have admin role of the cluster. 182 AdminGroupObjectIDs []string 183 } 184 185 // AddonProfile is the profile of a managed cluster add-on. 186 type AddonProfile struct { 187 Name string 188 Config map[string]string 189 Enabled bool 190 } 191 192 // SKU is an AKS SKU. 193 type SKU struct { 194 // Tier is the tier of a managed cluster SKU. 195 Tier string 196 } 197 198 // LoadBalancerProfile is the profile of the cluster load balancer. 199 type LoadBalancerProfile struct { 200 // Load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs. 201 // By default the AKS cluster automatically creates a public IP in the AKS-managed infrastructure resource group and assigns it to the load balancer outbound pool. 202 // Alternatively, you can assign your own custom public IP or public IP prefix at cluster creation time. 203 // See https://learn.microsoft.com/azure/aks/load-balancer-standard#provide-your-own-outbound-public-ips-or-prefixes 204 205 // ManagedOutboundIPs are the desired managed outbound IPs for the cluster load balancer. 206 ManagedOutboundIPs *int 207 208 // OutboundIPPrefixes are the desired outbound IP Prefix resources for the cluster load balancer. 209 OutboundIPPrefixes []string 210 211 // OutboundIPs are the desired outbound IP resources for the cluster load balancer. 212 OutboundIPs []string 213 214 // AllocatedOutboundPorts are the desired number of allocated SNAT ports per VM. Allowed values must be in the range of 0 to 64000 (inclusive). The default value is 0 which results in Azure dynamically allocating ports. 215 AllocatedOutboundPorts *int 216 217 // IdleTimeoutInMinutes are the desired outbound flow idle timeout in minutes. Allowed values must be in the range of 4 to 120 (inclusive). The default value is 30 minutes. 218 IdleTimeoutInMinutes *int 219 } 220 221 // APIServerAccessProfile is the access profile for AKS API server. 222 type APIServerAccessProfile struct { 223 // AuthorizedIPRanges are the authorized IP Ranges to kubernetes API server. 224 AuthorizedIPRanges []string 225 // EnablePrivateCluster defines hether to create the cluster as a private cluster or not. 226 EnablePrivateCluster *bool 227 // PrivateDNSZone is the private dns zone for private clusters. 228 PrivateDNSZone *string 229 // EnablePrivateClusterPublicFQDN defines whether to create additional public FQDN for private cluster or not. 230 EnablePrivateClusterPublicFQDN *bool 231 } 232 233 // AutoScalerProfile parameters to be applied to the cluster-autoscaler when enabled. 234 type AutoScalerProfile struct { 235 // BalanceSimilarNodeGroups - Valid values are 'true' and 'false' 236 BalanceSimilarNodeGroups *string 237 // Expander - If not specified, the default is 'random'. See [expanders](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-expanders) for more information. 238 Expander *string 239 // MaxEmptyBulkDelete - The default is 10. 240 MaxEmptyBulkDelete *string 241 // MaxGracefulTerminationSec - The default is 600. 242 MaxGracefulTerminationSec *string 243 // MaxNodeProvisionTime - The default is '15m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported. 244 MaxNodeProvisionTime *string 245 // MaxTotalUnreadyPercentage - The default is 45. The maximum is 100 and the minimum is 0. 246 MaxTotalUnreadyPercentage *string 247 // NewPodScaleUpDelay - For scenarios like burst/batch scale where you don't want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they're a certain age. The default is '0s'. Values must be an integer followed by a unit ('s' for seconds, 'm' for minutes, 'h' for hours, etc). 248 NewPodScaleUpDelay *string 249 // OkTotalUnreadyCount - This must be an integer. The default is 3. 250 OkTotalUnreadyCount *string 251 // ScanInterval - The default is '10s'. Values must be an integer number of seconds. 252 ScanInterval *string 253 // ScaleDownDelayAfterAdd - The default is '10m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported. 254 ScaleDownDelayAfterAdd *string 255 // ScaleDownDelayAfterDelete - The default is the scan-interval. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported. 256 ScaleDownDelayAfterDelete *string 257 // ScaleDownDelayAfterFailure - The default is '3m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported. 258 ScaleDownDelayAfterFailure *string 259 // ScaleDownUnneededTime - The default is '10m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported. 260 ScaleDownUnneededTime *string 261 // ScaleDownUnreadyTime - The default is '20m'. Values must be an integer followed by an 'm'. No unit of time other than minutes (m) is supported. 262 ScaleDownUnreadyTime *string 263 // ScaleDownUtilizationThreshold - The default is '0.5'. 264 ScaleDownUtilizationThreshold *string 265 // SkipNodesWithLocalStorage - The default is true. 266 SkipNodesWithLocalStorage *string 267 // SkipNodesWithSystemPods - The default is true. 268 SkipNodesWithSystemPods *string 269 } 270 271 // OIDCIssuerProfile is the OIDC issuer profile of the Managed Cluster. 272 type OIDCIssuerProfile struct { 273 // Enabled is whether the OIDC issuer is enabled. 274 Enabled *bool 275 } 276 277 // ManagedClusterSecurityProfile defines the security profile for the cluster. 278 type ManagedClusterSecurityProfile struct { 279 // AzureKeyVaultKms defines Azure Key Vault key management service settings for the security profile. 280 AzureKeyVaultKms *AzureKeyVaultKms 281 282 // Defender defines Microsoft Defender settings for the security profile. 283 Defender *ManagedClusterSecurityProfileDefender 284 285 // ImageCleaner settings for the security profile. 286 ImageCleaner *ManagedClusterSecurityProfileImageCleaner 287 288 // Workloadidentity enables Kubernetes applications to access Azure cloud resources securely with Azure AD. 289 WorkloadIdentity *ManagedClusterSecurityProfileWorkloadIdentity 290 } 291 292 // ManagedClusterSecurityProfileDefender defines Microsoft Defender settings for the security profile. 293 type ManagedClusterSecurityProfileDefender struct { 294 // LogAnalyticsWorkspaceResourceID is the ID of the Log Analytics workspace that has to be associated with Microsoft Defender. 295 // When Microsoft Defender is enabled, this field is required and must be a valid workspace resource ID. 296 LogAnalyticsWorkspaceResourceID *string 297 298 // SecurityMonitoring profile defines the Microsoft Defender threat detection for Cloud settings for the security profile. 299 SecurityMonitoring *ManagedClusterSecurityProfileDefenderSecurityMonitoring 300 } 301 302 // ManagedClusterSecurityProfileDefenderSecurityMonitoring settings for the security profile threat detection. 303 type ManagedClusterSecurityProfileDefenderSecurityMonitoring struct { 304 // Enabled enables Defender threat detection 305 Enabled *bool 306 } 307 308 // ManagedClusterSecurityProfileImageCleaner removes unused images from nodes, freeing up disk space and helping to reduce attack surface area. 309 type ManagedClusterSecurityProfileImageCleaner struct { 310 // Enabled enables Image Cleaner on AKS cluster. 311 Enabled *bool 312 313 // Image Cleaner scanning interval in hours. 314 IntervalHours *int 315 } 316 317 // ManagedClusterSecurityProfileWorkloadIdentity defines Workload identity settings for the security profile. 318 type ManagedClusterSecurityProfileWorkloadIdentity struct { 319 // Enabled enables workload identity. 320 Enabled *bool 321 } 322 323 // AzureKeyVaultKms Azure Key Vault key management service settings for the security profile. 324 type AzureKeyVaultKms struct { 325 // Enabled enables Azure Key Vault key management service. The default is false. 326 Enabled *bool 327 328 // KeyID defines the Identifier of Azure Key Vault key. 329 // When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier. 330 KeyID *string 331 332 // KeyVaultNetworkAccess defines the network access of key vault. 333 // The possible values are Public and Private. 334 // Public means the key vault allows public access from all networks. 335 // Private means the key vault disables public access and enables private link. The default value is Public. 336 KeyVaultNetworkAccess *infrav1.KeyVaultNetworkAccessTypes 337 338 // KeyVaultResourceID is the Resource ID of key vault. When keyVaultNetworkAccess is Private, this field is required and must be a valid resource ID. 339 KeyVaultResourceID *string 340 } 341 342 // buildAutoScalerProfile builds the AutoScalerProfile for the ManagedClusterProperties. 343 func buildAutoScalerProfile(autoScalerProfile *AutoScalerProfile) *asocontainerservicev1hub.ManagedClusterProperties_AutoScalerProfile { 344 if autoScalerProfile == nil { 345 return nil 346 } 347 348 mcAutoScalerProfile := &asocontainerservicev1hub.ManagedClusterProperties_AutoScalerProfile{ 349 BalanceSimilarNodeGroups: autoScalerProfile.BalanceSimilarNodeGroups, 350 MaxEmptyBulkDelete: autoScalerProfile.MaxEmptyBulkDelete, 351 MaxGracefulTerminationSec: autoScalerProfile.MaxGracefulTerminationSec, 352 MaxNodeProvisionTime: autoScalerProfile.MaxNodeProvisionTime, 353 MaxTotalUnreadyPercentage: autoScalerProfile.MaxTotalUnreadyPercentage, 354 NewPodScaleUpDelay: autoScalerProfile.NewPodScaleUpDelay, 355 OkTotalUnreadyCount: autoScalerProfile.OkTotalUnreadyCount, 356 ScanInterval: autoScalerProfile.ScanInterval, 357 ScaleDownDelayAfterAdd: autoScalerProfile.ScaleDownDelayAfterAdd, 358 ScaleDownDelayAfterDelete: autoScalerProfile.ScaleDownDelayAfterDelete, 359 ScaleDownDelayAfterFailure: autoScalerProfile.ScaleDownDelayAfterFailure, 360 ScaleDownUnneededTime: autoScalerProfile.ScaleDownUnneededTime, 361 ScaleDownUnreadyTime: autoScalerProfile.ScaleDownUnreadyTime, 362 ScaleDownUtilizationThreshold: autoScalerProfile.ScaleDownUtilizationThreshold, 363 SkipNodesWithLocalStorage: autoScalerProfile.SkipNodesWithLocalStorage, 364 SkipNodesWithSystemPods: autoScalerProfile.SkipNodesWithSystemPods, 365 } 366 if autoScalerProfile.Expander != nil { 367 mcAutoScalerProfile.Expander = ptr.To(string(asocontainerservicev1.ManagedClusterProperties_AutoScalerProfile_Expander(*autoScalerProfile.Expander))) 368 } 369 370 return mcAutoScalerProfile 371 } 372 373 // getManagedClusterVersion gets the desired managed k8s version. 374 // If autoupgrade channels is set to patch, stable or rapid, clusters can be upgraded to higher version by AKS. 375 // If autoupgrade is triggered, existing kubernetes version will be higher than the user desired kubernetes version. 376 // CAPZ should honour the upgrade and it should not downgrade to the lower desired version. 377 func (s *ManagedClusterSpec) getManagedClusterVersion(existing *asocontainerservicev1hub.ManagedCluster) string { 378 if existing == nil || existing.Status.CurrentKubernetesVersion == nil { 379 return s.Version 380 } 381 return versions.GetHigherK8sVersion(s.Version, *existing.Status.CurrentKubernetesVersion) 382 } 383 384 // ResourceRef implements azure.ASOResourceSpecGetter. 385 func (s *ManagedClusterSpec) ResourceRef() genruntime.MetaObject { 386 if s.Preview { 387 return &asocontainerservicev1preview.ManagedCluster{ 388 ObjectMeta: metav1.ObjectMeta{ 389 Name: azure.GetNormalizedKubernetesName(s.Name), 390 }, 391 } 392 } 393 return &asocontainerservicev1.ManagedCluster{ 394 ObjectMeta: metav1.ObjectMeta{ 395 Name: azure.GetNormalizedKubernetesName(s.Name), 396 }, 397 } 398 } 399 400 // Parameters returns the parameters for the managed clusters. 401 // 402 //nolint:gocyclo // Function requires a lot of nil checks that raise complexity. 403 func (s *ManagedClusterSpec) Parameters(ctx context.Context, existingObj genruntime.MetaObject) (params genruntime.MetaObject, err error) { 404 ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.Service.Parameters") 405 defer done() 406 407 // If existing is preview, convert to stable then back to preview at the end of the function. 408 var existing *asocontainerservicev1hub.ManagedCluster 409 var existingStatus asocontainerservicev1preview.ManagedCluster_STATUS 410 if existingObj != nil { 411 hub := &asocontainerservicev1hub.ManagedCluster{} 412 if err := existingObj.(conversion.Convertible).ConvertTo(hub); err != nil { 413 return nil, err 414 } 415 existing = hub 416 } 417 418 managedCluster := existing 419 if managedCluster == nil { 420 managedCluster = &asocontainerservicev1hub.ManagedCluster{ 421 Spec: asocontainerservicev1hub.ManagedCluster_Spec{ 422 Tags: infrav1.Build(infrav1.BuildParams{ 423 Lifecycle: infrav1.ResourceLifecycleOwned, 424 ClusterName: s.ClusterName, 425 Name: ptr.To(s.Name), 426 Role: ptr.To(infrav1.CommonRole), 427 // Additional tags managed separately 428 }), 429 }, 430 } 431 } 432 433 managedCluster.Spec.AzureName = s.Name 434 managedCluster.Spec.Owner = &genruntime.KnownResourceReference{ 435 Name: azure.GetNormalizedKubernetesName(s.ResourceGroup), 436 } 437 managedCluster.Spec.Identity = &asocontainerservicev1hub.ManagedClusterIdentity{ 438 Type: ptr.To(string(asocontainerservicev1.ManagedClusterIdentity_Type_SystemAssigned)), 439 } 440 managedCluster.Spec.Location = &s.Location 441 managedCluster.Spec.NodeResourceGroup = &s.NodeResourceGroup 442 managedCluster.Spec.EnableRBAC = ptr.To(true) 443 managedCluster.Spec.DnsPrefix = s.DNSPrefix 444 445 if kubernetesVersion := s.getManagedClusterVersion(existing); kubernetesVersion != "" { 446 managedCluster.Spec.KubernetesVersion = &kubernetesVersion 447 } 448 449 managedCluster.Spec.ServicePrincipalProfile = &asocontainerservicev1hub.ManagedClusterServicePrincipalProfile{ 450 ClientId: ptr.To("msi"), 451 } 452 managedCluster.Spec.NetworkProfile = &asocontainerservicev1hub.ContainerServiceNetworkProfile{ 453 NetworkPlugin: azure.AliasOrNil[string](ptr.To(s.NetworkPlugin)), 454 LoadBalancerSku: azure.AliasOrNil[string](ptr.To(s.LoadBalancerSKU)), 455 NetworkPolicy: azure.AliasOrNil[string](ptr.To(s.NetworkPolicy)), 456 } 457 if s.NetworkDataplane != nil { 458 managedCluster.Spec.NetworkProfile.NetworkDataplane = ptr.To(string(asocontainerservicev1.ContainerServiceNetworkProfile_NetworkDataplane(*s.NetworkDataplane))) 459 } 460 managedCluster.Spec.AutoScalerProfile = buildAutoScalerProfile(s.AutoScalerProfile) 461 462 var decodedSSHPublicKey []byte 463 if s.SSHPublicKey != "" { 464 decodedSSHPublicKey, err = base64.StdEncoding.DecodeString(s.SSHPublicKey) 465 if err != nil { 466 return nil, errors.Wrap(err, "failed to decode SSHPublicKey") 467 } 468 } 469 470 if decodedSSHPublicKey != nil { 471 managedCluster.Spec.LinuxProfile = &asocontainerservicev1hub.ContainerServiceLinuxProfile{ 472 AdminUsername: ptr.To(azure.DefaultAKSUserName), 473 Ssh: &asocontainerservicev1hub.ContainerServiceSshConfiguration{ 474 PublicKeys: []asocontainerservicev1hub.ContainerServiceSshPublicKey{ 475 { 476 KeyData: ptr.To(string(decodedSSHPublicKey)), 477 }, 478 }, 479 }, 480 } 481 } 482 483 if s.NetworkPluginMode != nil { 484 managedCluster.Spec.NetworkProfile.NetworkPluginMode = ptr.To(string(asocontainerservicev1.ContainerServiceNetworkProfile_NetworkPluginMode(*s.NetworkPluginMode))) 485 } 486 487 if s.PodCIDR != "" { 488 managedCluster.Spec.NetworkProfile.PodCidr = &s.PodCIDR 489 } 490 491 if s.ServiceCIDR != "" { 492 managedCluster.Spec.NetworkProfile.ServiceCidr = &s.ServiceCIDR 493 managedCluster.Spec.NetworkProfile.DnsServiceIP = s.DNSServiceIP 494 if s.DNSServiceIP == nil { 495 ip, _, err := net.ParseCIDR(s.ServiceCIDR) 496 if err != nil { 497 return nil, fmt.Errorf("failed to parse service cidr: %w", err) 498 } 499 // HACK: set the last octet of the IP to .10 500 // This ensures the dns IP is valid in the service cidr without forcing the user 501 // to specify it in both the Capi cluster and the Azure control plane. 502 // https://golang.org/src/net/ip.go#L48 503 ip[15] = byte(10) 504 dnsIP := ip.String() 505 managedCluster.Spec.NetworkProfile.DnsServiceIP = &dnsIP 506 } 507 } 508 509 // OperatorSpec defines how the Secrets generated by ASO should look for the AKS cluster kubeconfigs. 510 // There is no prescribed naming convention that must be followed. 511 managedCluster.Spec.OperatorSpec = &asocontainerservicev1hub.ManagedClusterOperatorSpec{ 512 Secrets: &asocontainerservicev1hub.ManagedClusterOperatorSecrets{ 513 AdminCredentials: &genruntime.SecretDestination{ 514 Name: adminKubeconfigSecretName(s.ClusterName), 515 Key: secret.KubeconfigDataName, 516 }, 517 }, 518 } 519 520 if s.AADProfile != nil { 521 managedCluster.Spec.AadProfile = &asocontainerservicev1hub.ManagedClusterAADProfile{ 522 Managed: &s.AADProfile.Managed, 523 EnableAzureRBAC: &s.AADProfile.EnableAzureRBAC, 524 AdminGroupObjectIDs: s.AADProfile.AdminGroupObjectIDs, 525 } 526 if s.DisableLocalAccounts != nil { 527 managedCluster.Spec.DisableLocalAccounts = s.DisableLocalAccounts 528 } 529 530 if ptr.Deref(s.DisableLocalAccounts, false) { 531 // admin credentials cannot be fetched when local accounts are disabled 532 managedCluster.Spec.OperatorSpec.Secrets.AdminCredentials = nil 533 } 534 if s.AADProfile.Managed { 535 managedCluster.Spec.OperatorSpec.Secrets.UserCredentials = &genruntime.SecretDestination{ 536 Name: userKubeconfigSecretName(s.ClusterName), 537 Key: secret.KubeconfigDataName, 538 } 539 } 540 } 541 542 for i := range s.AddonProfiles { 543 if managedCluster.Spec.AddonProfiles == nil { 544 managedCluster.Spec.AddonProfiles = map[string]asocontainerservicev1hub.ManagedClusterAddonProfile{} 545 } 546 item := s.AddonProfiles[i] 547 addonProfile := asocontainerservicev1hub.ManagedClusterAddonProfile{ 548 Enabled: &item.Enabled, 549 } 550 if item.Config != nil { 551 addonProfile.Config = item.Config 552 } 553 managedCluster.Spec.AddonProfiles[item.Name] = addonProfile 554 } 555 556 if s.SKU != nil { 557 tierName := asocontainerservicev1.ManagedClusterSKU_Tier(s.SKU.Tier) 558 managedCluster.Spec.Sku = &asocontainerservicev1hub.ManagedClusterSKU{ 559 Name: ptr.To(string(asocontainerservicev1.ManagedClusterSKU_Name("Base"))), 560 Tier: ptr.To(string(tierName)), 561 } 562 } 563 564 if s.LoadBalancerProfile != nil { 565 managedCluster.Spec.NetworkProfile.LoadBalancerProfile = s.GetLoadBalancerProfile() 566 } 567 568 if s.APIServerAccessProfile != nil { 569 managedCluster.Spec.ApiServerAccessProfile = &asocontainerservicev1hub.ManagedClusterAPIServerAccessProfile{ 570 EnablePrivateCluster: s.APIServerAccessProfile.EnablePrivateCluster, 571 PrivateDNSZone: s.APIServerAccessProfile.PrivateDNSZone, 572 EnablePrivateClusterPublicFQDN: s.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, 573 } 574 575 if s.APIServerAccessProfile.AuthorizedIPRanges != nil { 576 managedCluster.Spec.ApiServerAccessProfile.AuthorizedIPRanges = s.APIServerAccessProfile.AuthorizedIPRanges 577 } 578 } 579 580 if s.OutboundType != nil { 581 managedCluster.Spec.NetworkProfile.OutboundType = ptr.To(string(asocontainerservicev1.ContainerServiceNetworkProfile_OutboundType(*s.OutboundType))) 582 } 583 584 if s.Identity != nil { 585 managedCluster.Spec.Identity, err = getIdentity(s.Identity) 586 if err != nil { 587 return nil, errors.Wrapf(err, "Identity is not valid: %s", err) 588 } 589 } 590 591 if s.KubeletUserAssignedIdentity != "" { 592 managedCluster.Spec.IdentityProfile = map[string]asocontainerservicev1hub.UserAssignedIdentity{ 593 kubeletIdentityKey: { 594 ResourceReference: &genruntime.ResourceReference{ 595 ARMID: s.KubeletUserAssignedIdentity, 596 }, 597 }, 598 } 599 } 600 601 if s.HTTPProxyConfig != nil { 602 managedCluster.Spec.HttpProxyConfig = &asocontainerservicev1hub.ManagedClusterHTTPProxyConfig{ 603 HttpProxy: s.HTTPProxyConfig.HTTPProxy, 604 HttpsProxy: s.HTTPProxyConfig.HTTPSProxy, 605 TrustedCa: s.HTTPProxyConfig.TrustedCA, 606 } 607 608 if s.HTTPProxyConfig.NoProxy != nil { 609 managedCluster.Spec.HttpProxyConfig.NoProxy = s.HTTPProxyConfig.NoProxy 610 } 611 } 612 613 if s.OIDCIssuerProfile != nil { 614 managedCluster.Spec.OidcIssuerProfile = &asocontainerservicev1hub.ManagedClusterOIDCIssuerProfile{ 615 Enabled: s.OIDCIssuerProfile.Enabled, 616 } 617 if ptr.Deref(s.OIDCIssuerProfile.Enabled, false) { 618 managedCluster.Spec.OperatorSpec.ConfigMaps = &asocontainerservicev1hub.ManagedClusterOperatorConfigMaps{ 619 OIDCIssuerProfile: &genruntime.ConfigMapDestination{ 620 Name: oidcIssuerURLConfigMapName(s.ClusterName), 621 Key: oidcIssuerProfileURL, 622 }, 623 } 624 } 625 } 626 627 if s.AutoUpgradeProfile != nil { 628 managedCluster.Spec.AutoUpgradeProfile = &asocontainerservicev1hub.ManagedClusterAutoUpgradeProfile{ 629 UpgradeChannel: (*string)(s.AutoUpgradeProfile.UpgradeChannel), 630 } 631 } 632 633 if s.SecurityProfile != nil { 634 securityProfile := &asocontainerservicev1hub.ManagedClusterSecurityProfile{} 635 if s.SecurityProfile.AzureKeyVaultKms != nil { 636 securityProfile.AzureKeyVaultKms = &asocontainerservicev1hub.AzureKeyVaultKms{ 637 Enabled: s.SecurityProfile.AzureKeyVaultKms.Enabled, 638 KeyId: s.SecurityProfile.AzureKeyVaultKms.KeyID, 639 } 640 if s.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess != nil { 641 keyVaultNetworkAccess := string(*s.SecurityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess) 642 securityProfile.AzureKeyVaultKms.KeyVaultNetworkAccess = ptr.To(string(asocontainerservicev1.AzureKeyVaultKms_KeyVaultNetworkAccess(keyVaultNetworkAccess))) 643 } 644 if s.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID != nil { 645 securityProfile.AzureKeyVaultKms.KeyVaultResourceReference = &genruntime.ResourceReference{ 646 ARMID: *s.SecurityProfile.AzureKeyVaultKms.KeyVaultResourceID, 647 } 648 } 649 } 650 if s.SecurityProfile.Defender != nil { 651 securityProfile.Defender = &asocontainerservicev1hub.ManagedClusterSecurityProfileDefender{ 652 LogAnalyticsWorkspaceResourceReference: &genruntime.ResourceReference{ 653 ARMID: *s.SecurityProfile.Defender.LogAnalyticsWorkspaceResourceID, 654 }, 655 } 656 if s.SecurityProfile.Defender.SecurityMonitoring != nil { 657 securityProfile.Defender.SecurityMonitoring = &asocontainerservicev1hub.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ 658 Enabled: s.SecurityProfile.Defender.SecurityMonitoring.Enabled, 659 } 660 } 661 } 662 if s.SecurityProfile.ImageCleaner != nil { 663 securityProfile.ImageCleaner = &asocontainerservicev1hub.ManagedClusterSecurityProfileImageCleaner{ 664 Enabled: s.SecurityProfile.ImageCleaner.Enabled, 665 IntervalHours: s.SecurityProfile.ImageCleaner.IntervalHours, 666 } 667 } 668 if s.SecurityProfile.WorkloadIdentity != nil { 669 securityProfile.WorkloadIdentity = &asocontainerservicev1hub.ManagedClusterSecurityProfileWorkloadIdentity{ 670 Enabled: s.SecurityProfile.WorkloadIdentity.Enabled, 671 } 672 } 673 managedCluster.Spec.SecurityProfile = securityProfile 674 } 675 676 // Only include AgentPoolProfiles during initial cluster creation. Agent pools are managed solely by the 677 // AzureManagedMachinePool controller thereafter. 678 var prevAgentPoolProfiles []asocontainerservicev1hub.ManagedClusterAgentPoolProfile 679 managedCluster.Spec.AgentPoolProfiles = nil 680 if managedCluster.Status.AgentPoolProfiles == nil { 681 // Add all agent pools to cluster spec that will be submitted to the API 682 agentPoolSpecs, err := s.GetAllAgentPools() 683 if err != nil { 684 return nil, errors.Wrapf(err, "failed to get agent pool specs for managed cluster %s", s.Name) 685 } 686 687 scheme := runtime.NewScheme() 688 if err := asocontainerservicev1.AddToScheme(scheme); err != nil { 689 return nil, errors.Wrap(err, "error constructing scheme") 690 } 691 if err := asocontainerservicev1preview.AddToScheme(scheme); err != nil { 692 return nil, errors.Wrap(err, "error constructing scheme") 693 } 694 for _, agentPoolSpec := range agentPoolSpecs { 695 agentPool, err := aso.PatchedParameters(ctx, scheme, agentPoolSpec, nil) 696 if err != nil { 697 return nil, errors.Wrapf(err, "failed to get agent pool parameters for managed cluster %s", s.Name) 698 } 699 agentPoolSpecTyped := agentPoolSpec.(*agentpools.AgentPoolSpec) 700 hubAgentPool := &asocontainerservicev1hub.ManagedClustersAgentPool{} 701 if s.Preview { 702 agentPoolTyped := agentPool.(*asocontainerservicev1preview.ManagedClustersAgentPool) 703 if err := agentPoolTyped.ConvertTo(hubAgentPool); err != nil { 704 return nil, err 705 } 706 } else { 707 agentPoolTyped := agentPool.(*asocontainerservicev1.ManagedClustersAgentPool) 708 if err := agentPoolTyped.ConvertTo(hubAgentPool); err != nil { 709 return nil, err 710 } 711 } 712 hubAgentPool.Spec.AzureName = agentPoolSpecTyped.AzureName 713 profile := converters.AgentPoolToManagedClusterAgentPoolProfile(hubAgentPool) 714 prevAgentPoolProfiles = append(prevAgentPoolProfiles, profile) 715 } 716 } 717 718 managedCluster.Spec.AgentPoolProfiles = prevAgentPoolProfiles 719 720 if s.Preview { 721 prev := &asocontainerservicev1preview.ManagedCluster{} 722 if err := prev.ConvertFrom(managedCluster); err != nil { 723 return nil, err 724 } 725 if existing != nil { 726 prev.Status = existingStatus 727 } 728 return prev, nil 729 } 730 731 stable := &asocontainerservicev1.ManagedCluster{} 732 if err := stable.ConvertFrom(managedCluster); err != nil { 733 return nil, err 734 } 735 736 return stable, nil 737 } 738 739 // GetLoadBalancerProfile returns an asocontainerservicev1.ManagedClusterLoadBalancerProfile from the 740 // information present in ManagedClusterSpec.LoadBalancerProfile. 741 func (s *ManagedClusterSpec) GetLoadBalancerProfile() (loadBalancerProfile *asocontainerservicev1hub.ManagedClusterLoadBalancerProfile) { 742 loadBalancerProfile = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile{ 743 AllocatedOutboundPorts: s.LoadBalancerProfile.AllocatedOutboundPorts, 744 IdleTimeoutInMinutes: s.LoadBalancerProfile.IdleTimeoutInMinutes, 745 } 746 if s.LoadBalancerProfile.ManagedOutboundIPs != nil { 747 loadBalancerProfile.ManagedOutboundIPs = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile_ManagedOutboundIPs{Count: s.LoadBalancerProfile.ManagedOutboundIPs} 748 } 749 if len(s.LoadBalancerProfile.OutboundIPPrefixes) > 0 { 750 loadBalancerProfile.OutboundIPPrefixes = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile_OutboundIPPrefixes{ 751 PublicIPPrefixes: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPPrefixes), 752 } 753 } 754 if len(s.LoadBalancerProfile.OutboundIPs) > 0 { 755 loadBalancerProfile.OutboundIPs = &asocontainerservicev1hub.ManagedClusterLoadBalancerProfile_OutboundIPs{ 756 PublicIPs: convertToResourceReferences(s.LoadBalancerProfile.OutboundIPs), 757 } 758 } 759 return 760 } 761 762 func convertToResourceReferences(resources []string) []asocontainerservicev1hub.ResourceReference { 763 resourceReferences := make([]asocontainerservicev1hub.ResourceReference, len(resources)) 764 for i := range resources { 765 resourceReferences[i] = asocontainerservicev1hub.ResourceReference{ 766 Reference: &genruntime.ResourceReference{ 767 ARMID: resources[i], 768 }, 769 } 770 } 771 return resourceReferences 772 } 773 774 func getIdentity(identity *infrav1.Identity) (managedClusterIdentity *asocontainerservicev1hub.ManagedClusterIdentity, err error) { 775 if identity.Type == "" { 776 return 777 } 778 779 managedClusterIdentity = &asocontainerservicev1hub.ManagedClusterIdentity{ 780 Type: ptr.To(string(asocontainerservicev1.ManagedClusterIdentity_Type(identity.Type))), 781 } 782 if ptr.Deref(managedClusterIdentity.Type, "") == string(asocontainerservicev1.ManagedClusterIdentity_Type_UserAssigned) { 783 if identity.UserAssignedIdentityResourceID == "" { 784 err = errors.Errorf("Identity is set to \"UserAssigned\" but no UserAssignedIdentityResourceID is present") 785 return 786 } 787 managedClusterIdentity.UserAssignedIdentities = []asocontainerservicev1hub.UserAssignedIdentityDetails{ 788 { 789 Reference: genruntime.ResourceReference{ 790 ARMID: identity.UserAssignedIdentityResourceID, 791 }, 792 }, 793 } 794 } 795 return 796 } 797 798 func adminKubeconfigSecretName(clusterName string) string { 799 return secret.Name(clusterName+"-aso", secret.Kubeconfig) 800 } 801 802 func oidcIssuerURLConfigMapName(clusterName string) string { 803 return secret.Name(clusterName+"-aso", "oidc-issuer-profile") 804 } 805 806 func userKubeconfigSecretName(clusterName string) string { 807 return secret.Name(clusterName+"-user-aso", secret.Kubeconfig) 808 } 809 810 // WasManaged implements azure.ASOResourceSpecGetter. 811 func (s *ManagedClusterSpec) WasManaged(resource genruntime.MetaObject) bool { 812 // CAPZ has never supported BYO managed clusters. 813 return true 814 } 815 816 var _ aso.TagsGetterSetter[genruntime.MetaObject] = (*ManagedClusterSpec)(nil) 817 818 // GetAdditionalTags implements aso.TagsGetterSetter. 819 func (s *ManagedClusterSpec) GetAdditionalTags() infrav1.Tags { 820 return s.Tags 821 } 822 823 // GetDesiredTags implements aso.TagsGetterSetter. 824 func (s *ManagedClusterSpec) GetDesiredTags(resource genruntime.MetaObject) infrav1.Tags { 825 if s.Preview { 826 return resource.(*asocontainerservicev1preview.ManagedCluster).Spec.Tags 827 } 828 return resource.(*asocontainerservicev1.ManagedCluster).Spec.Tags 829 } 830 831 // SetTags implements aso.TagsGetterSetter. 832 func (s *ManagedClusterSpec) SetTags(resource genruntime.MetaObject, tags infrav1.Tags) { 833 if s.Preview { 834 resource.(*asocontainerservicev1preview.ManagedCluster).Spec.Tags = tags 835 return 836 } 837 resource.(*asocontainerservicev1.ManagedCluster).Spec.Tags = tags 838 } 839 840 var _ aso.Patcher = (*ManagedClusterSpec)(nil) 841 842 // ExtraPatches implements aso.Patcher. 843 func (s *ManagedClusterSpec) ExtraPatches() []string { 844 return s.Patches 845 }