sigs.k8s.io/cluster-api@v1.6.3/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package controllers 18 19 import ( 20 "bytes" 21 "context" 22 "fmt" 23 "reflect" 24 "testing" 25 "time" 26 27 ignition "github.com/flatcar/ignition/config/v2_3" 28 "github.com/go-logr/logr" 29 . "github.com/onsi/gomega" 30 corev1 "k8s.io/api/core/v1" 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 "k8s.io/apimachinery/pkg/types" 33 bootstrapapi "k8s.io/cluster-bootstrap/token/api" 34 "k8s.io/utils/pointer" 35 ctrl "sigs.k8s.io/controller-runtime" 36 "sigs.k8s.io/controller-runtime/pkg/client" 37 "sigs.k8s.io/controller-runtime/pkg/client/fake" 38 "sigs.k8s.io/controller-runtime/pkg/log" 39 "sigs.k8s.io/controller-runtime/pkg/reconcile" 40 "sigs.k8s.io/yaml" 41 42 clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 43 bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" 44 bootstrapbuilder "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/builder" 45 "sigs.k8s.io/cluster-api/controllers/remote" 46 expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" 47 "sigs.k8s.io/cluster-api/feature" 48 "sigs.k8s.io/cluster-api/internal/test/builder" 49 "sigs.k8s.io/cluster-api/util" 50 "sigs.k8s.io/cluster-api/util/conditions" 51 "sigs.k8s.io/cluster-api/util/patch" 52 "sigs.k8s.io/cluster-api/util/secret" 53 ) 54 55 // MachineToBootstrapMapFunc return kubeadm bootstrap configref name when configref exists. 56 func TestKubeadmConfigReconciler_MachineToBootstrapMapFuncReturn(t *testing.T) { 57 g := NewWithT(t) 58 cluster := builder.Cluster("my-cluster", metav1.NamespaceDefault).Build() 59 objs := []client.Object{cluster} 60 machineObjs := []client.Object{} 61 var expectedConfigName string 62 for i := 0; i < 3; i++ { 63 configName := fmt.Sprintf("my-config-%d", i) 64 m := builder.Machine(metav1.NamespaceDefault, fmt.Sprintf("my-machine-%d", i)). 65 WithVersion("v1.19.1"). 66 WithClusterName(cluster.Name). 67 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "").Unstructured()). 68 Build() 69 if i == 1 { 70 c := newKubeadmConfig(metav1.NamespaceDefault, configName) 71 addKubeadmConfigToMachine(c, m) 72 objs = append(objs, m, c) 73 expectedConfigName = configName 74 } else { 75 objs = append(objs, m) 76 } 77 machineObjs = append(machineObjs, m) 78 } 79 fakeClient := fake.NewClientBuilder().WithObjects(objs...).Build() 80 reconciler := &KubeadmConfigReconciler{ 81 Client: fakeClient, 82 SecretCachingClient: fakeClient, 83 } 84 for i := 0; i < 3; i++ { 85 o := machineObjs[i] 86 configs := reconciler.MachineToBootstrapMapFunc(ctx, o) 87 if i == 1 { 88 g.Expect(configs[0].Name).To(Equal(expectedConfigName)) 89 } else { 90 g.Expect(configs[0].Name).To(Equal("")) 91 } 92 } 93 } 94 95 // Reconcile returns early if the kubeadm config is ready because it should never re-generate bootstrap data. 96 func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfKubeadmConfigIsReady(t *testing.T) { 97 g := NewWithT(t) 98 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster1").Build() 99 cluster.Status.InfrastructureReady = true 100 machine := builder.Machine(metav1.NamespaceDefault, "m1").WithClusterName("cluster1").Build() 101 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 102 addKubeadmConfigToMachine(config, machine) 103 104 config.Status.Ready = true 105 106 objects := []client.Object{ 107 cluster, 108 machine, 109 config, 110 } 111 myclient := fake.NewClientBuilder().WithObjects(objects...).Build() 112 113 k := &KubeadmConfigReconciler{ 114 Client: myclient, 115 SecretCachingClient: myclient, 116 } 117 118 request := ctrl.Request{ 119 NamespacedName: client.ObjectKey{ 120 Namespace: metav1.NamespaceDefault, 121 Name: "cfg", 122 }, 123 } 124 result, err := k.Reconcile(ctx, request) 125 g.Expect(err).ToNot(HaveOccurred()) 126 g.Expect(result.Requeue).To(BeFalse()) 127 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 128 } 129 130 // Reconcile should update owner references on bootstrap secrets on creation and update. 131 func TestKubeadmConfigReconciler_TestSecretOwnerReferenceReconciliation(t *testing.T) { 132 g := NewWithT(t) 133 134 clusterName := "my-cluster" 135 cluster := builder.Cluster(metav1.NamespaceDefault, clusterName).Build() 136 machine := builder.Machine(metav1.NamespaceDefault, "machine"). 137 WithVersion("v1.19.1"). 138 WithClusterName(clusterName). 139 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 140 Build() 141 machine.Spec.Bootstrap.DataSecretName = pointer.String("something") 142 143 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 144 config.SetOwnerReferences(util.EnsureOwnerRef(config.GetOwnerReferences(), metav1.OwnerReference{ 145 APIVersion: machine.APIVersion, 146 Kind: machine.Kind, 147 Name: machine.Name, 148 UID: machine.UID, 149 })) 150 secret := &corev1.Secret{ 151 ObjectMeta: metav1.ObjectMeta{ 152 Name: config.Name, 153 Namespace: config.Namespace, 154 }, 155 Type: corev1.SecretTypeBootstrapToken, 156 } 157 config.Status.Ready = true 158 159 objects := []client.Object{ 160 config, 161 machine, 162 secret, 163 cluster, 164 } 165 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 166 167 k := &KubeadmConfigReconciler{ 168 Client: myclient, 169 SecretCachingClient: myclient, 170 } 171 172 request := ctrl.Request{ 173 NamespacedName: client.ObjectKey{ 174 Namespace: metav1.NamespaceDefault, 175 Name: "cfg", 176 }, 177 } 178 var err error 179 key := client.ObjectKeyFromObject(config) 180 actual := &corev1.Secret{} 181 182 t.Run("KubeadmConfig ownerReference is added on first reconcile", func(t *testing.T) { 183 _, err = k.Reconcile(ctx, request) 184 g.Expect(err).ToNot(HaveOccurred()) 185 186 g.Expect(myclient.Get(ctx, key, actual)).To(Succeed()) 187 188 controllerOwner := metav1.GetControllerOf(actual) 189 g.Expect(controllerOwner).To(Not(BeNil())) 190 g.Expect(controllerOwner.Kind).To(Equal(config.Kind)) 191 g.Expect(controllerOwner.Name).To(Equal(config.Name)) 192 }) 193 194 t.Run("KubeadmConfig ownerReference re-reconciled without error", func(t *testing.T) { 195 _, err = k.Reconcile(ctx, request) 196 g.Expect(err).ToNot(HaveOccurred()) 197 198 g.Expect(myclient.Get(ctx, key, actual)).To(Succeed()) 199 200 controllerOwner := metav1.GetControllerOf(actual) 201 g.Expect(controllerOwner).To(Not(BeNil())) 202 g.Expect(controllerOwner.Kind).To(Equal(config.Kind)) 203 g.Expect(controllerOwner.Name).To(Equal(config.Name)) 204 }) 205 t.Run("non-KubeadmConfig controller OwnerReference is replaced", func(t *testing.T) { 206 g.Expect(myclient.Get(ctx, key, actual)).To(Succeed()) 207 208 actual.SetOwnerReferences([]metav1.OwnerReference{ 209 { 210 APIVersion: machine.APIVersion, 211 Kind: machine.Kind, 212 Name: machine.Name, 213 UID: machine.UID, 214 Controller: pointer.Bool(true), 215 }}) 216 g.Expect(myclient.Update(ctx, actual)).To(Succeed()) 217 218 _, err = k.Reconcile(ctx, request) 219 g.Expect(err).ToNot(HaveOccurred()) 220 221 g.Expect(myclient.Get(ctx, key, actual)).To(Succeed()) 222 223 controllerOwner := metav1.GetControllerOf(actual) 224 g.Expect(controllerOwner).To(Not(BeNil())) 225 g.Expect(controllerOwner.Kind).To(Equal(config.Kind)) 226 g.Expect(controllerOwner.Name).To(Equal(config.Name)) 227 }) 228 } 229 230 // Reconcile returns nil if the referenced Machine cannot be found. 231 func TestKubeadmConfigReconciler_Reconcile_ReturnNilIfReferencedMachineIsNotFound(t *testing.T) { 232 g := NewWithT(t) 233 234 machine := builder.Machine(metav1.NamespaceDefault, "machine"). 235 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 236 WithVersion("v1.19.1"). 237 Build() 238 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 239 addKubeadmConfigToMachine(config, machine) 240 objects := []client.Object{ 241 // intentionally omitting machine 242 config, 243 } 244 myclient := fake.NewClientBuilder().WithObjects(objects...).Build() 245 246 k := &KubeadmConfigReconciler{ 247 Client: myclient, 248 SecretCachingClient: myclient, 249 } 250 251 request := ctrl.Request{ 252 NamespacedName: client.ObjectKey{ 253 Namespace: metav1.NamespaceDefault, 254 Name: "cfg", 255 }, 256 } 257 _, err := k.Reconcile(ctx, request) 258 g.Expect(err).ToNot(HaveOccurred()) 259 } 260 261 // If the machine has bootstrap data secret reference, there is no need to generate more bootstrap data. 262 func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfMachineHasDataSecretName(t *testing.T) { 263 g := NewWithT(t) 264 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster1").Build() 265 cluster.Status.InfrastructureReady = true 266 267 machine := builder.Machine(metav1.NamespaceDefault, "machine"). 268 WithVersion("v1.19.1"). 269 WithClusterName("cluster1"). 270 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 271 Build() 272 machine.Spec.Bootstrap.DataSecretName = pointer.String("something") 273 274 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 275 addKubeadmConfigToMachine(config, machine) 276 objects := []client.Object{ 277 cluster, 278 machine, 279 config, 280 } 281 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 282 283 k := &KubeadmConfigReconciler{ 284 Client: myclient, 285 SecretCachingClient: myclient, 286 } 287 288 request := ctrl.Request{ 289 NamespacedName: client.ObjectKey{ 290 Namespace: metav1.NamespaceDefault, 291 Name: "cfg", 292 }, 293 } 294 result, err := k.Reconcile(ctx, request) 295 actual := &bootstrapv1.KubeadmConfig{} 296 g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: config.Namespace, Name: config.Name}, actual)).To(Succeed()) 297 g.Expect(err).ToNot(HaveOccurred()) 298 g.Expect(result.Requeue).To(BeFalse()) 299 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 300 assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition) 301 } 302 303 func TestKubeadmConfigReconciler_ReturnEarlyIfClusterInfraNotReady(t *testing.T) { 304 g := NewWithT(t) 305 306 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 307 machine := builder.Machine(metav1.NamespaceDefault, "machine"). 308 WithVersion("v1.19.1"). 309 WithClusterName(cluster.Name). 310 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 311 Build() 312 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 313 addKubeadmConfigToMachine(config, machine) 314 315 // cluster infra not ready 316 cluster.Status = clusterv1.ClusterStatus{ 317 InfrastructureReady: false, 318 } 319 320 objects := []client.Object{ 321 cluster, 322 machine, 323 config, 324 } 325 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 326 327 k := &KubeadmConfigReconciler{ 328 Client: myclient, 329 SecretCachingClient: myclient, 330 } 331 332 request := ctrl.Request{ 333 NamespacedName: client.ObjectKey{ 334 Namespace: metav1.NamespaceDefault, 335 Name: "cfg", 336 }, 337 } 338 339 expectedResult := reconcile.Result{} 340 actualResult, actualError := k.Reconcile(ctx, request) 341 g.Expect(actualResult).To(BeComparableTo(expectedResult)) 342 g.Expect(actualError).ToNot(HaveOccurred()) 343 assertHasFalseCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition, clusterv1.ConditionSeverityInfo, bootstrapv1.WaitingForClusterInfrastructureReason) 344 } 345 346 // Return early If the owning machine does not have an associated cluster. 347 func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfMachineHasNoCluster(t *testing.T) { 348 g := NewWithT(t) 349 machine := builder.Machine(metav1.NamespaceDefault, "machine"). 350 WithVersion("v1.19.1"). 351 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 352 Build() 353 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 354 addKubeadmConfigToMachine(config, machine) 355 356 objects := []client.Object{ 357 machine, 358 config, 359 } 360 myclient := fake.NewClientBuilder().WithObjects(objects...).Build() 361 362 k := &KubeadmConfigReconciler{ 363 Client: myclient, 364 SecretCachingClient: myclient, 365 } 366 367 request := ctrl.Request{ 368 NamespacedName: client.ObjectKey{ 369 Namespace: metav1.NamespaceDefault, 370 Name: "cfg", 371 }, 372 } 373 _, err := k.Reconcile(ctx, request) 374 g.Expect(err).ToNot(HaveOccurred()) 375 } 376 377 // This does not expect an error, hoping that the associated cluster will be created. 378 func TestKubeadmConfigReconciler_Reconcile_ReturnNilIfAssociatedClusterIsNotFound(t *testing.T) { 379 g := NewWithT(t) 380 381 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 382 machine := builder.Machine(metav1.NamespaceDefault, "machine"). 383 WithVersion("v1.19.1"). 384 WithClusterName(cluster.Name). 385 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 386 Build() 387 config := newKubeadmConfig(metav1.NamespaceDefault, "cfg") 388 389 addKubeadmConfigToMachine(config, machine) 390 objects := []client.Object{ 391 // intentionally omitting cluster 392 machine, 393 config, 394 } 395 myclient := fake.NewClientBuilder().WithObjects(objects...).Build() 396 397 k := &KubeadmConfigReconciler{ 398 Client: myclient, 399 SecretCachingClient: myclient, 400 } 401 402 request := ctrl.Request{ 403 NamespacedName: client.ObjectKey{ 404 Namespace: metav1.NamespaceDefault, 405 Name: "cfg", 406 }, 407 } 408 _, err := k.Reconcile(ctx, request) 409 g.Expect(err).ToNot(HaveOccurred()) 410 } 411 412 // If the control plane isn't initialized then there is no cluster for either a worker or control plane node to join. 413 func TestKubeadmConfigReconciler_Reconcile_RequeueJoiningNodesIfControlPlaneNotInitialized(t *testing.T) { 414 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 415 cluster.Status.InfrastructureReady = true 416 417 workerMachine := newWorkerMachineForCluster(cluster) 418 workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg") 419 addKubeadmConfigToMachine(workerJoinConfig, workerMachine) 420 421 controlPlaneJoinMachine := newControlPlaneMachine(cluster, "control-plane-join-machine") 422 controlPlaneJoinConfig := newControlPlaneJoinKubeadmConfig(controlPlaneJoinMachine.Namespace, "control-plane-join-cfg") 423 addKubeadmConfigToMachine(controlPlaneJoinConfig, controlPlaneJoinMachine) 424 425 testcases := []struct { 426 name string 427 request ctrl.Request 428 objects []client.Object 429 }{ 430 { 431 name: "requeue worker when control plane is not yet initialized", 432 request: ctrl.Request{ 433 NamespacedName: client.ObjectKey{ 434 Namespace: workerJoinConfig.Namespace, 435 Name: workerJoinConfig.Name, 436 }, 437 }, 438 objects: []client.Object{ 439 cluster, 440 workerMachine, 441 workerJoinConfig, 442 }, 443 }, 444 { 445 name: "requeue a secondary control plane when the control plane is not yet initialized", 446 request: ctrl.Request{ 447 NamespacedName: client.ObjectKey{ 448 Namespace: controlPlaneJoinConfig.Namespace, 449 Name: controlPlaneJoinConfig.Name, 450 }, 451 }, 452 objects: []client.Object{ 453 cluster, 454 controlPlaneJoinMachine, 455 controlPlaneJoinConfig, 456 }, 457 }, 458 } 459 for _, tc := range testcases { 460 t.Run(tc.name, func(t *testing.T) { 461 g := NewWithT(t) 462 463 myclient := fake.NewClientBuilder().WithObjects(tc.objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 464 465 k := &KubeadmConfigReconciler{ 466 Client: myclient, 467 SecretCachingClient: myclient, 468 KubeadmInitLock: &myInitLocker{}, 469 } 470 471 result, err := k.Reconcile(ctx, tc.request) 472 g.Expect(err).ToNot(HaveOccurred()) 473 g.Expect(result.Requeue).To(BeFalse()) 474 g.Expect(result.RequeueAfter).To(Equal(30 * time.Second)) 475 assertHasFalseCondition(g, myclient, tc.request, bootstrapv1.DataSecretAvailableCondition, clusterv1.ConditionSeverityInfo, clusterv1.WaitingForControlPlaneAvailableReason) 476 }) 477 } 478 } 479 480 // This generates cloud-config data but does not test the validity of it. 481 func TestKubeadmConfigReconciler_Reconcile_GenerateCloudConfigData(t *testing.T) { 482 g := NewWithT(t) 483 484 configName := "control-plane-init-cfg" 485 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 486 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "validhost", Port: 6443} 487 cluster.Status.InfrastructureReady = true 488 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 489 490 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 491 controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, configName) 492 controlPlaneInitConfig.Spec.JoinConfiguration = &bootstrapv1.JoinConfiguration{} 493 controlPlaneInitConfig.Spec.JoinConfiguration.Discovery.BootstrapToken = &bootstrapv1.BootstrapTokenDiscovery{ 494 CACertHashes: []string{"...."}, 495 } 496 497 addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine) 498 499 objects := []client.Object{ 500 cluster, 501 controlPlaneInitMachine, 502 controlPlaneInitConfig, 503 } 504 objects = append(objects, createSecrets(t, cluster, controlPlaneInitConfig)...) 505 506 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 507 508 k := &KubeadmConfigReconciler{ 509 Client: myclient, 510 SecretCachingClient: myclient, 511 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 512 KubeadmInitLock: &myInitLocker{}, 513 } 514 515 request := ctrl.Request{ 516 NamespacedName: client.ObjectKey{ 517 Namespace: metav1.NamespaceDefault, 518 Name: "control-plane-init-cfg", 519 }, 520 } 521 s := &corev1.Secret{} 522 g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: configName}, s)).ToNot(Succeed()) 523 524 result, err := k.Reconcile(ctx, request) 525 g.Expect(err).ToNot(HaveOccurred()) 526 g.Expect(result.Requeue).To(BeFalse()) 527 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 528 529 cfg, err := getKubeadmConfig(myclient, "control-plane-init-cfg", metav1.NamespaceDefault) 530 g.Expect(err).ToNot(HaveOccurred()) 531 g.Expect(cfg.Status.Ready).To(BeTrue()) 532 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 533 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 534 assertHasTrueCondition(g, myclient, request, bootstrapv1.CertificatesAvailableCondition) 535 assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition) 536 537 // Expect the Secret to exist, and for it to contain some data under the "value" key. 538 g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: configName}, s)).To(Succeed()) 539 g.Expect(s.Data["value"]).ToNot(BeEmpty()) 540 // Ensure that we don't fail trying to refresh any bootstrap tokens 541 _, err = k.Reconcile(ctx, request) 542 g.Expect(err).ToNot(HaveOccurred()) 543 } 544 545 // If a control plane has no JoinConfiguration, then we will create a default and no error will occur. 546 func TestKubeadmConfigReconciler_Reconcile_ErrorIfJoiningControlPlaneHasInvalidConfiguration(t *testing.T) { 547 g := NewWithT(t) 548 // TODO: extract this kind of code into a setup function that puts the state of objects into an initialized controlplane (implies secrets exist) 549 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 550 cluster.Status.InfrastructureReady = true 551 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 552 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 553 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 554 controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-cfg") 555 addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine) 556 557 controlPlaneJoinMachine := newControlPlaneMachine(cluster, "control-plane-join-machine") 558 controlPlaneJoinConfig := newControlPlaneJoinKubeadmConfig(controlPlaneJoinMachine.Namespace, "control-plane-join-cfg") 559 controlPlaneJoinConfig.Spec.JoinConfiguration.ControlPlane = nil // Makes controlPlaneJoinConfig invalid for a control plane machine 560 addKubeadmConfigToMachine(controlPlaneJoinConfig, controlPlaneJoinMachine) 561 562 objects := []client.Object{ 563 cluster, 564 controlPlaneJoinMachine, 565 controlPlaneJoinConfig, 566 } 567 objects = append(objects, createSecrets(t, cluster, controlPlaneInitConfig)...) 568 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 569 570 k := &KubeadmConfigReconciler{ 571 Client: myclient, 572 SecretCachingClient: myclient, 573 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 574 KubeadmInitLock: &myInitLocker{}, 575 } 576 577 request := ctrl.Request{ 578 NamespacedName: client.ObjectKey{ 579 Namespace: metav1.NamespaceDefault, 580 Name: controlPlaneJoinConfig.Name, 581 }, 582 } 583 _, err := k.Reconcile(ctx, request) 584 g.Expect(err).ToNot(HaveOccurred()) 585 actualConfig := &bootstrapv1.KubeadmConfig{} 586 g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: controlPlaneJoinConfig.Namespace, Name: controlPlaneJoinConfig.Name}, actualConfig)).To(Succeed()) 587 assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition) 588 assertHasTrueCondition(g, myclient, request, bootstrapv1.CertificatesAvailableCondition) 589 } 590 591 // If there is no APIEndpoint but everything is ready then requeue in hopes of a new APIEndpoint showing up eventually. 592 func TestKubeadmConfigReconciler_Reconcile_RequeueIfControlPlaneIsMissingAPIEndpoints(t *testing.T) { 593 g := NewWithT(t) 594 595 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 596 cluster.Status.InfrastructureReady = true 597 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 598 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 599 controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-cfg") 600 addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine) 601 602 workerMachine := newWorkerMachineForCluster(cluster) 603 workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg") 604 addKubeadmConfigToMachine(workerJoinConfig, workerMachine) 605 606 objects := []client.Object{ 607 cluster, 608 workerMachine, 609 workerJoinConfig, 610 } 611 objects = append(objects, createSecrets(t, cluster, controlPlaneInitConfig)...) 612 613 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 614 615 k := &KubeadmConfigReconciler{ 616 Client: myclient, 617 SecretCachingClient: myclient, 618 KubeadmInitLock: &myInitLocker{}, 619 } 620 621 request := ctrl.Request{ 622 NamespacedName: client.ObjectKey{ 623 Namespace: metav1.NamespaceDefault, 624 Name: "worker-join-cfg", 625 }, 626 } 627 result, err := k.Reconcile(ctx, request) 628 g.Expect(err).ToNot(HaveOccurred()) 629 g.Expect(result.Requeue).To(BeFalse()) 630 g.Expect(result.RequeueAfter).To(Equal(10 * time.Second)) 631 632 actualConfig := &bootstrapv1.KubeadmConfig{} 633 g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: workerJoinConfig.Namespace, Name: workerJoinConfig.Name}, actualConfig)).To(Succeed()) 634 635 // At this point the DataSecretAvailableCondition should not be set. CertificatesAvailableCondition should be true. 636 g.Expect(conditions.Get(actualConfig, bootstrapv1.DataSecretAvailableCondition)).To(BeNil()) 637 assertHasTrueCondition(g, myclient, request, bootstrapv1.CertificatesAvailableCondition) 638 } 639 640 func TestReconcileIfJoinCertificatesAvailableConditioninNodesAndControlPlaneIsReady(t *testing.T) { 641 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 642 cluster.Status.InfrastructureReady = true 643 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 644 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 645 646 useCases := []struct { 647 name string 648 machine *clusterv1.Machine 649 configName string 650 configBuilder func(string, string) *bootstrapv1.KubeadmConfig 651 }{ 652 { 653 name: "Join a worker node with a fully compiled kubeadm config object", 654 machine: newWorkerMachineForCluster(cluster), 655 configName: "worker-join-cfg", 656 configBuilder: newWorkerJoinKubeadmConfig, 657 }, 658 { 659 name: "Join a worker node with an empty kubeadm config object (defaults apply)", 660 machine: newWorkerMachineForCluster(cluster), 661 configName: "worker-join-cfg", 662 configBuilder: newKubeadmConfig, 663 }, 664 { 665 name: "Join a control plane node with a fully compiled kubeadm config object", 666 machine: newControlPlaneMachine(cluster, "control-plane-join-machine"), 667 configName: "control-plane-join-cfg", 668 configBuilder: newControlPlaneJoinKubeadmConfig, 669 }, 670 { 671 name: "Join a control plane node with an empty kubeadm config object (defaults apply)", 672 machine: newControlPlaneMachine(cluster, "control-plane-join-machine"), 673 configName: "control-plane-join-cfg", 674 configBuilder: newKubeadmConfig, 675 }, 676 } 677 678 for _, rt := range useCases { 679 rt := rt // pin! 680 t.Run(rt.name, func(t *testing.T) { 681 g := NewWithT(t) 682 683 config := rt.configBuilder(rt.machine.Namespace, rt.configName) 684 addKubeadmConfigToMachine(config, rt.machine) 685 686 objects := []client.Object{ 687 cluster, 688 rt.machine, 689 config, 690 } 691 objects = append(objects, createSecrets(t, cluster, config)...) 692 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 693 k := &KubeadmConfigReconciler{ 694 Client: myclient, 695 SecretCachingClient: myclient, 696 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 697 KubeadmInitLock: &myInitLocker{}, 698 } 699 700 request := ctrl.Request{ 701 NamespacedName: client.ObjectKey{ 702 Namespace: config.GetNamespace(), 703 Name: rt.configName, 704 }, 705 } 706 result, err := k.Reconcile(ctx, request) 707 g.Expect(err).ToNot(HaveOccurred()) 708 g.Expect(result.Requeue).To(BeFalse()) 709 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 710 711 cfg, err := getKubeadmConfig(myclient, rt.configName, metav1.NamespaceDefault) 712 g.Expect(err).ToNot(HaveOccurred()) 713 g.Expect(cfg.Status.Ready).To(BeTrue()) 714 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 715 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 716 assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition) 717 718 l := &corev1.SecretList{} 719 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 720 g.Expect(err).ToNot(HaveOccurred()) 721 g.Expect(l.Items).To(HaveLen(1)) 722 }) 723 } 724 } 725 726 func TestReconcileIfJoinNodePoolsAndControlPlaneIsReady(t *testing.T) { 727 _ = feature.MutableGates.Set("MachinePool=true") 728 729 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 730 cluster.Status.InfrastructureReady = true 731 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 732 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 733 734 useCases := []struct { 735 name string 736 machinePool *expv1.MachinePool 737 configName string 738 configBuilder func(string, string) *bootstrapv1.KubeadmConfig 739 }{ 740 { 741 name: "Join a worker node with a fully compiled kubeadm config object", 742 machinePool: newWorkerMachinePoolForCluster(cluster), 743 configName: "workerpool-join-cfg", 744 configBuilder: func(namespace, name string) *bootstrapv1.KubeadmConfig { 745 return newWorkerJoinKubeadmConfig(namespace, "workerpool-join-cfg") 746 }, 747 }, 748 { 749 name: "Join a worker node with an empty kubeadm config object (defaults apply)", 750 machinePool: newWorkerMachinePoolForCluster(cluster), 751 configName: "workerpool-join-cfg", 752 configBuilder: newKubeadmConfig, 753 }, 754 } 755 756 for _, rt := range useCases { 757 rt := rt // pin! 758 t.Run(rt.name, func(t *testing.T) { 759 g := NewWithT(t) 760 761 config := rt.configBuilder(rt.machinePool.Namespace, rt.configName) 762 addKubeadmConfigToMachinePool(config, rt.machinePool) 763 764 objects := []client.Object{ 765 cluster, 766 rt.machinePool, 767 config, 768 } 769 objects = append(objects, createSecrets(t, cluster, config)...) 770 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 771 k := &KubeadmConfigReconciler{ 772 Client: myclient, 773 SecretCachingClient: myclient, 774 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 775 KubeadmInitLock: &myInitLocker{}, 776 } 777 778 request := ctrl.Request{ 779 NamespacedName: client.ObjectKey{ 780 Namespace: config.GetNamespace(), 781 Name: rt.configName, 782 }, 783 } 784 result, err := k.Reconcile(ctx, request) 785 g.Expect(err).ToNot(HaveOccurred()) 786 g.Expect(result.Requeue).To(BeFalse()) 787 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 788 789 cfg, err := getKubeadmConfig(myclient, rt.configName, metav1.NamespaceDefault) 790 g.Expect(err).ToNot(HaveOccurred()) 791 g.Expect(cfg.Status.Ready).To(BeTrue()) 792 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 793 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 794 795 l := &corev1.SecretList{} 796 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 797 g.Expect(err).ToNot(HaveOccurred()) 798 g.Expect(l.Items).To(HaveLen(1)) 799 }) 800 } 801 } 802 803 // Ensure bootstrap data is generated in the correct format based on the format specified in the 804 // KubeadmConfig resource. 805 func TestBootstrapDataFormat(t *testing.T) { 806 testcases := []struct { 807 name string 808 isWorker bool 809 format bootstrapv1.Format 810 clusterInitialized bool 811 }{ 812 { 813 name: "cloud-config init config", 814 format: bootstrapv1.CloudConfig, 815 }, 816 { 817 name: "Ignition init config", 818 format: bootstrapv1.Ignition, 819 }, 820 { 821 name: "Ignition control plane join config", 822 format: bootstrapv1.Ignition, 823 clusterInitialized: true, 824 }, 825 { 826 name: "Ignition worker join config", 827 isWorker: true, 828 format: bootstrapv1.Ignition, 829 clusterInitialized: true, 830 }, 831 { 832 name: "Empty format field", 833 }, 834 } 835 836 for _, tc := range testcases { 837 t.Run(tc.name, func(t *testing.T) { 838 g := NewWithT(t) 839 840 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 841 cluster.Status.InfrastructureReady = true 842 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 843 if tc.clusterInitialized { 844 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 845 } 846 847 var machine *clusterv1.Machine 848 var config *bootstrapv1.KubeadmConfig 849 var configName string 850 if tc.isWorker { 851 machine = newWorkerMachineForCluster(cluster) 852 configName = "worker-join-cfg" 853 config = newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, configName) 854 addKubeadmConfigToMachine(config, machine) 855 } else { 856 machine = newControlPlaneMachine(cluster, "machine") 857 configName = "cfg" 858 config = newControlPlaneInitKubeadmConfig(metav1.NamespaceDefault, configName) 859 addKubeadmConfigToMachine(config, machine) 860 } 861 config.Spec.Format = tc.format 862 863 objects := []client.Object{ 864 cluster, 865 machine, 866 config, 867 } 868 objects = append(objects, createSecrets(t, cluster, config)...) 869 870 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 871 872 k := &KubeadmConfigReconciler{ 873 Client: myclient, 874 SecretCachingClient: myclient, 875 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 876 KubeadmInitLock: &myInitLocker{}, 877 } 878 request := ctrl.Request{ 879 NamespacedName: client.ObjectKey{ 880 Namespace: metav1.NamespaceDefault, 881 Name: configName, 882 }, 883 } 884 885 // Reconcile the KubeadmConfig resource. 886 _, err := k.Reconcile(ctx, request) 887 g.Expect(err).ToNot(HaveOccurred()) 888 889 // Verify the KubeadmConfig resource state is correct. 890 cfg, err := getKubeadmConfig(myclient, configName, metav1.NamespaceDefault) 891 g.Expect(err).ToNot(HaveOccurred()) 892 g.Expect(cfg.Status.Ready).To(BeTrue()) 893 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 894 895 // Read the secret containing the bootstrap data which was generated by the 896 // KubeadmConfig controller. 897 key := client.ObjectKey{ 898 Namespace: metav1.NamespaceDefault, 899 Name: *cfg.Status.DataSecretName, 900 } 901 secret := &corev1.Secret{} 902 err = myclient.Get(ctx, key, secret) 903 g.Expect(err).ToNot(HaveOccurred()) 904 905 // Verify the format field of the bootstrap data secret is correct. 906 g.Expect(string(secret.Data["format"])).To(Equal(string(tc.format))) 907 908 // Verify the bootstrap data value is in the correct format. 909 data := secret.Data["value"] 910 switch tc.format { 911 case bootstrapv1.CloudConfig, "": 912 // Verify the bootstrap data is valid YAML. 913 // TODO: Verify the YAML document is valid cloud-config? 914 var out interface{} 915 err = yaml.Unmarshal(data, &out) 916 g.Expect(err).ToNot(HaveOccurred()) 917 case bootstrapv1.Ignition: 918 // Verify the bootstrap data is valid Ignition. 919 _, reports, err := ignition.Parse(data) 920 g.Expect(err).ToNot(HaveOccurred()) 921 g.Expect(reports.IsFatal()).NotTo(BeTrue()) 922 } 923 }) 924 } 925 } 926 927 // during kubeadmconfig reconcile it is possible that bootstrap secret gets created 928 // but kubeadmconfig is not patched, do not error if secret already exists. 929 // ignore the alreadyexists error and update the status to ready. 930 func TestKubeadmConfigSecretCreatedStatusNotPatched(t *testing.T) { 931 g := NewWithT(t) 932 933 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 934 cluster.Status.InfrastructureReady = true 935 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 936 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 937 938 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 939 initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config") 940 addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine) 941 942 workerMachine := newWorkerMachineForCluster(cluster) 943 workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg") 944 addKubeadmConfigToMachine(workerJoinConfig, workerMachine) 945 objects := []client.Object{ 946 cluster, 947 workerMachine, 948 workerJoinConfig, 949 } 950 951 objects = append(objects, createSecrets(t, cluster, initConfig)...) 952 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 953 k := &KubeadmConfigReconciler{ 954 Client: myclient, 955 SecretCachingClient: myclient, 956 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 957 KubeadmInitLock: &myInitLocker{}, 958 } 959 request := ctrl.Request{ 960 NamespacedName: client.ObjectKey{ 961 Namespace: metav1.NamespaceDefault, 962 Name: "worker-join-cfg", 963 }, 964 } 965 secret := &corev1.Secret{ 966 ObjectMeta: metav1.ObjectMeta{ 967 Name: workerJoinConfig.Name, 968 Namespace: workerJoinConfig.Namespace, 969 Labels: map[string]string{ 970 clusterv1.ClusterNameLabel: cluster.Name, 971 }, 972 OwnerReferences: []metav1.OwnerReference{ 973 { 974 APIVersion: bootstrapv1.GroupVersion.String(), 975 Kind: "KubeadmConfig", 976 Name: workerJoinConfig.Name, 977 UID: workerJoinConfig.UID, 978 Controller: pointer.Bool(true), 979 }, 980 }, 981 }, 982 Data: map[string][]byte{ 983 "value": nil, 984 }, 985 Type: clusterv1.ClusterSecretType, 986 } 987 988 err := myclient.Create(ctx, secret) 989 g.Expect(err).ToNot(HaveOccurred()) 990 result, err := k.Reconcile(ctx, request) 991 g.Expect(err).ToNot(HaveOccurred()) 992 g.Expect(result.Requeue).To(BeFalse()) 993 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 994 995 cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault) 996 g.Expect(err).ToNot(HaveOccurred()) 997 g.Expect(cfg.Status.Ready).To(BeTrue()) 998 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 999 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 1000 } 1001 1002 func TestBootstrapTokenTTLExtension(t *testing.T) { 1003 g := NewWithT(t) 1004 1005 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 1006 cluster.Status.InfrastructureReady = true 1007 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 1008 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 1009 1010 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 1011 initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config") 1012 addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine) 1013 1014 workerMachine := newWorkerMachineForCluster(cluster) 1015 workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg") 1016 addKubeadmConfigToMachine(workerJoinConfig, workerMachine) 1017 1018 controlPlaneJoinMachine := newControlPlaneMachine(cluster, "control-plane-join-machine") 1019 controlPlaneJoinConfig := newControlPlaneJoinKubeadmConfig(controlPlaneJoinMachine.Namespace, "control-plane-join-cfg") 1020 addKubeadmConfigToMachine(controlPlaneJoinConfig, controlPlaneJoinMachine) 1021 objects := []client.Object{ 1022 cluster, 1023 workerMachine, 1024 workerJoinConfig, 1025 controlPlaneJoinMachine, 1026 controlPlaneJoinConfig, 1027 } 1028 1029 objects = append(objects, createSecrets(t, cluster, initConfig)...) 1030 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &clusterv1.Machine{}).Build() 1031 k := &KubeadmConfigReconciler{ 1032 Client: myclient, 1033 SecretCachingClient: myclient, 1034 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 1035 KubeadmInitLock: &myInitLocker{}, 1036 TokenTTL: DefaultTokenTTL, 1037 } 1038 request := ctrl.Request{ 1039 NamespacedName: client.ObjectKey{ 1040 Namespace: metav1.NamespaceDefault, 1041 Name: "worker-join-cfg", 1042 }, 1043 } 1044 result, err := k.Reconcile(ctx, request) 1045 g.Expect(err).ToNot(HaveOccurred()) 1046 g.Expect(result.Requeue).To(BeFalse()) 1047 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1048 1049 cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault) 1050 g.Expect(err).ToNot(HaveOccurred()) 1051 g.Expect(cfg.Status.Ready).To(BeTrue()) 1052 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 1053 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 1054 1055 request = ctrl.Request{ 1056 NamespacedName: client.ObjectKey{ 1057 Namespace: metav1.NamespaceDefault, 1058 Name: "control-plane-join-cfg", 1059 }, 1060 } 1061 result, err = k.Reconcile(ctx, request) 1062 g.Expect(err).ToNot(HaveOccurred()) 1063 g.Expect(result.Requeue).To(BeFalse()) 1064 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1065 1066 cfg, err = getKubeadmConfig(myclient, "control-plane-join-cfg", metav1.NamespaceDefault) 1067 g.Expect(err).ToNot(HaveOccurred()) 1068 g.Expect(cfg.Status.Ready).To(BeTrue()) 1069 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 1070 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 1071 1072 l := &corev1.SecretList{} 1073 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1074 g.Expect(err).ToNot(HaveOccurred()) 1075 g.Expect(l.Items).To(HaveLen(2)) 1076 1077 // ensure that the token is refreshed... 1078 tokenExpires := make([][]byte, len(l.Items)) 1079 1080 for i, item := range l.Items { 1081 tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey] 1082 } 1083 1084 <-time.After(1 * time.Second) 1085 1086 for _, req := range []ctrl.Request{ 1087 { 1088 NamespacedName: client.ObjectKey{ 1089 Namespace: metav1.NamespaceDefault, 1090 Name: "worker-join-cfg", 1091 }, 1092 }, 1093 { 1094 NamespacedName: client.ObjectKey{ 1095 Namespace: metav1.NamespaceDefault, 1096 Name: "control-plane-join-cfg", 1097 }, 1098 }, 1099 } { 1100 result, err := k.Reconcile(ctx, req) 1101 g.Expect(err).ToNot(HaveOccurred()) 1102 g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL)) 1103 } 1104 1105 l = &corev1.SecretList{} 1106 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1107 g.Expect(err).ToNot(HaveOccurred()) 1108 g.Expect(l.Items).To(HaveLen(2)) 1109 1110 for i, item := range l.Items { 1111 g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse()) 1112 tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey] 1113 } 1114 1115 // ...the infrastructure is marked "ready", but token should still be refreshed... 1116 patchHelper, err := patch.NewHelper(workerMachine, myclient) 1117 g.Expect(err).ShouldNot(HaveOccurred()) 1118 workerMachine.Status.InfrastructureReady = true 1119 g.Expect(patchHelper.Patch(ctx, workerMachine)).To(Succeed()) 1120 1121 patchHelper, err = patch.NewHelper(controlPlaneJoinMachine, myclient) 1122 g.Expect(err).ShouldNot(HaveOccurred()) 1123 controlPlaneJoinMachine.Status.InfrastructureReady = true 1124 g.Expect(patchHelper.Patch(ctx, controlPlaneJoinMachine)).To(Succeed()) 1125 1126 <-time.After(1 * time.Second) 1127 1128 for _, req := range []ctrl.Request{ 1129 { 1130 NamespacedName: client.ObjectKey{ 1131 Namespace: metav1.NamespaceDefault, 1132 Name: "worker-join-cfg", 1133 }, 1134 }, 1135 { 1136 NamespacedName: client.ObjectKey{ 1137 Namespace: metav1.NamespaceDefault, 1138 Name: "control-plane-join-cfg", 1139 }, 1140 }, 1141 } { 1142 result, err := k.Reconcile(ctx, req) 1143 g.Expect(err).ToNot(HaveOccurred()) 1144 g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL)) 1145 } 1146 1147 l = &corev1.SecretList{} 1148 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1149 g.Expect(err).ToNot(HaveOccurred()) 1150 g.Expect(l.Items).To(HaveLen(2)) 1151 1152 for i, item := range l.Items { 1153 g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse()) 1154 tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey] 1155 } 1156 1157 // ...until the Nodes have actually joined the cluster and we get a nodeRef 1158 patchHelper, err = patch.NewHelper(workerMachine, myclient) 1159 g.Expect(err).ShouldNot(HaveOccurred()) 1160 workerMachine.Status.NodeRef = &corev1.ObjectReference{ 1161 APIVersion: "v1", 1162 Kind: "Node", 1163 Name: "worker-node", 1164 } 1165 g.Expect(patchHelper.Patch(ctx, workerMachine)).To(Succeed()) 1166 1167 patchHelper, err = patch.NewHelper(controlPlaneJoinMachine, myclient) 1168 g.Expect(err).ShouldNot(HaveOccurred()) 1169 controlPlaneJoinMachine.Status.NodeRef = &corev1.ObjectReference{ 1170 APIVersion: "v1", 1171 Kind: "Node", 1172 Name: "control-plane-node", 1173 } 1174 g.Expect(patchHelper.Patch(ctx, controlPlaneJoinMachine)).To(Succeed()) 1175 1176 <-time.After(1 * time.Second) 1177 1178 for _, req := range []ctrl.Request{ 1179 { 1180 NamespacedName: client.ObjectKey{ 1181 Namespace: metav1.NamespaceDefault, 1182 Name: "worker-join-cfg", 1183 }, 1184 }, 1185 { 1186 NamespacedName: client.ObjectKey{ 1187 Namespace: metav1.NamespaceDefault, 1188 Name: "control-plane-join-cfg", 1189 }, 1190 }, 1191 } { 1192 result, err := k.Reconcile(ctx, req) 1193 g.Expect(err).ToNot(HaveOccurred()) 1194 g.Expect(result.Requeue).To(BeFalse()) 1195 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1196 } 1197 1198 l = &corev1.SecretList{} 1199 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1200 g.Expect(err).ToNot(HaveOccurred()) 1201 g.Expect(l.Items).To(HaveLen(2)) 1202 1203 for i, item := range l.Items { 1204 g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeTrue()) 1205 } 1206 } 1207 1208 func TestBootstrapTokenRotationMachinePool(t *testing.T) { 1209 _ = feature.MutableGates.Set("MachinePool=true") 1210 g := NewWithT(t) 1211 1212 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 1213 cluster.Status.InfrastructureReady = true 1214 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 1215 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} 1216 1217 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 1218 initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config") 1219 1220 addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine) 1221 1222 workerMachinePool := newWorkerMachinePoolForCluster(cluster) 1223 workerJoinConfig := newWorkerJoinKubeadmConfig(workerMachinePool.Namespace, "workerpool-join-cfg") 1224 addKubeadmConfigToMachinePool(workerJoinConfig, workerMachinePool) 1225 objects := []client.Object{ 1226 cluster, 1227 workerMachinePool, 1228 workerJoinConfig, 1229 } 1230 1231 objects = append(objects, createSecrets(t, cluster, initConfig)...) 1232 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &expv1.MachinePool{}).Build() 1233 k := &KubeadmConfigReconciler{ 1234 Client: myclient, 1235 SecretCachingClient: myclient, 1236 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 1237 KubeadmInitLock: &myInitLocker{}, 1238 TokenTTL: DefaultTokenTTL, 1239 } 1240 request := ctrl.Request{ 1241 NamespacedName: client.ObjectKey{ 1242 Namespace: metav1.NamespaceDefault, 1243 Name: "workerpool-join-cfg", 1244 }, 1245 } 1246 result, err := k.Reconcile(ctx, request) 1247 g.Expect(err).ToNot(HaveOccurred()) 1248 g.Expect(result.Requeue).To(BeFalse()) 1249 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1250 1251 cfg, err := getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault) 1252 g.Expect(err).ToNot(HaveOccurred()) 1253 g.Expect(cfg.Status.Ready).To(BeTrue()) 1254 g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) 1255 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 1256 1257 l := &corev1.SecretList{} 1258 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1259 g.Expect(err).ToNot(HaveOccurred()) 1260 g.Expect(l.Items).To(HaveLen(1)) 1261 1262 // ensure that the token is refreshed... 1263 tokenExpires := make([][]byte, len(l.Items)) 1264 1265 for i, item := range l.Items { 1266 tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey] 1267 } 1268 1269 <-time.After(1 * time.Second) 1270 1271 for _, req := range []ctrl.Request{ 1272 { 1273 NamespacedName: client.ObjectKey{ 1274 Namespace: metav1.NamespaceDefault, 1275 Name: "workerpool-join-cfg", 1276 }, 1277 }, 1278 } { 1279 result, err := k.Reconcile(ctx, req) 1280 g.Expect(err).ToNot(HaveOccurred()) 1281 g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL)) 1282 } 1283 1284 l = &corev1.SecretList{} 1285 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1286 g.Expect(err).ToNot(HaveOccurred()) 1287 g.Expect(l.Items).To(HaveLen(1)) 1288 1289 for i, item := range l.Items { 1290 g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse()) 1291 tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey] 1292 } 1293 1294 // ...the infrastructure is marked "ready", but token should still be refreshed... 1295 patchHelper, err := patch.NewHelper(workerMachinePool, myclient) 1296 g.Expect(err).ShouldNot(HaveOccurred()) 1297 workerMachinePool.Status.InfrastructureReady = true 1298 g.Expect(patchHelper.Patch(ctx, workerMachinePool, patch.WithStatusObservedGeneration{})).To(Succeed()) 1299 1300 <-time.After(1 * time.Second) 1301 1302 request = ctrl.Request{ 1303 NamespacedName: client.ObjectKey{ 1304 Namespace: metav1.NamespaceDefault, 1305 Name: "workerpool-join-cfg", 1306 }, 1307 } 1308 result, err = k.Reconcile(ctx, request) 1309 g.Expect(err).ToNot(HaveOccurred()) 1310 g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL)) 1311 1312 l = &corev1.SecretList{} 1313 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1314 g.Expect(err).ToNot(HaveOccurred()) 1315 g.Expect(l.Items).To(HaveLen(1)) 1316 1317 for i, item := range l.Items { 1318 g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse()) 1319 tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey] 1320 } 1321 1322 // ...until all nodes have joined 1323 workerMachinePool.Status.NodeRefs = []corev1.ObjectReference{ 1324 { 1325 Kind: "Node", 1326 Namespace: metav1.NamespaceDefault, 1327 Name: "node-0", 1328 }, 1329 } 1330 g.Expect(patchHelper.Patch(ctx, workerMachinePool, patch.WithStatusObservedGeneration{})).To(Succeed()) 1331 1332 <-time.After(1 * time.Second) 1333 1334 request = ctrl.Request{ 1335 NamespacedName: client.ObjectKey{ 1336 Namespace: metav1.NamespaceDefault, 1337 Name: "workerpool-join-cfg", 1338 }, 1339 } 1340 result, err = k.Reconcile(ctx, request) 1341 g.Expect(err).ToNot(HaveOccurred()) 1342 g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3)) 1343 1344 l = &corev1.SecretList{} 1345 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1346 g.Expect(err).ToNot(HaveOccurred()) 1347 g.Expect(l.Items).To(HaveLen(1)) 1348 1349 for i, item := range l.Items { 1350 g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeTrue()) 1351 } 1352 1353 // before token expires, it should rotate it 1354 tokenExpires[0] = []byte(time.Now().UTC().Add(k.TokenTTL / 5).Format(time.RFC3339)) 1355 l.Items[0].Data[bootstrapapi.BootstrapTokenExpirationKey] = tokenExpires[0] 1356 err = myclient.Update(ctx, &l.Items[0]) 1357 g.Expect(err).ToNot(HaveOccurred()) 1358 1359 request = ctrl.Request{ 1360 NamespacedName: client.ObjectKey{ 1361 Namespace: metav1.NamespaceDefault, 1362 Name: "workerpool-join-cfg", 1363 }, 1364 } 1365 result, err = k.Reconcile(ctx, request) 1366 g.Expect(err).ToNot(HaveOccurred()) 1367 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1368 1369 l = &corev1.SecretList{} 1370 err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem))) 1371 g.Expect(err).ToNot(HaveOccurred()) 1372 g.Expect(l.Items).To(HaveLen(2)) 1373 foundOld := false 1374 foundNew := true 1375 for _, item := range l.Items { 1376 if bytes.Equal(item.Data[bootstrapapi.BootstrapTokenExpirationKey], tokenExpires[0]) { 1377 foundOld = true 1378 } else { 1379 expirationTime, err := time.Parse(time.RFC3339, string(item.Data[bootstrapapi.BootstrapTokenExpirationKey])) 1380 g.Expect(err).ToNot(HaveOccurred()) 1381 g.Expect(expirationTime).Should(BeTemporally("~", time.Now().UTC().Add(k.TokenTTL), 10*time.Second)) 1382 foundNew = true 1383 } 1384 } 1385 g.Expect(foundOld).To(BeTrue()) 1386 g.Expect(foundNew).To(BeTrue()) 1387 } 1388 1389 // Ensure the discovery portion of the JoinConfiguration gets generated correctly. 1390 func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileBehaviors(t *testing.T) { 1391 caHash := []string{"...."} 1392 bootstrapToken := bootstrapv1.Discovery{ 1393 BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{ 1394 CACertHashes: caHash, 1395 }, 1396 } 1397 goodcluster := &clusterv1.Cluster{ 1398 Spec: clusterv1.ClusterSpec{ 1399 ControlPlaneEndpoint: clusterv1.APIEndpoint{ 1400 Host: "example.com", 1401 Port: 6443, 1402 }, 1403 }, 1404 } 1405 testcases := []struct { 1406 name string 1407 cluster *clusterv1.Cluster 1408 config *bootstrapv1.KubeadmConfig 1409 validateDiscovery func(*WithT, *bootstrapv1.KubeadmConfig) error 1410 }{ 1411 { 1412 name: "Automatically generate token if discovery not specified", 1413 cluster: goodcluster, 1414 config: &bootstrapv1.KubeadmConfig{ 1415 Spec: bootstrapv1.KubeadmConfigSpec{ 1416 JoinConfiguration: &bootstrapv1.JoinConfiguration{ 1417 Discovery: bootstrapToken, 1418 }, 1419 }, 1420 }, 1421 validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error { 1422 d := c.Spec.JoinConfiguration.Discovery 1423 g.Expect(d.BootstrapToken).NotTo(BeNil()) 1424 g.Expect(d.BootstrapToken.Token).NotTo(Equal("")) 1425 g.Expect(d.BootstrapToken.APIServerEndpoint).To(Equal("example.com:6443")) 1426 g.Expect(d.BootstrapToken.UnsafeSkipCAVerification).To(BeFalse()) 1427 return nil 1428 }, 1429 }, 1430 { 1431 name: "Respect discoveryConfiguration.File", 1432 cluster: goodcluster, 1433 config: &bootstrapv1.KubeadmConfig{ 1434 Spec: bootstrapv1.KubeadmConfigSpec{ 1435 JoinConfiguration: &bootstrapv1.JoinConfiguration{ 1436 Discovery: bootstrapv1.Discovery{ 1437 File: &bootstrapv1.FileDiscovery{}, 1438 }, 1439 }, 1440 }, 1441 }, 1442 validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error { 1443 d := c.Spec.JoinConfiguration.Discovery 1444 g.Expect(d.BootstrapToken).To(BeNil()) 1445 return nil 1446 }, 1447 }, 1448 { 1449 name: "Respect discoveryConfiguration.BootstrapToken.APIServerEndpoint", 1450 cluster: goodcluster, 1451 config: &bootstrapv1.KubeadmConfig{ 1452 Spec: bootstrapv1.KubeadmConfigSpec{ 1453 JoinConfiguration: &bootstrapv1.JoinConfiguration{ 1454 Discovery: bootstrapv1.Discovery{ 1455 BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{ 1456 CACertHashes: caHash, 1457 APIServerEndpoint: "bar.com:6443", 1458 }, 1459 }, 1460 }, 1461 }, 1462 }, 1463 validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error { 1464 d := c.Spec.JoinConfiguration.Discovery 1465 g.Expect(d.BootstrapToken.APIServerEndpoint).To(Equal("bar.com:6443")) 1466 return nil 1467 }, 1468 }, 1469 { 1470 name: "Respect discoveryConfiguration.BootstrapToken.Token", 1471 cluster: goodcluster, 1472 config: &bootstrapv1.KubeadmConfig{ 1473 Spec: bootstrapv1.KubeadmConfigSpec{ 1474 JoinConfiguration: &bootstrapv1.JoinConfiguration{ 1475 Discovery: bootstrapv1.Discovery{ 1476 BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{ 1477 CACertHashes: caHash, 1478 Token: "abcdef.0123456789abcdef", 1479 }, 1480 }, 1481 }, 1482 }, 1483 }, 1484 validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error { 1485 d := c.Spec.JoinConfiguration.Discovery 1486 g.Expect(d.BootstrapToken.Token).To(Equal("abcdef.0123456789abcdef")) 1487 return nil 1488 }, 1489 }, 1490 { 1491 name: "Respect discoveryConfiguration.BootstrapToken.CACertHashes", 1492 cluster: goodcluster, 1493 config: &bootstrapv1.KubeadmConfig{ 1494 Spec: bootstrapv1.KubeadmConfigSpec{ 1495 JoinConfiguration: &bootstrapv1.JoinConfiguration{ 1496 Discovery: bootstrapv1.Discovery{ 1497 BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{ 1498 CACertHashes: caHash, 1499 }, 1500 }, 1501 }, 1502 }, 1503 }, 1504 validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error { 1505 d := c.Spec.JoinConfiguration.Discovery 1506 g.Expect(reflect.DeepEqual(d.BootstrapToken.CACertHashes, caHash)).To(BeTrue()) 1507 return nil 1508 }, 1509 }, 1510 } 1511 1512 for _, tc := range testcases { 1513 t.Run(tc.name, func(t *testing.T) { 1514 g := NewWithT(t) 1515 1516 fakeClient := fake.NewClientBuilder().Build() 1517 k := &KubeadmConfigReconciler{ 1518 Client: fakeClient, 1519 SecretCachingClient: fakeClient, 1520 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), fakeClient, fakeClient.Scheme(), client.ObjectKey{Name: tc.cluster.Name, Namespace: tc.cluster.Namespace}), 1521 KubeadmInitLock: &myInitLocker{}, 1522 } 1523 1524 res, err := k.reconcileDiscovery(ctx, tc.cluster, tc.config, secret.Certificates{}) 1525 g.Expect(res.IsZero()).To(BeTrue()) 1526 g.Expect(err).ToNot(HaveOccurred()) 1527 1528 err = tc.validateDiscovery(g, tc.config) 1529 g.Expect(err).ToNot(HaveOccurred()) 1530 }) 1531 } 1532 } 1533 1534 // Test failure cases for the discovery reconcile function. 1535 func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileFailureBehaviors(t *testing.T) { 1536 k := &KubeadmConfigReconciler{} 1537 1538 testcases := []struct { 1539 name string 1540 cluster *clusterv1.Cluster 1541 config *bootstrapv1.KubeadmConfig 1542 1543 result ctrl.Result 1544 err error 1545 }{ 1546 { 1547 name: "Should requeue if cluster has not ControlPlaneEndpoint", 1548 cluster: &clusterv1.Cluster{}, // cluster without endpoints 1549 config: &bootstrapv1.KubeadmConfig{ 1550 Spec: bootstrapv1.KubeadmConfigSpec{ 1551 JoinConfiguration: &bootstrapv1.JoinConfiguration{ 1552 Discovery: bootstrapv1.Discovery{ 1553 BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{ 1554 CACertHashes: []string{"item"}, 1555 }, 1556 }, 1557 }, 1558 }, 1559 }, 1560 result: ctrl.Result{RequeueAfter: 10 * time.Second}, 1561 }, 1562 } 1563 1564 for _, tc := range testcases { 1565 t.Run(tc.name, func(t *testing.T) { 1566 g := NewWithT(t) 1567 1568 res, err := k.reconcileDiscovery(ctx, tc.cluster, tc.config, secret.Certificates{}) 1569 g.Expect(res).To(BeComparableTo(tc.result)) 1570 if tc.err == nil { 1571 g.Expect(err).ToNot(HaveOccurred()) 1572 } else { 1573 g.Expect(err).To(Equal(tc.err)) 1574 } 1575 }) 1576 } 1577 } 1578 1579 // Set cluster configuration defaults based on dynamic values from the cluster object. 1580 func TestKubeadmConfigReconciler_Reconcile_DynamicDefaultsForClusterConfiguration(t *testing.T) { 1581 k := &KubeadmConfigReconciler{} 1582 1583 testcases := []struct { 1584 name string 1585 cluster *clusterv1.Cluster 1586 machine *clusterv1.Machine 1587 config *bootstrapv1.KubeadmConfig 1588 }{ 1589 { 1590 name: "Config settings have precedence", 1591 config: &bootstrapv1.KubeadmConfig{ 1592 Spec: bootstrapv1.KubeadmConfigSpec{ 1593 ClusterConfiguration: &bootstrapv1.ClusterConfiguration{ 1594 ClusterName: "mycluster", 1595 KubernetesVersion: "myversion", 1596 Networking: bootstrapv1.Networking{ 1597 PodSubnet: "myPodSubnet", 1598 ServiceSubnet: "myServiceSubnet", 1599 DNSDomain: "myDNSDomain", 1600 }, 1601 ControlPlaneEndpoint: "myControlPlaneEndpoint:6443", 1602 }, 1603 }, 1604 }, 1605 cluster: &clusterv1.Cluster{ 1606 ObjectMeta: metav1.ObjectMeta{ 1607 Name: "OtherName", 1608 }, 1609 Spec: clusterv1.ClusterSpec{ 1610 ClusterNetwork: &clusterv1.ClusterNetwork{ 1611 Services: &clusterv1.NetworkRanges{CIDRBlocks: []string{"otherServicesCidr"}}, 1612 Pods: &clusterv1.NetworkRanges{CIDRBlocks: []string{"otherPodsCidr"}}, 1613 ServiceDomain: "otherServiceDomain", 1614 }, 1615 ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: "otherVersion", Port: 0}, 1616 }, 1617 }, 1618 machine: &clusterv1.Machine{ 1619 Spec: clusterv1.MachineSpec{ 1620 Version: pointer.String("otherVersion"), 1621 }, 1622 }, 1623 }, 1624 { 1625 name: "Top level object settings are used in case config settings are missing", 1626 config: &bootstrapv1.KubeadmConfig{ 1627 Spec: bootstrapv1.KubeadmConfigSpec{ 1628 ClusterConfiguration: &bootstrapv1.ClusterConfiguration{}, 1629 }, 1630 }, 1631 cluster: &clusterv1.Cluster{ 1632 ObjectMeta: metav1.ObjectMeta{ 1633 Name: "mycluster", 1634 }, 1635 Spec: clusterv1.ClusterSpec{ 1636 ClusterNetwork: &clusterv1.ClusterNetwork{ 1637 Services: &clusterv1.NetworkRanges{CIDRBlocks: []string{"myServiceSubnet"}}, 1638 Pods: &clusterv1.NetworkRanges{CIDRBlocks: []string{"myPodSubnet"}}, 1639 ServiceDomain: "myDNSDomain", 1640 }, 1641 ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: "myControlPlaneEndpoint", Port: 6443}, 1642 }, 1643 }, 1644 machine: &clusterv1.Machine{ 1645 Spec: clusterv1.MachineSpec{ 1646 Version: pointer.String("myversion"), 1647 }, 1648 }, 1649 }, 1650 } 1651 1652 for _, tc := range testcases { 1653 t.Run(tc.name, func(t *testing.T) { 1654 g := NewWithT(t) 1655 1656 k.reconcileTopLevelObjectSettings(ctx, tc.cluster, tc.machine, tc.config) 1657 1658 g.Expect(tc.config.Spec.ClusterConfiguration.ControlPlaneEndpoint).To(Equal("myControlPlaneEndpoint:6443")) 1659 g.Expect(tc.config.Spec.ClusterConfiguration.ClusterName).To(Equal("mycluster")) 1660 g.Expect(tc.config.Spec.ClusterConfiguration.Networking.PodSubnet).To(Equal("myPodSubnet")) 1661 g.Expect(tc.config.Spec.ClusterConfiguration.Networking.ServiceSubnet).To(Equal("myServiceSubnet")) 1662 g.Expect(tc.config.Spec.ClusterConfiguration.Networking.DNSDomain).To(Equal("myDNSDomain")) 1663 g.Expect(tc.config.Spec.ClusterConfiguration.KubernetesVersion).To(Equal("myversion")) 1664 }) 1665 } 1666 } 1667 1668 // Allow users to skip CA Verification if they *really* want to. 1669 func TestKubeadmConfigReconciler_Reconcile_AlwaysCheckCAVerificationUnlessRequestedToSkip(t *testing.T) { 1670 // Setup work for an initialized cluster 1671 clusterName := "my-cluster" 1672 cluster := builder.Cluster(metav1.NamespaceDefault, clusterName).Build() 1673 conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) 1674 cluster.Status.InfrastructureReady = true 1675 cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ 1676 Host: "example.com", 1677 Port: 6443, 1678 } 1679 controlPlaneInitMachine := newControlPlaneMachine(cluster, "my-control-plane-init-machine") 1680 initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "my-control-plane-init-config") 1681 1682 controlPlaneMachineName := "my-machine" 1683 machine := builder.Machine(metav1.NamespaceDefault, controlPlaneMachineName). 1684 WithVersion("v1.19.1"). 1685 WithClusterName(cluster.Name). 1686 Build() 1687 1688 workerMachineName := "my-worker" 1689 workerMachine := builder.Machine(metav1.NamespaceDefault, workerMachineName). 1690 WithVersion("v1.19.1"). 1691 WithClusterName(cluster.Name). 1692 Build() 1693 1694 controlPlaneConfigName := "my-config" 1695 config := newKubeadmConfig(metav1.NamespaceDefault, controlPlaneConfigName) 1696 1697 objects := []client.Object{ 1698 cluster, machine, workerMachine, config, 1699 } 1700 objects = append(objects, createSecrets(t, cluster, initConfig)...) 1701 1702 testcases := []struct { 1703 name string 1704 discovery *bootstrapv1.BootstrapTokenDiscovery 1705 skipCAVerification bool 1706 }{ 1707 { 1708 name: "Do not skip CA verification by default", 1709 discovery: &bootstrapv1.BootstrapTokenDiscovery{}, 1710 skipCAVerification: false, 1711 }, 1712 { 1713 name: "Skip CA verification if requested by the user", 1714 discovery: &bootstrapv1.BootstrapTokenDiscovery{ 1715 UnsafeSkipCAVerification: true, 1716 }, 1717 skipCAVerification: true, 1718 }, 1719 { 1720 // skipCAVerification should be true since no Cert Hashes are provided, but reconcile will *always* get or create certs. 1721 // TODO: Certificate get/create behavior needs to be mocked to enable this test. 1722 name: "cannot test for defaulting behavior through the reconcile function", 1723 discovery: &bootstrapv1.BootstrapTokenDiscovery{ 1724 CACertHashes: []string{""}, 1725 }, 1726 skipCAVerification: false, 1727 }, 1728 } 1729 for _, tc := range testcases { 1730 t.Run(tc.name, func(t *testing.T) { 1731 g := NewWithT(t) 1732 1733 myclient := fake.NewClientBuilder().WithObjects(objects...).Build() 1734 reconciler := KubeadmConfigReconciler{ 1735 Client: myclient, 1736 SecretCachingClient: myclient, 1737 Tracker: remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}), 1738 KubeadmInitLock: &myInitLocker{}, 1739 } 1740 1741 wc := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg") 1742 wc.Spec.JoinConfiguration.Discovery.BootstrapToken = tc.discovery 1743 key := client.ObjectKey{Namespace: wc.Namespace, Name: wc.Name} 1744 err := myclient.Create(ctx, wc) 1745 g.Expect(err).ToNot(HaveOccurred()) 1746 1747 req := ctrl.Request{NamespacedName: key} 1748 _, err = reconciler.Reconcile(ctx, req) 1749 g.Expect(err).ToNot(HaveOccurred()) 1750 1751 cfg := &bootstrapv1.KubeadmConfig{} 1752 err = myclient.Get(ctx, key, cfg) 1753 g.Expect(err).ToNot(HaveOccurred()) 1754 g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.UnsafeSkipCAVerification).To(Equal(tc.skipCAVerification)) 1755 }) 1756 } 1757 } 1758 1759 // If a cluster object changes then all associated KubeadmConfigs should be re-reconciled. 1760 // This allows us to not requeue a kubeadm config while we wait for InfrastructureReady. 1761 func TestKubeadmConfigReconciler_ClusterToKubeadmConfigs(t *testing.T) { 1762 _ = feature.MutableGates.Set("MachinePool=true") 1763 g := NewWithT(t) 1764 1765 cluster := builder.Cluster(metav1.NamespaceDefault, "my-cluster").Build() 1766 objs := []client.Object{cluster} 1767 expectedNames := []string{} 1768 for i := 0; i < 3; i++ { 1769 configName := fmt.Sprintf("my-config-%d", i) 1770 m := builder.Machine(metav1.NamespaceDefault, fmt.Sprintf("my-machine-%d", i)). 1771 WithVersion("v1.19.1"). 1772 WithClusterName(cluster.Name). 1773 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, configName).Unstructured()). 1774 Build() 1775 c := newKubeadmConfig(metav1.NamespaceDefault, configName) 1776 addKubeadmConfigToMachine(c, m) 1777 expectedNames = append(expectedNames, configName) 1778 objs = append(objs, m, c) 1779 } 1780 for i := 3; i < 6; i++ { 1781 mp := newMachinePool(cluster, fmt.Sprintf("my-machinepool-%d", i)) 1782 configName := fmt.Sprintf("my-config-%d", i) 1783 c := newKubeadmConfig(mp.Namespace, configName) 1784 addKubeadmConfigToMachinePool(c, mp) 1785 expectedNames = append(expectedNames, configName) 1786 objs = append(objs, mp, c) 1787 } 1788 fakeClient := fake.NewClientBuilder().WithObjects(objs...).Build() 1789 reconciler := &KubeadmConfigReconciler{ 1790 Client: fakeClient, 1791 SecretCachingClient: fakeClient, 1792 } 1793 configs := reconciler.ClusterToKubeadmConfigs(ctx, cluster) 1794 names := make([]string, 6) 1795 for i := range configs { 1796 names[i] = configs[i].Name 1797 } 1798 for _, name := range expectedNames { 1799 found := false 1800 for _, foundName := range names { 1801 if foundName == name { 1802 found = true 1803 } 1804 } 1805 g.Expect(found).To(BeTrue()) 1806 } 1807 } 1808 1809 // Reconcile should not fail if the Etcd CA Secret already exists. 1810 func TestKubeadmConfigReconciler_Reconcile_DoesNotFailIfCASecretsAlreadyExist(t *testing.T) { 1811 g := NewWithT(t) 1812 1813 cluster := builder.Cluster(metav1.NamespaceDefault, "my-cluster").Build() 1814 cluster.Status.InfrastructureReady = true 1815 m := newControlPlaneMachine(cluster, "control-plane-machine") 1816 configName := "my-config" 1817 c := newControlPlaneInitKubeadmConfig(m.Namespace, configName) 1818 scrt := &corev1.Secret{ 1819 ObjectMeta: metav1.ObjectMeta{ 1820 Name: fmt.Sprintf("%s-%s", cluster.Name, secret.EtcdCA), 1821 Namespace: metav1.NamespaceDefault, 1822 }, 1823 Data: map[string][]byte{ 1824 "tls.crt": []byte("hello world"), 1825 "tls.key": []byte("hello world"), 1826 }, 1827 } 1828 fakec := fake.NewClientBuilder().WithObjects(cluster, m, c, scrt).Build() 1829 reconciler := &KubeadmConfigReconciler{ 1830 Client: fakec, 1831 SecretCachingClient: fakec, 1832 KubeadmInitLock: &myInitLocker{}, 1833 } 1834 req := ctrl.Request{ 1835 NamespacedName: client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: configName}, 1836 } 1837 _, err := reconciler.Reconcile(ctx, req) 1838 g.Expect(err).ToNot(HaveOccurred()) 1839 } 1840 1841 // Exactly one control plane machine initializes if there are multiple control plane machines defined. 1842 func TestKubeadmConfigReconciler_Reconcile_ExactlyOneControlPlaneMachineInitializes(t *testing.T) { 1843 g := NewWithT(t) 1844 1845 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 1846 cluster.Status.InfrastructureReady = true 1847 1848 controlPlaneInitMachineFirst := newControlPlaneMachine(cluster, "control-plane-init-machine-first") 1849 controlPlaneInitConfigFirst := newControlPlaneInitKubeadmConfig(controlPlaneInitMachineFirst.Namespace, "control-plane-init-cfg-first") 1850 addKubeadmConfigToMachine(controlPlaneInitConfigFirst, controlPlaneInitMachineFirst) 1851 1852 controlPlaneInitMachineSecond := newControlPlaneMachine(cluster, "control-plane-init-machine-second") 1853 controlPlaneInitConfigSecond := newControlPlaneInitKubeadmConfig(controlPlaneInitMachineSecond.Namespace, "control-plane-init-cfg-second") 1854 addKubeadmConfigToMachine(controlPlaneInitConfigSecond, controlPlaneInitMachineSecond) 1855 1856 objects := []client.Object{ 1857 cluster, 1858 controlPlaneInitMachineFirst, 1859 controlPlaneInitConfigFirst, 1860 controlPlaneInitMachineSecond, 1861 controlPlaneInitConfigSecond, 1862 } 1863 myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build() 1864 k := &KubeadmConfigReconciler{ 1865 Client: myclient, 1866 SecretCachingClient: myclient, 1867 KubeadmInitLock: &myInitLocker{}, 1868 } 1869 1870 request := ctrl.Request{ 1871 NamespacedName: client.ObjectKey{ 1872 Namespace: metav1.NamespaceDefault, 1873 Name: "control-plane-init-cfg-first", 1874 }, 1875 } 1876 result, err := k.Reconcile(ctx, request) 1877 g.Expect(err).ToNot(HaveOccurred()) 1878 g.Expect(result.Requeue).To(BeFalse()) 1879 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1880 1881 request = ctrl.Request{ 1882 NamespacedName: client.ObjectKey{ 1883 Namespace: metav1.NamespaceDefault, 1884 Name: "control-plane-init-cfg-second", 1885 }, 1886 } 1887 result, err = k.Reconcile(ctx, request) 1888 g.Expect(err).ToNot(HaveOccurred()) 1889 g.Expect(result.Requeue).To(BeFalse()) 1890 g.Expect(result.RequeueAfter).To(Equal(30 * time.Second)) 1891 confList := &bootstrapv1.KubeadmConfigList{} 1892 g.Expect(myclient.List(ctx, confList)).To(Succeed()) 1893 for _, c := range confList.Items { 1894 // Ensure the DataSecretName is only set for controlPlaneInitConfigFirst. 1895 if c.Name == controlPlaneInitConfigFirst.Name { 1896 g.Expect(*c.Status.DataSecretName).To(Not(BeEmpty())) 1897 } 1898 if c.Name == controlPlaneInitConfigSecond.Name { 1899 g.Expect(c.Status.DataSecretName).To(BeNil()) 1900 } 1901 } 1902 } 1903 1904 // Patch should be applied if there is an error in reconcile. 1905 func TestKubeadmConfigReconciler_Reconcile_PatchWhenErrorOccurred(t *testing.T) { 1906 g := NewWithT(t) 1907 1908 cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build() 1909 cluster.Status.InfrastructureReady = true 1910 1911 controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine") 1912 controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-cfg") 1913 addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine) 1914 // set InitConfiguration as nil, we will check this to determine if the kubeadm config has been patched 1915 controlPlaneInitConfig.Spec.InitConfiguration = nil 1916 1917 objects := []client.Object{ 1918 cluster, 1919 controlPlaneInitMachine, 1920 controlPlaneInitConfig, 1921 } 1922 1923 secrets := createSecrets(t, cluster, controlPlaneInitConfig) 1924 for _, obj := range secrets { 1925 s := obj.(*corev1.Secret) 1926 delete(s.Data, secret.TLSCrtDataName) // destroy the secrets, which will cause Reconcile to fail 1927 objects = append(objects, s) 1928 } 1929 1930 myclient := fake.NewClientBuilder().WithObjects(objects...).Build() 1931 k := &KubeadmConfigReconciler{ 1932 Client: myclient, 1933 SecretCachingClient: myclient, 1934 KubeadmInitLock: &myInitLocker{}, 1935 } 1936 1937 request := ctrl.Request{ 1938 NamespacedName: client.ObjectKey{ 1939 Namespace: metav1.NamespaceDefault, 1940 Name: "control-plane-init-cfg", 1941 }, 1942 } 1943 1944 result, err := k.Reconcile(ctx, request) 1945 g.Expect(err).To(HaveOccurred()) 1946 g.Expect(result.Requeue).To(BeFalse()) 1947 g.Expect(result.RequeueAfter).To(Equal(time.Duration(0))) 1948 1949 cfg, err := getKubeadmConfig(myclient, "control-plane-init-cfg", metav1.NamespaceDefault) 1950 g.Expect(err).ToNot(HaveOccurred()) 1951 // check if the kubeadm config has been patched 1952 g.Expect(cfg.Spec.InitConfiguration).ToNot(BeNil()) 1953 g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil()) 1954 } 1955 1956 func TestKubeadmConfigReconciler_ResolveFiles(t *testing.T) { 1957 testSecret := &corev1.Secret{ 1958 ObjectMeta: metav1.ObjectMeta{ 1959 Name: "source", 1960 }, 1961 Data: map[string][]byte{ 1962 "key": []byte("foo"), 1963 }, 1964 } 1965 1966 cases := map[string]struct { 1967 cfg *bootstrapv1.KubeadmConfig 1968 objects []client.Object 1969 expect []bootstrapv1.File 1970 }{ 1971 "content should pass through": { 1972 cfg: &bootstrapv1.KubeadmConfig{ 1973 Spec: bootstrapv1.KubeadmConfigSpec{ 1974 Files: []bootstrapv1.File{ 1975 { 1976 Content: "foo", 1977 Path: "/path", 1978 Owner: "root:root", 1979 Permissions: "0600", 1980 }, 1981 }, 1982 }, 1983 }, 1984 expect: []bootstrapv1.File{ 1985 { 1986 Content: "foo", 1987 Path: "/path", 1988 Owner: "root:root", 1989 Permissions: "0600", 1990 }, 1991 }, 1992 }, 1993 "contentFrom should convert correctly": { 1994 cfg: &bootstrapv1.KubeadmConfig{ 1995 Spec: bootstrapv1.KubeadmConfigSpec{ 1996 Files: []bootstrapv1.File{ 1997 { 1998 ContentFrom: &bootstrapv1.FileSource{ 1999 Secret: bootstrapv1.SecretFileSource{ 2000 Name: "source", 2001 Key: "key", 2002 }, 2003 }, 2004 Path: "/path", 2005 Owner: "root:root", 2006 Permissions: "0600", 2007 }, 2008 }, 2009 }, 2010 }, 2011 expect: []bootstrapv1.File{ 2012 { 2013 Content: "foo", 2014 Path: "/path", 2015 Owner: "root:root", 2016 Permissions: "0600", 2017 }, 2018 }, 2019 objects: []client.Object{testSecret}, 2020 }, 2021 "multiple files should work correctly": { 2022 cfg: &bootstrapv1.KubeadmConfig{ 2023 Spec: bootstrapv1.KubeadmConfigSpec{ 2024 Files: []bootstrapv1.File{ 2025 { 2026 Content: "bar", 2027 Path: "/bar", 2028 Owner: "root:root", 2029 Permissions: "0600", 2030 }, 2031 { 2032 ContentFrom: &bootstrapv1.FileSource{ 2033 Secret: bootstrapv1.SecretFileSource{ 2034 Name: "source", 2035 Key: "key", 2036 }, 2037 }, 2038 Path: "/path", 2039 Owner: "root:root", 2040 Permissions: "0600", 2041 }, 2042 }, 2043 }, 2044 }, 2045 expect: []bootstrapv1.File{ 2046 { 2047 Content: "bar", 2048 Path: "/bar", 2049 Owner: "root:root", 2050 Permissions: "0600", 2051 }, 2052 { 2053 Content: "foo", 2054 Path: "/path", 2055 Owner: "root:root", 2056 Permissions: "0600", 2057 }, 2058 }, 2059 objects: []client.Object{testSecret}, 2060 }, 2061 } 2062 2063 for name, tc := range cases { 2064 t.Run(name, func(t *testing.T) { 2065 g := NewWithT(t) 2066 2067 myclient := fake.NewClientBuilder().WithObjects(tc.objects...).Build() 2068 k := &KubeadmConfigReconciler{ 2069 Client: myclient, 2070 SecretCachingClient: myclient, 2071 KubeadmInitLock: &myInitLocker{}, 2072 } 2073 2074 // make a list of files we expect to be sourced from secrets 2075 // after we resolve files, assert that the original spec has 2076 // not been mutated and all paths we expected to be sourced 2077 // from secrets still are. 2078 contentFrom := map[string]bool{} 2079 for _, file := range tc.cfg.Spec.Files { 2080 if file.ContentFrom != nil { 2081 contentFrom[file.Path] = true 2082 } 2083 } 2084 2085 files, err := k.resolveFiles(ctx, tc.cfg) 2086 g.Expect(err).ToNot(HaveOccurred()) 2087 g.Expect(files).To(BeComparableTo(tc.expect)) 2088 for _, file := range tc.cfg.Spec.Files { 2089 if contentFrom[file.Path] { 2090 g.Expect(file.ContentFrom).NotTo(BeNil()) 2091 g.Expect(file.Content).To(Equal("")) 2092 } 2093 } 2094 }) 2095 } 2096 } 2097 2098 func TestKubeadmConfigReconciler_ResolveUsers(t *testing.T) { 2099 fakePasswd := "bar" 2100 testSecret := &corev1.Secret{ 2101 ObjectMeta: metav1.ObjectMeta{ 2102 Name: "source", 2103 }, 2104 Data: map[string][]byte{ 2105 "key": []byte(fakePasswd), 2106 }, 2107 } 2108 2109 cases := map[string]struct { 2110 cfg *bootstrapv1.KubeadmConfig 2111 objects []client.Object 2112 expect []bootstrapv1.User 2113 }{ 2114 "password should pass through": { 2115 cfg: &bootstrapv1.KubeadmConfig{ 2116 Spec: bootstrapv1.KubeadmConfigSpec{ 2117 Users: []bootstrapv1.User{ 2118 { 2119 Name: "foo", 2120 Passwd: &fakePasswd, 2121 }, 2122 }, 2123 }, 2124 }, 2125 expect: []bootstrapv1.User{ 2126 { 2127 Name: "foo", 2128 Passwd: &fakePasswd, 2129 }, 2130 }, 2131 }, 2132 "passwdFrom should convert correctly": { 2133 cfg: &bootstrapv1.KubeadmConfig{ 2134 Spec: bootstrapv1.KubeadmConfigSpec{ 2135 Users: []bootstrapv1.User{ 2136 { 2137 Name: "foo", 2138 PasswdFrom: &bootstrapv1.PasswdSource{ 2139 Secret: bootstrapv1.SecretPasswdSource{ 2140 Name: "source", 2141 Key: "key", 2142 }, 2143 }, 2144 }, 2145 }, 2146 }, 2147 }, 2148 expect: []bootstrapv1.User{ 2149 { 2150 Name: "foo", 2151 Passwd: &fakePasswd, 2152 }, 2153 }, 2154 objects: []client.Object{testSecret}, 2155 }, 2156 "multiple users should work correctly": { 2157 cfg: &bootstrapv1.KubeadmConfig{ 2158 Spec: bootstrapv1.KubeadmConfigSpec{ 2159 Users: []bootstrapv1.User{ 2160 { 2161 Name: "foo", 2162 Passwd: &fakePasswd, 2163 }, 2164 { 2165 Name: "bar", 2166 PasswdFrom: &bootstrapv1.PasswdSource{ 2167 Secret: bootstrapv1.SecretPasswdSource{ 2168 Name: "source", 2169 Key: "key", 2170 }, 2171 }, 2172 }, 2173 }, 2174 }, 2175 }, 2176 expect: []bootstrapv1.User{ 2177 { 2178 Name: "foo", 2179 Passwd: &fakePasswd, 2180 }, 2181 { 2182 Name: "bar", 2183 Passwd: &fakePasswd, 2184 }, 2185 }, 2186 objects: []client.Object{testSecret}, 2187 }, 2188 } 2189 2190 for name, tc := range cases { 2191 t.Run(name, func(t *testing.T) { 2192 g := NewWithT(t) 2193 2194 myclient := fake.NewClientBuilder().WithObjects(tc.objects...).Build() 2195 k := &KubeadmConfigReconciler{ 2196 Client: myclient, 2197 SecretCachingClient: myclient, 2198 KubeadmInitLock: &myInitLocker{}, 2199 } 2200 2201 // make a list of password we expect to be sourced from secrets 2202 // after we resolve users, assert that the original spec has 2203 // not been mutated and all password we expected to be sourced 2204 // from secret still are. 2205 passwdFrom := map[string]bool{} 2206 for _, user := range tc.cfg.Spec.Users { 2207 if user.PasswdFrom != nil { 2208 passwdFrom[user.Name] = true 2209 } 2210 } 2211 2212 users, err := k.resolveUsers(ctx, tc.cfg) 2213 g.Expect(err).ToNot(HaveOccurred()) 2214 g.Expect(users).To(BeComparableTo(tc.expect)) 2215 for _, user := range tc.cfg.Spec.Users { 2216 if passwdFrom[user.Name] { 2217 g.Expect(user.PasswdFrom).NotTo(BeNil()) 2218 g.Expect(user.Passwd).To(BeNil()) 2219 } 2220 } 2221 }) 2222 } 2223 } 2224 2225 // test utils. 2226 2227 // newWorkerMachineForCluster returns a Machine with the passed Cluster's information and a pre-configured name. 2228 func newWorkerMachineForCluster(cluster *clusterv1.Cluster) *clusterv1.Machine { 2229 return builder.Machine(cluster.Namespace, "worker-machine"). 2230 WithVersion("v1.19.1"). 2231 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(cluster.Namespace, "conf1").Unstructured()). 2232 WithClusterName(cluster.Name). 2233 Build() 2234 } 2235 2236 // newControlPlaneMachine returns a Machine with the passed Cluster information and a MachineControlPlaneLabel. 2237 func newControlPlaneMachine(cluster *clusterv1.Cluster, name string) *clusterv1.Machine { 2238 m := builder.Machine(cluster.Namespace, name). 2239 WithVersion("v1.19.1"). 2240 WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()). 2241 WithClusterName(cluster.Name). 2242 WithLabels(map[string]string{clusterv1.MachineControlPlaneLabel: ""}). 2243 Build() 2244 return m 2245 } 2246 2247 // newMachinePool return a MachinePool object with the passed Cluster information and a basic bootstrap template. 2248 func newMachinePool(cluster *clusterv1.Cluster, name string) *expv1.MachinePool { 2249 m := builder.MachinePool(cluster.Namespace, name). 2250 WithClusterName(cluster.Name). 2251 WithLabels(map[string]string{clusterv1.ClusterNameLabel: cluster.Name}). 2252 WithBootstrap(bootstrapbuilder.KubeadmConfig(cluster.Namespace, "conf1").Unstructured()). 2253 WithVersion("1.19.1"). 2254 Build() 2255 return m 2256 } 2257 2258 // newWorkerMachinePoolForCluster returns a MachinePool with the passed Cluster's information and a pre-configured name. 2259 func newWorkerMachinePoolForCluster(cluster *clusterv1.Cluster) *expv1.MachinePool { 2260 return newMachinePool(cluster, "worker-machinepool") 2261 } 2262 2263 // newKubeadmConfig return a CABPK KubeadmConfig object. 2264 func newKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig { 2265 return bootstrapbuilder.KubeadmConfig(namespace, name). 2266 Build() 2267 } 2268 2269 // newKubeadmConfig return a CABPK KubeadmConfig object with a worker JoinConfiguration. 2270 func newWorkerJoinKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig { 2271 return bootstrapbuilder.KubeadmConfig(namespace, name). 2272 WithJoinConfig(&bootstrapv1.JoinConfiguration{ 2273 ControlPlane: nil, 2274 }). 2275 Build() 2276 } 2277 2278 // newKubeadmConfig returns a CABPK KubeadmConfig object with a ControlPlane JoinConfiguration. 2279 func newControlPlaneJoinKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig { 2280 return bootstrapbuilder.KubeadmConfig(namespace, name). 2281 WithJoinConfig(&bootstrapv1.JoinConfiguration{ 2282 ControlPlane: &bootstrapv1.JoinControlPlane{}, 2283 }). 2284 Build() 2285 } 2286 2287 // newControlPlaneJoinConfig returns a CABPK KubeadmConfig object with a ControlPlane InitConfiguration and ClusterConfiguration. 2288 func newControlPlaneInitKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig { 2289 return bootstrapbuilder.KubeadmConfig(namespace, name). 2290 WithInitConfig(&bootstrapv1.InitConfiguration{}). 2291 WithClusterConfig(&bootstrapv1.ClusterConfiguration{}). 2292 Build() 2293 } 2294 2295 // addKubeadmConfigToMachine adds the config details to the passed Machine, and adds the Machine to the KubeadmConfig as an ownerReference. 2296 func addKubeadmConfigToMachine(config *bootstrapv1.KubeadmConfig, machine *clusterv1.Machine) { 2297 if machine == nil { 2298 panic("no machine passed to function") 2299 } 2300 config.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ 2301 { 2302 Kind: "Machine", 2303 APIVersion: clusterv1.GroupVersion.String(), 2304 Name: machine.Name, 2305 UID: types.UID(fmt.Sprintf("%s uid", machine.Name)), 2306 }, 2307 } 2308 2309 if machine.Spec.Bootstrap.ConfigRef == nil { 2310 machine.Spec.Bootstrap.ConfigRef = &corev1.ObjectReference{} 2311 } 2312 2313 machine.Spec.Bootstrap.ConfigRef.Name = config.Name 2314 machine.Spec.Bootstrap.ConfigRef.Namespace = config.Namespace 2315 } 2316 2317 // addKubeadmConfigToMachine adds the config details to the passed MachinePool and adds the Machine to the KubeadmConfig as an ownerReference. 2318 func addKubeadmConfigToMachinePool(config *bootstrapv1.KubeadmConfig, machinePool *expv1.MachinePool) { 2319 if machinePool == nil { 2320 panic("no machinePool passed to function") 2321 } 2322 config.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ 2323 { 2324 Kind: "MachinePool", 2325 APIVersion: expv1.GroupVersion.String(), 2326 Name: machinePool.Name, 2327 UID: types.UID(fmt.Sprintf("%s uid", machinePool.Name)), 2328 }, 2329 } 2330 machinePool.Spec.Template.Spec.Bootstrap.ConfigRef.Name = config.Name 2331 machinePool.Spec.Template.Spec.Bootstrap.ConfigRef.Namespace = config.Namespace 2332 } 2333 2334 func createSecrets(t *testing.T, cluster *clusterv1.Cluster, config *bootstrapv1.KubeadmConfig) []client.Object { 2335 t.Helper() 2336 2337 out := []client.Object{} 2338 if config.Spec.ClusterConfiguration == nil { 2339 config.Spec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{} 2340 } 2341 certificates := secret.NewCertificatesForInitialControlPlane(config.Spec.ClusterConfiguration) 2342 if err := certificates.Generate(); err != nil { 2343 t.Fatal(err) 2344 } 2345 for _, certificate := range certificates { 2346 out = append(out, certificate.AsSecret(util.ObjectKey(cluster), *metav1.NewControllerRef(config, bootstrapv1.GroupVersion.WithKind("KubeadmConfig")))) 2347 } 2348 return out 2349 } 2350 2351 type myInitLocker struct { 2352 locked bool 2353 } 2354 2355 func (m *myInitLocker) Lock(_ context.Context, _ *clusterv1.Cluster, _ *clusterv1.Machine) bool { 2356 if !m.locked { 2357 m.locked = true 2358 return true 2359 } 2360 return false 2361 } 2362 2363 func (m *myInitLocker) Unlock(_ context.Context, _ *clusterv1.Cluster) bool { 2364 if m.locked { 2365 m.locked = false 2366 } 2367 return true 2368 } 2369 2370 func assertHasFalseCondition(g *WithT, myclient client.Client, req ctrl.Request, t clusterv1.ConditionType, s clusterv1.ConditionSeverity, r string) { 2371 config := &bootstrapv1.KubeadmConfig{ 2372 ObjectMeta: metav1.ObjectMeta{ 2373 Name: req.Name, 2374 Namespace: req.Namespace, 2375 }, 2376 } 2377 2378 configKey := client.ObjectKeyFromObject(config) 2379 g.Expect(myclient.Get(ctx, configKey, config)).To(Succeed()) 2380 c := conditions.Get(config, t) 2381 g.Expect(c).ToNot(BeNil()) 2382 g.Expect(c.Status).To(Equal(corev1.ConditionFalse)) 2383 g.Expect(c.Severity).To(Equal(s)) 2384 g.Expect(c.Reason).To(Equal(r)) 2385 } 2386 2387 func assertHasTrueCondition(g *WithT, myclient client.Client, req ctrl.Request, t clusterv1.ConditionType) { 2388 config := &bootstrapv1.KubeadmConfig{ 2389 ObjectMeta: metav1.ObjectMeta{ 2390 Name: req.Name, 2391 Namespace: req.Namespace, 2392 }, 2393 } 2394 configKey := client.ObjectKeyFromObject(config) 2395 g.Expect(myclient.Get(ctx, configKey, config)).To(Succeed()) 2396 c := conditions.Get(config, t) 2397 g.Expect(c).ToNot(BeNil()) 2398 g.Expect(c.Status).To(Equal(corev1.ConditionTrue)) 2399 }