sigs.k8s.io/cluster-api@v1.6.3/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go (about)

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controllers
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"fmt"
    23  	"reflect"
    24  	"testing"
    25  	"time"
    26  
    27  	ignition "github.com/flatcar/ignition/config/v2_3"
    28  	"github.com/go-logr/logr"
    29  	. "github.com/onsi/gomega"
    30  	corev1 "k8s.io/api/core/v1"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/types"
    33  	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
    34  	"k8s.io/utils/pointer"
    35  	ctrl "sigs.k8s.io/controller-runtime"
    36  	"sigs.k8s.io/controller-runtime/pkg/client"
    37  	"sigs.k8s.io/controller-runtime/pkg/client/fake"
    38  	"sigs.k8s.io/controller-runtime/pkg/log"
    39  	"sigs.k8s.io/controller-runtime/pkg/reconcile"
    40  	"sigs.k8s.io/yaml"
    41  
    42  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    43  	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
    44  	bootstrapbuilder "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/builder"
    45  	"sigs.k8s.io/cluster-api/controllers/remote"
    46  	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
    47  	"sigs.k8s.io/cluster-api/feature"
    48  	"sigs.k8s.io/cluster-api/internal/test/builder"
    49  	"sigs.k8s.io/cluster-api/util"
    50  	"sigs.k8s.io/cluster-api/util/conditions"
    51  	"sigs.k8s.io/cluster-api/util/patch"
    52  	"sigs.k8s.io/cluster-api/util/secret"
    53  )
    54  
    55  // MachineToBootstrapMapFunc return kubeadm bootstrap configref name when configref exists.
    56  func TestKubeadmConfigReconciler_MachineToBootstrapMapFuncReturn(t *testing.T) {
    57  	g := NewWithT(t)
    58  	cluster := builder.Cluster("my-cluster", metav1.NamespaceDefault).Build()
    59  	objs := []client.Object{cluster}
    60  	machineObjs := []client.Object{}
    61  	var expectedConfigName string
    62  	for i := 0; i < 3; i++ {
    63  		configName := fmt.Sprintf("my-config-%d", i)
    64  		m := builder.Machine(metav1.NamespaceDefault, fmt.Sprintf("my-machine-%d", i)).
    65  			WithVersion("v1.19.1").
    66  			WithClusterName(cluster.Name).
    67  			WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "").Unstructured()).
    68  			Build()
    69  		if i == 1 {
    70  			c := newKubeadmConfig(metav1.NamespaceDefault, configName)
    71  			addKubeadmConfigToMachine(c, m)
    72  			objs = append(objs, m, c)
    73  			expectedConfigName = configName
    74  		} else {
    75  			objs = append(objs, m)
    76  		}
    77  		machineObjs = append(machineObjs, m)
    78  	}
    79  	fakeClient := fake.NewClientBuilder().WithObjects(objs...).Build()
    80  	reconciler := &KubeadmConfigReconciler{
    81  		Client:              fakeClient,
    82  		SecretCachingClient: fakeClient,
    83  	}
    84  	for i := 0; i < 3; i++ {
    85  		o := machineObjs[i]
    86  		configs := reconciler.MachineToBootstrapMapFunc(ctx, o)
    87  		if i == 1 {
    88  			g.Expect(configs[0].Name).To(Equal(expectedConfigName))
    89  		} else {
    90  			g.Expect(configs[0].Name).To(Equal(""))
    91  		}
    92  	}
    93  }
    94  
    95  // Reconcile returns early if the kubeadm config is ready because it should never re-generate bootstrap data.
    96  func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfKubeadmConfigIsReady(t *testing.T) {
    97  	g := NewWithT(t)
    98  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster1").Build()
    99  	cluster.Status.InfrastructureReady = true
   100  	machine := builder.Machine(metav1.NamespaceDefault, "m1").WithClusterName("cluster1").Build()
   101  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   102  	addKubeadmConfigToMachine(config, machine)
   103  
   104  	config.Status.Ready = true
   105  
   106  	objects := []client.Object{
   107  		cluster,
   108  		machine,
   109  		config,
   110  	}
   111  	myclient := fake.NewClientBuilder().WithObjects(objects...).Build()
   112  
   113  	k := &KubeadmConfigReconciler{
   114  		Client:              myclient,
   115  		SecretCachingClient: myclient,
   116  	}
   117  
   118  	request := ctrl.Request{
   119  		NamespacedName: client.ObjectKey{
   120  			Namespace: metav1.NamespaceDefault,
   121  			Name:      "cfg",
   122  		},
   123  	}
   124  	result, err := k.Reconcile(ctx, request)
   125  	g.Expect(err).ToNot(HaveOccurred())
   126  	g.Expect(result.Requeue).To(BeFalse())
   127  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
   128  }
   129  
   130  // Reconcile should update owner references on bootstrap secrets on creation and update.
   131  func TestKubeadmConfigReconciler_TestSecretOwnerReferenceReconciliation(t *testing.T) {
   132  	g := NewWithT(t)
   133  
   134  	clusterName := "my-cluster"
   135  	cluster := builder.Cluster(metav1.NamespaceDefault, clusterName).Build()
   136  	machine := builder.Machine(metav1.NamespaceDefault, "machine").
   137  		WithVersion("v1.19.1").
   138  		WithClusterName(clusterName).
   139  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
   140  		Build()
   141  	machine.Spec.Bootstrap.DataSecretName = pointer.String("something")
   142  
   143  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   144  	config.SetOwnerReferences(util.EnsureOwnerRef(config.GetOwnerReferences(), metav1.OwnerReference{
   145  		APIVersion: machine.APIVersion,
   146  		Kind:       machine.Kind,
   147  		Name:       machine.Name,
   148  		UID:        machine.UID,
   149  	}))
   150  	secret := &corev1.Secret{
   151  		ObjectMeta: metav1.ObjectMeta{
   152  			Name:      config.Name,
   153  			Namespace: config.Namespace,
   154  		},
   155  		Type: corev1.SecretTypeBootstrapToken,
   156  	}
   157  	config.Status.Ready = true
   158  
   159  	objects := []client.Object{
   160  		config,
   161  		machine,
   162  		secret,
   163  		cluster,
   164  	}
   165  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   166  
   167  	k := &KubeadmConfigReconciler{
   168  		Client:              myclient,
   169  		SecretCachingClient: myclient,
   170  	}
   171  
   172  	request := ctrl.Request{
   173  		NamespacedName: client.ObjectKey{
   174  			Namespace: metav1.NamespaceDefault,
   175  			Name:      "cfg",
   176  		},
   177  	}
   178  	var err error
   179  	key := client.ObjectKeyFromObject(config)
   180  	actual := &corev1.Secret{}
   181  
   182  	t.Run("KubeadmConfig ownerReference is added on first reconcile", func(t *testing.T) {
   183  		_, err = k.Reconcile(ctx, request)
   184  		g.Expect(err).ToNot(HaveOccurred())
   185  
   186  		g.Expect(myclient.Get(ctx, key, actual)).To(Succeed())
   187  
   188  		controllerOwner := metav1.GetControllerOf(actual)
   189  		g.Expect(controllerOwner).To(Not(BeNil()))
   190  		g.Expect(controllerOwner.Kind).To(Equal(config.Kind))
   191  		g.Expect(controllerOwner.Name).To(Equal(config.Name))
   192  	})
   193  
   194  	t.Run("KubeadmConfig ownerReference re-reconciled without error", func(t *testing.T) {
   195  		_, err = k.Reconcile(ctx, request)
   196  		g.Expect(err).ToNot(HaveOccurred())
   197  
   198  		g.Expect(myclient.Get(ctx, key, actual)).To(Succeed())
   199  
   200  		controllerOwner := metav1.GetControllerOf(actual)
   201  		g.Expect(controllerOwner).To(Not(BeNil()))
   202  		g.Expect(controllerOwner.Kind).To(Equal(config.Kind))
   203  		g.Expect(controllerOwner.Name).To(Equal(config.Name))
   204  	})
   205  	t.Run("non-KubeadmConfig controller OwnerReference is replaced", func(t *testing.T) {
   206  		g.Expect(myclient.Get(ctx, key, actual)).To(Succeed())
   207  
   208  		actual.SetOwnerReferences([]metav1.OwnerReference{
   209  			{
   210  				APIVersion: machine.APIVersion,
   211  				Kind:       machine.Kind,
   212  				Name:       machine.Name,
   213  				UID:        machine.UID,
   214  				Controller: pointer.Bool(true),
   215  			}})
   216  		g.Expect(myclient.Update(ctx, actual)).To(Succeed())
   217  
   218  		_, err = k.Reconcile(ctx, request)
   219  		g.Expect(err).ToNot(HaveOccurred())
   220  
   221  		g.Expect(myclient.Get(ctx, key, actual)).To(Succeed())
   222  
   223  		controllerOwner := metav1.GetControllerOf(actual)
   224  		g.Expect(controllerOwner).To(Not(BeNil()))
   225  		g.Expect(controllerOwner.Kind).To(Equal(config.Kind))
   226  		g.Expect(controllerOwner.Name).To(Equal(config.Name))
   227  	})
   228  }
   229  
   230  // Reconcile returns nil if the referenced Machine cannot be found.
   231  func TestKubeadmConfigReconciler_Reconcile_ReturnNilIfReferencedMachineIsNotFound(t *testing.T) {
   232  	g := NewWithT(t)
   233  
   234  	machine := builder.Machine(metav1.NamespaceDefault, "machine").
   235  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
   236  		WithVersion("v1.19.1").
   237  		Build()
   238  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   239  	addKubeadmConfigToMachine(config, machine)
   240  	objects := []client.Object{
   241  		// intentionally omitting machine
   242  		config,
   243  	}
   244  	myclient := fake.NewClientBuilder().WithObjects(objects...).Build()
   245  
   246  	k := &KubeadmConfigReconciler{
   247  		Client:              myclient,
   248  		SecretCachingClient: myclient,
   249  	}
   250  
   251  	request := ctrl.Request{
   252  		NamespacedName: client.ObjectKey{
   253  			Namespace: metav1.NamespaceDefault,
   254  			Name:      "cfg",
   255  		},
   256  	}
   257  	_, err := k.Reconcile(ctx, request)
   258  	g.Expect(err).ToNot(HaveOccurred())
   259  }
   260  
   261  // If the machine has bootstrap data secret reference, there is no need to generate more bootstrap data.
   262  func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfMachineHasDataSecretName(t *testing.T) {
   263  	g := NewWithT(t)
   264  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster1").Build()
   265  	cluster.Status.InfrastructureReady = true
   266  
   267  	machine := builder.Machine(metav1.NamespaceDefault, "machine").
   268  		WithVersion("v1.19.1").
   269  		WithClusterName("cluster1").
   270  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
   271  		Build()
   272  	machine.Spec.Bootstrap.DataSecretName = pointer.String("something")
   273  
   274  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   275  	addKubeadmConfigToMachine(config, machine)
   276  	objects := []client.Object{
   277  		cluster,
   278  		machine,
   279  		config,
   280  	}
   281  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   282  
   283  	k := &KubeadmConfigReconciler{
   284  		Client:              myclient,
   285  		SecretCachingClient: myclient,
   286  	}
   287  
   288  	request := ctrl.Request{
   289  		NamespacedName: client.ObjectKey{
   290  			Namespace: metav1.NamespaceDefault,
   291  			Name:      "cfg",
   292  		},
   293  	}
   294  	result, err := k.Reconcile(ctx, request)
   295  	actual := &bootstrapv1.KubeadmConfig{}
   296  	g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: config.Namespace, Name: config.Name}, actual)).To(Succeed())
   297  	g.Expect(err).ToNot(HaveOccurred())
   298  	g.Expect(result.Requeue).To(BeFalse())
   299  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
   300  	assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition)
   301  }
   302  
   303  func TestKubeadmConfigReconciler_ReturnEarlyIfClusterInfraNotReady(t *testing.T) {
   304  	g := NewWithT(t)
   305  
   306  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   307  	machine := builder.Machine(metav1.NamespaceDefault, "machine").
   308  		WithVersion("v1.19.1").
   309  		WithClusterName(cluster.Name).
   310  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
   311  		Build()
   312  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   313  	addKubeadmConfigToMachine(config, machine)
   314  
   315  	// cluster infra not ready
   316  	cluster.Status = clusterv1.ClusterStatus{
   317  		InfrastructureReady: false,
   318  	}
   319  
   320  	objects := []client.Object{
   321  		cluster,
   322  		machine,
   323  		config,
   324  	}
   325  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   326  
   327  	k := &KubeadmConfigReconciler{
   328  		Client:              myclient,
   329  		SecretCachingClient: myclient,
   330  	}
   331  
   332  	request := ctrl.Request{
   333  		NamespacedName: client.ObjectKey{
   334  			Namespace: metav1.NamespaceDefault,
   335  			Name:      "cfg",
   336  		},
   337  	}
   338  
   339  	expectedResult := reconcile.Result{}
   340  	actualResult, actualError := k.Reconcile(ctx, request)
   341  	g.Expect(actualResult).To(BeComparableTo(expectedResult))
   342  	g.Expect(actualError).ToNot(HaveOccurred())
   343  	assertHasFalseCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition, clusterv1.ConditionSeverityInfo, bootstrapv1.WaitingForClusterInfrastructureReason)
   344  }
   345  
   346  // Return early If the owning machine does not have an associated cluster.
   347  func TestKubeadmConfigReconciler_Reconcile_ReturnEarlyIfMachineHasNoCluster(t *testing.T) {
   348  	g := NewWithT(t)
   349  	machine := builder.Machine(metav1.NamespaceDefault, "machine").
   350  		WithVersion("v1.19.1").
   351  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
   352  		Build()
   353  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   354  	addKubeadmConfigToMachine(config, machine)
   355  
   356  	objects := []client.Object{
   357  		machine,
   358  		config,
   359  	}
   360  	myclient := fake.NewClientBuilder().WithObjects(objects...).Build()
   361  
   362  	k := &KubeadmConfigReconciler{
   363  		Client:              myclient,
   364  		SecretCachingClient: myclient,
   365  	}
   366  
   367  	request := ctrl.Request{
   368  		NamespacedName: client.ObjectKey{
   369  			Namespace: metav1.NamespaceDefault,
   370  			Name:      "cfg",
   371  		},
   372  	}
   373  	_, err := k.Reconcile(ctx, request)
   374  	g.Expect(err).ToNot(HaveOccurred())
   375  }
   376  
   377  // This does not expect an error, hoping that the associated cluster will be created.
   378  func TestKubeadmConfigReconciler_Reconcile_ReturnNilIfAssociatedClusterIsNotFound(t *testing.T) {
   379  	g := NewWithT(t)
   380  
   381  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   382  	machine := builder.Machine(metav1.NamespaceDefault, "machine").
   383  		WithVersion("v1.19.1").
   384  		WithClusterName(cluster.Name).
   385  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
   386  		Build()
   387  	config := newKubeadmConfig(metav1.NamespaceDefault, "cfg")
   388  
   389  	addKubeadmConfigToMachine(config, machine)
   390  	objects := []client.Object{
   391  		// intentionally omitting cluster
   392  		machine,
   393  		config,
   394  	}
   395  	myclient := fake.NewClientBuilder().WithObjects(objects...).Build()
   396  
   397  	k := &KubeadmConfigReconciler{
   398  		Client:              myclient,
   399  		SecretCachingClient: myclient,
   400  	}
   401  
   402  	request := ctrl.Request{
   403  		NamespacedName: client.ObjectKey{
   404  			Namespace: metav1.NamespaceDefault,
   405  			Name:      "cfg",
   406  		},
   407  	}
   408  	_, err := k.Reconcile(ctx, request)
   409  	g.Expect(err).ToNot(HaveOccurred())
   410  }
   411  
   412  // If the control plane isn't initialized then there is no cluster for either a worker or control plane node to join.
   413  func TestKubeadmConfigReconciler_Reconcile_RequeueJoiningNodesIfControlPlaneNotInitialized(t *testing.T) {
   414  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   415  	cluster.Status.InfrastructureReady = true
   416  
   417  	workerMachine := newWorkerMachineForCluster(cluster)
   418  	workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
   419  	addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
   420  
   421  	controlPlaneJoinMachine := newControlPlaneMachine(cluster, "control-plane-join-machine")
   422  	controlPlaneJoinConfig := newControlPlaneJoinKubeadmConfig(controlPlaneJoinMachine.Namespace, "control-plane-join-cfg")
   423  	addKubeadmConfigToMachine(controlPlaneJoinConfig, controlPlaneJoinMachine)
   424  
   425  	testcases := []struct {
   426  		name    string
   427  		request ctrl.Request
   428  		objects []client.Object
   429  	}{
   430  		{
   431  			name: "requeue worker when control plane is not yet initialized",
   432  			request: ctrl.Request{
   433  				NamespacedName: client.ObjectKey{
   434  					Namespace: workerJoinConfig.Namespace,
   435  					Name:      workerJoinConfig.Name,
   436  				},
   437  			},
   438  			objects: []client.Object{
   439  				cluster,
   440  				workerMachine,
   441  				workerJoinConfig,
   442  			},
   443  		},
   444  		{
   445  			name: "requeue a secondary control plane when the control plane is not yet initialized",
   446  			request: ctrl.Request{
   447  				NamespacedName: client.ObjectKey{
   448  					Namespace: controlPlaneJoinConfig.Namespace,
   449  					Name:      controlPlaneJoinConfig.Name,
   450  				},
   451  			},
   452  			objects: []client.Object{
   453  				cluster,
   454  				controlPlaneJoinMachine,
   455  				controlPlaneJoinConfig,
   456  			},
   457  		},
   458  	}
   459  	for _, tc := range testcases {
   460  		t.Run(tc.name, func(t *testing.T) {
   461  			g := NewWithT(t)
   462  
   463  			myclient := fake.NewClientBuilder().WithObjects(tc.objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   464  
   465  			k := &KubeadmConfigReconciler{
   466  				Client:              myclient,
   467  				SecretCachingClient: myclient,
   468  				KubeadmInitLock:     &myInitLocker{},
   469  			}
   470  
   471  			result, err := k.Reconcile(ctx, tc.request)
   472  			g.Expect(err).ToNot(HaveOccurred())
   473  			g.Expect(result.Requeue).To(BeFalse())
   474  			g.Expect(result.RequeueAfter).To(Equal(30 * time.Second))
   475  			assertHasFalseCondition(g, myclient, tc.request, bootstrapv1.DataSecretAvailableCondition, clusterv1.ConditionSeverityInfo, clusterv1.WaitingForControlPlaneAvailableReason)
   476  		})
   477  	}
   478  }
   479  
   480  // This generates cloud-config data but does not test the validity of it.
   481  func TestKubeadmConfigReconciler_Reconcile_GenerateCloudConfigData(t *testing.T) {
   482  	g := NewWithT(t)
   483  
   484  	configName := "control-plane-init-cfg"
   485  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   486  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "validhost", Port: 6443}
   487  	cluster.Status.InfrastructureReady = true
   488  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   489  
   490  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
   491  	controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, configName)
   492  	controlPlaneInitConfig.Spec.JoinConfiguration = &bootstrapv1.JoinConfiguration{}
   493  	controlPlaneInitConfig.Spec.JoinConfiguration.Discovery.BootstrapToken = &bootstrapv1.BootstrapTokenDiscovery{
   494  		CACertHashes: []string{"...."},
   495  	}
   496  
   497  	addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine)
   498  
   499  	objects := []client.Object{
   500  		cluster,
   501  		controlPlaneInitMachine,
   502  		controlPlaneInitConfig,
   503  	}
   504  	objects = append(objects, createSecrets(t, cluster, controlPlaneInitConfig)...)
   505  
   506  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   507  
   508  	k := &KubeadmConfigReconciler{
   509  		Client:              myclient,
   510  		SecretCachingClient: myclient,
   511  		Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
   512  		KubeadmInitLock:     &myInitLocker{},
   513  	}
   514  
   515  	request := ctrl.Request{
   516  		NamespacedName: client.ObjectKey{
   517  			Namespace: metav1.NamespaceDefault,
   518  			Name:      "control-plane-init-cfg",
   519  		},
   520  	}
   521  	s := &corev1.Secret{}
   522  	g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: configName}, s)).ToNot(Succeed())
   523  
   524  	result, err := k.Reconcile(ctx, request)
   525  	g.Expect(err).ToNot(HaveOccurred())
   526  	g.Expect(result.Requeue).To(BeFalse())
   527  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
   528  
   529  	cfg, err := getKubeadmConfig(myclient, "control-plane-init-cfg", metav1.NamespaceDefault)
   530  	g.Expect(err).ToNot(HaveOccurred())
   531  	g.Expect(cfg.Status.Ready).To(BeTrue())
   532  	g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
   533  	g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
   534  	assertHasTrueCondition(g, myclient, request, bootstrapv1.CertificatesAvailableCondition)
   535  	assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition)
   536  
   537  	// Expect the Secret to exist, and for it to contain some data under the "value" key.
   538  	g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: configName}, s)).To(Succeed())
   539  	g.Expect(s.Data["value"]).ToNot(BeEmpty())
   540  	// Ensure that we don't fail trying to refresh any bootstrap tokens
   541  	_, err = k.Reconcile(ctx, request)
   542  	g.Expect(err).ToNot(HaveOccurred())
   543  }
   544  
   545  // If a control plane has no JoinConfiguration, then we will create a default and no error will occur.
   546  func TestKubeadmConfigReconciler_Reconcile_ErrorIfJoiningControlPlaneHasInvalidConfiguration(t *testing.T) {
   547  	g := NewWithT(t)
   548  	// TODO: extract this kind of code into a setup function that puts the state of objects into an initialized controlplane (implies secrets exist)
   549  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   550  	cluster.Status.InfrastructureReady = true
   551  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   552  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
   553  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
   554  	controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-cfg")
   555  	addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine)
   556  
   557  	controlPlaneJoinMachine := newControlPlaneMachine(cluster, "control-plane-join-machine")
   558  	controlPlaneJoinConfig := newControlPlaneJoinKubeadmConfig(controlPlaneJoinMachine.Namespace, "control-plane-join-cfg")
   559  	controlPlaneJoinConfig.Spec.JoinConfiguration.ControlPlane = nil // Makes controlPlaneJoinConfig invalid for a control plane machine
   560  	addKubeadmConfigToMachine(controlPlaneJoinConfig, controlPlaneJoinMachine)
   561  
   562  	objects := []client.Object{
   563  		cluster,
   564  		controlPlaneJoinMachine,
   565  		controlPlaneJoinConfig,
   566  	}
   567  	objects = append(objects, createSecrets(t, cluster, controlPlaneInitConfig)...)
   568  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   569  
   570  	k := &KubeadmConfigReconciler{
   571  		Client:              myclient,
   572  		SecretCachingClient: myclient,
   573  		Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
   574  		KubeadmInitLock:     &myInitLocker{},
   575  	}
   576  
   577  	request := ctrl.Request{
   578  		NamespacedName: client.ObjectKey{
   579  			Namespace: metav1.NamespaceDefault,
   580  			Name:      controlPlaneJoinConfig.Name,
   581  		},
   582  	}
   583  	_, err := k.Reconcile(ctx, request)
   584  	g.Expect(err).ToNot(HaveOccurred())
   585  	actualConfig := &bootstrapv1.KubeadmConfig{}
   586  	g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: controlPlaneJoinConfig.Namespace, Name: controlPlaneJoinConfig.Name}, actualConfig)).To(Succeed())
   587  	assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition)
   588  	assertHasTrueCondition(g, myclient, request, bootstrapv1.CertificatesAvailableCondition)
   589  }
   590  
   591  // If there is no APIEndpoint but everything is ready then requeue in hopes of a new APIEndpoint showing up eventually.
   592  func TestKubeadmConfigReconciler_Reconcile_RequeueIfControlPlaneIsMissingAPIEndpoints(t *testing.T) {
   593  	g := NewWithT(t)
   594  
   595  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   596  	cluster.Status.InfrastructureReady = true
   597  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   598  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
   599  	controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-cfg")
   600  	addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine)
   601  
   602  	workerMachine := newWorkerMachineForCluster(cluster)
   603  	workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
   604  	addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
   605  
   606  	objects := []client.Object{
   607  		cluster,
   608  		workerMachine,
   609  		workerJoinConfig,
   610  	}
   611  	objects = append(objects, createSecrets(t, cluster, controlPlaneInitConfig)...)
   612  
   613  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   614  
   615  	k := &KubeadmConfigReconciler{
   616  		Client:              myclient,
   617  		SecretCachingClient: myclient,
   618  		KubeadmInitLock:     &myInitLocker{},
   619  	}
   620  
   621  	request := ctrl.Request{
   622  		NamespacedName: client.ObjectKey{
   623  			Namespace: metav1.NamespaceDefault,
   624  			Name:      "worker-join-cfg",
   625  		},
   626  	}
   627  	result, err := k.Reconcile(ctx, request)
   628  	g.Expect(err).ToNot(HaveOccurred())
   629  	g.Expect(result.Requeue).To(BeFalse())
   630  	g.Expect(result.RequeueAfter).To(Equal(10 * time.Second))
   631  
   632  	actualConfig := &bootstrapv1.KubeadmConfig{}
   633  	g.Expect(myclient.Get(ctx, client.ObjectKey{Namespace: workerJoinConfig.Namespace, Name: workerJoinConfig.Name}, actualConfig)).To(Succeed())
   634  
   635  	// At this point the DataSecretAvailableCondition should not be set. CertificatesAvailableCondition should be true.
   636  	g.Expect(conditions.Get(actualConfig, bootstrapv1.DataSecretAvailableCondition)).To(BeNil())
   637  	assertHasTrueCondition(g, myclient, request, bootstrapv1.CertificatesAvailableCondition)
   638  }
   639  
   640  func TestReconcileIfJoinCertificatesAvailableConditioninNodesAndControlPlaneIsReady(t *testing.T) {
   641  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   642  	cluster.Status.InfrastructureReady = true
   643  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   644  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
   645  
   646  	useCases := []struct {
   647  		name          string
   648  		machine       *clusterv1.Machine
   649  		configName    string
   650  		configBuilder func(string, string) *bootstrapv1.KubeadmConfig
   651  	}{
   652  		{
   653  			name:          "Join a worker node with a fully compiled kubeadm config object",
   654  			machine:       newWorkerMachineForCluster(cluster),
   655  			configName:    "worker-join-cfg",
   656  			configBuilder: newWorkerJoinKubeadmConfig,
   657  		},
   658  		{
   659  			name:          "Join a worker node  with an empty kubeadm config object (defaults apply)",
   660  			machine:       newWorkerMachineForCluster(cluster),
   661  			configName:    "worker-join-cfg",
   662  			configBuilder: newKubeadmConfig,
   663  		},
   664  		{
   665  			name:          "Join a control plane node with a fully compiled kubeadm config object",
   666  			machine:       newControlPlaneMachine(cluster, "control-plane-join-machine"),
   667  			configName:    "control-plane-join-cfg",
   668  			configBuilder: newControlPlaneJoinKubeadmConfig,
   669  		},
   670  		{
   671  			name:          "Join a control plane node with an empty kubeadm config object (defaults apply)",
   672  			machine:       newControlPlaneMachine(cluster, "control-plane-join-machine"),
   673  			configName:    "control-plane-join-cfg",
   674  			configBuilder: newKubeadmConfig,
   675  		},
   676  	}
   677  
   678  	for _, rt := range useCases {
   679  		rt := rt // pin!
   680  		t.Run(rt.name, func(t *testing.T) {
   681  			g := NewWithT(t)
   682  
   683  			config := rt.configBuilder(rt.machine.Namespace, rt.configName)
   684  			addKubeadmConfigToMachine(config, rt.machine)
   685  
   686  			objects := []client.Object{
   687  				cluster,
   688  				rt.machine,
   689  				config,
   690  			}
   691  			objects = append(objects, createSecrets(t, cluster, config)...)
   692  			myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   693  			k := &KubeadmConfigReconciler{
   694  				Client:              myclient,
   695  				SecretCachingClient: myclient,
   696  				Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
   697  				KubeadmInitLock:     &myInitLocker{},
   698  			}
   699  
   700  			request := ctrl.Request{
   701  				NamespacedName: client.ObjectKey{
   702  					Namespace: config.GetNamespace(),
   703  					Name:      rt.configName,
   704  				},
   705  			}
   706  			result, err := k.Reconcile(ctx, request)
   707  			g.Expect(err).ToNot(HaveOccurred())
   708  			g.Expect(result.Requeue).To(BeFalse())
   709  			g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
   710  
   711  			cfg, err := getKubeadmConfig(myclient, rt.configName, metav1.NamespaceDefault)
   712  			g.Expect(err).ToNot(HaveOccurred())
   713  			g.Expect(cfg.Status.Ready).To(BeTrue())
   714  			g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
   715  			g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
   716  			assertHasTrueCondition(g, myclient, request, bootstrapv1.DataSecretAvailableCondition)
   717  
   718  			l := &corev1.SecretList{}
   719  			err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
   720  			g.Expect(err).ToNot(HaveOccurred())
   721  			g.Expect(l.Items).To(HaveLen(1))
   722  		})
   723  	}
   724  }
   725  
   726  func TestReconcileIfJoinNodePoolsAndControlPlaneIsReady(t *testing.T) {
   727  	_ = feature.MutableGates.Set("MachinePool=true")
   728  
   729  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   730  	cluster.Status.InfrastructureReady = true
   731  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   732  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
   733  
   734  	useCases := []struct {
   735  		name          string
   736  		machinePool   *expv1.MachinePool
   737  		configName    string
   738  		configBuilder func(string, string) *bootstrapv1.KubeadmConfig
   739  	}{
   740  		{
   741  			name:        "Join a worker node with a fully compiled kubeadm config object",
   742  			machinePool: newWorkerMachinePoolForCluster(cluster),
   743  			configName:  "workerpool-join-cfg",
   744  			configBuilder: func(namespace, name string) *bootstrapv1.KubeadmConfig {
   745  				return newWorkerJoinKubeadmConfig(namespace, "workerpool-join-cfg")
   746  			},
   747  		},
   748  		{
   749  			name:          "Join a worker node  with an empty kubeadm config object (defaults apply)",
   750  			machinePool:   newWorkerMachinePoolForCluster(cluster),
   751  			configName:    "workerpool-join-cfg",
   752  			configBuilder: newKubeadmConfig,
   753  		},
   754  	}
   755  
   756  	for _, rt := range useCases {
   757  		rt := rt // pin!
   758  		t.Run(rt.name, func(t *testing.T) {
   759  			g := NewWithT(t)
   760  
   761  			config := rt.configBuilder(rt.machinePool.Namespace, rt.configName)
   762  			addKubeadmConfigToMachinePool(config, rt.machinePool)
   763  
   764  			objects := []client.Object{
   765  				cluster,
   766  				rt.machinePool,
   767  				config,
   768  			}
   769  			objects = append(objects, createSecrets(t, cluster, config)...)
   770  			myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   771  			k := &KubeadmConfigReconciler{
   772  				Client:              myclient,
   773  				SecretCachingClient: myclient,
   774  				Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
   775  				KubeadmInitLock:     &myInitLocker{},
   776  			}
   777  
   778  			request := ctrl.Request{
   779  				NamespacedName: client.ObjectKey{
   780  					Namespace: config.GetNamespace(),
   781  					Name:      rt.configName,
   782  				},
   783  			}
   784  			result, err := k.Reconcile(ctx, request)
   785  			g.Expect(err).ToNot(HaveOccurred())
   786  			g.Expect(result.Requeue).To(BeFalse())
   787  			g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
   788  
   789  			cfg, err := getKubeadmConfig(myclient, rt.configName, metav1.NamespaceDefault)
   790  			g.Expect(err).ToNot(HaveOccurred())
   791  			g.Expect(cfg.Status.Ready).To(BeTrue())
   792  			g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
   793  			g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
   794  
   795  			l := &corev1.SecretList{}
   796  			err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
   797  			g.Expect(err).ToNot(HaveOccurred())
   798  			g.Expect(l.Items).To(HaveLen(1))
   799  		})
   800  	}
   801  }
   802  
   803  // Ensure bootstrap data is generated in the correct format based on the format specified in the
   804  // KubeadmConfig resource.
   805  func TestBootstrapDataFormat(t *testing.T) {
   806  	testcases := []struct {
   807  		name               string
   808  		isWorker           bool
   809  		format             bootstrapv1.Format
   810  		clusterInitialized bool
   811  	}{
   812  		{
   813  			name:   "cloud-config init config",
   814  			format: bootstrapv1.CloudConfig,
   815  		},
   816  		{
   817  			name:   "Ignition init config",
   818  			format: bootstrapv1.Ignition,
   819  		},
   820  		{
   821  			name:               "Ignition control plane join config",
   822  			format:             bootstrapv1.Ignition,
   823  			clusterInitialized: true,
   824  		},
   825  		{
   826  			name:               "Ignition worker join config",
   827  			isWorker:           true,
   828  			format:             bootstrapv1.Ignition,
   829  			clusterInitialized: true,
   830  		},
   831  		{
   832  			name: "Empty format field",
   833  		},
   834  	}
   835  
   836  	for _, tc := range testcases {
   837  		t.Run(tc.name, func(t *testing.T) {
   838  			g := NewWithT(t)
   839  
   840  			cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   841  			cluster.Status.InfrastructureReady = true
   842  			cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
   843  			if tc.clusterInitialized {
   844  				conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   845  			}
   846  
   847  			var machine *clusterv1.Machine
   848  			var config *bootstrapv1.KubeadmConfig
   849  			var configName string
   850  			if tc.isWorker {
   851  				machine = newWorkerMachineForCluster(cluster)
   852  				configName = "worker-join-cfg"
   853  				config = newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, configName)
   854  				addKubeadmConfigToMachine(config, machine)
   855  			} else {
   856  				machine = newControlPlaneMachine(cluster, "machine")
   857  				configName = "cfg"
   858  				config = newControlPlaneInitKubeadmConfig(metav1.NamespaceDefault, configName)
   859  				addKubeadmConfigToMachine(config, machine)
   860  			}
   861  			config.Spec.Format = tc.format
   862  
   863  			objects := []client.Object{
   864  				cluster,
   865  				machine,
   866  				config,
   867  			}
   868  			objects = append(objects, createSecrets(t, cluster, config)...)
   869  
   870  			myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   871  
   872  			k := &KubeadmConfigReconciler{
   873  				Client:              myclient,
   874  				SecretCachingClient: myclient,
   875  				Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
   876  				KubeadmInitLock:     &myInitLocker{},
   877  			}
   878  			request := ctrl.Request{
   879  				NamespacedName: client.ObjectKey{
   880  					Namespace: metav1.NamespaceDefault,
   881  					Name:      configName,
   882  				},
   883  			}
   884  
   885  			// Reconcile the KubeadmConfig resource.
   886  			_, err := k.Reconcile(ctx, request)
   887  			g.Expect(err).ToNot(HaveOccurred())
   888  
   889  			// Verify the KubeadmConfig resource state is correct.
   890  			cfg, err := getKubeadmConfig(myclient, configName, metav1.NamespaceDefault)
   891  			g.Expect(err).ToNot(HaveOccurred())
   892  			g.Expect(cfg.Status.Ready).To(BeTrue())
   893  			g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
   894  
   895  			// Read the secret containing the bootstrap data which was generated by the
   896  			// KubeadmConfig controller.
   897  			key := client.ObjectKey{
   898  				Namespace: metav1.NamespaceDefault,
   899  				Name:      *cfg.Status.DataSecretName,
   900  			}
   901  			secret := &corev1.Secret{}
   902  			err = myclient.Get(ctx, key, secret)
   903  			g.Expect(err).ToNot(HaveOccurred())
   904  
   905  			// Verify the format field of the bootstrap data secret is correct.
   906  			g.Expect(string(secret.Data["format"])).To(Equal(string(tc.format)))
   907  
   908  			// Verify the bootstrap data value is in the correct format.
   909  			data := secret.Data["value"]
   910  			switch tc.format {
   911  			case bootstrapv1.CloudConfig, "":
   912  				// Verify the bootstrap data is valid YAML.
   913  				// TODO: Verify the YAML document is valid cloud-config?
   914  				var out interface{}
   915  				err = yaml.Unmarshal(data, &out)
   916  				g.Expect(err).ToNot(HaveOccurred())
   917  			case bootstrapv1.Ignition:
   918  				// Verify the bootstrap data is valid Ignition.
   919  				_, reports, err := ignition.Parse(data)
   920  				g.Expect(err).ToNot(HaveOccurred())
   921  				g.Expect(reports.IsFatal()).NotTo(BeTrue())
   922  			}
   923  		})
   924  	}
   925  }
   926  
   927  // during kubeadmconfig reconcile it is possible that bootstrap secret gets created
   928  // but kubeadmconfig is not patched, do not error if secret already exists.
   929  // ignore the alreadyexists error and update the status to ready.
   930  func TestKubeadmConfigSecretCreatedStatusNotPatched(t *testing.T) {
   931  	g := NewWithT(t)
   932  
   933  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
   934  	cluster.Status.InfrastructureReady = true
   935  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
   936  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
   937  
   938  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
   939  	initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
   940  	addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
   941  
   942  	workerMachine := newWorkerMachineForCluster(cluster)
   943  	workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
   944  	addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
   945  	objects := []client.Object{
   946  		cluster,
   947  		workerMachine,
   948  		workerJoinConfig,
   949  	}
   950  
   951  	objects = append(objects, createSecrets(t, cluster, initConfig)...)
   952  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
   953  	k := &KubeadmConfigReconciler{
   954  		Client:              myclient,
   955  		SecretCachingClient: myclient,
   956  		Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
   957  		KubeadmInitLock:     &myInitLocker{},
   958  	}
   959  	request := ctrl.Request{
   960  		NamespacedName: client.ObjectKey{
   961  			Namespace: metav1.NamespaceDefault,
   962  			Name:      "worker-join-cfg",
   963  		},
   964  	}
   965  	secret := &corev1.Secret{
   966  		ObjectMeta: metav1.ObjectMeta{
   967  			Name:      workerJoinConfig.Name,
   968  			Namespace: workerJoinConfig.Namespace,
   969  			Labels: map[string]string{
   970  				clusterv1.ClusterNameLabel: cluster.Name,
   971  			},
   972  			OwnerReferences: []metav1.OwnerReference{
   973  				{
   974  					APIVersion: bootstrapv1.GroupVersion.String(),
   975  					Kind:       "KubeadmConfig",
   976  					Name:       workerJoinConfig.Name,
   977  					UID:        workerJoinConfig.UID,
   978  					Controller: pointer.Bool(true),
   979  				},
   980  			},
   981  		},
   982  		Data: map[string][]byte{
   983  			"value": nil,
   984  		},
   985  		Type: clusterv1.ClusterSecretType,
   986  	}
   987  
   988  	err := myclient.Create(ctx, secret)
   989  	g.Expect(err).ToNot(HaveOccurred())
   990  	result, err := k.Reconcile(ctx, request)
   991  	g.Expect(err).ToNot(HaveOccurred())
   992  	g.Expect(result.Requeue).To(BeFalse())
   993  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
   994  
   995  	cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
   996  	g.Expect(err).ToNot(HaveOccurred())
   997  	g.Expect(cfg.Status.Ready).To(BeTrue())
   998  	g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
   999  	g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
  1000  }
  1001  
  1002  func TestBootstrapTokenTTLExtension(t *testing.T) {
  1003  	g := NewWithT(t)
  1004  
  1005  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
  1006  	cluster.Status.InfrastructureReady = true
  1007  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
  1008  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
  1009  
  1010  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
  1011  	initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
  1012  	addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
  1013  
  1014  	workerMachine := newWorkerMachineForCluster(cluster)
  1015  	workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
  1016  	addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
  1017  
  1018  	controlPlaneJoinMachine := newControlPlaneMachine(cluster, "control-plane-join-machine")
  1019  	controlPlaneJoinConfig := newControlPlaneJoinKubeadmConfig(controlPlaneJoinMachine.Namespace, "control-plane-join-cfg")
  1020  	addKubeadmConfigToMachine(controlPlaneJoinConfig, controlPlaneJoinMachine)
  1021  	objects := []client.Object{
  1022  		cluster,
  1023  		workerMachine,
  1024  		workerJoinConfig,
  1025  		controlPlaneJoinMachine,
  1026  		controlPlaneJoinConfig,
  1027  	}
  1028  
  1029  	objects = append(objects, createSecrets(t, cluster, initConfig)...)
  1030  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &clusterv1.Machine{}).Build()
  1031  	k := &KubeadmConfigReconciler{
  1032  		Client:              myclient,
  1033  		SecretCachingClient: myclient,
  1034  		Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
  1035  		KubeadmInitLock:     &myInitLocker{},
  1036  		TokenTTL:            DefaultTokenTTL,
  1037  	}
  1038  	request := ctrl.Request{
  1039  		NamespacedName: client.ObjectKey{
  1040  			Namespace: metav1.NamespaceDefault,
  1041  			Name:      "worker-join-cfg",
  1042  		},
  1043  	}
  1044  	result, err := k.Reconcile(ctx, request)
  1045  	g.Expect(err).ToNot(HaveOccurred())
  1046  	g.Expect(result.Requeue).To(BeFalse())
  1047  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1048  
  1049  	cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
  1050  	g.Expect(err).ToNot(HaveOccurred())
  1051  	g.Expect(cfg.Status.Ready).To(BeTrue())
  1052  	g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
  1053  	g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
  1054  
  1055  	request = ctrl.Request{
  1056  		NamespacedName: client.ObjectKey{
  1057  			Namespace: metav1.NamespaceDefault,
  1058  			Name:      "control-plane-join-cfg",
  1059  		},
  1060  	}
  1061  	result, err = k.Reconcile(ctx, request)
  1062  	g.Expect(err).ToNot(HaveOccurred())
  1063  	g.Expect(result.Requeue).To(BeFalse())
  1064  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1065  
  1066  	cfg, err = getKubeadmConfig(myclient, "control-plane-join-cfg", metav1.NamespaceDefault)
  1067  	g.Expect(err).ToNot(HaveOccurred())
  1068  	g.Expect(cfg.Status.Ready).To(BeTrue())
  1069  	g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
  1070  	g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
  1071  
  1072  	l := &corev1.SecretList{}
  1073  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1074  	g.Expect(err).ToNot(HaveOccurred())
  1075  	g.Expect(l.Items).To(HaveLen(2))
  1076  
  1077  	// ensure that the token is refreshed...
  1078  	tokenExpires := make([][]byte, len(l.Items))
  1079  
  1080  	for i, item := range l.Items {
  1081  		tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey]
  1082  	}
  1083  
  1084  	<-time.After(1 * time.Second)
  1085  
  1086  	for _, req := range []ctrl.Request{
  1087  		{
  1088  			NamespacedName: client.ObjectKey{
  1089  				Namespace: metav1.NamespaceDefault,
  1090  				Name:      "worker-join-cfg",
  1091  			},
  1092  		},
  1093  		{
  1094  			NamespacedName: client.ObjectKey{
  1095  				Namespace: metav1.NamespaceDefault,
  1096  				Name:      "control-plane-join-cfg",
  1097  			},
  1098  		},
  1099  	} {
  1100  		result, err := k.Reconcile(ctx, req)
  1101  		g.Expect(err).ToNot(HaveOccurred())
  1102  		g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL))
  1103  	}
  1104  
  1105  	l = &corev1.SecretList{}
  1106  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1107  	g.Expect(err).ToNot(HaveOccurred())
  1108  	g.Expect(l.Items).To(HaveLen(2))
  1109  
  1110  	for i, item := range l.Items {
  1111  		g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse())
  1112  		tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey]
  1113  	}
  1114  
  1115  	// ...the infrastructure is marked "ready", but token should still be refreshed...
  1116  	patchHelper, err := patch.NewHelper(workerMachine, myclient)
  1117  	g.Expect(err).ShouldNot(HaveOccurred())
  1118  	workerMachine.Status.InfrastructureReady = true
  1119  	g.Expect(patchHelper.Patch(ctx, workerMachine)).To(Succeed())
  1120  
  1121  	patchHelper, err = patch.NewHelper(controlPlaneJoinMachine, myclient)
  1122  	g.Expect(err).ShouldNot(HaveOccurred())
  1123  	controlPlaneJoinMachine.Status.InfrastructureReady = true
  1124  	g.Expect(patchHelper.Patch(ctx, controlPlaneJoinMachine)).To(Succeed())
  1125  
  1126  	<-time.After(1 * time.Second)
  1127  
  1128  	for _, req := range []ctrl.Request{
  1129  		{
  1130  			NamespacedName: client.ObjectKey{
  1131  				Namespace: metav1.NamespaceDefault,
  1132  				Name:      "worker-join-cfg",
  1133  			},
  1134  		},
  1135  		{
  1136  			NamespacedName: client.ObjectKey{
  1137  				Namespace: metav1.NamespaceDefault,
  1138  				Name:      "control-plane-join-cfg",
  1139  			},
  1140  		},
  1141  	} {
  1142  		result, err := k.Reconcile(ctx, req)
  1143  		g.Expect(err).ToNot(HaveOccurred())
  1144  		g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL))
  1145  	}
  1146  
  1147  	l = &corev1.SecretList{}
  1148  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1149  	g.Expect(err).ToNot(HaveOccurred())
  1150  	g.Expect(l.Items).To(HaveLen(2))
  1151  
  1152  	for i, item := range l.Items {
  1153  		g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse())
  1154  		tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey]
  1155  	}
  1156  
  1157  	// ...until the Nodes have actually joined the cluster and we get a nodeRef
  1158  	patchHelper, err = patch.NewHelper(workerMachine, myclient)
  1159  	g.Expect(err).ShouldNot(HaveOccurred())
  1160  	workerMachine.Status.NodeRef = &corev1.ObjectReference{
  1161  		APIVersion: "v1",
  1162  		Kind:       "Node",
  1163  		Name:       "worker-node",
  1164  	}
  1165  	g.Expect(patchHelper.Patch(ctx, workerMachine)).To(Succeed())
  1166  
  1167  	patchHelper, err = patch.NewHelper(controlPlaneJoinMachine, myclient)
  1168  	g.Expect(err).ShouldNot(HaveOccurred())
  1169  	controlPlaneJoinMachine.Status.NodeRef = &corev1.ObjectReference{
  1170  		APIVersion: "v1",
  1171  		Kind:       "Node",
  1172  		Name:       "control-plane-node",
  1173  	}
  1174  	g.Expect(patchHelper.Patch(ctx, controlPlaneJoinMachine)).To(Succeed())
  1175  
  1176  	<-time.After(1 * time.Second)
  1177  
  1178  	for _, req := range []ctrl.Request{
  1179  		{
  1180  			NamespacedName: client.ObjectKey{
  1181  				Namespace: metav1.NamespaceDefault,
  1182  				Name:      "worker-join-cfg",
  1183  			},
  1184  		},
  1185  		{
  1186  			NamespacedName: client.ObjectKey{
  1187  				Namespace: metav1.NamespaceDefault,
  1188  				Name:      "control-plane-join-cfg",
  1189  			},
  1190  		},
  1191  	} {
  1192  		result, err := k.Reconcile(ctx, req)
  1193  		g.Expect(err).ToNot(HaveOccurred())
  1194  		g.Expect(result.Requeue).To(BeFalse())
  1195  		g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1196  	}
  1197  
  1198  	l = &corev1.SecretList{}
  1199  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1200  	g.Expect(err).ToNot(HaveOccurred())
  1201  	g.Expect(l.Items).To(HaveLen(2))
  1202  
  1203  	for i, item := range l.Items {
  1204  		g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeTrue())
  1205  	}
  1206  }
  1207  
  1208  func TestBootstrapTokenRotationMachinePool(t *testing.T) {
  1209  	_ = feature.MutableGates.Set("MachinePool=true")
  1210  	g := NewWithT(t)
  1211  
  1212  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
  1213  	cluster.Status.InfrastructureReady = true
  1214  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
  1215  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
  1216  
  1217  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
  1218  	initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
  1219  
  1220  	addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
  1221  
  1222  	workerMachinePool := newWorkerMachinePoolForCluster(cluster)
  1223  	workerJoinConfig := newWorkerJoinKubeadmConfig(workerMachinePool.Namespace, "workerpool-join-cfg")
  1224  	addKubeadmConfigToMachinePool(workerJoinConfig, workerMachinePool)
  1225  	objects := []client.Object{
  1226  		cluster,
  1227  		workerMachinePool,
  1228  		workerJoinConfig,
  1229  	}
  1230  
  1231  	objects = append(objects, createSecrets(t, cluster, initConfig)...)
  1232  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &expv1.MachinePool{}).Build()
  1233  	k := &KubeadmConfigReconciler{
  1234  		Client:              myclient,
  1235  		SecretCachingClient: myclient,
  1236  		Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
  1237  		KubeadmInitLock:     &myInitLocker{},
  1238  		TokenTTL:            DefaultTokenTTL,
  1239  	}
  1240  	request := ctrl.Request{
  1241  		NamespacedName: client.ObjectKey{
  1242  			Namespace: metav1.NamespaceDefault,
  1243  			Name:      "workerpool-join-cfg",
  1244  		},
  1245  	}
  1246  	result, err := k.Reconcile(ctx, request)
  1247  	g.Expect(err).ToNot(HaveOccurred())
  1248  	g.Expect(result.Requeue).To(BeFalse())
  1249  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1250  
  1251  	cfg, err := getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
  1252  	g.Expect(err).ToNot(HaveOccurred())
  1253  	g.Expect(cfg.Status.Ready).To(BeTrue())
  1254  	g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
  1255  	g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
  1256  
  1257  	l := &corev1.SecretList{}
  1258  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1259  	g.Expect(err).ToNot(HaveOccurred())
  1260  	g.Expect(l.Items).To(HaveLen(1))
  1261  
  1262  	// ensure that the token is refreshed...
  1263  	tokenExpires := make([][]byte, len(l.Items))
  1264  
  1265  	for i, item := range l.Items {
  1266  		tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey]
  1267  	}
  1268  
  1269  	<-time.After(1 * time.Second)
  1270  
  1271  	for _, req := range []ctrl.Request{
  1272  		{
  1273  			NamespacedName: client.ObjectKey{
  1274  				Namespace: metav1.NamespaceDefault,
  1275  				Name:      "workerpool-join-cfg",
  1276  			},
  1277  		},
  1278  	} {
  1279  		result, err := k.Reconcile(ctx, req)
  1280  		g.Expect(err).ToNot(HaveOccurred())
  1281  		g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL))
  1282  	}
  1283  
  1284  	l = &corev1.SecretList{}
  1285  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1286  	g.Expect(err).ToNot(HaveOccurred())
  1287  	g.Expect(l.Items).To(HaveLen(1))
  1288  
  1289  	for i, item := range l.Items {
  1290  		g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse())
  1291  		tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey]
  1292  	}
  1293  
  1294  	// ...the infrastructure is marked "ready", but token should still be refreshed...
  1295  	patchHelper, err := patch.NewHelper(workerMachinePool, myclient)
  1296  	g.Expect(err).ShouldNot(HaveOccurred())
  1297  	workerMachinePool.Status.InfrastructureReady = true
  1298  	g.Expect(patchHelper.Patch(ctx, workerMachinePool, patch.WithStatusObservedGeneration{})).To(Succeed())
  1299  
  1300  	<-time.After(1 * time.Second)
  1301  
  1302  	request = ctrl.Request{
  1303  		NamespacedName: client.ObjectKey{
  1304  			Namespace: metav1.NamespaceDefault,
  1305  			Name:      "workerpool-join-cfg",
  1306  		},
  1307  	}
  1308  	result, err = k.Reconcile(ctx, request)
  1309  	g.Expect(err).ToNot(HaveOccurred())
  1310  	g.Expect(result.RequeueAfter).NotTo(BeNumerically(">=", k.TokenTTL))
  1311  
  1312  	l = &corev1.SecretList{}
  1313  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1314  	g.Expect(err).ToNot(HaveOccurred())
  1315  	g.Expect(l.Items).To(HaveLen(1))
  1316  
  1317  	for i, item := range l.Items {
  1318  		g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeFalse())
  1319  		tokenExpires[i] = item.Data[bootstrapapi.BootstrapTokenExpirationKey]
  1320  	}
  1321  
  1322  	// ...until all nodes have joined
  1323  	workerMachinePool.Status.NodeRefs = []corev1.ObjectReference{
  1324  		{
  1325  			Kind:      "Node",
  1326  			Namespace: metav1.NamespaceDefault,
  1327  			Name:      "node-0",
  1328  		},
  1329  	}
  1330  	g.Expect(patchHelper.Patch(ctx, workerMachinePool, patch.WithStatusObservedGeneration{})).To(Succeed())
  1331  
  1332  	<-time.After(1 * time.Second)
  1333  
  1334  	request = ctrl.Request{
  1335  		NamespacedName: client.ObjectKey{
  1336  			Namespace: metav1.NamespaceDefault,
  1337  			Name:      "workerpool-join-cfg",
  1338  		},
  1339  	}
  1340  	result, err = k.Reconcile(ctx, request)
  1341  	g.Expect(err).ToNot(HaveOccurred())
  1342  	g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
  1343  
  1344  	l = &corev1.SecretList{}
  1345  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1346  	g.Expect(err).ToNot(HaveOccurred())
  1347  	g.Expect(l.Items).To(HaveLen(1))
  1348  
  1349  	for i, item := range l.Items {
  1350  		g.Expect(bytes.Equal(tokenExpires[i], item.Data[bootstrapapi.BootstrapTokenExpirationKey])).To(BeTrue())
  1351  	}
  1352  
  1353  	// before token expires, it should rotate it
  1354  	tokenExpires[0] = []byte(time.Now().UTC().Add(k.TokenTTL / 5).Format(time.RFC3339))
  1355  	l.Items[0].Data[bootstrapapi.BootstrapTokenExpirationKey] = tokenExpires[0]
  1356  	err = myclient.Update(ctx, &l.Items[0])
  1357  	g.Expect(err).ToNot(HaveOccurred())
  1358  
  1359  	request = ctrl.Request{
  1360  		NamespacedName: client.ObjectKey{
  1361  			Namespace: metav1.NamespaceDefault,
  1362  			Name:      "workerpool-join-cfg",
  1363  		},
  1364  	}
  1365  	result, err = k.Reconcile(ctx, request)
  1366  	g.Expect(err).ToNot(HaveOccurred())
  1367  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1368  
  1369  	l = &corev1.SecretList{}
  1370  	err = myclient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))
  1371  	g.Expect(err).ToNot(HaveOccurred())
  1372  	g.Expect(l.Items).To(HaveLen(2))
  1373  	foundOld := false
  1374  	foundNew := true
  1375  	for _, item := range l.Items {
  1376  		if bytes.Equal(item.Data[bootstrapapi.BootstrapTokenExpirationKey], tokenExpires[0]) {
  1377  			foundOld = true
  1378  		} else {
  1379  			expirationTime, err := time.Parse(time.RFC3339, string(item.Data[bootstrapapi.BootstrapTokenExpirationKey]))
  1380  			g.Expect(err).ToNot(HaveOccurred())
  1381  			g.Expect(expirationTime).Should(BeTemporally("~", time.Now().UTC().Add(k.TokenTTL), 10*time.Second))
  1382  			foundNew = true
  1383  		}
  1384  	}
  1385  	g.Expect(foundOld).To(BeTrue())
  1386  	g.Expect(foundNew).To(BeTrue())
  1387  }
  1388  
  1389  // Ensure the discovery portion of the JoinConfiguration gets generated correctly.
  1390  func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileBehaviors(t *testing.T) {
  1391  	caHash := []string{"...."}
  1392  	bootstrapToken := bootstrapv1.Discovery{
  1393  		BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{
  1394  			CACertHashes: caHash,
  1395  		},
  1396  	}
  1397  	goodcluster := &clusterv1.Cluster{
  1398  		Spec: clusterv1.ClusterSpec{
  1399  			ControlPlaneEndpoint: clusterv1.APIEndpoint{
  1400  				Host: "example.com",
  1401  				Port: 6443,
  1402  			},
  1403  		},
  1404  	}
  1405  	testcases := []struct {
  1406  		name              string
  1407  		cluster           *clusterv1.Cluster
  1408  		config            *bootstrapv1.KubeadmConfig
  1409  		validateDiscovery func(*WithT, *bootstrapv1.KubeadmConfig) error
  1410  	}{
  1411  		{
  1412  			name:    "Automatically generate token if discovery not specified",
  1413  			cluster: goodcluster,
  1414  			config: &bootstrapv1.KubeadmConfig{
  1415  				Spec: bootstrapv1.KubeadmConfigSpec{
  1416  					JoinConfiguration: &bootstrapv1.JoinConfiguration{
  1417  						Discovery: bootstrapToken,
  1418  					},
  1419  				},
  1420  			},
  1421  			validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error {
  1422  				d := c.Spec.JoinConfiguration.Discovery
  1423  				g.Expect(d.BootstrapToken).NotTo(BeNil())
  1424  				g.Expect(d.BootstrapToken.Token).NotTo(Equal(""))
  1425  				g.Expect(d.BootstrapToken.APIServerEndpoint).To(Equal("example.com:6443"))
  1426  				g.Expect(d.BootstrapToken.UnsafeSkipCAVerification).To(BeFalse())
  1427  				return nil
  1428  			},
  1429  		},
  1430  		{
  1431  			name:    "Respect discoveryConfiguration.File",
  1432  			cluster: goodcluster,
  1433  			config: &bootstrapv1.KubeadmConfig{
  1434  				Spec: bootstrapv1.KubeadmConfigSpec{
  1435  					JoinConfiguration: &bootstrapv1.JoinConfiguration{
  1436  						Discovery: bootstrapv1.Discovery{
  1437  							File: &bootstrapv1.FileDiscovery{},
  1438  						},
  1439  					},
  1440  				},
  1441  			},
  1442  			validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error {
  1443  				d := c.Spec.JoinConfiguration.Discovery
  1444  				g.Expect(d.BootstrapToken).To(BeNil())
  1445  				return nil
  1446  			},
  1447  		},
  1448  		{
  1449  			name:    "Respect discoveryConfiguration.BootstrapToken.APIServerEndpoint",
  1450  			cluster: goodcluster,
  1451  			config: &bootstrapv1.KubeadmConfig{
  1452  				Spec: bootstrapv1.KubeadmConfigSpec{
  1453  					JoinConfiguration: &bootstrapv1.JoinConfiguration{
  1454  						Discovery: bootstrapv1.Discovery{
  1455  							BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{
  1456  								CACertHashes:      caHash,
  1457  								APIServerEndpoint: "bar.com:6443",
  1458  							},
  1459  						},
  1460  					},
  1461  				},
  1462  			},
  1463  			validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error {
  1464  				d := c.Spec.JoinConfiguration.Discovery
  1465  				g.Expect(d.BootstrapToken.APIServerEndpoint).To(Equal("bar.com:6443"))
  1466  				return nil
  1467  			},
  1468  		},
  1469  		{
  1470  			name:    "Respect discoveryConfiguration.BootstrapToken.Token",
  1471  			cluster: goodcluster,
  1472  			config: &bootstrapv1.KubeadmConfig{
  1473  				Spec: bootstrapv1.KubeadmConfigSpec{
  1474  					JoinConfiguration: &bootstrapv1.JoinConfiguration{
  1475  						Discovery: bootstrapv1.Discovery{
  1476  							BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{
  1477  								CACertHashes: caHash,
  1478  								Token:        "abcdef.0123456789abcdef",
  1479  							},
  1480  						},
  1481  					},
  1482  				},
  1483  			},
  1484  			validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error {
  1485  				d := c.Spec.JoinConfiguration.Discovery
  1486  				g.Expect(d.BootstrapToken.Token).To(Equal("abcdef.0123456789abcdef"))
  1487  				return nil
  1488  			},
  1489  		},
  1490  		{
  1491  			name:    "Respect discoveryConfiguration.BootstrapToken.CACertHashes",
  1492  			cluster: goodcluster,
  1493  			config: &bootstrapv1.KubeadmConfig{
  1494  				Spec: bootstrapv1.KubeadmConfigSpec{
  1495  					JoinConfiguration: &bootstrapv1.JoinConfiguration{
  1496  						Discovery: bootstrapv1.Discovery{
  1497  							BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{
  1498  								CACertHashes: caHash,
  1499  							},
  1500  						},
  1501  					},
  1502  				},
  1503  			},
  1504  			validateDiscovery: func(g *WithT, c *bootstrapv1.KubeadmConfig) error {
  1505  				d := c.Spec.JoinConfiguration.Discovery
  1506  				g.Expect(reflect.DeepEqual(d.BootstrapToken.CACertHashes, caHash)).To(BeTrue())
  1507  				return nil
  1508  			},
  1509  		},
  1510  	}
  1511  
  1512  	for _, tc := range testcases {
  1513  		t.Run(tc.name, func(t *testing.T) {
  1514  			g := NewWithT(t)
  1515  
  1516  			fakeClient := fake.NewClientBuilder().Build()
  1517  			k := &KubeadmConfigReconciler{
  1518  				Client:              fakeClient,
  1519  				SecretCachingClient: fakeClient,
  1520  				Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), fakeClient, fakeClient.Scheme(), client.ObjectKey{Name: tc.cluster.Name, Namespace: tc.cluster.Namespace}),
  1521  				KubeadmInitLock:     &myInitLocker{},
  1522  			}
  1523  
  1524  			res, err := k.reconcileDiscovery(ctx, tc.cluster, tc.config, secret.Certificates{})
  1525  			g.Expect(res.IsZero()).To(BeTrue())
  1526  			g.Expect(err).ToNot(HaveOccurred())
  1527  
  1528  			err = tc.validateDiscovery(g, tc.config)
  1529  			g.Expect(err).ToNot(HaveOccurred())
  1530  		})
  1531  	}
  1532  }
  1533  
  1534  // Test failure cases for the discovery reconcile function.
  1535  func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileFailureBehaviors(t *testing.T) {
  1536  	k := &KubeadmConfigReconciler{}
  1537  
  1538  	testcases := []struct {
  1539  		name    string
  1540  		cluster *clusterv1.Cluster
  1541  		config  *bootstrapv1.KubeadmConfig
  1542  
  1543  		result ctrl.Result
  1544  		err    error
  1545  	}{
  1546  		{
  1547  			name:    "Should requeue if cluster has not ControlPlaneEndpoint",
  1548  			cluster: &clusterv1.Cluster{}, // cluster without endpoints
  1549  			config: &bootstrapv1.KubeadmConfig{
  1550  				Spec: bootstrapv1.KubeadmConfigSpec{
  1551  					JoinConfiguration: &bootstrapv1.JoinConfiguration{
  1552  						Discovery: bootstrapv1.Discovery{
  1553  							BootstrapToken: &bootstrapv1.BootstrapTokenDiscovery{
  1554  								CACertHashes: []string{"item"},
  1555  							},
  1556  						},
  1557  					},
  1558  				},
  1559  			},
  1560  			result: ctrl.Result{RequeueAfter: 10 * time.Second},
  1561  		},
  1562  	}
  1563  
  1564  	for _, tc := range testcases {
  1565  		t.Run(tc.name, func(t *testing.T) {
  1566  			g := NewWithT(t)
  1567  
  1568  			res, err := k.reconcileDiscovery(ctx, tc.cluster, tc.config, secret.Certificates{})
  1569  			g.Expect(res).To(BeComparableTo(tc.result))
  1570  			if tc.err == nil {
  1571  				g.Expect(err).ToNot(HaveOccurred())
  1572  			} else {
  1573  				g.Expect(err).To(Equal(tc.err))
  1574  			}
  1575  		})
  1576  	}
  1577  }
  1578  
  1579  // Set cluster configuration defaults based on dynamic values from the cluster object.
  1580  func TestKubeadmConfigReconciler_Reconcile_DynamicDefaultsForClusterConfiguration(t *testing.T) {
  1581  	k := &KubeadmConfigReconciler{}
  1582  
  1583  	testcases := []struct {
  1584  		name    string
  1585  		cluster *clusterv1.Cluster
  1586  		machine *clusterv1.Machine
  1587  		config  *bootstrapv1.KubeadmConfig
  1588  	}{
  1589  		{
  1590  			name: "Config settings have precedence",
  1591  			config: &bootstrapv1.KubeadmConfig{
  1592  				Spec: bootstrapv1.KubeadmConfigSpec{
  1593  					ClusterConfiguration: &bootstrapv1.ClusterConfiguration{
  1594  						ClusterName:       "mycluster",
  1595  						KubernetesVersion: "myversion",
  1596  						Networking: bootstrapv1.Networking{
  1597  							PodSubnet:     "myPodSubnet",
  1598  							ServiceSubnet: "myServiceSubnet",
  1599  							DNSDomain:     "myDNSDomain",
  1600  						},
  1601  						ControlPlaneEndpoint: "myControlPlaneEndpoint:6443",
  1602  					},
  1603  				},
  1604  			},
  1605  			cluster: &clusterv1.Cluster{
  1606  				ObjectMeta: metav1.ObjectMeta{
  1607  					Name: "OtherName",
  1608  				},
  1609  				Spec: clusterv1.ClusterSpec{
  1610  					ClusterNetwork: &clusterv1.ClusterNetwork{
  1611  						Services:      &clusterv1.NetworkRanges{CIDRBlocks: []string{"otherServicesCidr"}},
  1612  						Pods:          &clusterv1.NetworkRanges{CIDRBlocks: []string{"otherPodsCidr"}},
  1613  						ServiceDomain: "otherServiceDomain",
  1614  					},
  1615  					ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: "otherVersion", Port: 0},
  1616  				},
  1617  			},
  1618  			machine: &clusterv1.Machine{
  1619  				Spec: clusterv1.MachineSpec{
  1620  					Version: pointer.String("otherVersion"),
  1621  				},
  1622  			},
  1623  		},
  1624  		{
  1625  			name: "Top level object settings are used in case config settings are missing",
  1626  			config: &bootstrapv1.KubeadmConfig{
  1627  				Spec: bootstrapv1.KubeadmConfigSpec{
  1628  					ClusterConfiguration: &bootstrapv1.ClusterConfiguration{},
  1629  				},
  1630  			},
  1631  			cluster: &clusterv1.Cluster{
  1632  				ObjectMeta: metav1.ObjectMeta{
  1633  					Name: "mycluster",
  1634  				},
  1635  				Spec: clusterv1.ClusterSpec{
  1636  					ClusterNetwork: &clusterv1.ClusterNetwork{
  1637  						Services:      &clusterv1.NetworkRanges{CIDRBlocks: []string{"myServiceSubnet"}},
  1638  						Pods:          &clusterv1.NetworkRanges{CIDRBlocks: []string{"myPodSubnet"}},
  1639  						ServiceDomain: "myDNSDomain",
  1640  					},
  1641  					ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: "myControlPlaneEndpoint", Port: 6443},
  1642  				},
  1643  			},
  1644  			machine: &clusterv1.Machine{
  1645  				Spec: clusterv1.MachineSpec{
  1646  					Version: pointer.String("myversion"),
  1647  				},
  1648  			},
  1649  		},
  1650  	}
  1651  
  1652  	for _, tc := range testcases {
  1653  		t.Run(tc.name, func(t *testing.T) {
  1654  			g := NewWithT(t)
  1655  
  1656  			k.reconcileTopLevelObjectSettings(ctx, tc.cluster, tc.machine, tc.config)
  1657  
  1658  			g.Expect(tc.config.Spec.ClusterConfiguration.ControlPlaneEndpoint).To(Equal("myControlPlaneEndpoint:6443"))
  1659  			g.Expect(tc.config.Spec.ClusterConfiguration.ClusterName).To(Equal("mycluster"))
  1660  			g.Expect(tc.config.Spec.ClusterConfiguration.Networking.PodSubnet).To(Equal("myPodSubnet"))
  1661  			g.Expect(tc.config.Spec.ClusterConfiguration.Networking.ServiceSubnet).To(Equal("myServiceSubnet"))
  1662  			g.Expect(tc.config.Spec.ClusterConfiguration.Networking.DNSDomain).To(Equal("myDNSDomain"))
  1663  			g.Expect(tc.config.Spec.ClusterConfiguration.KubernetesVersion).To(Equal("myversion"))
  1664  		})
  1665  	}
  1666  }
  1667  
  1668  // Allow users to skip CA Verification if they *really* want to.
  1669  func TestKubeadmConfigReconciler_Reconcile_AlwaysCheckCAVerificationUnlessRequestedToSkip(t *testing.T) {
  1670  	// Setup work for an initialized cluster
  1671  	clusterName := "my-cluster"
  1672  	cluster := builder.Cluster(metav1.NamespaceDefault, clusterName).Build()
  1673  	conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
  1674  	cluster.Status.InfrastructureReady = true
  1675  	cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
  1676  		Host: "example.com",
  1677  		Port: 6443,
  1678  	}
  1679  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "my-control-plane-init-machine")
  1680  	initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "my-control-plane-init-config")
  1681  
  1682  	controlPlaneMachineName := "my-machine"
  1683  	machine := builder.Machine(metav1.NamespaceDefault, controlPlaneMachineName).
  1684  		WithVersion("v1.19.1").
  1685  		WithClusterName(cluster.Name).
  1686  		Build()
  1687  
  1688  	workerMachineName := "my-worker"
  1689  	workerMachine := builder.Machine(metav1.NamespaceDefault, workerMachineName).
  1690  		WithVersion("v1.19.1").
  1691  		WithClusterName(cluster.Name).
  1692  		Build()
  1693  
  1694  	controlPlaneConfigName := "my-config"
  1695  	config := newKubeadmConfig(metav1.NamespaceDefault, controlPlaneConfigName)
  1696  
  1697  	objects := []client.Object{
  1698  		cluster, machine, workerMachine, config,
  1699  	}
  1700  	objects = append(objects, createSecrets(t, cluster, initConfig)...)
  1701  
  1702  	testcases := []struct {
  1703  		name               string
  1704  		discovery          *bootstrapv1.BootstrapTokenDiscovery
  1705  		skipCAVerification bool
  1706  	}{
  1707  		{
  1708  			name:               "Do not skip CA verification by default",
  1709  			discovery:          &bootstrapv1.BootstrapTokenDiscovery{},
  1710  			skipCAVerification: false,
  1711  		},
  1712  		{
  1713  			name: "Skip CA verification if requested by the user",
  1714  			discovery: &bootstrapv1.BootstrapTokenDiscovery{
  1715  				UnsafeSkipCAVerification: true,
  1716  			},
  1717  			skipCAVerification: true,
  1718  		},
  1719  		{
  1720  			// skipCAVerification should be true since no Cert Hashes are provided, but reconcile will *always* get or create certs.
  1721  			// TODO: Certificate get/create behavior needs to be mocked to enable this test.
  1722  			name: "cannot test for defaulting behavior through the reconcile function",
  1723  			discovery: &bootstrapv1.BootstrapTokenDiscovery{
  1724  				CACertHashes: []string{""},
  1725  			},
  1726  			skipCAVerification: false,
  1727  		},
  1728  	}
  1729  	for _, tc := range testcases {
  1730  		t.Run(tc.name, func(t *testing.T) {
  1731  			g := NewWithT(t)
  1732  
  1733  			myclient := fake.NewClientBuilder().WithObjects(objects...).Build()
  1734  			reconciler := KubeadmConfigReconciler{
  1735  				Client:              myclient,
  1736  				SecretCachingClient: myclient,
  1737  				Tracker:             remote.NewTestClusterCacheTracker(logr.New(log.NullLogSink{}), myclient, myclient.Scheme(), client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
  1738  				KubeadmInitLock:     &myInitLocker{},
  1739  			}
  1740  
  1741  			wc := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
  1742  			wc.Spec.JoinConfiguration.Discovery.BootstrapToken = tc.discovery
  1743  			key := client.ObjectKey{Namespace: wc.Namespace, Name: wc.Name}
  1744  			err := myclient.Create(ctx, wc)
  1745  			g.Expect(err).ToNot(HaveOccurred())
  1746  
  1747  			req := ctrl.Request{NamespacedName: key}
  1748  			_, err = reconciler.Reconcile(ctx, req)
  1749  			g.Expect(err).ToNot(HaveOccurred())
  1750  
  1751  			cfg := &bootstrapv1.KubeadmConfig{}
  1752  			err = myclient.Get(ctx, key, cfg)
  1753  			g.Expect(err).ToNot(HaveOccurred())
  1754  			g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.UnsafeSkipCAVerification).To(Equal(tc.skipCAVerification))
  1755  		})
  1756  	}
  1757  }
  1758  
  1759  // If a cluster object changes then all associated KubeadmConfigs should be re-reconciled.
  1760  // This allows us to not requeue a kubeadm config while we wait for InfrastructureReady.
  1761  func TestKubeadmConfigReconciler_ClusterToKubeadmConfigs(t *testing.T) {
  1762  	_ = feature.MutableGates.Set("MachinePool=true")
  1763  	g := NewWithT(t)
  1764  
  1765  	cluster := builder.Cluster(metav1.NamespaceDefault, "my-cluster").Build()
  1766  	objs := []client.Object{cluster}
  1767  	expectedNames := []string{}
  1768  	for i := 0; i < 3; i++ {
  1769  		configName := fmt.Sprintf("my-config-%d", i)
  1770  		m := builder.Machine(metav1.NamespaceDefault, fmt.Sprintf("my-machine-%d", i)).
  1771  			WithVersion("v1.19.1").
  1772  			WithClusterName(cluster.Name).
  1773  			WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, configName).Unstructured()).
  1774  			Build()
  1775  		c := newKubeadmConfig(metav1.NamespaceDefault, configName)
  1776  		addKubeadmConfigToMachine(c, m)
  1777  		expectedNames = append(expectedNames, configName)
  1778  		objs = append(objs, m, c)
  1779  	}
  1780  	for i := 3; i < 6; i++ {
  1781  		mp := newMachinePool(cluster, fmt.Sprintf("my-machinepool-%d", i))
  1782  		configName := fmt.Sprintf("my-config-%d", i)
  1783  		c := newKubeadmConfig(mp.Namespace, configName)
  1784  		addKubeadmConfigToMachinePool(c, mp)
  1785  		expectedNames = append(expectedNames, configName)
  1786  		objs = append(objs, mp, c)
  1787  	}
  1788  	fakeClient := fake.NewClientBuilder().WithObjects(objs...).Build()
  1789  	reconciler := &KubeadmConfigReconciler{
  1790  		Client:              fakeClient,
  1791  		SecretCachingClient: fakeClient,
  1792  	}
  1793  	configs := reconciler.ClusterToKubeadmConfigs(ctx, cluster)
  1794  	names := make([]string, 6)
  1795  	for i := range configs {
  1796  		names[i] = configs[i].Name
  1797  	}
  1798  	for _, name := range expectedNames {
  1799  		found := false
  1800  		for _, foundName := range names {
  1801  			if foundName == name {
  1802  				found = true
  1803  			}
  1804  		}
  1805  		g.Expect(found).To(BeTrue())
  1806  	}
  1807  }
  1808  
  1809  // Reconcile should not fail if the Etcd CA Secret already exists.
  1810  func TestKubeadmConfigReconciler_Reconcile_DoesNotFailIfCASecretsAlreadyExist(t *testing.T) {
  1811  	g := NewWithT(t)
  1812  
  1813  	cluster := builder.Cluster(metav1.NamespaceDefault, "my-cluster").Build()
  1814  	cluster.Status.InfrastructureReady = true
  1815  	m := newControlPlaneMachine(cluster, "control-plane-machine")
  1816  	configName := "my-config"
  1817  	c := newControlPlaneInitKubeadmConfig(m.Namespace, configName)
  1818  	scrt := &corev1.Secret{
  1819  		ObjectMeta: metav1.ObjectMeta{
  1820  			Name:      fmt.Sprintf("%s-%s", cluster.Name, secret.EtcdCA),
  1821  			Namespace: metav1.NamespaceDefault,
  1822  		},
  1823  		Data: map[string][]byte{
  1824  			"tls.crt": []byte("hello world"),
  1825  			"tls.key": []byte("hello world"),
  1826  		},
  1827  	}
  1828  	fakec := fake.NewClientBuilder().WithObjects(cluster, m, c, scrt).Build()
  1829  	reconciler := &KubeadmConfigReconciler{
  1830  		Client:              fakec,
  1831  		SecretCachingClient: fakec,
  1832  		KubeadmInitLock:     &myInitLocker{},
  1833  	}
  1834  	req := ctrl.Request{
  1835  		NamespacedName: client.ObjectKey{Namespace: metav1.NamespaceDefault, Name: configName},
  1836  	}
  1837  	_, err := reconciler.Reconcile(ctx, req)
  1838  	g.Expect(err).ToNot(HaveOccurred())
  1839  }
  1840  
  1841  // Exactly one control plane machine initializes if there are multiple control plane machines defined.
  1842  func TestKubeadmConfigReconciler_Reconcile_ExactlyOneControlPlaneMachineInitializes(t *testing.T) {
  1843  	g := NewWithT(t)
  1844  
  1845  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
  1846  	cluster.Status.InfrastructureReady = true
  1847  
  1848  	controlPlaneInitMachineFirst := newControlPlaneMachine(cluster, "control-plane-init-machine-first")
  1849  	controlPlaneInitConfigFirst := newControlPlaneInitKubeadmConfig(controlPlaneInitMachineFirst.Namespace, "control-plane-init-cfg-first")
  1850  	addKubeadmConfigToMachine(controlPlaneInitConfigFirst, controlPlaneInitMachineFirst)
  1851  
  1852  	controlPlaneInitMachineSecond := newControlPlaneMachine(cluster, "control-plane-init-machine-second")
  1853  	controlPlaneInitConfigSecond := newControlPlaneInitKubeadmConfig(controlPlaneInitMachineSecond.Namespace, "control-plane-init-cfg-second")
  1854  	addKubeadmConfigToMachine(controlPlaneInitConfigSecond, controlPlaneInitMachineSecond)
  1855  
  1856  	objects := []client.Object{
  1857  		cluster,
  1858  		controlPlaneInitMachineFirst,
  1859  		controlPlaneInitConfigFirst,
  1860  		controlPlaneInitMachineSecond,
  1861  		controlPlaneInitConfigSecond,
  1862  	}
  1863  	myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
  1864  	k := &KubeadmConfigReconciler{
  1865  		Client:              myclient,
  1866  		SecretCachingClient: myclient,
  1867  		KubeadmInitLock:     &myInitLocker{},
  1868  	}
  1869  
  1870  	request := ctrl.Request{
  1871  		NamespacedName: client.ObjectKey{
  1872  			Namespace: metav1.NamespaceDefault,
  1873  			Name:      "control-plane-init-cfg-first",
  1874  		},
  1875  	}
  1876  	result, err := k.Reconcile(ctx, request)
  1877  	g.Expect(err).ToNot(HaveOccurred())
  1878  	g.Expect(result.Requeue).To(BeFalse())
  1879  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1880  
  1881  	request = ctrl.Request{
  1882  		NamespacedName: client.ObjectKey{
  1883  			Namespace: metav1.NamespaceDefault,
  1884  			Name:      "control-plane-init-cfg-second",
  1885  		},
  1886  	}
  1887  	result, err = k.Reconcile(ctx, request)
  1888  	g.Expect(err).ToNot(HaveOccurred())
  1889  	g.Expect(result.Requeue).To(BeFalse())
  1890  	g.Expect(result.RequeueAfter).To(Equal(30 * time.Second))
  1891  	confList := &bootstrapv1.KubeadmConfigList{}
  1892  	g.Expect(myclient.List(ctx, confList)).To(Succeed())
  1893  	for _, c := range confList.Items {
  1894  		// Ensure the DataSecretName is only set for controlPlaneInitConfigFirst.
  1895  		if c.Name == controlPlaneInitConfigFirst.Name {
  1896  			g.Expect(*c.Status.DataSecretName).To(Not(BeEmpty()))
  1897  		}
  1898  		if c.Name == controlPlaneInitConfigSecond.Name {
  1899  			g.Expect(c.Status.DataSecretName).To(BeNil())
  1900  		}
  1901  	}
  1902  }
  1903  
  1904  // Patch should be applied if there is an error in reconcile.
  1905  func TestKubeadmConfigReconciler_Reconcile_PatchWhenErrorOccurred(t *testing.T) {
  1906  	g := NewWithT(t)
  1907  
  1908  	cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
  1909  	cluster.Status.InfrastructureReady = true
  1910  
  1911  	controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
  1912  	controlPlaneInitConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-cfg")
  1913  	addKubeadmConfigToMachine(controlPlaneInitConfig, controlPlaneInitMachine)
  1914  	// set InitConfiguration as nil, we will check this to determine if the kubeadm config has been patched
  1915  	controlPlaneInitConfig.Spec.InitConfiguration = nil
  1916  
  1917  	objects := []client.Object{
  1918  		cluster,
  1919  		controlPlaneInitMachine,
  1920  		controlPlaneInitConfig,
  1921  	}
  1922  
  1923  	secrets := createSecrets(t, cluster, controlPlaneInitConfig)
  1924  	for _, obj := range secrets {
  1925  		s := obj.(*corev1.Secret)
  1926  		delete(s.Data, secret.TLSCrtDataName) // destroy the secrets, which will cause Reconcile to fail
  1927  		objects = append(objects, s)
  1928  	}
  1929  
  1930  	myclient := fake.NewClientBuilder().WithObjects(objects...).Build()
  1931  	k := &KubeadmConfigReconciler{
  1932  		Client:              myclient,
  1933  		SecretCachingClient: myclient,
  1934  		KubeadmInitLock:     &myInitLocker{},
  1935  	}
  1936  
  1937  	request := ctrl.Request{
  1938  		NamespacedName: client.ObjectKey{
  1939  			Namespace: metav1.NamespaceDefault,
  1940  			Name:      "control-plane-init-cfg",
  1941  		},
  1942  	}
  1943  
  1944  	result, err := k.Reconcile(ctx, request)
  1945  	g.Expect(err).To(HaveOccurred())
  1946  	g.Expect(result.Requeue).To(BeFalse())
  1947  	g.Expect(result.RequeueAfter).To(Equal(time.Duration(0)))
  1948  
  1949  	cfg, err := getKubeadmConfig(myclient, "control-plane-init-cfg", metav1.NamespaceDefault)
  1950  	g.Expect(err).ToNot(HaveOccurred())
  1951  	// check if the kubeadm config has been patched
  1952  	g.Expect(cfg.Spec.InitConfiguration).ToNot(BeNil())
  1953  	g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
  1954  }
  1955  
  1956  func TestKubeadmConfigReconciler_ResolveFiles(t *testing.T) {
  1957  	testSecret := &corev1.Secret{
  1958  		ObjectMeta: metav1.ObjectMeta{
  1959  			Name: "source",
  1960  		},
  1961  		Data: map[string][]byte{
  1962  			"key": []byte("foo"),
  1963  		},
  1964  	}
  1965  
  1966  	cases := map[string]struct {
  1967  		cfg     *bootstrapv1.KubeadmConfig
  1968  		objects []client.Object
  1969  		expect  []bootstrapv1.File
  1970  	}{
  1971  		"content should pass through": {
  1972  			cfg: &bootstrapv1.KubeadmConfig{
  1973  				Spec: bootstrapv1.KubeadmConfigSpec{
  1974  					Files: []bootstrapv1.File{
  1975  						{
  1976  							Content:     "foo",
  1977  							Path:        "/path",
  1978  							Owner:       "root:root",
  1979  							Permissions: "0600",
  1980  						},
  1981  					},
  1982  				},
  1983  			},
  1984  			expect: []bootstrapv1.File{
  1985  				{
  1986  					Content:     "foo",
  1987  					Path:        "/path",
  1988  					Owner:       "root:root",
  1989  					Permissions: "0600",
  1990  				},
  1991  			},
  1992  		},
  1993  		"contentFrom should convert correctly": {
  1994  			cfg: &bootstrapv1.KubeadmConfig{
  1995  				Spec: bootstrapv1.KubeadmConfigSpec{
  1996  					Files: []bootstrapv1.File{
  1997  						{
  1998  							ContentFrom: &bootstrapv1.FileSource{
  1999  								Secret: bootstrapv1.SecretFileSource{
  2000  									Name: "source",
  2001  									Key:  "key",
  2002  								},
  2003  							},
  2004  							Path:        "/path",
  2005  							Owner:       "root:root",
  2006  							Permissions: "0600",
  2007  						},
  2008  					},
  2009  				},
  2010  			},
  2011  			expect: []bootstrapv1.File{
  2012  				{
  2013  					Content:     "foo",
  2014  					Path:        "/path",
  2015  					Owner:       "root:root",
  2016  					Permissions: "0600",
  2017  				},
  2018  			},
  2019  			objects: []client.Object{testSecret},
  2020  		},
  2021  		"multiple files should work correctly": {
  2022  			cfg: &bootstrapv1.KubeadmConfig{
  2023  				Spec: bootstrapv1.KubeadmConfigSpec{
  2024  					Files: []bootstrapv1.File{
  2025  						{
  2026  							Content:     "bar",
  2027  							Path:        "/bar",
  2028  							Owner:       "root:root",
  2029  							Permissions: "0600",
  2030  						},
  2031  						{
  2032  							ContentFrom: &bootstrapv1.FileSource{
  2033  								Secret: bootstrapv1.SecretFileSource{
  2034  									Name: "source",
  2035  									Key:  "key",
  2036  								},
  2037  							},
  2038  							Path:        "/path",
  2039  							Owner:       "root:root",
  2040  							Permissions: "0600",
  2041  						},
  2042  					},
  2043  				},
  2044  			},
  2045  			expect: []bootstrapv1.File{
  2046  				{
  2047  					Content:     "bar",
  2048  					Path:        "/bar",
  2049  					Owner:       "root:root",
  2050  					Permissions: "0600",
  2051  				},
  2052  				{
  2053  					Content:     "foo",
  2054  					Path:        "/path",
  2055  					Owner:       "root:root",
  2056  					Permissions: "0600",
  2057  				},
  2058  			},
  2059  			objects: []client.Object{testSecret},
  2060  		},
  2061  	}
  2062  
  2063  	for name, tc := range cases {
  2064  		t.Run(name, func(t *testing.T) {
  2065  			g := NewWithT(t)
  2066  
  2067  			myclient := fake.NewClientBuilder().WithObjects(tc.objects...).Build()
  2068  			k := &KubeadmConfigReconciler{
  2069  				Client:              myclient,
  2070  				SecretCachingClient: myclient,
  2071  				KubeadmInitLock:     &myInitLocker{},
  2072  			}
  2073  
  2074  			// make a list of files we expect to be sourced from secrets
  2075  			// after we resolve files, assert that the original spec has
  2076  			// not been mutated and all paths we expected to be sourced
  2077  			// from secrets still are.
  2078  			contentFrom := map[string]bool{}
  2079  			for _, file := range tc.cfg.Spec.Files {
  2080  				if file.ContentFrom != nil {
  2081  					contentFrom[file.Path] = true
  2082  				}
  2083  			}
  2084  
  2085  			files, err := k.resolveFiles(ctx, tc.cfg)
  2086  			g.Expect(err).ToNot(HaveOccurred())
  2087  			g.Expect(files).To(BeComparableTo(tc.expect))
  2088  			for _, file := range tc.cfg.Spec.Files {
  2089  				if contentFrom[file.Path] {
  2090  					g.Expect(file.ContentFrom).NotTo(BeNil())
  2091  					g.Expect(file.Content).To(Equal(""))
  2092  				}
  2093  			}
  2094  		})
  2095  	}
  2096  }
  2097  
  2098  func TestKubeadmConfigReconciler_ResolveUsers(t *testing.T) {
  2099  	fakePasswd := "bar"
  2100  	testSecret := &corev1.Secret{
  2101  		ObjectMeta: metav1.ObjectMeta{
  2102  			Name: "source",
  2103  		},
  2104  		Data: map[string][]byte{
  2105  			"key": []byte(fakePasswd),
  2106  		},
  2107  	}
  2108  
  2109  	cases := map[string]struct {
  2110  		cfg     *bootstrapv1.KubeadmConfig
  2111  		objects []client.Object
  2112  		expect  []bootstrapv1.User
  2113  	}{
  2114  		"password should pass through": {
  2115  			cfg: &bootstrapv1.KubeadmConfig{
  2116  				Spec: bootstrapv1.KubeadmConfigSpec{
  2117  					Users: []bootstrapv1.User{
  2118  						{
  2119  							Name:   "foo",
  2120  							Passwd: &fakePasswd,
  2121  						},
  2122  					},
  2123  				},
  2124  			},
  2125  			expect: []bootstrapv1.User{
  2126  				{
  2127  					Name:   "foo",
  2128  					Passwd: &fakePasswd,
  2129  				},
  2130  			},
  2131  		},
  2132  		"passwdFrom should convert correctly": {
  2133  			cfg: &bootstrapv1.KubeadmConfig{
  2134  				Spec: bootstrapv1.KubeadmConfigSpec{
  2135  					Users: []bootstrapv1.User{
  2136  						{
  2137  							Name: "foo",
  2138  							PasswdFrom: &bootstrapv1.PasswdSource{
  2139  								Secret: bootstrapv1.SecretPasswdSource{
  2140  									Name: "source",
  2141  									Key:  "key",
  2142  								},
  2143  							},
  2144  						},
  2145  					},
  2146  				},
  2147  			},
  2148  			expect: []bootstrapv1.User{
  2149  				{
  2150  					Name:   "foo",
  2151  					Passwd: &fakePasswd,
  2152  				},
  2153  			},
  2154  			objects: []client.Object{testSecret},
  2155  		},
  2156  		"multiple users should work correctly": {
  2157  			cfg: &bootstrapv1.KubeadmConfig{
  2158  				Spec: bootstrapv1.KubeadmConfigSpec{
  2159  					Users: []bootstrapv1.User{
  2160  						{
  2161  							Name:   "foo",
  2162  							Passwd: &fakePasswd,
  2163  						},
  2164  						{
  2165  							Name: "bar",
  2166  							PasswdFrom: &bootstrapv1.PasswdSource{
  2167  								Secret: bootstrapv1.SecretPasswdSource{
  2168  									Name: "source",
  2169  									Key:  "key",
  2170  								},
  2171  							},
  2172  						},
  2173  					},
  2174  				},
  2175  			},
  2176  			expect: []bootstrapv1.User{
  2177  				{
  2178  					Name:   "foo",
  2179  					Passwd: &fakePasswd,
  2180  				},
  2181  				{
  2182  					Name:   "bar",
  2183  					Passwd: &fakePasswd,
  2184  				},
  2185  			},
  2186  			objects: []client.Object{testSecret},
  2187  		},
  2188  	}
  2189  
  2190  	for name, tc := range cases {
  2191  		t.Run(name, func(t *testing.T) {
  2192  			g := NewWithT(t)
  2193  
  2194  			myclient := fake.NewClientBuilder().WithObjects(tc.objects...).Build()
  2195  			k := &KubeadmConfigReconciler{
  2196  				Client:              myclient,
  2197  				SecretCachingClient: myclient,
  2198  				KubeadmInitLock:     &myInitLocker{},
  2199  			}
  2200  
  2201  			// make a list of password we expect to be sourced from secrets
  2202  			// after we resolve users, assert that the original spec has
  2203  			// not been mutated and all password we expected to be sourced
  2204  			// from secret still are.
  2205  			passwdFrom := map[string]bool{}
  2206  			for _, user := range tc.cfg.Spec.Users {
  2207  				if user.PasswdFrom != nil {
  2208  					passwdFrom[user.Name] = true
  2209  				}
  2210  			}
  2211  
  2212  			users, err := k.resolveUsers(ctx, tc.cfg)
  2213  			g.Expect(err).ToNot(HaveOccurred())
  2214  			g.Expect(users).To(BeComparableTo(tc.expect))
  2215  			for _, user := range tc.cfg.Spec.Users {
  2216  				if passwdFrom[user.Name] {
  2217  					g.Expect(user.PasswdFrom).NotTo(BeNil())
  2218  					g.Expect(user.Passwd).To(BeNil())
  2219  				}
  2220  			}
  2221  		})
  2222  	}
  2223  }
  2224  
  2225  // test utils.
  2226  
  2227  // newWorkerMachineForCluster returns a Machine with the passed Cluster's information and a pre-configured name.
  2228  func newWorkerMachineForCluster(cluster *clusterv1.Cluster) *clusterv1.Machine {
  2229  	return builder.Machine(cluster.Namespace, "worker-machine").
  2230  		WithVersion("v1.19.1").
  2231  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(cluster.Namespace, "conf1").Unstructured()).
  2232  		WithClusterName(cluster.Name).
  2233  		Build()
  2234  }
  2235  
  2236  // newControlPlaneMachine returns a Machine with the passed Cluster information and a MachineControlPlaneLabel.
  2237  func newControlPlaneMachine(cluster *clusterv1.Cluster, name string) *clusterv1.Machine {
  2238  	m := builder.Machine(cluster.Namespace, name).
  2239  		WithVersion("v1.19.1").
  2240  		WithBootstrapTemplate(bootstrapbuilder.KubeadmConfig(metav1.NamespaceDefault, "cfg").Unstructured()).
  2241  		WithClusterName(cluster.Name).
  2242  		WithLabels(map[string]string{clusterv1.MachineControlPlaneLabel: ""}).
  2243  		Build()
  2244  	return m
  2245  }
  2246  
  2247  // newMachinePool return a MachinePool object with the passed Cluster information and a basic bootstrap template.
  2248  func newMachinePool(cluster *clusterv1.Cluster, name string) *expv1.MachinePool {
  2249  	m := builder.MachinePool(cluster.Namespace, name).
  2250  		WithClusterName(cluster.Name).
  2251  		WithLabels(map[string]string{clusterv1.ClusterNameLabel: cluster.Name}).
  2252  		WithBootstrap(bootstrapbuilder.KubeadmConfig(cluster.Namespace, "conf1").Unstructured()).
  2253  		WithVersion("1.19.1").
  2254  		Build()
  2255  	return m
  2256  }
  2257  
  2258  // newWorkerMachinePoolForCluster returns a MachinePool with the passed Cluster's information and a pre-configured name.
  2259  func newWorkerMachinePoolForCluster(cluster *clusterv1.Cluster) *expv1.MachinePool {
  2260  	return newMachinePool(cluster, "worker-machinepool")
  2261  }
  2262  
  2263  // newKubeadmConfig return a CABPK KubeadmConfig object.
  2264  func newKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig {
  2265  	return bootstrapbuilder.KubeadmConfig(namespace, name).
  2266  		Build()
  2267  }
  2268  
  2269  // newKubeadmConfig return a CABPK KubeadmConfig object with a worker JoinConfiguration.
  2270  func newWorkerJoinKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig {
  2271  	return bootstrapbuilder.KubeadmConfig(namespace, name).
  2272  		WithJoinConfig(&bootstrapv1.JoinConfiguration{
  2273  			ControlPlane: nil,
  2274  		}).
  2275  		Build()
  2276  }
  2277  
  2278  // newKubeadmConfig returns a CABPK KubeadmConfig object with a ControlPlane JoinConfiguration.
  2279  func newControlPlaneJoinKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig {
  2280  	return bootstrapbuilder.KubeadmConfig(namespace, name).
  2281  		WithJoinConfig(&bootstrapv1.JoinConfiguration{
  2282  			ControlPlane: &bootstrapv1.JoinControlPlane{},
  2283  		}).
  2284  		Build()
  2285  }
  2286  
  2287  // newControlPlaneJoinConfig returns a CABPK KubeadmConfig object with a ControlPlane InitConfiguration and ClusterConfiguration.
  2288  func newControlPlaneInitKubeadmConfig(namespace, name string) *bootstrapv1.KubeadmConfig {
  2289  	return bootstrapbuilder.KubeadmConfig(namespace, name).
  2290  		WithInitConfig(&bootstrapv1.InitConfiguration{}).
  2291  		WithClusterConfig(&bootstrapv1.ClusterConfiguration{}).
  2292  		Build()
  2293  }
  2294  
  2295  // addKubeadmConfigToMachine adds the config details to the passed Machine, and adds the Machine to the KubeadmConfig as an ownerReference.
  2296  func addKubeadmConfigToMachine(config *bootstrapv1.KubeadmConfig, machine *clusterv1.Machine) {
  2297  	if machine == nil {
  2298  		panic("no machine passed to function")
  2299  	}
  2300  	config.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
  2301  		{
  2302  			Kind:       "Machine",
  2303  			APIVersion: clusterv1.GroupVersion.String(),
  2304  			Name:       machine.Name,
  2305  			UID:        types.UID(fmt.Sprintf("%s uid", machine.Name)),
  2306  		},
  2307  	}
  2308  
  2309  	if machine.Spec.Bootstrap.ConfigRef == nil {
  2310  		machine.Spec.Bootstrap.ConfigRef = &corev1.ObjectReference{}
  2311  	}
  2312  
  2313  	machine.Spec.Bootstrap.ConfigRef.Name = config.Name
  2314  	machine.Spec.Bootstrap.ConfigRef.Namespace = config.Namespace
  2315  }
  2316  
  2317  // addKubeadmConfigToMachine adds the config details to the passed MachinePool and adds the Machine to the KubeadmConfig as an ownerReference.
  2318  func addKubeadmConfigToMachinePool(config *bootstrapv1.KubeadmConfig, machinePool *expv1.MachinePool) {
  2319  	if machinePool == nil {
  2320  		panic("no machinePool passed to function")
  2321  	}
  2322  	config.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
  2323  		{
  2324  			Kind:       "MachinePool",
  2325  			APIVersion: expv1.GroupVersion.String(),
  2326  			Name:       machinePool.Name,
  2327  			UID:        types.UID(fmt.Sprintf("%s uid", machinePool.Name)),
  2328  		},
  2329  	}
  2330  	machinePool.Spec.Template.Spec.Bootstrap.ConfigRef.Name = config.Name
  2331  	machinePool.Spec.Template.Spec.Bootstrap.ConfigRef.Namespace = config.Namespace
  2332  }
  2333  
  2334  func createSecrets(t *testing.T, cluster *clusterv1.Cluster, config *bootstrapv1.KubeadmConfig) []client.Object {
  2335  	t.Helper()
  2336  
  2337  	out := []client.Object{}
  2338  	if config.Spec.ClusterConfiguration == nil {
  2339  		config.Spec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{}
  2340  	}
  2341  	certificates := secret.NewCertificatesForInitialControlPlane(config.Spec.ClusterConfiguration)
  2342  	if err := certificates.Generate(); err != nil {
  2343  		t.Fatal(err)
  2344  	}
  2345  	for _, certificate := range certificates {
  2346  		out = append(out, certificate.AsSecret(util.ObjectKey(cluster), *metav1.NewControllerRef(config, bootstrapv1.GroupVersion.WithKind("KubeadmConfig"))))
  2347  	}
  2348  	return out
  2349  }
  2350  
  2351  type myInitLocker struct {
  2352  	locked bool
  2353  }
  2354  
  2355  func (m *myInitLocker) Lock(_ context.Context, _ *clusterv1.Cluster, _ *clusterv1.Machine) bool {
  2356  	if !m.locked {
  2357  		m.locked = true
  2358  		return true
  2359  	}
  2360  	return false
  2361  }
  2362  
  2363  func (m *myInitLocker) Unlock(_ context.Context, _ *clusterv1.Cluster) bool {
  2364  	if m.locked {
  2365  		m.locked = false
  2366  	}
  2367  	return true
  2368  }
  2369  
  2370  func assertHasFalseCondition(g *WithT, myclient client.Client, req ctrl.Request, t clusterv1.ConditionType, s clusterv1.ConditionSeverity, r string) {
  2371  	config := &bootstrapv1.KubeadmConfig{
  2372  		ObjectMeta: metav1.ObjectMeta{
  2373  			Name:      req.Name,
  2374  			Namespace: req.Namespace,
  2375  		},
  2376  	}
  2377  
  2378  	configKey := client.ObjectKeyFromObject(config)
  2379  	g.Expect(myclient.Get(ctx, configKey, config)).To(Succeed())
  2380  	c := conditions.Get(config, t)
  2381  	g.Expect(c).ToNot(BeNil())
  2382  	g.Expect(c.Status).To(Equal(corev1.ConditionFalse))
  2383  	g.Expect(c.Severity).To(Equal(s))
  2384  	g.Expect(c.Reason).To(Equal(r))
  2385  }
  2386  
  2387  func assertHasTrueCondition(g *WithT, myclient client.Client, req ctrl.Request, t clusterv1.ConditionType) {
  2388  	config := &bootstrapv1.KubeadmConfig{
  2389  		ObjectMeta: metav1.ObjectMeta{
  2390  			Name:      req.Name,
  2391  			Namespace: req.Namespace,
  2392  		},
  2393  	}
  2394  	configKey := client.ObjectKeyFromObject(config)
  2395  	g.Expect(myclient.Get(ctx, configKey, config)).To(Succeed())
  2396  	c := conditions.Get(config, t)
  2397  	g.Expect(c).ToNot(BeNil())
  2398  	g.Expect(c.Status).To(Equal(corev1.ConditionTrue))
  2399  }