sigs.k8s.io/cluster-api@v1.6.3/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go (about)

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controllers
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strconv"
    23  	"time"
    24  
    25  	"github.com/blang/semver/v4"
    26  	"github.com/go-logr/logr"
    27  	"github.com/pkg/errors"
    28  	corev1 "k8s.io/api/core/v1"
    29  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/runtime"
    32  	"k8s.io/apimachinery/pkg/types"
    33  	kerrors "k8s.io/apimachinery/pkg/util/errors"
    34  	"k8s.io/klog/v2"
    35  	"k8s.io/utils/pointer"
    36  	ctrl "sigs.k8s.io/controller-runtime"
    37  	"sigs.k8s.io/controller-runtime/pkg/builder"
    38  	"sigs.k8s.io/controller-runtime/pkg/client"
    39  	"sigs.k8s.io/controller-runtime/pkg/controller"
    40  	"sigs.k8s.io/controller-runtime/pkg/handler"
    41  
    42  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    43  	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
    44  	"sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit"
    45  	"sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition"
    46  	"sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/locking"
    47  	kubeadmtypes "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types"
    48  	bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
    49  	"sigs.k8s.io/cluster-api/controllers/remote"
    50  	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
    51  	"sigs.k8s.io/cluster-api/feature"
    52  	"sigs.k8s.io/cluster-api/internal/util/taints"
    53  	"sigs.k8s.io/cluster-api/util"
    54  	"sigs.k8s.io/cluster-api/util/annotations"
    55  	"sigs.k8s.io/cluster-api/util/conditions"
    56  	clog "sigs.k8s.io/cluster-api/util/log"
    57  	"sigs.k8s.io/cluster-api/util/patch"
    58  	"sigs.k8s.io/cluster-api/util/predicates"
    59  	"sigs.k8s.io/cluster-api/util/secret"
    60  )
    61  
    62  const (
    63  	// DefaultTokenTTL is the default TTL used for tokens.
    64  	DefaultTokenTTL = 15 * time.Minute
    65  )
    66  
    67  // InitLocker is a lock that is used around kubeadm init.
    68  type InitLocker interface {
    69  	Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool
    70  	Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool
    71  }
    72  
    73  // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kubeadmconfigs;kubeadmconfigs/status;kubeadmconfigs/finalizers,verbs=get;list;watch;create;update;patch;delete
    74  // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machinesets;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch
    75  // +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete
    76  
    77  // KubeadmConfigReconciler reconciles a KubeadmConfig object.
    78  type KubeadmConfigReconciler struct {
    79  	Client              client.Client
    80  	SecretCachingClient client.Client
    81  	Tracker             *remote.ClusterCacheTracker
    82  	KubeadmInitLock     InitLocker
    83  
    84  	// WatchFilterValue is the label value used to filter events prior to reconciliation.
    85  	WatchFilterValue string
    86  
    87  	// TokenTTL is the amount of time a bootstrap token (and therefore a KubeadmConfig) will be valid.
    88  	TokenTTL time.Duration
    89  }
    90  
    91  // Scope is a scoped struct used during reconciliation.
    92  type Scope struct {
    93  	logr.Logger
    94  	Config      *bootstrapv1.KubeadmConfig
    95  	ConfigOwner *bsutil.ConfigOwner
    96  	Cluster     *clusterv1.Cluster
    97  }
    98  
    99  // SetupWithManager sets up the reconciler with the Manager.
   100  func (r *KubeadmConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
   101  	if r.KubeadmInitLock == nil {
   102  		r.KubeadmInitLock = locking.NewControlPlaneInitMutex(mgr.GetClient())
   103  	}
   104  	if r.TokenTTL == 0 {
   105  		r.TokenTTL = DefaultTokenTTL
   106  	}
   107  
   108  	b := ctrl.NewControllerManagedBy(mgr).
   109  		For(&bootstrapv1.KubeadmConfig{}).
   110  		WithOptions(options).
   111  		Watches(
   112  			&clusterv1.Machine{},
   113  			handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc),
   114  		).WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue))
   115  
   116  	if feature.Gates.Enabled(feature.MachinePool) {
   117  		b = b.Watches(
   118  			&expv1.MachinePool{},
   119  			handler.EnqueueRequestsFromMapFunc(r.MachinePoolToBootstrapMapFunc),
   120  		)
   121  	}
   122  
   123  	b = b.Watches(
   124  		&clusterv1.Cluster{},
   125  		handler.EnqueueRequestsFromMapFunc(r.ClusterToKubeadmConfigs),
   126  		builder.WithPredicates(
   127  			predicates.All(ctrl.LoggerFrom(ctx),
   128  				predicates.ClusterUnpausedAndInfrastructureReady(ctrl.LoggerFrom(ctx)),
   129  				predicates.ResourceHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue),
   130  			),
   131  		),
   132  	)
   133  
   134  	if err := b.Complete(r); err != nil {
   135  		return errors.Wrap(err, "failed setting up with a controller manager")
   136  	}
   137  	return nil
   138  }
   139  
   140  // Reconcile handles KubeadmConfig events.
   141  func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, rerr error) {
   142  	log := ctrl.LoggerFrom(ctx)
   143  
   144  	// Look up the kubeadm config
   145  	config := &bootstrapv1.KubeadmConfig{}
   146  	if err := r.Client.Get(ctx, req.NamespacedName, config); err != nil {
   147  		if apierrors.IsNotFound(err) {
   148  			return ctrl.Result{}, nil
   149  		}
   150  		return ctrl.Result{}, err
   151  	}
   152  
   153  	// Look up the owner of this kubeadm config if there is one
   154  	configOwner, err := bsutil.GetTypedConfigOwner(ctx, r.Client, config)
   155  	if apierrors.IsNotFound(err) {
   156  		// Could not find the owner yet, this is not an error and will rereconcile when the owner gets set.
   157  		return ctrl.Result{}, nil
   158  	}
   159  	if err != nil {
   160  		return ctrl.Result{}, errors.Wrapf(err, "failed to get owner")
   161  	}
   162  	if configOwner == nil {
   163  		return ctrl.Result{}, nil
   164  	}
   165  	log = log.WithValues(configOwner.GetKind(), klog.KRef(configOwner.GetNamespace(), configOwner.GetName()), "resourceVersion", configOwner.GetResourceVersion())
   166  	ctx = ctrl.LoggerInto(ctx, log)
   167  
   168  	if configOwner.GetKind() == "Machine" {
   169  		// AddOwners adds the owners of Machine as k/v pairs to the logger.
   170  		// Specifically, it will add KubeadmControlPlane, MachineSet and MachineDeployment.
   171  		ctx, log, err = clog.AddOwners(ctx, r.Client, configOwner)
   172  		if err != nil {
   173  			return ctrl.Result{}, err
   174  		}
   175  	}
   176  
   177  	log = log.WithValues("Cluster", klog.KRef(configOwner.GetNamespace(), configOwner.ClusterName()))
   178  	ctx = ctrl.LoggerInto(ctx, log)
   179  
   180  	// Lookup the cluster the config owner is associated with
   181  	cluster, err := util.GetClusterByName(ctx, r.Client, configOwner.GetNamespace(), configOwner.ClusterName())
   182  	if err != nil {
   183  		if errors.Cause(err) == util.ErrNoCluster {
   184  			log.Info(fmt.Sprintf("%s does not belong to a cluster yet, waiting until it's part of a cluster", configOwner.GetKind()))
   185  			return ctrl.Result{}, nil
   186  		}
   187  
   188  		if apierrors.IsNotFound(err) {
   189  			log.Info("Cluster does not exist yet, waiting until it is created")
   190  			return ctrl.Result{}, nil
   191  		}
   192  		log.Error(err, "Could not get cluster with metadata")
   193  		return ctrl.Result{}, err
   194  	}
   195  
   196  	if annotations.IsPaused(cluster, config) {
   197  		log.Info("Reconciliation is paused for this object")
   198  		return ctrl.Result{}, nil
   199  	}
   200  
   201  	scope := &Scope{
   202  		Logger:      log,
   203  		Config:      config,
   204  		ConfigOwner: configOwner,
   205  		Cluster:     cluster,
   206  	}
   207  
   208  	// Initialize the patch helper.
   209  	patchHelper, err := patch.NewHelper(config, r.Client)
   210  	if err != nil {
   211  		return ctrl.Result{}, err
   212  	}
   213  
   214  	// Attempt to Patch the KubeadmConfig object and status after each reconciliation if no error occurs.
   215  	defer func() {
   216  		// always update the readyCondition; the summary is represented using the "1 of x completed" notation.
   217  		conditions.SetSummary(config,
   218  			conditions.WithConditions(
   219  				bootstrapv1.DataSecretAvailableCondition,
   220  				bootstrapv1.CertificatesAvailableCondition,
   221  			),
   222  		)
   223  		// Patch ObservedGeneration only if the reconciliation completed successfully
   224  		patchOpts := []patch.Option{}
   225  		if rerr == nil {
   226  			patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
   227  		}
   228  		if err := patchHelper.Patch(ctx, config, patchOpts...); err != nil {
   229  			rerr = kerrors.NewAggregate([]error{rerr, errors.Wrapf(err, "failed to patch %s", klog.KObj(config))})
   230  		}
   231  	}()
   232  
   233  	// Ignore deleted KubeadmConfigs.
   234  	if !config.DeletionTimestamp.IsZero() {
   235  		return ctrl.Result{}, nil
   236  	}
   237  
   238  	res, err := r.reconcile(ctx, scope, cluster, config, configOwner)
   239  	if err != nil && errors.Is(err, remote.ErrClusterLocked) {
   240  		// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
   241  		// the current cluster because of concurrent access.
   242  		log.V(5).Info("Requeuing because another worker has the lock on the ClusterCacheTracker")
   243  		return ctrl.Result{Requeue: true}, nil
   244  	}
   245  	return res, err
   246  }
   247  
   248  func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, cluster *clusterv1.Cluster, config *bootstrapv1.KubeadmConfig, configOwner *bsutil.ConfigOwner) (ctrl.Result, error) {
   249  	log := ctrl.LoggerFrom(ctx)
   250  
   251  	// Ensure the bootstrap secret associated with this KubeadmConfig has the correct ownerReference.
   252  	if err := r.ensureBootstrapSecretOwnersRef(ctx, scope); err != nil {
   253  		return ctrl.Result{}, err
   254  	}
   255  	switch {
   256  	// Wait for the infrastructure to be ready.
   257  	case !cluster.Status.InfrastructureReady:
   258  		log.Info("Cluster infrastructure is not ready, waiting")
   259  		conditions.MarkFalse(config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.WaitingForClusterInfrastructureReason, clusterv1.ConditionSeverityInfo, "")
   260  		return ctrl.Result{}, nil
   261  	// Reconcile status for machines that already have a secret reference, but our status isn't up to date.
   262  	// This case solves the pivoting scenario (or a backup restore) which doesn't preserve the status subresource on objects.
   263  	case configOwner.DataSecretName() != nil && (!config.Status.Ready || config.Status.DataSecretName == nil):
   264  		config.Status.Ready = true
   265  		config.Status.DataSecretName = configOwner.DataSecretName()
   266  		conditions.MarkTrue(config, bootstrapv1.DataSecretAvailableCondition)
   267  		return ctrl.Result{}, nil
   268  	// Status is ready means a config has been generated.
   269  	case config.Status.Ready:
   270  		if config.Spec.JoinConfiguration != nil && config.Spec.JoinConfiguration.Discovery.BootstrapToken != nil {
   271  			if !configOwner.HasNodeRefs() {
   272  				// If the BootstrapToken has been generated for a join but the config owner has no nodeRefs,
   273  				// this indicates that the node has not yet joined and the token in the join config has not
   274  				// been consumed and it may need a refresh.
   275  				return r.refreshBootstrapToken(ctx, config, cluster)
   276  			}
   277  			if configOwner.IsMachinePool() {
   278  				// If the BootstrapToken has been generated and infrastructure is ready but the configOwner is a MachinePool,
   279  				// we rotate the token to keep it fresh for future scale ups.
   280  				return r.rotateMachinePoolBootstrapToken(ctx, config, cluster, scope)
   281  			}
   282  		}
   283  		// In any other case just return as the config is already generated and need not be generated again.
   284  		return ctrl.Result{}, nil
   285  	}
   286  
   287  	// Note: can't use IsFalse here because we need to handle the absence of the condition as well as false.
   288  	if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) {
   289  		return r.handleClusterNotInitialized(ctx, scope)
   290  	}
   291  
   292  	// Every other case it's a join scenario
   293  	// Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK simply ignore them
   294  
   295  	// Unlock any locks that might have been set during init process
   296  	r.KubeadmInitLock.Unlock(ctx, cluster)
   297  
   298  	// if the JoinConfiguration is missing, create a default one
   299  	if config.Spec.JoinConfiguration == nil {
   300  		log.Info("Creating default JoinConfiguration")
   301  		config.Spec.JoinConfiguration = &bootstrapv1.JoinConfiguration{}
   302  	}
   303  
   304  	// it's a control plane join
   305  	if configOwner.IsControlPlaneMachine() {
   306  		return r.joinControlplane(ctx, scope)
   307  	}
   308  
   309  	// It's a worker join
   310  	return r.joinWorker(ctx, scope)
   311  }
   312  
   313  func (r *KubeadmConfigReconciler) refreshBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster) (ctrl.Result, error) {
   314  	log := ctrl.LoggerFrom(ctx)
   315  	token := config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
   316  
   317  	remoteClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
   318  	if err != nil {
   319  		return ctrl.Result{}, err
   320  	}
   321  
   322  	log.Info("Refreshing token until the infrastructure has a chance to consume it")
   323  	if err := refreshToken(ctx, remoteClient, token, r.TokenTTL); err != nil {
   324  		return ctrl.Result{}, errors.Wrapf(err, "failed to refresh bootstrap token")
   325  	}
   326  	return ctrl.Result{
   327  		RequeueAfter: r.TokenTTL / 2,
   328  	}, nil
   329  }
   330  
   331  func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
   332  	log := ctrl.LoggerFrom(ctx)
   333  	log.V(2).Info("Config is owned by a MachinePool, checking if token should be rotated")
   334  	remoteClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
   335  	if err != nil {
   336  		return ctrl.Result{}, err
   337  	}
   338  
   339  	token := config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
   340  	shouldRotate, err := shouldRotate(ctx, remoteClient, token, r.TokenTTL)
   341  	if err != nil {
   342  		return ctrl.Result{}, err
   343  	}
   344  	if shouldRotate {
   345  		log.Info("Creating new bootstrap token, the existing one should be rotated")
   346  		token, err := createToken(ctx, remoteClient, r.TokenTTL)
   347  		if err != nil {
   348  			return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
   349  		}
   350  
   351  		config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
   352  		log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
   353  
   354  		// update the bootstrap data
   355  		return r.joinWorker(ctx, scope)
   356  	}
   357  	return ctrl.Result{
   358  		RequeueAfter: r.TokenTTL / 3,
   359  	}, nil
   360  }
   361  
   362  func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) {
   363  	// initialize the DataSecretAvailableCondition if missing.
   364  	// this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
   365  	// using the DataSecretGeneratedFailedReason
   366  	if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason {
   367  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "")
   368  	}
   369  
   370  	// if it's NOT a control plane machine, requeue
   371  	if !scope.ConfigOwner.IsControlPlaneMachine() {
   372  		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
   373  	}
   374  
   375  	// if the machine has not ClusterConfiguration and InitConfiguration, requeue
   376  	if scope.Config.Spec.InitConfiguration == nil && scope.Config.Spec.ClusterConfiguration == nil {
   377  		scope.Info("Control plane is not ready, requeuing joining control planes until ready.")
   378  		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
   379  	}
   380  
   381  	machine := &clusterv1.Machine{}
   382  	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil {
   383  		return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind())
   384  	}
   385  
   386  	// acquire the init lock so that only the first machine configured
   387  	// as control plane get processed here
   388  	// if not the first, requeue
   389  	if !r.KubeadmInitLock.Lock(ctx, scope.Cluster, machine) {
   390  		scope.Info("A control plane is already being initialized, requeuing until control plane is ready")
   391  		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
   392  	}
   393  
   394  	defer func() {
   395  		if reterr != nil {
   396  			if !r.KubeadmInitLock.Unlock(ctx, scope.Cluster) {
   397  				reterr = kerrors.NewAggregate([]error{reterr, errors.New("failed to unlock the kubeadm init lock")})
   398  			}
   399  		}
   400  	}()
   401  
   402  	scope.Info("Creating BootstrapData for the first control plane")
   403  
   404  	// Nb. in this case JoinConfiguration should not be defined by users, but in case of misconfigurations, CABPK simply ignore it
   405  
   406  	// get both of ClusterConfiguration and InitConfiguration strings to pass to the cloud init control plane generator
   407  	// kubeadm allows one of these values to be empty; CABPK replace missing values with an empty config, so the cloud init generation
   408  	// should not handle special cases.
   409  
   410  	kubernetesVersion := scope.ConfigOwner.KubernetesVersion()
   411  	parsedVersion, err := semver.ParseTolerant(kubernetesVersion)
   412  	if err != nil {
   413  		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion)
   414  	}
   415  
   416  	if scope.Config.Spec.InitConfiguration == nil {
   417  		scope.Config.Spec.InitConfiguration = &bootstrapv1.InitConfiguration{
   418  			TypeMeta: metav1.TypeMeta{
   419  				APIVersion: "kubeadm.k8s.io/v1beta1",
   420  				Kind:       "InitConfiguration",
   421  			},
   422  		}
   423  	}
   424  
   425  	initdata, err := kubeadmtypes.MarshalInitConfigurationForVersion(scope.Config.Spec.InitConfiguration, parsedVersion)
   426  	if err != nil {
   427  		scope.Error(err, "Failed to marshal init configuration")
   428  		return ctrl.Result{}, err
   429  	}
   430  
   431  	if scope.Config.Spec.ClusterConfiguration == nil {
   432  		scope.Config.Spec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{
   433  			TypeMeta: metav1.TypeMeta{
   434  				APIVersion: "kubeadm.k8s.io/v1beta1",
   435  				Kind:       "ClusterConfiguration",
   436  			},
   437  		}
   438  	}
   439  
   440  	// injects into config.ClusterConfiguration values from top level object
   441  	r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config)
   442  
   443  	clusterdata, err := kubeadmtypes.MarshalClusterConfigurationForVersion(scope.Config.Spec.ClusterConfiguration, parsedVersion)
   444  	if err != nil {
   445  		scope.Error(err, "Failed to marshal cluster configuration")
   446  		return ctrl.Result{}, err
   447  	}
   448  
   449  	certificates := secret.NewCertificatesForInitialControlPlane(scope.Config.Spec.ClusterConfiguration)
   450  
   451  	// If the Cluster does not have a ControlPlane reference look up and generate the certificates.
   452  	// Otherwise rely on certificates generated by the ControlPlane controller.
   453  	// Note: A cluster does not have a ControlPlane reference when using standalone CP machines.
   454  	if scope.Cluster.Spec.ControlPlaneRef == nil {
   455  		err = certificates.LookupOrGenerateCached(
   456  			ctx,
   457  			r.SecretCachingClient,
   458  			r.Client,
   459  			util.ObjectKey(scope.Cluster),
   460  			*metav1.NewControllerRef(scope.Config, bootstrapv1.GroupVersion.WithKind("KubeadmConfig")))
   461  	} else {
   462  		err = certificates.LookupCached(ctx,
   463  			r.SecretCachingClient,
   464  			r.Client,
   465  			util.ObjectKey(scope.Cluster))
   466  	}
   467  	if err != nil {
   468  		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   469  		return ctrl.Result{}, err
   470  	}
   471  
   472  	conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition)
   473  
   474  	verbosityFlag := ""
   475  	if scope.Config.Spec.Verbosity != nil {
   476  		verbosityFlag = fmt.Sprintf("--v %s", strconv.Itoa(int(*scope.Config.Spec.Verbosity)))
   477  	}
   478  
   479  	files, err := r.resolveFiles(ctx, scope.Config)
   480  	if err != nil {
   481  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   482  		return ctrl.Result{}, err
   483  	}
   484  
   485  	users, err := r.resolveUsers(ctx, scope.Config)
   486  	if err != nil {
   487  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   488  		return ctrl.Result{}, err
   489  	}
   490  
   491  	controlPlaneInput := &cloudinit.ControlPlaneInput{
   492  		BaseUserData: cloudinit.BaseUserData{
   493  			AdditionalFiles:     files,
   494  			NTP:                 scope.Config.Spec.NTP,
   495  			PreKubeadmCommands:  scope.Config.Spec.PreKubeadmCommands,
   496  			PostKubeadmCommands: scope.Config.Spec.PostKubeadmCommands,
   497  			Users:               users,
   498  			Mounts:              scope.Config.Spec.Mounts,
   499  			DiskSetup:           scope.Config.Spec.DiskSetup,
   500  			KubeadmVerbosity:    verbosityFlag,
   501  		},
   502  		InitConfiguration:    initdata,
   503  		ClusterConfiguration: clusterdata,
   504  		Certificates:         certificates,
   505  	}
   506  
   507  	var bootstrapInitData []byte
   508  	switch scope.Config.Spec.Format {
   509  	case bootstrapv1.Ignition:
   510  		bootstrapInitData, _, err = ignition.NewInitControlPlane(&ignition.ControlPlaneInput{
   511  			ControlPlaneInput: controlPlaneInput,
   512  			Ignition:          scope.Config.Spec.Ignition,
   513  		})
   514  	default:
   515  		bootstrapInitData, err = cloudinit.NewInitControlPlane(controlPlaneInput)
   516  	}
   517  
   518  	if err != nil {
   519  		scope.Error(err, "Failed to generate user data for bootstrap control plane")
   520  		return ctrl.Result{}, err
   521  	}
   522  
   523  	if err := r.storeBootstrapData(ctx, scope, bootstrapInitData); err != nil {
   524  		scope.Error(err, "Failed to store bootstrap data")
   525  		return ctrl.Result{}, err
   526  	}
   527  
   528  	return ctrl.Result{}, nil
   529  }
   530  
   531  func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) {
   532  	scope.Info("Creating BootstrapData for the worker node")
   533  
   534  	certificates := secret.NewCertificatesForWorker(scope.Config.Spec.JoinConfiguration.CACertPath)
   535  	err := certificates.LookupCached(
   536  		ctx,
   537  		r.SecretCachingClient,
   538  		r.Client,
   539  		util.ObjectKey(scope.Cluster),
   540  	)
   541  	if err != nil {
   542  		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesCorruptedReason, clusterv1.ConditionSeverityError, err.Error())
   543  		return ctrl.Result{}, err
   544  	}
   545  	if err := certificates.EnsureAllExist(); err != nil {
   546  		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesCorruptedReason, clusterv1.ConditionSeverityError, err.Error())
   547  		return ctrl.Result{}, err
   548  	}
   549  	conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition)
   550  
   551  	// Ensure that joinConfiguration.Discovery is properly set for joining node on the current cluster.
   552  	if res, err := r.reconcileDiscovery(ctx, scope.Cluster, scope.Config, certificates); err != nil {
   553  		return ctrl.Result{}, err
   554  	} else if !res.IsZero() {
   555  		return res, nil
   556  	}
   557  
   558  	kubernetesVersion := scope.ConfigOwner.KubernetesVersion()
   559  	parsedVersion, err := semver.ParseTolerant(kubernetesVersion)
   560  	if err != nil {
   561  		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion)
   562  	}
   563  
   564  	// Add the node uninitialized taint to the list of taints.
   565  	// DeepCopy the JoinConfiguration to prevent updating the actual KubeadmConfig.
   566  	// Do not modify the KubeadmConfig in etcd as this is a temporary taint that will be dropped after the node
   567  	// is initialized by ClusterAPI.
   568  	joinConfiguration := scope.Config.Spec.JoinConfiguration.DeepCopy()
   569  	if !taints.HasTaint(joinConfiguration.NodeRegistration.Taints, clusterv1.NodeUninitializedTaint) {
   570  		joinConfiguration.NodeRegistration.Taints = append(joinConfiguration.NodeRegistration.Taints, clusterv1.NodeUninitializedTaint)
   571  	}
   572  
   573  	joinData, err := kubeadmtypes.MarshalJoinConfigurationForVersion(joinConfiguration, parsedVersion)
   574  	if err != nil {
   575  		scope.Error(err, "Failed to marshal join configuration")
   576  		return ctrl.Result{}, err
   577  	}
   578  
   579  	if scope.Config.Spec.JoinConfiguration.ControlPlane != nil {
   580  		return ctrl.Result{}, errors.New("Machine is a Worker, but JoinConfiguration.ControlPlane is set in the KubeadmConfig object")
   581  	}
   582  
   583  	verbosityFlag := ""
   584  	if scope.Config.Spec.Verbosity != nil {
   585  		verbosityFlag = fmt.Sprintf("--v %s", strconv.Itoa(int(*scope.Config.Spec.Verbosity)))
   586  	}
   587  
   588  	files, err := r.resolveFiles(ctx, scope.Config)
   589  	if err != nil {
   590  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   591  		return ctrl.Result{}, err
   592  	}
   593  
   594  	users, err := r.resolveUsers(ctx, scope.Config)
   595  	if err != nil {
   596  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   597  		return ctrl.Result{}, err
   598  	}
   599  
   600  	nodeInput := &cloudinit.NodeInput{
   601  		BaseUserData: cloudinit.BaseUserData{
   602  			AdditionalFiles:      files,
   603  			NTP:                  scope.Config.Spec.NTP,
   604  			PreKubeadmCommands:   scope.Config.Spec.PreKubeadmCommands,
   605  			PostKubeadmCommands:  scope.Config.Spec.PostKubeadmCommands,
   606  			Users:                users,
   607  			Mounts:               scope.Config.Spec.Mounts,
   608  			DiskSetup:            scope.Config.Spec.DiskSetup,
   609  			KubeadmVerbosity:     verbosityFlag,
   610  			UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin,
   611  		},
   612  		JoinConfiguration: joinData,
   613  	}
   614  
   615  	var bootstrapJoinData []byte
   616  	switch scope.Config.Spec.Format {
   617  	case bootstrapv1.Ignition:
   618  		bootstrapJoinData, _, err = ignition.NewNode(&ignition.NodeInput{
   619  			NodeInput: nodeInput,
   620  			Ignition:  scope.Config.Spec.Ignition,
   621  		})
   622  	default:
   623  		bootstrapJoinData, err = cloudinit.NewNode(nodeInput)
   624  	}
   625  
   626  	if err != nil {
   627  		scope.Error(err, "Failed to create a worker join configuration")
   628  		return ctrl.Result{}, err
   629  	}
   630  
   631  	if err := r.storeBootstrapData(ctx, scope, bootstrapJoinData); err != nil {
   632  		scope.Error(err, "Failed to store bootstrap data")
   633  		return ctrl.Result{}, err
   634  	}
   635  	return ctrl.Result{}, nil
   636  }
   637  
   638  func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *Scope) (ctrl.Result, error) {
   639  	scope.Info("Creating BootstrapData for the joining control plane")
   640  
   641  	if !scope.ConfigOwner.IsControlPlaneMachine() {
   642  		return ctrl.Result{}, fmt.Errorf("%s is not a valid control plane kind, only Machine is supported", scope.ConfigOwner.GetKind())
   643  	}
   644  
   645  	if scope.Config.Spec.JoinConfiguration.ControlPlane == nil {
   646  		scope.Config.Spec.JoinConfiguration.ControlPlane = &bootstrapv1.JoinControlPlane{}
   647  	}
   648  
   649  	certificates := secret.NewControlPlaneJoinCerts(scope.Config.Spec.ClusterConfiguration)
   650  	err := certificates.LookupCached(
   651  		ctx,
   652  		r.SecretCachingClient,
   653  		r.Client,
   654  		util.ObjectKey(scope.Cluster),
   655  	)
   656  	if err != nil {
   657  		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesCorruptedReason, clusterv1.ConditionSeverityError, err.Error())
   658  		return ctrl.Result{}, err
   659  	}
   660  	if err := certificates.EnsureAllExist(); err != nil {
   661  		conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesCorruptedReason, clusterv1.ConditionSeverityError, err.Error())
   662  		return ctrl.Result{}, err
   663  	}
   664  
   665  	conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition)
   666  
   667  	// Ensure that joinConfiguration.Discovery is properly set for joining node on the current cluster.
   668  	if res, err := r.reconcileDiscovery(ctx, scope.Cluster, scope.Config, certificates); err != nil {
   669  		return ctrl.Result{}, err
   670  	} else if !res.IsZero() {
   671  		return res, nil
   672  	}
   673  
   674  	kubernetesVersion := scope.ConfigOwner.KubernetesVersion()
   675  	parsedVersion, err := semver.ParseTolerant(kubernetesVersion)
   676  	if err != nil {
   677  		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion)
   678  	}
   679  
   680  	joinData, err := kubeadmtypes.MarshalJoinConfigurationForVersion(scope.Config.Spec.JoinConfiguration, parsedVersion)
   681  	if err != nil {
   682  		scope.Error(err, "Failed to marshal join configuration")
   683  		return ctrl.Result{}, err
   684  	}
   685  
   686  	verbosityFlag := ""
   687  	if scope.Config.Spec.Verbosity != nil {
   688  		verbosityFlag = fmt.Sprintf("--v %s", strconv.Itoa(int(*scope.Config.Spec.Verbosity)))
   689  	}
   690  
   691  	files, err := r.resolveFiles(ctx, scope.Config)
   692  	if err != nil {
   693  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   694  		return ctrl.Result{}, err
   695  	}
   696  
   697  	users, err := r.resolveUsers(ctx, scope.Config)
   698  	if err != nil {
   699  		conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
   700  		return ctrl.Result{}, err
   701  	}
   702  
   703  	controlPlaneJoinInput := &cloudinit.ControlPlaneJoinInput{
   704  		JoinConfiguration: joinData,
   705  		Certificates:      certificates,
   706  		BaseUserData: cloudinit.BaseUserData{
   707  			AdditionalFiles:      files,
   708  			NTP:                  scope.Config.Spec.NTP,
   709  			PreKubeadmCommands:   scope.Config.Spec.PreKubeadmCommands,
   710  			PostKubeadmCommands:  scope.Config.Spec.PostKubeadmCommands,
   711  			Users:                users,
   712  			Mounts:               scope.Config.Spec.Mounts,
   713  			DiskSetup:            scope.Config.Spec.DiskSetup,
   714  			KubeadmVerbosity:     verbosityFlag,
   715  			UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin,
   716  		},
   717  	}
   718  
   719  	var bootstrapJoinData []byte
   720  	switch scope.Config.Spec.Format {
   721  	case bootstrapv1.Ignition:
   722  		bootstrapJoinData, _, err = ignition.NewJoinControlPlane(&ignition.ControlPlaneJoinInput{
   723  			ControlPlaneJoinInput: controlPlaneJoinInput,
   724  			Ignition:              scope.Config.Spec.Ignition,
   725  		})
   726  	default:
   727  		bootstrapJoinData, err = cloudinit.NewJoinControlPlane(controlPlaneJoinInput)
   728  	}
   729  
   730  	if err != nil {
   731  		scope.Error(err, "Failed to create a control plane join configuration")
   732  		return ctrl.Result{}, err
   733  	}
   734  
   735  	if err := r.storeBootstrapData(ctx, scope, bootstrapJoinData); err != nil {
   736  		scope.Error(err, "Failed to store bootstrap data")
   737  		return ctrl.Result{}, err
   738  	}
   739  
   740  	return ctrl.Result{}, nil
   741  }
   742  
   743  // resolveFiles maps .Spec.Files into cloudinit.Files, resolving any object references
   744  // along the way.
   745  func (r *KubeadmConfigReconciler) resolveFiles(ctx context.Context, cfg *bootstrapv1.KubeadmConfig) ([]bootstrapv1.File, error) {
   746  	collected := make([]bootstrapv1.File, 0, len(cfg.Spec.Files))
   747  
   748  	for i := range cfg.Spec.Files {
   749  		in := cfg.Spec.Files[i]
   750  		if in.ContentFrom != nil {
   751  			data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, in)
   752  			if err != nil {
   753  				return nil, errors.Wrapf(err, "failed to resolve file source")
   754  			}
   755  			in.ContentFrom = nil
   756  			in.Content = string(data)
   757  		}
   758  		collected = append(collected, in)
   759  	}
   760  
   761  	return collected, nil
   762  }
   763  
   764  // resolveSecretFileContent returns file content fetched from a referenced secret object.
   765  func (r *KubeadmConfigReconciler) resolveSecretFileContent(ctx context.Context, ns string, source bootstrapv1.File) ([]byte, error) {
   766  	secret := &corev1.Secret{}
   767  	key := types.NamespacedName{Namespace: ns, Name: source.ContentFrom.Secret.Name}
   768  	if err := r.Client.Get(ctx, key, secret); err != nil {
   769  		if apierrors.IsNotFound(err) {
   770  			return nil, errors.Wrapf(err, "secret not found: %s", key)
   771  		}
   772  		return nil, errors.Wrapf(err, "failed to retrieve Secret %q", key)
   773  	}
   774  	data, ok := secret.Data[source.ContentFrom.Secret.Key]
   775  	if !ok {
   776  		return nil, errors.Errorf("secret references non-existent secret key: %q", source.ContentFrom.Secret.Key)
   777  	}
   778  	return data, nil
   779  }
   780  
   781  // resolveUsers maps .Spec.Users into cloudinit.Users, resolving any object references
   782  // along the way.
   783  func (r *KubeadmConfigReconciler) resolveUsers(ctx context.Context, cfg *bootstrapv1.KubeadmConfig) ([]bootstrapv1.User, error) {
   784  	collected := make([]bootstrapv1.User, 0, len(cfg.Spec.Users))
   785  
   786  	for i := range cfg.Spec.Users {
   787  		in := cfg.Spec.Users[i]
   788  		if in.PasswdFrom != nil {
   789  			data, err := r.resolveSecretPasswordContent(ctx, cfg.Namespace, in)
   790  			if err != nil {
   791  				return nil, errors.Wrapf(err, "failed to resolve passwd source")
   792  			}
   793  			in.PasswdFrom = nil
   794  			passwdContent := string(data)
   795  			in.Passwd = &passwdContent
   796  		}
   797  		collected = append(collected, in)
   798  	}
   799  
   800  	return collected, nil
   801  }
   802  
   803  // resolveSecretUserContent returns passwd fetched from a referenced secret object.
   804  func (r *KubeadmConfigReconciler) resolveSecretPasswordContent(ctx context.Context, ns string, source bootstrapv1.User) ([]byte, error) {
   805  	secret := &corev1.Secret{}
   806  	key := types.NamespacedName{Namespace: ns, Name: source.PasswdFrom.Secret.Name}
   807  	if err := r.Client.Get(ctx, key, secret); err != nil {
   808  		if apierrors.IsNotFound(err) {
   809  			return nil, errors.Wrapf(err, "secret not found: %s", key)
   810  		}
   811  		return nil, errors.Wrapf(err, "failed to retrieve Secret %q", key)
   812  	}
   813  	data, ok := secret.Data[source.PasswdFrom.Secret.Key]
   814  	if !ok {
   815  		return nil, errors.Errorf("secret references non-existent secret key: %q", source.PasswdFrom.Secret.Key)
   816  	}
   817  	return data, nil
   818  }
   819  
   820  // ClusterToKubeadmConfigs is a handler.ToRequestsFunc to be used to enqueue
   821  // requests for reconciliation of KubeadmConfigs.
   822  func (r *KubeadmConfigReconciler) ClusterToKubeadmConfigs(ctx context.Context, o client.Object) []ctrl.Request {
   823  	result := []ctrl.Request{}
   824  
   825  	c, ok := o.(*clusterv1.Cluster)
   826  	if !ok {
   827  		panic(fmt.Sprintf("Expected a Cluster but got a %T", o))
   828  	}
   829  
   830  	selectors := []client.ListOption{
   831  		client.InNamespace(c.Namespace),
   832  		client.MatchingLabels{
   833  			clusterv1.ClusterNameLabel: c.Name,
   834  		},
   835  	}
   836  
   837  	machineList := &clusterv1.MachineList{}
   838  	if err := r.Client.List(ctx, machineList, selectors...); err != nil {
   839  		return nil
   840  	}
   841  
   842  	for _, m := range machineList.Items {
   843  		if m.Spec.Bootstrap.ConfigRef != nil &&
   844  			m.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == bootstrapv1.GroupVersion.WithKind("KubeadmConfig").GroupKind() {
   845  			name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name}
   846  			result = append(result, ctrl.Request{NamespacedName: name})
   847  		}
   848  	}
   849  
   850  	if feature.Gates.Enabled(feature.MachinePool) {
   851  		machinePoolList := &expv1.MachinePoolList{}
   852  		if err := r.Client.List(ctx, machinePoolList, selectors...); err != nil {
   853  			return nil
   854  		}
   855  
   856  		for _, mp := range machinePoolList.Items {
   857  			if mp.Spec.Template.Spec.Bootstrap.ConfigRef != nil &&
   858  				mp.Spec.Template.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == bootstrapv1.GroupVersion.WithKind("KubeadmConfig").GroupKind() {
   859  				name := client.ObjectKey{Namespace: mp.Namespace, Name: mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name}
   860  				result = append(result, ctrl.Request{NamespacedName: name})
   861  			}
   862  		}
   863  	}
   864  
   865  	return result
   866  }
   867  
   868  // MachineToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue
   869  // request for reconciliation of KubeadmConfig.
   870  func (r *KubeadmConfigReconciler) MachineToBootstrapMapFunc(_ context.Context, o client.Object) []ctrl.Request {
   871  	m, ok := o.(*clusterv1.Machine)
   872  	if !ok {
   873  		panic(fmt.Sprintf("Expected a Machine but got a %T", o))
   874  	}
   875  
   876  	result := []ctrl.Request{}
   877  	if m.Spec.Bootstrap.ConfigRef != nil && m.Spec.Bootstrap.ConfigRef.GroupVersionKind() == bootstrapv1.GroupVersion.WithKind("KubeadmConfig") {
   878  		name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name}
   879  		result = append(result, ctrl.Request{NamespacedName: name})
   880  	}
   881  	return result
   882  }
   883  
   884  // MachinePoolToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue
   885  // request for reconciliation of KubeadmConfig.
   886  func (r *KubeadmConfigReconciler) MachinePoolToBootstrapMapFunc(_ context.Context, o client.Object) []ctrl.Request {
   887  	m, ok := o.(*expv1.MachinePool)
   888  	if !ok {
   889  		panic(fmt.Sprintf("Expected a MachinePool but got a %T", o))
   890  	}
   891  
   892  	result := []ctrl.Request{}
   893  	configRef := m.Spec.Template.Spec.Bootstrap.ConfigRef
   894  	if configRef != nil && configRef.GroupVersionKind().GroupKind() == bootstrapv1.GroupVersion.WithKind("KubeadmConfig").GroupKind() {
   895  		name := client.ObjectKey{Namespace: m.Namespace, Name: configRef.Name}
   896  		result = append(result, ctrl.Request{NamespacedName: name})
   897  	}
   898  	return result
   899  }
   900  
   901  // reconcileDiscovery ensures that config.JoinConfiguration.Discovery is properly set for the joining node.
   902  // The implementation func respect user provided discovery configurations, but in case some of them are missing, a valid BootstrapToken object
   903  // is automatically injected into config.JoinConfiguration.Discovery.
   904  // This allows to simplify configuration UX, by providing the option to delegate to CABPK the configuration of kubeadm join discovery.
   905  func (r *KubeadmConfigReconciler) reconcileDiscovery(ctx context.Context, cluster *clusterv1.Cluster, config *bootstrapv1.KubeadmConfig, certificates secret.Certificates) (ctrl.Result, error) {
   906  	log := ctrl.LoggerFrom(ctx)
   907  
   908  	// if config already contains a file discovery configuration, respect it without further validations
   909  	if config.Spec.JoinConfiguration.Discovery.File != nil {
   910  		return ctrl.Result{}, nil
   911  	}
   912  
   913  	// otherwise it is necessary to ensure token discovery is properly configured
   914  	if config.Spec.JoinConfiguration.Discovery.BootstrapToken == nil {
   915  		config.Spec.JoinConfiguration.Discovery.BootstrapToken = &bootstrapv1.BootstrapTokenDiscovery{}
   916  	}
   917  
   918  	// calculate the ca cert hashes if they are not already set
   919  	if len(config.Spec.JoinConfiguration.Discovery.BootstrapToken.CACertHashes) == 0 {
   920  		hashes, err := certificates.GetByPurpose(secret.ClusterCA).Hashes()
   921  		if err != nil {
   922  			log.Error(err, "Unable to generate Cluster CA certificate hashes")
   923  			return ctrl.Result{}, err
   924  		}
   925  		config.Spec.JoinConfiguration.Discovery.BootstrapToken.CACertHashes = hashes
   926  	}
   927  
   928  	// if BootstrapToken already contains an APIServerEndpoint, respect it; otherwise inject the APIServerEndpoint endpoint defined in cluster status
   929  	apiServerEndpoint := config.Spec.JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint
   930  	if apiServerEndpoint == "" {
   931  		if !cluster.Spec.ControlPlaneEndpoint.IsValid() {
   932  			log.V(1).Info("Waiting for Cluster Controller to set Cluster.Spec.ControlPlaneEndpoint")
   933  			return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
   934  		}
   935  
   936  		apiServerEndpoint = cluster.Spec.ControlPlaneEndpoint.String()
   937  		config.Spec.JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint = apiServerEndpoint
   938  		log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint", "APIServerEndpoint", apiServerEndpoint)
   939  	}
   940  
   941  	// if BootstrapToken already contains a token, respect it; otherwise create a new bootstrap token for the node to join
   942  	if config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token == "" {
   943  		remoteClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
   944  		if err != nil {
   945  			return ctrl.Result{}, err
   946  		}
   947  
   948  		token, err := createToken(ctx, remoteClient, r.TokenTTL)
   949  		if err != nil {
   950  			return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
   951  		}
   952  
   953  		config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
   954  		log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
   955  	}
   956  
   957  	// If the BootstrapToken does not contain any CACertHashes then force skip CA Verification
   958  	if len(config.Spec.JoinConfiguration.Discovery.BootstrapToken.CACertHashes) == 0 {
   959  		log.Info("No CAs were provided. Falling back to insecure discover method by skipping CA Cert validation")
   960  		config.Spec.JoinConfiguration.Discovery.BootstrapToken.UnsafeSkipCAVerification = true
   961  	}
   962  
   963  	return ctrl.Result{}, nil
   964  }
   965  
   966  // reconcileTopLevelObjectSettings injects into config.ClusterConfiguration values from top level objects like cluster and machine.
   967  // The implementation func respect user provided config values, but in case some of them are missing, values from top level objects are used.
   968  func (r *KubeadmConfigReconciler) reconcileTopLevelObjectSettings(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, config *bootstrapv1.KubeadmConfig) {
   969  	log := ctrl.LoggerFrom(ctx)
   970  
   971  	// If there is no ControlPlaneEndpoint defined in ClusterConfiguration but
   972  	// there is a ControlPlaneEndpoint defined at Cluster level (e.g. the load balancer endpoint),
   973  	// then use Cluster's ControlPlaneEndpoint as a control plane endpoint for the Kubernetes cluster.
   974  	if config.Spec.ClusterConfiguration.ControlPlaneEndpoint == "" && cluster.Spec.ControlPlaneEndpoint.IsValid() {
   975  		config.Spec.ClusterConfiguration.ControlPlaneEndpoint = cluster.Spec.ControlPlaneEndpoint.String()
   976  		log.V(3).Info("Altering ClusterConfiguration.ControlPlaneEndpoint", "ControlPlaneEndpoint", config.Spec.ClusterConfiguration.ControlPlaneEndpoint)
   977  	}
   978  
   979  	// If there are no ClusterName defined in ClusterConfiguration, use Cluster.Name
   980  	if config.Spec.ClusterConfiguration.ClusterName == "" {
   981  		config.Spec.ClusterConfiguration.ClusterName = cluster.Name
   982  		log.V(3).Info("Altering ClusterConfiguration.ClusterName", "ClusterName", config.Spec.ClusterConfiguration.ClusterName)
   983  	}
   984  
   985  	// If there are no Network settings defined in ClusterConfiguration, use ClusterNetwork settings, if defined
   986  	if cluster.Spec.ClusterNetwork != nil {
   987  		if config.Spec.ClusterConfiguration.Networking.DNSDomain == "" && cluster.Spec.ClusterNetwork.ServiceDomain != "" {
   988  			config.Spec.ClusterConfiguration.Networking.DNSDomain = cluster.Spec.ClusterNetwork.ServiceDomain
   989  			log.V(3).Info("Altering ClusterConfiguration.Networking.DNSDomain", "DNSDomain", config.Spec.ClusterConfiguration.Networking.DNSDomain)
   990  		}
   991  		if config.Spec.ClusterConfiguration.Networking.ServiceSubnet == "" &&
   992  			cluster.Spec.ClusterNetwork.Services != nil &&
   993  			len(cluster.Spec.ClusterNetwork.Services.CIDRBlocks) > 0 {
   994  			config.Spec.ClusterConfiguration.Networking.ServiceSubnet = cluster.Spec.ClusterNetwork.Services.String()
   995  			log.V(3).Info("Altering ClusterConfiguration.Networking.ServiceSubnet", "ServiceSubnet", config.Spec.ClusterConfiguration.Networking.ServiceSubnet)
   996  		}
   997  		if config.Spec.ClusterConfiguration.Networking.PodSubnet == "" &&
   998  			cluster.Spec.ClusterNetwork.Pods != nil &&
   999  			len(cluster.Spec.ClusterNetwork.Pods.CIDRBlocks) > 0 {
  1000  			config.Spec.ClusterConfiguration.Networking.PodSubnet = cluster.Spec.ClusterNetwork.Pods.String()
  1001  			log.V(3).Info("Altering ClusterConfiguration.Networking.PodSubnet", "PodSubnet", config.Spec.ClusterConfiguration.Networking.PodSubnet)
  1002  		}
  1003  	}
  1004  
  1005  	// If there are no KubernetesVersion settings defined in ClusterConfiguration, use Version from machine, if defined
  1006  	if config.Spec.ClusterConfiguration.KubernetesVersion == "" && machine.Spec.Version != nil {
  1007  		config.Spec.ClusterConfiguration.KubernetesVersion = *machine.Spec.Version
  1008  		log.V(3).Info("Altering ClusterConfiguration.KubernetesVersion", "KubernetesVersion", config.Spec.ClusterConfiguration.KubernetesVersion)
  1009  	}
  1010  }
  1011  
  1012  // storeBootstrapData creates a new secret with the data passed in as input,
  1013  // sets the reference in the configuration status and ready to true.
  1014  func (r *KubeadmConfigReconciler) storeBootstrapData(ctx context.Context, scope *Scope, data []byte) error {
  1015  	log := ctrl.LoggerFrom(ctx)
  1016  
  1017  	secret := &corev1.Secret{
  1018  		ObjectMeta: metav1.ObjectMeta{
  1019  			Name:      scope.Config.Name,
  1020  			Namespace: scope.Config.Namespace,
  1021  			Labels: map[string]string{
  1022  				clusterv1.ClusterNameLabel: scope.Cluster.Name,
  1023  			},
  1024  			OwnerReferences: []metav1.OwnerReference{
  1025  				{
  1026  					APIVersion: bootstrapv1.GroupVersion.String(),
  1027  					Kind:       "KubeadmConfig",
  1028  					Name:       scope.Config.Name,
  1029  					UID:        scope.Config.UID,
  1030  					Controller: pointer.Bool(true),
  1031  				},
  1032  			},
  1033  		},
  1034  		Data: map[string][]byte{
  1035  			"value":  data,
  1036  			"format": []byte(scope.Config.Spec.Format),
  1037  		},
  1038  		Type: clusterv1.ClusterSecretType,
  1039  	}
  1040  
  1041  	// as secret creation and scope.Config status patch are not atomic operations
  1042  	// it is possible that secret creation happens but the config.Status patches are not applied
  1043  	if err := r.Client.Create(ctx, secret); err != nil {
  1044  		if !apierrors.IsAlreadyExists(err) {
  1045  			return errors.Wrapf(err, "failed to create bootstrap data secret for KubeadmConfig %s/%s", scope.Config.Namespace, scope.Config.Name)
  1046  		}
  1047  		log.Info("Bootstrap data secret for KubeadmConfig already exists, updating", "Secret", klog.KObj(secret))
  1048  		if err := r.Client.Update(ctx, secret); err != nil {
  1049  			return errors.Wrapf(err, "failed to update bootstrap data secret for KubeadmConfig %s/%s", scope.Config.Namespace, scope.Config.Name)
  1050  		}
  1051  	}
  1052  	scope.Config.Status.DataSecretName = pointer.String(secret.Name)
  1053  	scope.Config.Status.Ready = true
  1054  	conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition)
  1055  	return nil
  1056  }
  1057  
  1058  // Ensure the bootstrap secret has the KubeadmConfig as a controller OwnerReference.
  1059  func (r *KubeadmConfigReconciler) ensureBootstrapSecretOwnersRef(ctx context.Context, scope *Scope) error {
  1060  	secret := &corev1.Secret{}
  1061  	err := r.SecretCachingClient.Get(ctx, client.ObjectKey{Namespace: scope.Config.Namespace, Name: scope.Config.Name}, secret)
  1062  	if err != nil {
  1063  		// If the secret has not been created yet return early.
  1064  		if apierrors.IsNotFound(err) {
  1065  			return nil
  1066  		}
  1067  		return errors.Wrapf(err, "failed to add KubeadmConfig %s as ownerReference to bootstrap Secret %s", scope.ConfigOwner.GetName(), secret.GetName())
  1068  	}
  1069  	patchHelper, err := patch.NewHelper(secret, r.Client)
  1070  	if err != nil {
  1071  		return errors.Wrapf(err, "failed to add KubeadmConfig %s as ownerReference to bootstrap Secret %s", scope.ConfigOwner.GetName(), secret.GetName())
  1072  	}
  1073  	if c := metav1.GetControllerOf(secret); c != nil && c.Kind != "KubeadmConfig" {
  1074  		secret.SetOwnerReferences(util.RemoveOwnerRef(secret.GetOwnerReferences(), *c))
  1075  	}
  1076  	secret.SetOwnerReferences(util.EnsureOwnerRef(secret.GetOwnerReferences(), metav1.OwnerReference{
  1077  		APIVersion: bootstrapv1.GroupVersion.String(),
  1078  		Kind:       "KubeadmConfig",
  1079  		UID:        scope.Config.UID,
  1080  		Name:       scope.Config.Name,
  1081  		Controller: pointer.Bool(true),
  1082  	}))
  1083  	err = patchHelper.Patch(ctx, secret)
  1084  	if err != nil {
  1085  		return errors.Wrapf(err, "could not add KubeadmConfig %s as ownerReference to bootstrap Secret %s", scope.ConfigOwner.GetName(), secret.GetName())
  1086  	}
  1087  	return nil
  1088  }