github.com/stefanmcshane/helm@v0.0.0-20221213002717-88a4a2c6e77d/pkg/kube/ready.go (about)

     1  /*
     2  Copyright The Helm Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kube // import "github.com/stefanmcshane/helm/pkg/kube"
    18  
    19  import (
    20  	"context"
    21  
    22  	appsv1 "k8s.io/api/apps/v1"
    23  	appsv1beta1 "k8s.io/api/apps/v1beta1"
    24  	appsv1beta2 "k8s.io/api/apps/v1beta2"
    25  	batchv1 "k8s.io/api/batch/v1"
    26  	corev1 "k8s.io/api/core/v1"
    27  	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
    28  	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    29  	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/runtime"
    32  	"k8s.io/apimachinery/pkg/util/intstr"
    33  	"k8s.io/cli-runtime/pkg/resource"
    34  	"k8s.io/client-go/kubernetes"
    35  	"k8s.io/client-go/kubernetes/scheme"
    36  
    37  	deploymentutil "github.com/stefanmcshane/helm/internal/third_party/k8s.io/kubernetes/deployment/util"
    38  )
    39  
    40  // ReadyCheckerOption is a function that configures a ReadyChecker.
    41  type ReadyCheckerOption func(*ReadyChecker)
    42  
    43  // PausedAsReady returns a ReadyCheckerOption that configures a ReadyChecker
    44  // to consider paused resources to be ready. For example a Deployment
    45  // with spec.paused equal to true would be considered ready.
    46  func PausedAsReady(pausedAsReady bool) ReadyCheckerOption {
    47  	return func(c *ReadyChecker) {
    48  		c.pausedAsReady = pausedAsReady
    49  	}
    50  }
    51  
    52  // CheckJobs returns a ReadyCheckerOption that configures a ReadyChecker
    53  // to consider readiness of Job resources.
    54  func CheckJobs(checkJobs bool) ReadyCheckerOption {
    55  	return func(c *ReadyChecker) {
    56  		c.checkJobs = checkJobs
    57  	}
    58  }
    59  
    60  // NewReadyChecker creates a new checker. Passed ReadyCheckerOptions can
    61  // be used to override defaults.
    62  func NewReadyChecker(cl kubernetes.Interface, log func(string, ...interface{}), opts ...ReadyCheckerOption) ReadyChecker {
    63  	c := ReadyChecker{
    64  		client: cl,
    65  		log:    log,
    66  	}
    67  	if c.log == nil {
    68  		c.log = nopLogger
    69  	}
    70  	for _, opt := range opts {
    71  		opt(&c)
    72  	}
    73  	return c
    74  }
    75  
    76  // ReadyChecker is a type that can check core Kubernetes types for readiness.
    77  type ReadyChecker struct {
    78  	client        kubernetes.Interface
    79  	log           func(string, ...interface{})
    80  	checkJobs     bool
    81  	pausedAsReady bool
    82  }
    83  
    84  // IsReady checks if v is ready. It supports checking readiness for pods,
    85  // deployments, persistent volume claims, services, daemon sets, custom
    86  // resource definitions, stateful sets, replication controllers, and replica
    87  // sets. All other resource kinds are always considered ready.
    88  //
    89  // IsReady will fetch the latest state of the object from the server prior to
    90  // performing readiness checks, and it will return any error encountered.
    91  func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, error) {
    92  	switch value := AsVersioned(v).(type) {
    93  	case *corev1.Pod:
    94  		pod, err := c.client.CoreV1().Pods(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
    95  		if err != nil || !c.isPodReady(pod) {
    96  			return false, err
    97  		}
    98  	case *batchv1.Job:
    99  		if c.checkJobs {
   100  			job, err := c.client.BatchV1().Jobs(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   101  			if err != nil || !c.jobReady(job) {
   102  				return false, err
   103  			}
   104  		}
   105  	case *appsv1.Deployment, *appsv1beta1.Deployment, *appsv1beta2.Deployment, *extensionsv1beta1.Deployment:
   106  		currentDeployment, err := c.client.AppsV1().Deployments(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   107  		if err != nil {
   108  			return false, err
   109  		}
   110  		// If paused deployment will never be ready
   111  		if currentDeployment.Spec.Paused {
   112  			return c.pausedAsReady, nil
   113  		}
   114  		// Find RS associated with deployment
   115  		newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, c.client.AppsV1())
   116  		if err != nil || newReplicaSet == nil {
   117  			return false, err
   118  		}
   119  		if !c.deploymentReady(newReplicaSet, currentDeployment) {
   120  			return false, nil
   121  		}
   122  	case *corev1.PersistentVolumeClaim:
   123  		claim, err := c.client.CoreV1().PersistentVolumeClaims(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   124  		if err != nil {
   125  			return false, err
   126  		}
   127  		if !c.volumeReady(claim) {
   128  			return false, nil
   129  		}
   130  	case *corev1.Service:
   131  		svc, err := c.client.CoreV1().Services(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   132  		if err != nil {
   133  			return false, err
   134  		}
   135  		if !c.serviceReady(svc) {
   136  			return false, nil
   137  		}
   138  	case *extensionsv1beta1.DaemonSet, *appsv1.DaemonSet, *appsv1beta2.DaemonSet:
   139  		ds, err := c.client.AppsV1().DaemonSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   140  		if err != nil {
   141  			return false, err
   142  		}
   143  		if !c.daemonSetReady(ds) {
   144  			return false, nil
   145  		}
   146  	case *apiextv1beta1.CustomResourceDefinition:
   147  		if err := v.Get(); err != nil {
   148  			return false, err
   149  		}
   150  		crd := &apiextv1beta1.CustomResourceDefinition{}
   151  		if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
   152  			return false, err
   153  		}
   154  		if !c.crdBetaReady(*crd) {
   155  			return false, nil
   156  		}
   157  	case *apiextv1.CustomResourceDefinition:
   158  		if err := v.Get(); err != nil {
   159  			return false, err
   160  		}
   161  		crd := &apiextv1.CustomResourceDefinition{}
   162  		if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
   163  			return false, err
   164  		}
   165  		if !c.crdReady(*crd) {
   166  			return false, nil
   167  		}
   168  	case *appsv1.StatefulSet, *appsv1beta1.StatefulSet, *appsv1beta2.StatefulSet:
   169  		sts, err := c.client.AppsV1().StatefulSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   170  		if err != nil {
   171  			return false, err
   172  		}
   173  		if !c.statefulSetReady(sts) {
   174  			return false, nil
   175  		}
   176  	case *corev1.ReplicationController:
   177  		rc, err := c.client.CoreV1().ReplicationControllers(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   178  		if err != nil {
   179  			return false, err
   180  		}
   181  		if !c.replicationControllerReady(rc) {
   182  			return false, nil
   183  		}
   184  		ready, err := c.podsReadyForObject(ctx, v.Namespace, value)
   185  		if !ready || err != nil {
   186  			return false, err
   187  		}
   188  	case *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet:
   189  		rs, err := c.client.AppsV1().ReplicaSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
   190  		if err != nil {
   191  			return false, err
   192  		}
   193  		if !c.replicaSetReady(rs) {
   194  			return false, nil
   195  		}
   196  		ready, err := c.podsReadyForObject(ctx, v.Namespace, value)
   197  		if !ready || err != nil {
   198  			return false, err
   199  		}
   200  	}
   201  	return true, nil
   202  }
   203  
   204  func (c *ReadyChecker) podsReadyForObject(ctx context.Context, namespace string, obj runtime.Object) (bool, error) {
   205  	pods, err := c.podsforObject(ctx, namespace, obj)
   206  	if err != nil {
   207  		return false, err
   208  	}
   209  	for _, pod := range pods {
   210  		if !c.isPodReady(&pod) {
   211  			return false, nil
   212  		}
   213  	}
   214  	return true, nil
   215  }
   216  
   217  func (c *ReadyChecker) podsforObject(ctx context.Context, namespace string, obj runtime.Object) ([]corev1.Pod, error) {
   218  	selector, err := SelectorsForObject(obj)
   219  	if err != nil {
   220  		return nil, err
   221  	}
   222  	list, err := getPods(ctx, c.client, namespace, selector.String())
   223  	return list, err
   224  }
   225  
   226  // isPodReady returns true if a pod is ready; false otherwise.
   227  func (c *ReadyChecker) isPodReady(pod *corev1.Pod) bool {
   228  	for _, c := range pod.Status.Conditions {
   229  		if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
   230  			return true
   231  		}
   232  	}
   233  	c.log("Pod is not ready: %s/%s", pod.GetNamespace(), pod.GetName())
   234  	return false
   235  }
   236  
   237  func (c *ReadyChecker) jobReady(job *batchv1.Job) bool {
   238  	if job.Status.Failed > *job.Spec.BackoffLimit {
   239  		c.log("Job is failed: %s/%s", job.GetNamespace(), job.GetName())
   240  		return false
   241  	}
   242  	if job.Spec.Completions != nil && job.Status.Succeeded < *job.Spec.Completions {
   243  		c.log("Job is not completed: %s/%s", job.GetNamespace(), job.GetName())
   244  		return false
   245  	}
   246  	return true
   247  }
   248  
   249  func (c *ReadyChecker) serviceReady(s *corev1.Service) bool {
   250  	// ExternalName Services are external to cluster so helm shouldn't be checking to see if they're 'ready' (i.e. have an IP Set)
   251  	if s.Spec.Type == corev1.ServiceTypeExternalName {
   252  		return true
   253  	}
   254  
   255  	// Ensure that the service cluster IP is not empty
   256  	if s.Spec.ClusterIP == "" {
   257  		c.log("Service does not have cluster IP address: %s/%s", s.GetNamespace(), s.GetName())
   258  		return false
   259  	}
   260  
   261  	// This checks if the service has a LoadBalancer and that balancer has an Ingress defined
   262  	if s.Spec.Type == corev1.ServiceTypeLoadBalancer {
   263  		// do not wait when at least 1 external IP is set
   264  		if len(s.Spec.ExternalIPs) > 0 {
   265  			c.log("Service %s/%s has external IP addresses (%v), marking as ready", s.GetNamespace(), s.GetName(), s.Spec.ExternalIPs)
   266  			return true
   267  		}
   268  
   269  		if s.Status.LoadBalancer.Ingress == nil {
   270  			c.log("Service does not have load balancer ingress IP address: %s/%s", s.GetNamespace(), s.GetName())
   271  			return false
   272  		}
   273  	}
   274  
   275  	return true
   276  }
   277  
   278  func (c *ReadyChecker) volumeReady(v *corev1.PersistentVolumeClaim) bool {
   279  	if v.Status.Phase != corev1.ClaimBound {
   280  		c.log("PersistentVolumeClaim is not bound: %s/%s", v.GetNamespace(), v.GetName())
   281  		return false
   282  	}
   283  	return true
   284  }
   285  
   286  func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool {
   287  	// Verify the replicaset readiness
   288  	if !c.replicaSetReady(rs) {
   289  		return false
   290  	}
   291  	// Verify the generation observed by the deployment controller matches the spec generation
   292  	if dep.Status.ObservedGeneration != dep.ObjectMeta.Generation {
   293  		c.log("Deployment is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", dep.Namespace, dep.Name, dep.Status.ObservedGeneration, dep.ObjectMeta.Generation)
   294  		return false
   295  	}
   296  
   297  	expectedReady := *dep.Spec.Replicas - deploymentutil.MaxUnavailable(*dep)
   298  	if !(rs.Status.ReadyReplicas >= expectedReady) {
   299  		c.log("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", dep.Namespace, dep.Name, rs.Status.ReadyReplicas, expectedReady)
   300  		return false
   301  	}
   302  	return true
   303  }
   304  
   305  func (c *ReadyChecker) daemonSetReady(ds *appsv1.DaemonSet) bool {
   306  	// Verify the generation observed by the daemonSet controller matches the spec generation
   307  	if ds.Status.ObservedGeneration != ds.ObjectMeta.Generation {
   308  		c.log("DaemonSet is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", ds.Namespace, ds.Name, ds.Status.ObservedGeneration, ds.ObjectMeta.Generation)
   309  		return false
   310  	}
   311  
   312  	// If the update strategy is not a rolling update, there will be nothing to wait for
   313  	if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType {
   314  		return true
   315  	}
   316  
   317  	// Make sure all the updated pods have been scheduled
   318  	if ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled {
   319  		c.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", ds.Namespace, ds.Name, ds.Status.UpdatedNumberScheduled, ds.Status.DesiredNumberScheduled)
   320  		return false
   321  	}
   322  	maxUnavailable, err := intstr.GetValueFromIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, int(ds.Status.DesiredNumberScheduled), true)
   323  	if err != nil {
   324  		// If for some reason the value is invalid, set max unavailable to the
   325  		// number of desired replicas. This is the same behavior as the
   326  		// `MaxUnavailable` function in deploymentutil
   327  		maxUnavailable = int(ds.Status.DesiredNumberScheduled)
   328  	}
   329  
   330  	expectedReady := int(ds.Status.DesiredNumberScheduled) - maxUnavailable
   331  	if !(int(ds.Status.NumberReady) >= expectedReady) {
   332  		c.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods are ready", ds.Namespace, ds.Name, ds.Status.NumberReady, expectedReady)
   333  		return false
   334  	}
   335  	return true
   336  }
   337  
   338  // Because the v1 extensions API is not available on all supported k8s versions
   339  // yet and because Go doesn't support generics, we need to have a duplicate
   340  // function to support the v1beta1 types
   341  func (c *ReadyChecker) crdBetaReady(crd apiextv1beta1.CustomResourceDefinition) bool {
   342  	for _, cond := range crd.Status.Conditions {
   343  		switch cond.Type {
   344  		case apiextv1beta1.Established:
   345  			if cond.Status == apiextv1beta1.ConditionTrue {
   346  				return true
   347  			}
   348  		case apiextv1beta1.NamesAccepted:
   349  			if cond.Status == apiextv1beta1.ConditionFalse {
   350  				// This indicates a naming conflict, but it's probably not the
   351  				// job of this function to fail because of that. Instead,
   352  				// we treat it as a success, since the process should be able to
   353  				// continue.
   354  				return true
   355  			}
   356  		}
   357  	}
   358  	return false
   359  }
   360  
   361  func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool {
   362  	for _, cond := range crd.Status.Conditions {
   363  		switch cond.Type {
   364  		case apiextv1.Established:
   365  			if cond.Status == apiextv1.ConditionTrue {
   366  				return true
   367  			}
   368  		case apiextv1.NamesAccepted:
   369  			if cond.Status == apiextv1.ConditionFalse {
   370  				// This indicates a naming conflict, but it's probably not the
   371  				// job of this function to fail because of that. Instead,
   372  				// we treat it as a success, since the process should be able to
   373  				// continue.
   374  				return true
   375  			}
   376  		}
   377  	}
   378  	return false
   379  }
   380  
   381  func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool {
   382  	// Verify the generation observed by the statefulSet controller matches the spec generation
   383  	if sts.Status.ObservedGeneration != sts.ObjectMeta.Generation {
   384  		c.log("Statefulset is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", sts.Namespace, sts.Name, sts.Status.ObservedGeneration, sts.ObjectMeta.Generation)
   385  		return false
   386  	}
   387  
   388  	// If the update strategy is not a rolling update, there will be nothing to wait for
   389  	if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
   390  		c.log("StatefulSet skipped ready check: %s/%s. updateStrategy is %v", sts.Namespace, sts.Name, sts.Spec.UpdateStrategy.Type)
   391  		return true
   392  	}
   393  
   394  	// Dereference all the pointers because StatefulSets like them
   395  	var partition int
   396  	// 1 is the default for replicas if not set
   397  	var replicas = 1
   398  	// For some reason, even if the update strategy is a rolling update, the
   399  	// actual rollingUpdate field can be nil. If it is, we can safely assume
   400  	// there is no partition value
   401  	if sts.Spec.UpdateStrategy.RollingUpdate != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
   402  		partition = int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition)
   403  	}
   404  	if sts.Spec.Replicas != nil {
   405  		replicas = int(*sts.Spec.Replicas)
   406  	}
   407  
   408  	// Because an update strategy can use partitioning, we need to calculate the
   409  	// number of updated replicas we should have. For example, if the replicas
   410  	// is set to 3 and the partition is 2, we'd expect only one pod to be
   411  	// updated
   412  	expectedReplicas := replicas - partition
   413  
   414  	// Make sure all the updated pods have been scheduled
   415  	if int(sts.Status.UpdatedReplicas) < expectedReplicas {
   416  		c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", sts.Namespace, sts.Name, sts.Status.UpdatedReplicas, expectedReplicas)
   417  		return false
   418  	}
   419  
   420  	if int(sts.Status.ReadyReplicas) != replicas {
   421  		c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas)
   422  		return false
   423  	}
   424  
   425  	if sts.Status.CurrentRevision != sts.Status.UpdateRevision {
   426  		c.log("StatefulSet is not ready: %s/%s. currentRevision %s does not yet match updateRevision %s", sts.Namespace, sts.Name, sts.Status.CurrentRevision, sts.Status.UpdateRevision)
   427  		return false
   428  	}
   429  
   430  	c.log("StatefulSet is ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas)
   431  	return true
   432  }
   433  
   434  func (c *ReadyChecker) replicationControllerReady(rc *corev1.ReplicationController) bool {
   435  	// Verify the generation observed by the replicationController controller matches the spec generation
   436  	if rc.Status.ObservedGeneration != rc.ObjectMeta.Generation {
   437  		c.log("ReplicationController is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", rc.Namespace, rc.Name, rc.Status.ObservedGeneration, rc.ObjectMeta.Generation)
   438  		return false
   439  	}
   440  	return true
   441  }
   442  
   443  func (c *ReadyChecker) replicaSetReady(rs *appsv1.ReplicaSet) bool {
   444  	// Verify the generation observed by the replicaSet controller matches the spec generation
   445  	if rs.Status.ObservedGeneration != rs.ObjectMeta.Generation {
   446  		c.log("ReplicaSet is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", rs.Namespace, rs.Name, rs.Status.ObservedGeneration, rs.ObjectMeta.Generation)
   447  		return false
   448  	}
   449  	return true
   450  }
   451  
   452  func getPods(ctx context.Context, client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) {
   453  	list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
   454  		LabelSelector: selector,
   455  	})
   456  	return list.Items, err
   457  }