github.com/zsuzhengdu/helm@v3.0.0-beta.3+incompatible/pkg/kube/wait.go (about)

     1  /*
     2  Copyright The Helm Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kube // import "helm.sh/helm/pkg/kube"
    18  
    19  import (
    20  	"fmt"
    21  	"time"
    22  
    23  	"github.com/pkg/errors"
    24  	appsv1 "k8s.io/api/apps/v1"
    25  	appsv1beta1 "k8s.io/api/apps/v1beta1"
    26  	appsv1beta2 "k8s.io/api/apps/v1beta2"
    27  	batchv1 "k8s.io/api/batch/v1"
    28  	corev1 "k8s.io/api/core/v1"
    29  	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
    30  	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/labels"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	"k8s.io/apimachinery/pkg/util/intstr"
    35  	"k8s.io/apimachinery/pkg/util/wait"
    36  	"k8s.io/client-go/kubernetes"
    37  	"k8s.io/client-go/kubernetes/scheme"
    38  	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
    39  )
    40  
    41  type waiter struct {
    42  	c       kubernetes.Interface
    43  	timeout time.Duration
    44  	log     func(string, ...interface{})
    45  }
    46  
    47  // waitForResources polls to get the current status of all pods, PVCs, and Services
    48  // until all are ready or a timeout is reached
    49  func (w *waiter) waitForResources(created ResourceList) error {
    50  	w.log("beginning wait for %d resources with timeout of %v", len(created), w.timeout)
    51  
    52  	return wait.Poll(2*time.Second, w.timeout, func() (bool, error) {
    53  		for _, v := range created {
    54  			var (
    55  				// This defaults to true, otherwise we get to a point where
    56  				// things will always return false unless one of the objects
    57  				// that manages pods has been hit
    58  				ok  = true
    59  				err error
    60  			)
    61  			switch value := AsVersioned(v).(type) {
    62  			case *corev1.Pod:
    63  				pod, err := w.c.CoreV1().Pods(value.Namespace).Get(value.Name, metav1.GetOptions{})
    64  				if err != nil || !w.isPodReady(pod) {
    65  					return false, err
    66  				}
    67  			case *appsv1.Deployment:
    68  				currentDeployment, err := w.c.AppsV1().Deployments(value.Namespace).Get(value.Name, metav1.GetOptions{})
    69  				if err != nil {
    70  					return false, err
    71  				}
    72  				// Find RS associated with deployment
    73  				newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, w.c.AppsV1())
    74  				if err != nil || newReplicaSet == nil {
    75  					return false, err
    76  				}
    77  				if !w.deploymentReady(newReplicaSet, currentDeployment) {
    78  					return false, nil
    79  				}
    80  			case *appsv1beta1.Deployment:
    81  				currentDeployment, err := w.c.AppsV1().Deployments(value.Namespace).Get(value.Name, metav1.GetOptions{})
    82  				if err != nil {
    83  					return false, err
    84  				}
    85  				// Find RS associated with deployment
    86  				newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, w.c.AppsV1())
    87  				if err != nil || newReplicaSet == nil {
    88  					return false, err
    89  				}
    90  				if !w.deploymentReady(newReplicaSet, currentDeployment) {
    91  					return false, nil
    92  				}
    93  			case *appsv1beta2.Deployment:
    94  				currentDeployment, err := w.c.AppsV1().Deployments(value.Namespace).Get(value.Name, metav1.GetOptions{})
    95  				if err != nil {
    96  					return false, err
    97  				}
    98  				// Find RS associated with deployment
    99  				newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, w.c.AppsV1())
   100  				if err != nil || newReplicaSet == nil {
   101  					return false, err
   102  				}
   103  				if !w.deploymentReady(newReplicaSet, currentDeployment) {
   104  					return false, nil
   105  				}
   106  			case *extensionsv1beta1.Deployment:
   107  				currentDeployment, err := w.c.AppsV1().Deployments(value.Namespace).Get(value.Name, metav1.GetOptions{})
   108  				if err != nil {
   109  					return false, err
   110  				}
   111  				// Find RS associated with deployment
   112  				newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, w.c.AppsV1())
   113  				if err != nil || newReplicaSet == nil {
   114  					return false, err
   115  				}
   116  				if !w.deploymentReady(newReplicaSet, currentDeployment) {
   117  					return false, nil
   118  				}
   119  			case *corev1.PersistentVolumeClaim:
   120  				claim, err := w.c.CoreV1().PersistentVolumeClaims(value.Namespace).Get(value.Name, metav1.GetOptions{})
   121  				if err != nil {
   122  					return false, err
   123  				}
   124  				if !w.volumeReady(claim) {
   125  					return false, nil
   126  				}
   127  			case *corev1.Service:
   128  				svc, err := w.c.CoreV1().Services(value.Namespace).Get(value.Name, metav1.GetOptions{})
   129  				if err != nil {
   130  					return false, err
   131  				}
   132  				if !w.serviceReady(svc) {
   133  					return false, nil
   134  				}
   135  			case *extensionsv1beta1.DaemonSet, *appsv1.DaemonSet, *appsv1beta2.DaemonSet:
   136  				ds, err := w.c.AppsV1().DaemonSets(v.Namespace).Get(v.Name, metav1.GetOptions{})
   137  				if err != nil {
   138  					return false, err
   139  				}
   140  				if !w.daemonSetReady(ds) {
   141  					return false, nil
   142  				}
   143  			case *apiextv1beta1.CustomResourceDefinition:
   144  				if err := v.Get(); err != nil {
   145  					return false, err
   146  				}
   147  				crd := &apiextv1beta1.CustomResourceDefinition{}
   148  				if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
   149  					return false, err
   150  				}
   151  				if !w.crdReady(*crd) {
   152  					return false, nil
   153  				}
   154  			case *appsv1.StatefulSet, *appsv1beta1.StatefulSet, *appsv1beta2.StatefulSet:
   155  				sts, err := w.c.AppsV1().StatefulSets(v.Namespace).Get(v.Name, metav1.GetOptions{})
   156  				if err != nil {
   157  					return false, err
   158  				}
   159  				if !w.statefulSetReady(sts) {
   160  					return false, nil
   161  				}
   162  
   163  			case *corev1.ReplicationController:
   164  				ok, err = w.podsReadyForObject(value.Namespace, value)
   165  			case *extensionsv1beta1.ReplicaSet:
   166  				ok, err = w.podsReadyForObject(value.Namespace, value)
   167  			case *appsv1beta2.ReplicaSet:
   168  				ok, err = w.podsReadyForObject(value.Namespace, value)
   169  			case *appsv1.ReplicaSet:
   170  				ok, err = w.podsReadyForObject(value.Namespace, value)
   171  			}
   172  			if !ok || err != nil {
   173  				return false, err
   174  			}
   175  		}
   176  		return true, nil
   177  	})
   178  }
   179  
   180  func (w *waiter) podsReadyForObject(namespace string, obj runtime.Object) (bool, error) {
   181  	pods, err := w.podsforObject(namespace, obj)
   182  	if err != nil {
   183  		return false, err
   184  	}
   185  	for _, pod := range pods {
   186  		if !w.isPodReady(&pod) {
   187  			return false, nil
   188  		}
   189  	}
   190  	return true, nil
   191  }
   192  
   193  func (w *waiter) podsforObject(namespace string, obj runtime.Object) ([]corev1.Pod, error) {
   194  	selector, err := SelectorsForObject(obj)
   195  	if err != nil {
   196  		return nil, err
   197  	}
   198  	list, err := getPods(w.c, namespace, selector.String())
   199  	return list, err
   200  }
   201  
   202  // isPodReady returns true if a pod is ready; false otherwise.
   203  func (w *waiter) isPodReady(pod *corev1.Pod) bool {
   204  	for _, c := range pod.Status.Conditions {
   205  		if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
   206  			return true
   207  		}
   208  	}
   209  	w.log("Pod is not ready: %s/%s", pod.GetNamespace(), pod.GetName())
   210  	return false
   211  }
   212  
   213  func (w *waiter) serviceReady(s *corev1.Service) bool {
   214  	// ExternalName Services are external to cluster so helm shouldn't be checking to see if they're 'ready' (i.e. have an IP Set)
   215  	if s.Spec.Type == corev1.ServiceTypeExternalName {
   216  		return true
   217  	}
   218  
   219  	// Make sure the service is not explicitly set to "None" before checking the IP
   220  	if (s.Spec.ClusterIP != corev1.ClusterIPNone && s.Spec.ClusterIP == "") ||
   221  		// This checks if the service has a LoadBalancer and that balancer has an Ingress defined
   222  		(s.Spec.Type == corev1.ServiceTypeLoadBalancer && s.Status.LoadBalancer.Ingress == nil) {
   223  		w.log("Service does not have IP address: %s/%s", s.GetNamespace(), s.GetName())
   224  		return false
   225  	}
   226  	return true
   227  }
   228  
   229  func (w *waiter) volumeReady(v *corev1.PersistentVolumeClaim) bool {
   230  	if v.Status.Phase != corev1.ClaimBound {
   231  		w.log("PersistentVolumeClaim is not bound: %s/%s", v.GetNamespace(), v.GetName())
   232  		return false
   233  	}
   234  	return true
   235  }
   236  
   237  func (w *waiter) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool {
   238  	expectedReady := *dep.Spec.Replicas - deploymentutil.MaxUnavailable(*dep)
   239  	if !(rs.Status.ReadyReplicas >= expectedReady) {
   240  		w.log("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", dep.Namespace, dep.Name, rs.Status.ReadyReplicas, expectedReady)
   241  		return false
   242  	}
   243  	return true
   244  }
   245  
   246  func (w *waiter) daemonSetReady(ds *appsv1.DaemonSet) bool {
   247  	// If the update strategy is not a rolling update, there will be nothing to wait for
   248  	if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType {
   249  		return true
   250  	}
   251  
   252  	// Make sure all the updated pods have been scheduled
   253  	if ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled {
   254  		w.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", ds.Namespace, ds.Name, ds.Status.UpdatedNumberScheduled, ds.Status.DesiredNumberScheduled)
   255  		return false
   256  	}
   257  	maxUnavailable, err := intstr.GetValueFromIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, int(ds.Status.DesiredNumberScheduled), true)
   258  	if err != nil {
   259  		// If for some reason the value is invalid, set max unavailable to the
   260  		// number of desired replicas. This is the same behavior as the
   261  		// `MaxUnavailable` function in deploymentutil
   262  		maxUnavailable = int(ds.Status.DesiredNumberScheduled)
   263  	}
   264  
   265  	expectedReady := int(ds.Status.DesiredNumberScheduled) - maxUnavailable
   266  	if !(int(ds.Status.NumberReady) >= expectedReady) {
   267  		w.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods are ready", ds.Namespace, ds.Name, ds.Status.NumberReady, expectedReady)
   268  		return false
   269  	}
   270  	return true
   271  }
   272  
   273  func (w *waiter) crdReady(crd apiextv1beta1.CustomResourceDefinition) bool {
   274  	for _, cond := range crd.Status.Conditions {
   275  		switch cond.Type {
   276  		case apiextv1beta1.Established:
   277  			if cond.Status == apiextv1beta1.ConditionTrue {
   278  				return true
   279  			}
   280  		case apiextv1beta1.NamesAccepted:
   281  			if cond.Status == apiextv1beta1.ConditionFalse {
   282  				// This indicates a naming conflict, but it's probably not the
   283  				// job of this function to fail because of that. Instead,
   284  				// we treat it as a success, since the process should be able to
   285  				// continue.
   286  				return true
   287  			}
   288  		}
   289  	}
   290  	return false
   291  }
   292  
   293  func (w *waiter) statefulSetReady(sts *appsv1.StatefulSet) bool {
   294  	// If the update strategy is not a rolling update, there will be nothing to wait for
   295  	if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
   296  		return true
   297  	}
   298  
   299  	// Dereference all the pointers because StatefulSets like them
   300  	var partition int
   301  	// 1 is the default for replicas if not set
   302  	var replicas = 1
   303  	// For some reason, even if the update strategy is a rolling update, the
   304  	// actual rollingUpdate field can be nil. If it is, we can safely assume
   305  	// there is no partition value
   306  	if sts.Spec.UpdateStrategy.RollingUpdate != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
   307  		partition = int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition)
   308  	}
   309  	if sts.Spec.Replicas != nil {
   310  		replicas = int(*sts.Spec.Replicas)
   311  	}
   312  
   313  	// Because an update strategy can use partitioning, we need to calculate the
   314  	// number of updated replicas we should have. For example, if the replicas
   315  	// is set to 3 and the partition is 2, we'd expect only one pod to be
   316  	// updated
   317  	expectedReplicas := replicas - partition
   318  
   319  	// Make sure all the updated pods have been scheduled
   320  	if int(sts.Status.UpdatedReplicas) != expectedReplicas {
   321  		w.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", sts.Namespace, sts.Name, sts.Status.UpdatedReplicas, expectedReplicas)
   322  		return false
   323  	}
   324  
   325  	if int(sts.Status.ReadyReplicas) != replicas {
   326  		w.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas)
   327  		return false
   328  	}
   329  	return true
   330  }
   331  
   332  func getPods(client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) {
   333  	list, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{
   334  		LabelSelector: selector,
   335  	})
   336  	return list.Items, err
   337  }
   338  
   339  // SelectorsForObject returns the pod label selector for a given object
   340  //
   341  // Modified version of https://github.com/kubernetes/kubernetes/blob/v1.14.1/pkg/kubectl/polymorphichelpers/helpers.go#L84
   342  func SelectorsForObject(object runtime.Object) (selector labels.Selector, err error) {
   343  	switch t := object.(type) {
   344  	case *extensionsv1beta1.ReplicaSet:
   345  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   346  	case *appsv1.ReplicaSet:
   347  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   348  	case *appsv1beta2.ReplicaSet:
   349  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   350  	case *corev1.ReplicationController:
   351  		selector = labels.SelectorFromSet(t.Spec.Selector)
   352  	case *appsv1.StatefulSet:
   353  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   354  	case *appsv1beta1.StatefulSet:
   355  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   356  	case *appsv1beta2.StatefulSet:
   357  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   358  	case *extensionsv1beta1.DaemonSet:
   359  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   360  	case *appsv1.DaemonSet:
   361  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   362  	case *appsv1beta2.DaemonSet:
   363  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   364  	case *extensionsv1beta1.Deployment:
   365  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   366  	case *appsv1.Deployment:
   367  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   368  	case *appsv1beta1.Deployment:
   369  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   370  	case *appsv1beta2.Deployment:
   371  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   372  	case *batchv1.Job:
   373  		selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
   374  	case *corev1.Service:
   375  		if t.Spec.Selector == nil || len(t.Spec.Selector) == 0 {
   376  			return nil, fmt.Errorf("invalid service '%s': Service is defined without a selector", t.Name)
   377  		}
   378  		selector = labels.SelectorFromSet(t.Spec.Selector)
   379  
   380  	default:
   381  		return nil, fmt.Errorf("selector for %T not implemented", object)
   382  	}
   383  
   384  	return selector, errors.Wrap(err, "invalid label selector")
   385  }