github.com/stefanmcshane/helm@v0.0.0-20221213002717-88a4a2c6e77d/pkg/kube/ready.go (about) 1 /* 2 Copyright The Helm Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package kube // import "github.com/stefanmcshane/helm/pkg/kube" 18 19 import ( 20 "context" 21 22 appsv1 "k8s.io/api/apps/v1" 23 appsv1beta1 "k8s.io/api/apps/v1beta1" 24 appsv1beta2 "k8s.io/api/apps/v1beta2" 25 batchv1 "k8s.io/api/batch/v1" 26 corev1 "k8s.io/api/core/v1" 27 extensionsv1beta1 "k8s.io/api/extensions/v1beta1" 28 apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 29 apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/runtime" 32 "k8s.io/apimachinery/pkg/util/intstr" 33 "k8s.io/cli-runtime/pkg/resource" 34 "k8s.io/client-go/kubernetes" 35 "k8s.io/client-go/kubernetes/scheme" 36 37 deploymentutil "github.com/stefanmcshane/helm/internal/third_party/k8s.io/kubernetes/deployment/util" 38 ) 39 40 // ReadyCheckerOption is a function that configures a ReadyChecker. 41 type ReadyCheckerOption func(*ReadyChecker) 42 43 // PausedAsReady returns a ReadyCheckerOption that configures a ReadyChecker 44 // to consider paused resources to be ready. For example a Deployment 45 // with spec.paused equal to true would be considered ready. 46 func PausedAsReady(pausedAsReady bool) ReadyCheckerOption { 47 return func(c *ReadyChecker) { 48 c.pausedAsReady = pausedAsReady 49 } 50 } 51 52 // CheckJobs returns a ReadyCheckerOption that configures a ReadyChecker 53 // to consider readiness of Job resources. 54 func CheckJobs(checkJobs bool) ReadyCheckerOption { 55 return func(c *ReadyChecker) { 56 c.checkJobs = checkJobs 57 } 58 } 59 60 // NewReadyChecker creates a new checker. Passed ReadyCheckerOptions can 61 // be used to override defaults. 62 func NewReadyChecker(cl kubernetes.Interface, log func(string, ...interface{}), opts ...ReadyCheckerOption) ReadyChecker { 63 c := ReadyChecker{ 64 client: cl, 65 log: log, 66 } 67 if c.log == nil { 68 c.log = nopLogger 69 } 70 for _, opt := range opts { 71 opt(&c) 72 } 73 return c 74 } 75 76 // ReadyChecker is a type that can check core Kubernetes types for readiness. 77 type ReadyChecker struct { 78 client kubernetes.Interface 79 log func(string, ...interface{}) 80 checkJobs bool 81 pausedAsReady bool 82 } 83 84 // IsReady checks if v is ready. It supports checking readiness for pods, 85 // deployments, persistent volume claims, services, daemon sets, custom 86 // resource definitions, stateful sets, replication controllers, and replica 87 // sets. All other resource kinds are always considered ready. 88 // 89 // IsReady will fetch the latest state of the object from the server prior to 90 // performing readiness checks, and it will return any error encountered. 91 func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, error) { 92 switch value := AsVersioned(v).(type) { 93 case *corev1.Pod: 94 pod, err := c.client.CoreV1().Pods(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 95 if err != nil || !c.isPodReady(pod) { 96 return false, err 97 } 98 case *batchv1.Job: 99 if c.checkJobs { 100 job, err := c.client.BatchV1().Jobs(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 101 if err != nil || !c.jobReady(job) { 102 return false, err 103 } 104 } 105 case *appsv1.Deployment, *appsv1beta1.Deployment, *appsv1beta2.Deployment, *extensionsv1beta1.Deployment: 106 currentDeployment, err := c.client.AppsV1().Deployments(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 107 if err != nil { 108 return false, err 109 } 110 // If paused deployment will never be ready 111 if currentDeployment.Spec.Paused { 112 return c.pausedAsReady, nil 113 } 114 // Find RS associated with deployment 115 newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, c.client.AppsV1()) 116 if err != nil || newReplicaSet == nil { 117 return false, err 118 } 119 if !c.deploymentReady(newReplicaSet, currentDeployment) { 120 return false, nil 121 } 122 case *corev1.PersistentVolumeClaim: 123 claim, err := c.client.CoreV1().PersistentVolumeClaims(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 124 if err != nil { 125 return false, err 126 } 127 if !c.volumeReady(claim) { 128 return false, nil 129 } 130 case *corev1.Service: 131 svc, err := c.client.CoreV1().Services(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 132 if err != nil { 133 return false, err 134 } 135 if !c.serviceReady(svc) { 136 return false, nil 137 } 138 case *extensionsv1beta1.DaemonSet, *appsv1.DaemonSet, *appsv1beta2.DaemonSet: 139 ds, err := c.client.AppsV1().DaemonSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 140 if err != nil { 141 return false, err 142 } 143 if !c.daemonSetReady(ds) { 144 return false, nil 145 } 146 case *apiextv1beta1.CustomResourceDefinition: 147 if err := v.Get(); err != nil { 148 return false, err 149 } 150 crd := &apiextv1beta1.CustomResourceDefinition{} 151 if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil { 152 return false, err 153 } 154 if !c.crdBetaReady(*crd) { 155 return false, nil 156 } 157 case *apiextv1.CustomResourceDefinition: 158 if err := v.Get(); err != nil { 159 return false, err 160 } 161 crd := &apiextv1.CustomResourceDefinition{} 162 if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil { 163 return false, err 164 } 165 if !c.crdReady(*crd) { 166 return false, nil 167 } 168 case *appsv1.StatefulSet, *appsv1beta1.StatefulSet, *appsv1beta2.StatefulSet: 169 sts, err := c.client.AppsV1().StatefulSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 170 if err != nil { 171 return false, err 172 } 173 if !c.statefulSetReady(sts) { 174 return false, nil 175 } 176 case *corev1.ReplicationController: 177 rc, err := c.client.CoreV1().ReplicationControllers(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 178 if err != nil { 179 return false, err 180 } 181 if !c.replicationControllerReady(rc) { 182 return false, nil 183 } 184 ready, err := c.podsReadyForObject(ctx, v.Namespace, value) 185 if !ready || err != nil { 186 return false, err 187 } 188 case *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet: 189 rs, err := c.client.AppsV1().ReplicaSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{}) 190 if err != nil { 191 return false, err 192 } 193 if !c.replicaSetReady(rs) { 194 return false, nil 195 } 196 ready, err := c.podsReadyForObject(ctx, v.Namespace, value) 197 if !ready || err != nil { 198 return false, err 199 } 200 } 201 return true, nil 202 } 203 204 func (c *ReadyChecker) podsReadyForObject(ctx context.Context, namespace string, obj runtime.Object) (bool, error) { 205 pods, err := c.podsforObject(ctx, namespace, obj) 206 if err != nil { 207 return false, err 208 } 209 for _, pod := range pods { 210 if !c.isPodReady(&pod) { 211 return false, nil 212 } 213 } 214 return true, nil 215 } 216 217 func (c *ReadyChecker) podsforObject(ctx context.Context, namespace string, obj runtime.Object) ([]corev1.Pod, error) { 218 selector, err := SelectorsForObject(obj) 219 if err != nil { 220 return nil, err 221 } 222 list, err := getPods(ctx, c.client, namespace, selector.String()) 223 return list, err 224 } 225 226 // isPodReady returns true if a pod is ready; false otherwise. 227 func (c *ReadyChecker) isPodReady(pod *corev1.Pod) bool { 228 for _, c := range pod.Status.Conditions { 229 if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { 230 return true 231 } 232 } 233 c.log("Pod is not ready: %s/%s", pod.GetNamespace(), pod.GetName()) 234 return false 235 } 236 237 func (c *ReadyChecker) jobReady(job *batchv1.Job) bool { 238 if job.Status.Failed > *job.Spec.BackoffLimit { 239 c.log("Job is failed: %s/%s", job.GetNamespace(), job.GetName()) 240 return false 241 } 242 if job.Spec.Completions != nil && job.Status.Succeeded < *job.Spec.Completions { 243 c.log("Job is not completed: %s/%s", job.GetNamespace(), job.GetName()) 244 return false 245 } 246 return true 247 } 248 249 func (c *ReadyChecker) serviceReady(s *corev1.Service) bool { 250 // ExternalName Services are external to cluster so helm shouldn't be checking to see if they're 'ready' (i.e. have an IP Set) 251 if s.Spec.Type == corev1.ServiceTypeExternalName { 252 return true 253 } 254 255 // Ensure that the service cluster IP is not empty 256 if s.Spec.ClusterIP == "" { 257 c.log("Service does not have cluster IP address: %s/%s", s.GetNamespace(), s.GetName()) 258 return false 259 } 260 261 // This checks if the service has a LoadBalancer and that balancer has an Ingress defined 262 if s.Spec.Type == corev1.ServiceTypeLoadBalancer { 263 // do not wait when at least 1 external IP is set 264 if len(s.Spec.ExternalIPs) > 0 { 265 c.log("Service %s/%s has external IP addresses (%v), marking as ready", s.GetNamespace(), s.GetName(), s.Spec.ExternalIPs) 266 return true 267 } 268 269 if s.Status.LoadBalancer.Ingress == nil { 270 c.log("Service does not have load balancer ingress IP address: %s/%s", s.GetNamespace(), s.GetName()) 271 return false 272 } 273 } 274 275 return true 276 } 277 278 func (c *ReadyChecker) volumeReady(v *corev1.PersistentVolumeClaim) bool { 279 if v.Status.Phase != corev1.ClaimBound { 280 c.log("PersistentVolumeClaim is not bound: %s/%s", v.GetNamespace(), v.GetName()) 281 return false 282 } 283 return true 284 } 285 286 func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool { 287 // Verify the replicaset readiness 288 if !c.replicaSetReady(rs) { 289 return false 290 } 291 // Verify the generation observed by the deployment controller matches the spec generation 292 if dep.Status.ObservedGeneration != dep.ObjectMeta.Generation { 293 c.log("Deployment is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", dep.Namespace, dep.Name, dep.Status.ObservedGeneration, dep.ObjectMeta.Generation) 294 return false 295 } 296 297 expectedReady := *dep.Spec.Replicas - deploymentutil.MaxUnavailable(*dep) 298 if !(rs.Status.ReadyReplicas >= expectedReady) { 299 c.log("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", dep.Namespace, dep.Name, rs.Status.ReadyReplicas, expectedReady) 300 return false 301 } 302 return true 303 } 304 305 func (c *ReadyChecker) daemonSetReady(ds *appsv1.DaemonSet) bool { 306 // Verify the generation observed by the daemonSet controller matches the spec generation 307 if ds.Status.ObservedGeneration != ds.ObjectMeta.Generation { 308 c.log("DaemonSet is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", ds.Namespace, ds.Name, ds.Status.ObservedGeneration, ds.ObjectMeta.Generation) 309 return false 310 } 311 312 // If the update strategy is not a rolling update, there will be nothing to wait for 313 if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { 314 return true 315 } 316 317 // Make sure all the updated pods have been scheduled 318 if ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled { 319 c.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", ds.Namespace, ds.Name, ds.Status.UpdatedNumberScheduled, ds.Status.DesiredNumberScheduled) 320 return false 321 } 322 maxUnavailable, err := intstr.GetValueFromIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, int(ds.Status.DesiredNumberScheduled), true) 323 if err != nil { 324 // If for some reason the value is invalid, set max unavailable to the 325 // number of desired replicas. This is the same behavior as the 326 // `MaxUnavailable` function in deploymentutil 327 maxUnavailable = int(ds.Status.DesiredNumberScheduled) 328 } 329 330 expectedReady := int(ds.Status.DesiredNumberScheduled) - maxUnavailable 331 if !(int(ds.Status.NumberReady) >= expectedReady) { 332 c.log("DaemonSet is not ready: %s/%s. %d out of %d expected pods are ready", ds.Namespace, ds.Name, ds.Status.NumberReady, expectedReady) 333 return false 334 } 335 return true 336 } 337 338 // Because the v1 extensions API is not available on all supported k8s versions 339 // yet and because Go doesn't support generics, we need to have a duplicate 340 // function to support the v1beta1 types 341 func (c *ReadyChecker) crdBetaReady(crd apiextv1beta1.CustomResourceDefinition) bool { 342 for _, cond := range crd.Status.Conditions { 343 switch cond.Type { 344 case apiextv1beta1.Established: 345 if cond.Status == apiextv1beta1.ConditionTrue { 346 return true 347 } 348 case apiextv1beta1.NamesAccepted: 349 if cond.Status == apiextv1beta1.ConditionFalse { 350 // This indicates a naming conflict, but it's probably not the 351 // job of this function to fail because of that. Instead, 352 // we treat it as a success, since the process should be able to 353 // continue. 354 return true 355 } 356 } 357 } 358 return false 359 } 360 361 func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool { 362 for _, cond := range crd.Status.Conditions { 363 switch cond.Type { 364 case apiextv1.Established: 365 if cond.Status == apiextv1.ConditionTrue { 366 return true 367 } 368 case apiextv1.NamesAccepted: 369 if cond.Status == apiextv1.ConditionFalse { 370 // This indicates a naming conflict, but it's probably not the 371 // job of this function to fail because of that. Instead, 372 // we treat it as a success, since the process should be able to 373 // continue. 374 return true 375 } 376 } 377 } 378 return false 379 } 380 381 func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool { 382 // Verify the generation observed by the statefulSet controller matches the spec generation 383 if sts.Status.ObservedGeneration != sts.ObjectMeta.Generation { 384 c.log("Statefulset is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", sts.Namespace, sts.Name, sts.Status.ObservedGeneration, sts.ObjectMeta.Generation) 385 return false 386 } 387 388 // If the update strategy is not a rolling update, there will be nothing to wait for 389 if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType { 390 c.log("StatefulSet skipped ready check: %s/%s. updateStrategy is %v", sts.Namespace, sts.Name, sts.Spec.UpdateStrategy.Type) 391 return true 392 } 393 394 // Dereference all the pointers because StatefulSets like them 395 var partition int 396 // 1 is the default for replicas if not set 397 var replicas = 1 398 // For some reason, even if the update strategy is a rolling update, the 399 // actual rollingUpdate field can be nil. If it is, we can safely assume 400 // there is no partition value 401 if sts.Spec.UpdateStrategy.RollingUpdate != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil { 402 partition = int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition) 403 } 404 if sts.Spec.Replicas != nil { 405 replicas = int(*sts.Spec.Replicas) 406 } 407 408 // Because an update strategy can use partitioning, we need to calculate the 409 // number of updated replicas we should have. For example, if the replicas 410 // is set to 3 and the partition is 2, we'd expect only one pod to be 411 // updated 412 expectedReplicas := replicas - partition 413 414 // Make sure all the updated pods have been scheduled 415 if int(sts.Status.UpdatedReplicas) < expectedReplicas { 416 c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", sts.Namespace, sts.Name, sts.Status.UpdatedReplicas, expectedReplicas) 417 return false 418 } 419 420 if int(sts.Status.ReadyReplicas) != replicas { 421 c.log("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas) 422 return false 423 } 424 425 if sts.Status.CurrentRevision != sts.Status.UpdateRevision { 426 c.log("StatefulSet is not ready: %s/%s. currentRevision %s does not yet match updateRevision %s", sts.Namespace, sts.Name, sts.Status.CurrentRevision, sts.Status.UpdateRevision) 427 return false 428 } 429 430 c.log("StatefulSet is ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas) 431 return true 432 } 433 434 func (c *ReadyChecker) replicationControllerReady(rc *corev1.ReplicationController) bool { 435 // Verify the generation observed by the replicationController controller matches the spec generation 436 if rc.Status.ObservedGeneration != rc.ObjectMeta.Generation { 437 c.log("ReplicationController is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", rc.Namespace, rc.Name, rc.Status.ObservedGeneration, rc.ObjectMeta.Generation) 438 return false 439 } 440 return true 441 } 442 443 func (c *ReadyChecker) replicaSetReady(rs *appsv1.ReplicaSet) bool { 444 // Verify the generation observed by the replicaSet controller matches the spec generation 445 if rs.Status.ObservedGeneration != rs.ObjectMeta.Generation { 446 c.log("ReplicaSet is not ready: %s/%s. observedGeneration (%s) does not match spec generation (%s).", rs.Namespace, rs.Name, rs.Status.ObservedGeneration, rs.ObjectMeta.Generation) 447 return false 448 } 449 return true 450 } 451 452 func getPods(ctx context.Context, client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) { 453 list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ 454 LabelSelector: selector, 455 }) 456 return list.Items, err 457 }