github.com/cloudposse/helm@v2.2.3+incompatible/pkg/kube/client.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors All rights reserved.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kube // import "k8s.io/helm/pkg/kube"
    18  
    19  import (
    20  	"bytes"
    21  	"encoding/json"
    22  	goerrors "errors"
    23  	"fmt"
    24  	"io"
    25  	"log"
    26  	"strings"
    27  	"time"
    28  
    29  	jsonpatch "github.com/evanphx/json-patch"
    30  	"k8s.io/kubernetes/pkg/api"
    31  	"k8s.io/kubernetes/pkg/api/errors"
    32  	"k8s.io/kubernetes/pkg/api/meta"
    33  	"k8s.io/kubernetes/pkg/api/v1"
    34  	apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
    35  	batchinternal "k8s.io/kubernetes/pkg/apis/batch"
    36  	batch "k8s.io/kubernetes/pkg/apis/batch/v1"
    37  	extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
    38  	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
    39  	conditions "k8s.io/kubernetes/pkg/client/unversioned"
    40  	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
    41  	"k8s.io/kubernetes/pkg/fields"
    42  	"k8s.io/kubernetes/pkg/kubectl"
    43  	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
    44  	"k8s.io/kubernetes/pkg/kubectl/resource"
    45  	"k8s.io/kubernetes/pkg/labels"
    46  	"k8s.io/kubernetes/pkg/runtime"
    47  	"k8s.io/kubernetes/pkg/util/strategicpatch"
    48  	"k8s.io/kubernetes/pkg/util/wait"
    49  	"k8s.io/kubernetes/pkg/watch"
    50  )
    51  
    52  // ErrNoObjectsVisited indicates that during a visit operation, no matching objects were found.
    53  var ErrNoObjectsVisited = goerrors.New("no objects visited")
    54  
    55  // Client represents a client capable of communicating with the Kubernetes API.
    56  type Client struct {
    57  	cmdutil.Factory
    58  	// SchemaCacheDir is the path for loading cached schema.
    59  	SchemaCacheDir string
    60  }
    61  
    62  // New create a new Client
    63  func New(config clientcmd.ClientConfig) *Client {
    64  	return &Client{
    65  		Factory:        cmdutil.NewFactory(config),
    66  		SchemaCacheDir: clientcmd.RecommendedSchemaFile,
    67  	}
    68  }
    69  
    70  // ResourceActorFunc performs an action on a single resource.
    71  type ResourceActorFunc func(*resource.Info) error
    72  
    73  // Create creates kubernetes resources from an io.reader
    74  //
    75  // Namespace will set the namespace
    76  func (c *Client) Create(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
    77  	client, err := c.ClientSet()
    78  	if err != nil {
    79  		return err
    80  	}
    81  	if err := ensureNamespace(client, namespace); err != nil {
    82  		return err
    83  	}
    84  	infos, buildErr := c.BuildUnstructured(namespace, reader)
    85  	if buildErr != nil {
    86  		return buildErr
    87  	}
    88  	if err := perform(c, namespace, infos, createResource); err != nil {
    89  		return err
    90  	}
    91  	if shouldWait {
    92  		return c.waitForResources(time.Duration(timeout)*time.Second, infos)
    93  	}
    94  	return nil
    95  }
    96  
    97  func (c *Client) newBuilder(namespace string, reader io.Reader) *resource.Result {
    98  	schema, err := c.Validator(true, c.SchemaCacheDir)
    99  	if err != nil {
   100  		log.Printf("warning: failed to load schema: %s", err)
   101  	}
   102  	return c.NewBuilder().
   103  		ContinueOnError().
   104  		Schema(schema).
   105  		NamespaceParam(namespace).
   106  		DefaultNamespace().
   107  		Stream(reader, "").
   108  		Flatten().
   109  		Do()
   110  }
   111  
   112  // BuildUnstructured validates for Kubernetes objects and returns unstructured infos.
   113  func (c *Client) BuildUnstructured(namespace string, reader io.Reader) (Result, error) {
   114  	schema, err := c.Validator(true, c.SchemaCacheDir)
   115  	if err != nil {
   116  		log.Printf("warning: failed to load schema: %s", err)
   117  	}
   118  
   119  	mapper, typer, err := c.UnstructuredObject()
   120  	if err != nil {
   121  		log.Printf("failed to load mapper: %s", err)
   122  		return nil, err
   123  	}
   124  	var result Result
   125  	result, err = resource.NewBuilder(mapper, typer, resource.ClientMapperFunc(c.UnstructuredClientForMapping), runtime.UnstructuredJSONScheme).
   126  		ContinueOnError().
   127  		Schema(schema).
   128  		NamespaceParam(namespace).
   129  		DefaultNamespace().
   130  		Stream(reader, "").
   131  		Flatten().
   132  		Do().Infos()
   133  	return result, scrubValidationError(err)
   134  }
   135  
   136  // Build validates for Kubernetes objects and returns resource Infos from a io.Reader.
   137  func (c *Client) Build(namespace string, reader io.Reader) (Result, error) {
   138  	var result Result
   139  	result, err := c.newBuilder(namespace, reader).Infos()
   140  	return result, scrubValidationError(err)
   141  }
   142  
   143  // Get gets kubernetes resources as pretty printed string
   144  //
   145  // Namespace will set the namespace
   146  func (c *Client) Get(namespace string, reader io.Reader) (string, error) {
   147  	// Since we don't know what order the objects come in, let's group them by the types, so
   148  	// that when we print them, they come looking good (headers apply to subgroups, etc.)
   149  	objs := make(map[string][]runtime.Object)
   150  	infos, err := c.BuildUnstructured(namespace, reader)
   151  	if err != nil {
   152  		return "", err
   153  	}
   154  	err = perform(c, namespace, infos, func(info *resource.Info) error {
   155  		log.Printf("Doing get for: '%s'", info.Name)
   156  		obj, err := resource.NewHelper(info.Client, info.Mapping).Get(info.Namespace, info.Name, info.Export)
   157  		if err != nil {
   158  			return err
   159  		}
   160  		// We need to grab the ObjectReference so we can correctly group the objects.
   161  		or, err := api.GetReference(obj)
   162  		if err != nil {
   163  			log.Printf("FAILED GetReference for: %#v\n%v", obj, err)
   164  			return err
   165  		}
   166  
   167  		// Use APIVersion/Kind as grouping mechanism. I'm not sure if you can have multiple
   168  		// versions per cluster, but this certainly won't hurt anything, so let's be safe.
   169  		objType := or.APIVersion + "/" + or.Kind
   170  		objs[objType] = append(objs[objType], obj)
   171  		return nil
   172  	})
   173  	if err != nil {
   174  		return "", err
   175  	}
   176  
   177  	// Ok, now we have all the objects grouped by types (say, by v1/Pod, v1/Service, etc.), so
   178  	// spin through them and print them. Printer is cool since it prints the header only when
   179  	// an object type changes, so we can just rely on that. Problem is it doesn't seem to keep
   180  	// track of tab widths
   181  	buf := new(bytes.Buffer)
   182  	p := kubectl.NewHumanReadablePrinter(kubectl.PrintOptions{})
   183  	for t, ot := range objs {
   184  		if _, err = buf.WriteString("==> " + t + "\n"); err != nil {
   185  			return "", err
   186  		}
   187  		for _, o := range ot {
   188  			if err := p.PrintObj(o, buf); err != nil {
   189  				log.Printf("failed to print object type '%s', object: '%s' :\n %v", t, o, err)
   190  				return "", err
   191  			}
   192  		}
   193  		if _, err := buf.WriteString("\n"); err != nil {
   194  			return "", err
   195  		}
   196  	}
   197  	return buf.String(), nil
   198  }
   199  
   200  // Update reads in the current configuration and a target configuration from io.reader
   201  //  and creates resources that don't already exists, updates resources that have been modified
   202  //  in the target configuration and deletes resources from the current configuration that are
   203  //  not present in the target configuration
   204  //
   205  // Namespace will set the namespaces
   206  func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, recreate bool, timeout int64, shouldWait bool) error {
   207  	original, err := c.BuildUnstructured(namespace, originalReader)
   208  	if err != nil {
   209  		return fmt.Errorf("failed decoding reader into objects: %s", err)
   210  	}
   211  
   212  	target, err := c.BuildUnstructured(namespace, targetReader)
   213  	if err != nil {
   214  		return fmt.Errorf("failed decoding reader into objects: %s", err)
   215  	}
   216  
   217  	updateErrors := []string{}
   218  
   219  	err = target.Visit(func(info *resource.Info, err error) error {
   220  		if err != nil {
   221  			return err
   222  		}
   223  
   224  		helper := resource.NewHelper(info.Client, info.Mapping)
   225  		if _, err := helper.Get(info.Namespace, info.Name, info.Export); err != nil {
   226  			if !errors.IsNotFound(err) {
   227  				return fmt.Errorf("Could not get information about the resource: err: %s", err)
   228  			}
   229  
   230  			// Since the resource does not exist, create it.
   231  			if err := createResource(info); err != nil {
   232  				return fmt.Errorf("failed to create resource: %s", err)
   233  			}
   234  
   235  			kind := info.Mapping.GroupVersionKind.Kind
   236  			log.Printf("Created a new %s called %s\n", kind, info.Name)
   237  			return nil
   238  		}
   239  
   240  		originalInfo := original.Get(info)
   241  		if originalInfo == nil {
   242  			return fmt.Errorf("no resource with the name %s found", info.Name)
   243  		}
   244  
   245  		if err := updateResource(c, info, originalInfo.Object, recreate); err != nil {
   246  			log.Printf("error updating the resource %s:\n\t %v", info.Name, err)
   247  			updateErrors = append(updateErrors, err.Error())
   248  		}
   249  
   250  		return nil
   251  	})
   252  
   253  	switch {
   254  	case err != nil:
   255  		return err
   256  	case len(updateErrors) != 0:
   257  		return fmt.Errorf(strings.Join(updateErrors, " && "))
   258  	}
   259  
   260  	for _, info := range original.Difference(target) {
   261  		log.Printf("Deleting %s in %s...", info.Name, info.Namespace)
   262  		if err := deleteResource(c, info); err != nil {
   263  			log.Printf("Failed to delete %s, err: %s", info.Name, err)
   264  		}
   265  	}
   266  	if shouldWait {
   267  		return c.waitForResources(time.Duration(timeout)*time.Second, target)
   268  	}
   269  	return nil
   270  }
   271  
   272  // Delete deletes kubernetes resources from an io.reader
   273  //
   274  // Namespace will set the namespace
   275  func (c *Client) Delete(namespace string, reader io.Reader) error {
   276  	infos, err := c.BuildUnstructured(namespace, reader)
   277  	if err != nil {
   278  		return err
   279  	}
   280  	return perform(c, namespace, infos, func(info *resource.Info) error {
   281  		log.Printf("Starting delete for %s %s", info.Name, info.Mapping.GroupVersionKind.Kind)
   282  		err := deleteResource(c, info)
   283  		return skipIfNotFound(err)
   284  	})
   285  }
   286  
   287  func skipIfNotFound(err error) error {
   288  	if errors.IsNotFound(err) {
   289  		log.Printf("%v", err)
   290  		return nil
   291  	}
   292  	return err
   293  }
   294  
   295  func watchTimeout(t time.Duration) ResourceActorFunc {
   296  	return func(info *resource.Info) error {
   297  		return watchUntilReady(t, info)
   298  	}
   299  }
   300  
   301  // WatchUntilReady watches the resource given in the reader, and waits until it is ready.
   302  //
   303  // This function is mainly for hook implementations. It watches for a resource to
   304  // hit a particular milestone. The milestone depends on the Kind.
   305  //
   306  // For most kinds, it checks to see if the resource is marked as Added or Modified
   307  // by the Kubernetes event stream. For some kinds, it does more:
   308  //
   309  // - Jobs: A job is marked "Ready" when it has successfully completed. This is
   310  //   ascertained by watching the Status fields in a job's output.
   311  //
   312  // Handling for other kinds will be added as necessary.
   313  func (c *Client) WatchUntilReady(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
   314  	infos, err := c.Build(namespace, reader)
   315  	if err != nil {
   316  		return err
   317  	}
   318  	// For jobs, there's also the option to do poll c.Jobs(namespace).Get():
   319  	// https://github.com/adamreese/kubernetes/blob/master/test/e2e/job.go#L291-L300
   320  	return perform(c, namespace, infos, watchTimeout(time.Duration(timeout)*time.Second))
   321  }
   322  
   323  func perform(c *Client, namespace string, infos Result, fn ResourceActorFunc) error {
   324  	if len(infos) == 0 {
   325  		return ErrNoObjectsVisited
   326  	}
   327  
   328  	for _, info := range infos {
   329  		if err := fn(info); err != nil {
   330  			return err
   331  		}
   332  	}
   333  	return nil
   334  }
   335  
   336  func createResource(info *resource.Info) error {
   337  	obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object)
   338  	if err != nil {
   339  		return err
   340  	}
   341  	return info.Refresh(obj, true)
   342  }
   343  
   344  func deleteResource(c *Client, info *resource.Info) error {
   345  	reaper, err := c.Reaper(info.Mapping)
   346  	if err != nil {
   347  		// If there is no reaper for this resources, delete it.
   348  		if kubectl.IsNoSuchReaperError(err) {
   349  			return resource.NewHelper(info.Client, info.Mapping).Delete(info.Namespace, info.Name)
   350  		}
   351  		return err
   352  	}
   353  	log.Printf("Using reaper for deleting %s", info.Name)
   354  	return reaper.Stop(info.Namespace, info.Name, 0, nil)
   355  }
   356  
   357  func createPatch(mapping *meta.RESTMapping, target, current runtime.Object) ([]byte, api.PatchType, error) {
   358  	oldData, err := json.Marshal(current)
   359  	if err != nil {
   360  		return nil, api.StrategicMergePatchType, fmt.Errorf("serializing current configuration: %s", err)
   361  	}
   362  	newData, err := json.Marshal(target)
   363  	if err != nil {
   364  		return nil, api.StrategicMergePatchType, fmt.Errorf("serializing target configuration: %s", err)
   365  	}
   366  
   367  	if api.Semantic.DeepEqual(oldData, newData) {
   368  		return nil, api.StrategicMergePatchType, nil
   369  	}
   370  
   371  	// Get a versioned object
   372  	versionedObject, err := api.Scheme.New(mapping.GroupVersionKind)
   373  	switch {
   374  	case runtime.IsNotRegisteredError(err):
   375  		// fall back to generic JSON merge patch
   376  		patch, err := jsonpatch.CreateMergePatch(oldData, newData)
   377  		return patch, api.MergePatchType, err
   378  	case err != nil:
   379  		return nil, api.StrategicMergePatchType, fmt.Errorf("failed to get versionedObject: %s", err)
   380  	default:
   381  		log.Printf("generating strategic merge patch for %T", target)
   382  		patch, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, versionedObject)
   383  		return patch, api.StrategicMergePatchType, err
   384  	}
   385  }
   386  
   387  func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, recreate bool) error {
   388  	patch, patchType, err := createPatch(target.Mapping, target.Object, currentObj)
   389  	if err != nil {
   390  		return fmt.Errorf("failed to create patch: %s", err)
   391  	}
   392  	if patch == nil {
   393  		log.Printf("Looks like there are no changes for %s", target.Name)
   394  		// This needs to happen to make sure that tiller has the latest info from the API
   395  		// Otherwise there will be no labels and other functions that use labels will panic
   396  		if err := target.Get(); err != nil {
   397  			return fmt.Errorf("error trying to refresh resource information: %v", err)
   398  		}
   399  		return nil
   400  	}
   401  
   402  	// send patch to server
   403  	helper := resource.NewHelper(target.Client, target.Mapping)
   404  	obj, err := helper.Patch(target.Namespace, target.Name, patchType, patch)
   405  	if err != nil {
   406  		return err
   407  	}
   408  
   409  	target.Refresh(obj, true)
   410  
   411  	if !recreate {
   412  		return nil
   413  	}
   414  
   415  	versioned, err := c.AsVersionedObject(target.Object)
   416  	if runtime.IsNotRegisteredError(err) {
   417  		return nil
   418  	}
   419  	if err != nil {
   420  		return err
   421  	}
   422  
   423  	selector, err := getSelectorFromObject(versioned)
   424  	if err != nil {
   425  		return nil
   426  	}
   427  	client, _ := c.ClientSet()
   428  	return recreatePods(client, target.Namespace, selector)
   429  }
   430  
   431  func getSelectorFromObject(obj runtime.Object) (map[string]string, error) {
   432  	switch typed := obj.(type) {
   433  	case *v1.ReplicationController:
   434  		return typed.Spec.Selector, nil
   435  	case *extensions.ReplicaSet:
   436  		return typed.Spec.Selector.MatchLabels, nil
   437  	case *extensions.Deployment:
   438  		return typed.Spec.Selector.MatchLabels, nil
   439  	case *extensions.DaemonSet:
   440  		return typed.Spec.Selector.MatchLabels, nil
   441  	case *batch.Job:
   442  		return typed.Spec.Selector.MatchLabels, nil
   443  	case *apps.StatefulSet:
   444  		return typed.Spec.Selector.MatchLabels, nil
   445  	default:
   446  		return nil, fmt.Errorf("Unsupported kind when getting selector: %v", obj)
   447  	}
   448  }
   449  
   450  func recreatePods(client *internalclientset.Clientset, namespace string, selector map[string]string) error {
   451  	pods, err := client.Pods(namespace).List(api.ListOptions{
   452  		FieldSelector: fields.Everything(),
   453  		LabelSelector: labels.Set(selector).AsSelector(),
   454  	})
   455  	if err != nil {
   456  		return err
   457  	}
   458  
   459  	// Restart pods
   460  	for _, pod := range pods.Items {
   461  		log.Printf("Restarting pod: %v/%v", pod.Namespace, pod.Name)
   462  
   463  		// Delete each pod for get them restarted with changed spec.
   464  		if err := client.Pods(pod.Namespace).Delete(pod.Name, api.NewPreconditionDeleteOptions(string(pod.UID))); err != nil {
   465  			return err
   466  		}
   467  	}
   468  	return nil
   469  }
   470  
   471  func watchUntilReady(timeout time.Duration, info *resource.Info) error {
   472  	w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion)
   473  	if err != nil {
   474  		return err
   475  	}
   476  
   477  	kind := info.Mapping.GroupVersionKind.Kind
   478  	log.Printf("Watching for changes to %s %s with timeout of %v", kind, info.Name, timeout)
   479  
   480  	// What we watch for depends on the Kind.
   481  	// - For a Job, we watch for completion.
   482  	// - For all else, we watch until Ready.
   483  	// In the future, we might want to add some special logic for types
   484  	// like Ingress, Volume, etc.
   485  
   486  	_, err = watch.Until(timeout, w, func(e watch.Event) (bool, error) {
   487  		switch e.Type {
   488  		case watch.Added, watch.Modified:
   489  			// For things like a secret or a config map, this is the best indicator
   490  			// we get. We care mostly about jobs, where what we want to see is
   491  			// the status go into a good state. For other types, like ReplicaSet
   492  			// we don't really do anything to support these as hooks.
   493  			log.Printf("Add/Modify event for %s: %v", info.Name, e.Type)
   494  			if kind == "Job" {
   495  				return waitForJob(e, info.Name)
   496  			}
   497  			return true, nil
   498  		case watch.Deleted:
   499  			log.Printf("Deleted event for %s", info.Name)
   500  			return true, nil
   501  		case watch.Error:
   502  			// Handle error and return with an error.
   503  			log.Printf("Error event for %s", info.Name)
   504  			return true, fmt.Errorf("Failed to deploy %s", info.Name)
   505  		default:
   506  			return false, nil
   507  		}
   508  	})
   509  	return err
   510  }
   511  
   512  func podsReady(pods []api.Pod) bool {
   513  	for _, pod := range pods {
   514  		if !api.IsPodReady(&pod) {
   515  			return false
   516  		}
   517  	}
   518  	return true
   519  }
   520  
   521  func servicesReady(svc []api.Service) bool {
   522  	for _, s := range svc {
   523  		// Make sure the service is not explicitly set to "None" before checking the IP
   524  		if s.Spec.ClusterIP != api.ClusterIPNone && !api.IsServiceIPSet(&s) {
   525  			return false
   526  		}
   527  		// This checks if the service has a LoadBalancer and that balancer has an Ingress defined
   528  		if s.Spec.Type == api.ServiceTypeLoadBalancer && s.Status.LoadBalancer.Ingress == nil {
   529  			return false
   530  		}
   531  	}
   532  	return true
   533  }
   534  
   535  func volumesReady(vols []api.PersistentVolumeClaim) bool {
   536  	for _, v := range vols {
   537  		if v.Status.Phase != api.ClaimBound {
   538  			return false
   539  		}
   540  	}
   541  	return true
   542  }
   543  
   544  func getPods(client *internalclientset.Clientset, namespace string, selector map[string]string) ([]api.Pod, error) {
   545  	list, err := client.Pods(namespace).List(api.ListOptions{
   546  		FieldSelector: fields.Everything(),
   547  		LabelSelector: labels.Set(selector).AsSelector(),
   548  	})
   549  	return list.Items, err
   550  }
   551  
   552  // AsVersionedObject converts a runtime.object to a versioned object.
   553  func (c *Client) AsVersionedObject(obj runtime.Object) (runtime.Object, error) {
   554  	json, err := runtime.Encode(runtime.UnstructuredJSONScheme, obj)
   555  	if err != nil {
   556  		return nil, err
   557  	}
   558  	versions := &runtime.VersionedObjects{}
   559  	err = runtime.DecodeInto(c.Decoder(true), json, versions)
   560  	return versions.First(), err
   561  }
   562  
   563  // waitForResources polls to get the current status of all pods, PVCs, and Services
   564  // until all are ready or a timeout is reached
   565  func (c *Client) waitForResources(timeout time.Duration, created Result) error {
   566  	log.Printf("beginning wait for resources with timeout of %v", timeout)
   567  	client, _ := c.ClientSet()
   568  	return wait.Poll(2*time.Second, timeout, func() (bool, error) {
   569  		pods := []api.Pod{}
   570  		services := []api.Service{}
   571  		pvc := []api.PersistentVolumeClaim{}
   572  		for _, v := range created {
   573  			obj, err := c.AsVersionedObject(v.Object)
   574  			if err != nil && !runtime.IsNotRegisteredError(err) {
   575  				return false, err
   576  			}
   577  			switch value := obj.(type) {
   578  			case (*v1.ReplicationController):
   579  				list, err := getPods(client, value.Namespace, value.Spec.Selector)
   580  				if err != nil {
   581  					return false, err
   582  				}
   583  				pods = append(pods, list...)
   584  			case (*v1.Pod):
   585  				pod, err := client.Pods(value.Namespace).Get(value.Name)
   586  				if err != nil {
   587  					return false, err
   588  				}
   589  				pods = append(pods, *pod)
   590  			case (*extensions.Deployment):
   591  				// Get the RS children first
   592  				rs, err := client.ReplicaSets(value.Namespace).List(api.ListOptions{
   593  					FieldSelector: fields.Everything(),
   594  					LabelSelector: labels.Set(value.Spec.Selector.MatchLabels).AsSelector(),
   595  				})
   596  				if err != nil {
   597  					return false, err
   598  				}
   599  				for _, r := range rs.Items {
   600  					list, err := getPods(client, value.Namespace, r.Spec.Selector.MatchLabels)
   601  					if err != nil {
   602  						return false, err
   603  					}
   604  					pods = append(pods, list...)
   605  				}
   606  			case (*extensions.DaemonSet):
   607  				list, err := getPods(client, value.Namespace, value.Spec.Selector.MatchLabels)
   608  				if err != nil {
   609  					return false, err
   610  				}
   611  				pods = append(pods, list...)
   612  			case (*apps.StatefulSet):
   613  				list, err := getPods(client, value.Namespace, value.Spec.Selector.MatchLabels)
   614  				if err != nil {
   615  					return false, err
   616  				}
   617  				pods = append(pods, list...)
   618  			case (*extensions.ReplicaSet):
   619  				list, err := getPods(client, value.Namespace, value.Spec.Selector.MatchLabels)
   620  				if err != nil {
   621  					return false, err
   622  				}
   623  				pods = append(pods, list...)
   624  			case (*v1.PersistentVolumeClaim):
   625  				claim, err := client.PersistentVolumeClaims(value.Namespace).Get(value.Name)
   626  				if err != nil {
   627  					return false, err
   628  				}
   629  				pvc = append(pvc, *claim)
   630  			case (*v1.Service):
   631  				svc, err := client.Services(value.Namespace).Get(value.Name)
   632  				if err != nil {
   633  					return false, err
   634  				}
   635  				services = append(services, *svc)
   636  			}
   637  		}
   638  		return podsReady(pods) && servicesReady(services) && volumesReady(pvc), nil
   639  	})
   640  }
   641  
   642  // waitForJob is a helper that waits for a job to complete.
   643  //
   644  // This operates on an event returned from a watcher.
   645  func waitForJob(e watch.Event, name string) (bool, error) {
   646  	o, ok := e.Object.(*batchinternal.Job)
   647  	if !ok {
   648  		return true, fmt.Errorf("Expected %s to be a *batch.Job, got %T", name, e.Object)
   649  	}
   650  
   651  	for _, c := range o.Status.Conditions {
   652  		if c.Type == batchinternal.JobComplete && c.Status == api.ConditionTrue {
   653  			return true, nil
   654  		} else if c.Type == batchinternal.JobFailed && c.Status == api.ConditionTrue {
   655  			return true, fmt.Errorf("Job failed: %s", c.Reason)
   656  		}
   657  	}
   658  
   659  	log.Printf("%s: Jobs active: %d, jobs failed: %d, jobs succeeded: %d", name, o.Status.Active, o.Status.Failed, o.Status.Succeeded)
   660  	return false, nil
   661  }
   662  
   663  // scrubValidationError removes kubectl info from the message
   664  func scrubValidationError(err error) error {
   665  	if err == nil {
   666  		return nil
   667  	}
   668  	const stopValidateMessage = "if you choose to ignore these errors, turn validation off with --validate=false"
   669  
   670  	if strings.Contains(err.Error(), stopValidateMessage) {
   671  		return goerrors.New(strings.Replace(err.Error(), "; "+stopValidateMessage, "", -1))
   672  	}
   673  	return err
   674  }
   675  
   676  // WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase
   677  // and returns said phase (PodSucceeded or PodFailed qualify)
   678  func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (api.PodPhase, error) {
   679  	infos, err := c.Build(namespace, reader)
   680  	if err != nil {
   681  		return api.PodUnknown, err
   682  	}
   683  	info := infos[0]
   684  
   685  	kind := info.Mapping.GroupVersionKind.Kind
   686  	if kind != "Pod" {
   687  		return api.PodUnknown, fmt.Errorf("%s is not a Pod", info.Name)
   688  	}
   689  
   690  	if err := watchPodUntilComplete(timeout, info); err != nil {
   691  		return api.PodUnknown, err
   692  	}
   693  
   694  	if err := info.Get(); err != nil {
   695  		return api.PodUnknown, err
   696  	}
   697  	status := info.Object.(*api.Pod).Status.Phase
   698  
   699  	return status, nil
   700  }
   701  
   702  func watchPodUntilComplete(timeout time.Duration, info *resource.Info) error {
   703  	w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion)
   704  	if err != nil {
   705  		return err
   706  	}
   707  
   708  	log.Printf("Watching pod %s for completion with timeout of %v", info.Name, timeout)
   709  	_, err = watch.Until(timeout, w, func(e watch.Event) (bool, error) {
   710  		return conditions.PodCompleted(e)
   711  	})
   712  
   713  	return err
   714  }