github.com/caicloud/helm@v2.5.0+incompatible/pkg/kube/client.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors All rights reserved.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kube // import "k8s.io/helm/pkg/kube"
    18  
    19  import (
    20  	"bytes"
    21  	"encoding/json"
    22  	goerrors "errors"
    23  	"fmt"
    24  	"io"
    25  	"log"
    26  	"strings"
    27  	"time"
    28  
    29  	jsonpatch "github.com/evanphx/json-patch"
    30  	"k8s.io/apimachinery/pkg/api/errors"
    31  	"k8s.io/apimachinery/pkg/api/meta"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    34  	"k8s.io/apimachinery/pkg/fields"
    35  	"k8s.io/apimachinery/pkg/labels"
    36  	"k8s.io/apimachinery/pkg/runtime"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	"k8s.io/apimachinery/pkg/util/strategicpatch"
    39  	"k8s.io/apimachinery/pkg/watch"
    40  	"k8s.io/client-go/tools/clientcmd"
    41  	"k8s.io/kubernetes/pkg/api"
    42  	"k8s.io/kubernetes/pkg/api/v1"
    43  	apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
    44  	batchinternal "k8s.io/kubernetes/pkg/apis/batch"
    45  	batch "k8s.io/kubernetes/pkg/apis/batch/v1"
    46  	"k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
    47  	conditions "k8s.io/kubernetes/pkg/client/unversioned"
    48  	"k8s.io/kubernetes/pkg/kubectl"
    49  	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
    50  	"k8s.io/kubernetes/pkg/kubectl/resource"
    51  	"k8s.io/kubernetes/pkg/printers"
    52  
    53  	"k8s.io/helm/pkg/releaseutil"
    54  )
    55  
    56  // ErrNoObjectsVisited indicates that during a visit operation, no matching objects were found.
    57  var ErrNoObjectsVisited = goerrors.New("no objects visited")
    58  
    59  // Client represents a client capable of communicating with the Kubernetes API.
    60  type Client struct {
    61  	cmdutil.Factory
    62  	// SchemaCacheDir is the path for loading cached schema.
    63  	SchemaCacheDir string
    64  
    65  	Log func(string, ...interface{})
    66  }
    67  
    68  // New create a new Client
    69  func New(config clientcmd.ClientConfig) *Client {
    70  	return &Client{
    71  		Factory:        cmdutil.NewFactory(config),
    72  		SchemaCacheDir: clientcmd.RecommendedSchemaFile,
    73  		Log:            func(_ string, _ ...interface{}) {},
    74  	}
    75  }
    76  
    77  // ResourceActorFunc performs an action on a single resource.
    78  type ResourceActorFunc func(*resource.Info) error
    79  
    80  // Create creates kubernetes resources from an io.reader
    81  //
    82  // Namespace will set the namespace
    83  func (c *Client) Create(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
    84  	client, err := c.ClientSet()
    85  	if err != nil {
    86  		return err
    87  	}
    88  	if err := ensureNamespace(client, namespace); err != nil {
    89  		return err
    90  	}
    91  	c.Log("building resources from manifest")
    92  	infos, buildErr := c.BuildUnstructured(namespace, reader)
    93  	if buildErr != nil {
    94  		return buildErr
    95  	}
    96  	c.Log("creating %d resource(s)", len(infos))
    97  	if err := perform(infos, createResource); err != nil {
    98  		return err
    99  	}
   100  	if shouldWait {
   101  		return c.waitForResources(time.Duration(timeout)*time.Second, infos)
   102  	}
   103  	return nil
   104  }
   105  
   106  func (c *Client) newBuilder(namespace string, reader io.Reader) *resource.Result {
   107  	schema, err := c.Validator(true, c.SchemaCacheDir)
   108  	if err != nil {
   109  		c.Log("warning: failed to load schema: %s", err)
   110  	}
   111  	return c.NewBuilder().
   112  		ContinueOnError().
   113  		Schema(schema).
   114  		NamespaceParam(namespace).
   115  		DefaultNamespace().
   116  		Stream(reader, "").
   117  		Flatten().
   118  		Do()
   119  }
   120  
   121  // BuildUnstructured validates for Kubernetes objects and returns unstructured infos.
   122  func (c *Client) BuildUnstructured(namespace string, reader io.Reader) (Result, error) {
   123  	schema, err := c.Validator(true, c.SchemaCacheDir)
   124  	if err != nil {
   125  		c.Log("warning: failed to load schema: %s", err)
   126  	}
   127  
   128  	mapper, typer, err := c.UnstructuredObject()
   129  	if err != nil {
   130  		c.Log("failed to load mapper: %s", err)
   131  		return nil, err
   132  	}
   133  	var result Result
   134  	result, err = resource.NewBuilder(mapper, typer, resource.ClientMapperFunc(c.UnstructuredClientForMapping), unstructured.UnstructuredJSONScheme).
   135  		ContinueOnError().
   136  		Schema(schema).
   137  		NamespaceParam(namespace).
   138  		DefaultNamespace().
   139  		Stream(reader, "").
   140  		Flatten().
   141  		Do().Infos()
   142  	return result, scrubValidationError(err)
   143  }
   144  
   145  // Build validates for Kubernetes objects and returns resource Infos from a io.Reader.
   146  func (c *Client) Build(namespace string, reader io.Reader) (Result, error) {
   147  	var result Result
   148  	result, err := c.newBuilder(namespace, reader).Infos()
   149  	return result, scrubValidationError(err)
   150  }
   151  
   152  // Get gets kubernetes resources as pretty printed string
   153  //
   154  // Namespace will set the namespace
   155  func (c *Client) Get(namespace string, reader io.Reader) (string, error) {
   156  	// Since we don't know what order the objects come in, let's group them by the types, so
   157  	// that when we print them, they come looking good (headers apply to subgroups, etc.)
   158  	objs := make(map[string][]runtime.Object)
   159  	infos, err := c.BuildUnstructured(namespace, reader)
   160  	if err != nil {
   161  		return "", err
   162  	}
   163  	missing := []string{}
   164  	err = perform(infos, func(info *resource.Info) error {
   165  		c.Log("Doing get for %s: %q", info.Mapping.GroupVersionKind.Kind, info.Name)
   166  		if err := info.Get(); err != nil {
   167  			c.Log("WARNING: Failed Get for resource %q: %s", info.Name, err)
   168  			missing = append(missing, fmt.Sprintf("%v\t\t%s", info.Mapping.Resource, info.Name))
   169  			return nil
   170  		}
   171  
   172  		// Use APIVersion/Kind as grouping mechanism. I'm not sure if you can have multiple
   173  		// versions per cluster, but this certainly won't hurt anything, so let's be safe.
   174  		gvk := info.ResourceMapping().GroupVersionKind
   175  		vk := gvk.Version + "/" + gvk.Kind
   176  		objs[vk] = append(objs[vk], info.Object)
   177  		return nil
   178  	})
   179  	if err != nil {
   180  		return "", err
   181  	}
   182  
   183  	// Ok, now we have all the objects grouped by types (say, by v1/Pod, v1/Service, etc.), so
   184  	// spin through them and print them. Printer is cool since it prints the header only when
   185  	// an object type changes, so we can just rely on that. Problem is it doesn't seem to keep
   186  	// track of tab widths
   187  	buf := new(bytes.Buffer)
   188  	p, _ := c.Printer(nil, printers.PrintOptions{})
   189  	for t, ot := range objs {
   190  		if _, err = buf.WriteString("==> " + t + "\n"); err != nil {
   191  			return "", err
   192  		}
   193  		for _, o := range ot {
   194  			if err := p.PrintObj(o, buf); err != nil {
   195  				c.Log("failed to print object type %s, object: %q :\n %v", t, o, err)
   196  				return "", err
   197  			}
   198  		}
   199  		if _, err := buf.WriteString("\n"); err != nil {
   200  			return "", err
   201  		}
   202  	}
   203  	if len(missing) > 0 {
   204  		buf.WriteString("==> MISSING\nKIND\t\tNAME\n")
   205  		for _, s := range missing {
   206  			fmt.Fprintln(buf, s)
   207  		}
   208  	}
   209  	return buf.String(), nil
   210  }
   211  
   212  // Update reads in the current configuration and a target configuration from io.reader
   213  //  and creates resources that don't already exists, updates resources that have been modified
   214  //  in the target configuration and deletes resources from the current configuration that are
   215  //  not present in the target configuration
   216  //
   217  // Namespace will set the namespaces
   218  func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error {
   219  	original, err := c.BuildUnstructured(namespace, originalReader)
   220  	if err != nil {
   221  		return fmt.Errorf("failed decoding reader into objects: %s", err)
   222  	}
   223  
   224  	c.Log("building resources from updated manifest")
   225  	target, err := c.BuildUnstructured(namespace, targetReader)
   226  	if err != nil {
   227  		return fmt.Errorf("failed decoding reader into objects: %s", err)
   228  	}
   229  
   230  	updateErrors := []string{}
   231  
   232  	c.Log("checking %d resources for changes", len(target))
   233  	err = target.Visit(func(info *resource.Info, err error) error {
   234  		if err != nil {
   235  			return err
   236  		}
   237  
   238  		helper := resource.NewHelper(info.Client, info.Mapping)
   239  		if _, err := helper.Get(info.Namespace, info.Name, info.Export); err != nil {
   240  			if !errors.IsNotFound(err) {
   241  				return fmt.Errorf("Could not get information about the resource: %s", err)
   242  			}
   243  
   244  			// Since the resource does not exist, create it.
   245  			if err := createResource(info); err != nil {
   246  				return fmt.Errorf("failed to create resource: %s", err)
   247  			}
   248  
   249  			kind := info.Mapping.GroupVersionKind.Kind
   250  			c.Log("Created a new %s called %q\n", kind, info.Name)
   251  			return nil
   252  		}
   253  
   254  		originalInfo := original.Get(info)
   255  		if originalInfo == nil {
   256  			return fmt.Errorf("no resource with the name %q found", info.Name)
   257  		}
   258  
   259  		if err := updateResource(c, info, originalInfo.Object, force, recreate); err != nil {
   260  			c.Log("error updating the resource %q:\n\t %v", info.Name, err)
   261  			updateErrors = append(updateErrors, err.Error())
   262  		}
   263  
   264  		return nil
   265  	})
   266  
   267  	switch {
   268  	case err != nil:
   269  		return err
   270  	case len(updateErrors) != 0:
   271  		return fmt.Errorf(strings.Join(updateErrors, " && "))
   272  	}
   273  
   274  	for _, info := range original.Difference(target) {
   275  		c.Log("Deleting %q in %s...", info.Name, info.Namespace)
   276  		if err := deleteResource(c, info); err != nil {
   277  			c.Log("Failed to delete %q, err: %s", info.Name, err)
   278  		}
   279  	}
   280  	if shouldWait {
   281  		return c.waitForResources(time.Duration(timeout)*time.Second, target)
   282  	}
   283  	return nil
   284  }
   285  
   286  // Delete deletes kubernetes resources from an io.reader
   287  //
   288  // Namespace will set the namespace
   289  func (c *Client) Delete(namespace string, reader io.Reader) error {
   290  	infos, err := c.BuildUnstructured(namespace, reader)
   291  	if err != nil {
   292  		return err
   293  	}
   294  	return perform(infos, func(info *resource.Info) error {
   295  		c.Log("Starting delete for %q %s", info.Name, info.Mapping.GroupVersionKind.Kind)
   296  		err := deleteResource(c, info)
   297  		return c.skipIfNotFound(err)
   298  	})
   299  }
   300  
   301  func (c *Client) skipIfNotFound(err error) error {
   302  	if errors.IsNotFound(err) {
   303  		c.Log("%v", err)
   304  		return nil
   305  	}
   306  	return err
   307  }
   308  
   309  func (c *Client) watchTimeout(t time.Duration) ResourceActorFunc {
   310  	return func(info *resource.Info) error {
   311  		return c.watchUntilReady(t, info)
   312  	}
   313  }
   314  
   315  // WatchUntilReady watches the resource given in the reader, and waits until it is ready.
   316  //
   317  // This function is mainly for hook implementations. It watches for a resource to
   318  // hit a particular milestone. The milestone depends on the Kind.
   319  //
   320  // For most kinds, it checks to see if the resource is marked as Added or Modified
   321  // by the Kubernetes event stream. For some kinds, it does more:
   322  //
   323  // - Jobs: A job is marked "Ready" when it has successfully completed. This is
   324  //   ascertained by watching the Status fields in a job's output.
   325  //
   326  // Handling for other kinds will be added as necessary.
   327  func (c *Client) WatchUntilReady(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
   328  	infos, err := c.Build(namespace, reader)
   329  	if err != nil {
   330  		return err
   331  	}
   332  	// For jobs, there's also the option to do poll c.Jobs(namespace).Get():
   333  	// https://github.com/adamreese/kubernetes/blob/master/test/e2e/job.go#L291-L300
   334  	return perform(infos, c.watchTimeout(time.Duration(timeout)*time.Second))
   335  }
   336  
   337  func perform(infos Result, fn ResourceActorFunc) error {
   338  	if len(infos) == 0 {
   339  		return ErrNoObjectsVisited
   340  	}
   341  
   342  	for _, info := range infos {
   343  		if err := fn(info); err != nil {
   344  			return err
   345  		}
   346  	}
   347  	return nil
   348  }
   349  
   350  func createResource(info *resource.Info) error {
   351  	obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object)
   352  	if err != nil {
   353  		return err
   354  	}
   355  	return info.Refresh(obj, true)
   356  }
   357  
   358  func deleteResource(c *Client, info *resource.Info) error {
   359  	helper := resource.NewHelper(info.Client, info.Mapping)
   360  	runningObj, err := helper.Get(info.Namespace, info.Name, info.Export)
   361  	if err != nil {
   362  		if errors.IsNotFound(err) {
   363  			return nil
   364  		}
   365  		return err
   366  	}
   367  	if !releaseutil.MatchRelease(info.VersionedObject, runningObj) {
   368  		log.Printf("Delete: Ignore unmatched object: %s/%s", info.Namespace, info.Name)
   369  		return nil
   370  	}
   371  	reaper, err := c.Reaper(info.Mapping)
   372  	if err != nil {
   373  		// If there is no reaper for this resources, delete it.
   374  		if kubectl.IsNoSuchReaperError(err) {
   375  			return helper.Delete(info.Namespace, info.Name)
   376  		}
   377  		return err
   378  	}
   379  	c.Log("Using reaper for deleting %q", info.Name)
   380  	return reaper.Stop(info.Namespace, info.Name, 0, nil)
   381  }
   382  
   383  func createPatch(mapping *meta.RESTMapping, target, current runtime.Object) ([]byte, types.PatchType, error) {
   384  	oldData, err := json.Marshal(current)
   385  	if err != nil {
   386  		return nil, types.StrategicMergePatchType, fmt.Errorf("serializing current configuration: %s", err)
   387  	}
   388  	newData, err := json.Marshal(target)
   389  	if err != nil {
   390  		return nil, types.StrategicMergePatchType, fmt.Errorf("serializing target configuration: %s", err)
   391  	}
   392  
   393  	if api.Semantic.DeepEqual(oldData, newData) {
   394  		return nil, types.StrategicMergePatchType, nil
   395  	}
   396  
   397  	// Get a versioned object
   398  	versionedObject, err := api.Scheme.New(mapping.GroupVersionKind)
   399  	switch {
   400  	case runtime.IsNotRegisteredError(err):
   401  		// fall back to generic JSON merge patch
   402  		patch, err := jsonpatch.CreateMergePatch(oldData, newData)
   403  		return patch, types.MergePatchType, err
   404  	case err != nil:
   405  		return nil, types.StrategicMergePatchType, fmt.Errorf("failed to get versionedObject: %s", err)
   406  	default:
   407  		patch, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, versionedObject)
   408  		return patch, types.StrategicMergePatchType, err
   409  	}
   410  }
   411  
   412  func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, force bool, recreate bool) error {
   413  	helper := resource.NewHelper(target.Client, target.Mapping)
   414  	runningObj, err := helper.Get(target.Namespace, target.Name, target.Export)
   415  	if err != nil {
   416  		return err
   417  	}
   418  	if !releaseutil.MatchRelease(target.VersionedObject, runningObj) {
   419  		log.Printf("Update: Find unmatched object: %s/%s", target.Namespace, target.Name)
   420  		return fmt.Errorf("Update: Unmatched object: %s/%s", target.Namespace, target.Name)
   421  	}
   422  	patch, patchType, err := createPatch(target.Mapping, target.Object, currentObj)
   423  	if err != nil {
   424  		return fmt.Errorf("failed to create patch: %s", err)
   425  	}
   426  	if patch == nil {
   427  		c.Log("Looks like there are no changes for %s %q", target.Mapping.GroupVersionKind.Kind, target.Name)
   428  		// This needs to happen to make sure that tiller has the latest info from the API
   429  		// Otherwise there will be no labels and other functions that use labels will panic
   430  		if err := target.Get(); err != nil {
   431  			return fmt.Errorf("error trying to refresh resource information: %v", err)
   432  		}
   433  		return nil
   434  	}
   435  
   436  	obj, err := helper.Patch(target.Namespace, target.Name, patchType, patch)
   437  	if err != nil {
   438  		kind := target.Mapping.GroupVersionKind.Kind
   439  		log.Printf("Cannot patch %s: %q (%v)", kind, target.Name, err)
   440  
   441  		if force {
   442  			// Attempt to delete...
   443  			if err := deleteResource(c, target); err != nil {
   444  				return err
   445  			}
   446  			log.Printf("Deleted %s: %q", kind, target.Name)
   447  
   448  			// ... and recreate
   449  			if err := createResource(target); err != nil {
   450  				return fmt.Errorf("Failed to recreate resource: %s", err)
   451  			}
   452  			log.Printf("Created a new %s called %q\n", kind, target.Name)
   453  
   454  			// No need to refresh the target, as we recreated the resource based
   455  			// on it. In addition, it might not exist yet and a call to `Refresh`
   456  			// may fail.
   457  		} else {
   458  			log.Print("Use --force to force recreation of the resource")
   459  			return err
   460  		}
   461  	} else {
   462  		// When patch succeeds without needing to recreate, refresh target.
   463  		target.Refresh(obj, true)
   464  	}
   465  
   466  	if !recreate {
   467  		return nil
   468  	}
   469  
   470  	versioned, err := c.AsVersionedObject(target.Object)
   471  	if runtime.IsNotRegisteredError(err) {
   472  		return nil
   473  	}
   474  	if err != nil {
   475  		return err
   476  	}
   477  
   478  	selector, err := getSelectorFromObject(versioned)
   479  	if err != nil {
   480  		return nil
   481  	}
   482  
   483  	client, err := c.ClientSet()
   484  	if err != nil {
   485  		return err
   486  	}
   487  
   488  	pods, err := client.Core().Pods(target.Namespace).List(metav1.ListOptions{
   489  		FieldSelector: fields.Everything().String(),
   490  		LabelSelector: labels.Set(selector).AsSelector().String(),
   491  	})
   492  	if err != nil {
   493  		return err
   494  	}
   495  
   496  	// Restart pods
   497  	for _, pod := range pods.Items {
   498  		c.Log("Restarting pod: %v/%v", pod.Namespace, pod.Name)
   499  
   500  		// Delete each pod for get them restarted with changed spec.
   501  		if err := client.Core().Pods(pod.Namespace).Delete(pod.Name, metav1.NewPreconditionDeleteOptions(string(pod.UID))); err != nil {
   502  			return err
   503  		}
   504  	}
   505  	return nil
   506  }
   507  
   508  func getSelectorFromObject(obj runtime.Object) (map[string]string, error) {
   509  	switch typed := obj.(type) {
   510  	case *v1.ReplicationController:
   511  		return typed.Spec.Selector, nil
   512  	case *v1beta1.ReplicaSet:
   513  		return typed.Spec.Selector.MatchLabels, nil
   514  	case *v1beta1.Deployment:
   515  		return typed.Spec.Selector.MatchLabels, nil
   516  	case *v1beta1.DaemonSet:
   517  		return typed.Spec.Selector.MatchLabels, nil
   518  	case *batch.Job:
   519  		return typed.Spec.Selector.MatchLabels, nil
   520  	case *apps.StatefulSet:
   521  		return typed.Spec.Selector.MatchLabels, nil
   522  	default:
   523  		return nil, fmt.Errorf("Unsupported kind when getting selector: %v", obj)
   524  	}
   525  }
   526  
   527  func (c *Client) watchUntilReady(timeout time.Duration, info *resource.Info) error {
   528  	w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion)
   529  	if err != nil {
   530  		return err
   531  	}
   532  
   533  	kind := info.Mapping.GroupVersionKind.Kind
   534  	c.Log("Watching for changes to %s %s with timeout of %v", kind, info.Name, timeout)
   535  
   536  	// What we watch for depends on the Kind.
   537  	// - For a Job, we watch for completion.
   538  	// - For all else, we watch until Ready.
   539  	// In the future, we might want to add some special logic for types
   540  	// like Ingress, Volume, etc.
   541  
   542  	_, err = watch.Until(timeout, w, func(e watch.Event) (bool, error) {
   543  		switch e.Type {
   544  		case watch.Added, watch.Modified:
   545  			// For things like a secret or a config map, this is the best indicator
   546  			// we get. We care mostly about jobs, where what we want to see is
   547  			// the status go into a good state. For other types, like ReplicaSet
   548  			// we don't really do anything to support these as hooks.
   549  			c.Log("Add/Modify event for %s: %v", info.Name, e.Type)
   550  			if kind == "Job" {
   551  				return c.waitForJob(e, info.Name)
   552  			}
   553  			return true, nil
   554  		case watch.Deleted:
   555  			c.Log("Deleted event for %s", info.Name)
   556  			return true, nil
   557  		case watch.Error:
   558  			// Handle error and return with an error.
   559  			c.Log("Error event for %s", info.Name)
   560  			return true, fmt.Errorf("Failed to deploy %s", info.Name)
   561  		default:
   562  			return false, nil
   563  		}
   564  	})
   565  	return err
   566  }
   567  
   568  // AsVersionedObject converts a runtime.object to a versioned object.
   569  func (c *Client) AsVersionedObject(obj runtime.Object) (runtime.Object, error) {
   570  	json, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
   571  	if err != nil {
   572  		return nil, err
   573  	}
   574  	versions := &runtime.VersionedObjects{}
   575  	err = runtime.DecodeInto(c.Decoder(true), json, versions)
   576  	return versions.First(), err
   577  }
   578  
   579  // waitForJob is a helper that waits for a job to complete.
   580  //
   581  // This operates on an event returned from a watcher.
   582  func (c *Client) waitForJob(e watch.Event, name string) (bool, error) {
   583  	o, ok := e.Object.(*batchinternal.Job)
   584  	if !ok {
   585  		return true, fmt.Errorf("Expected %s to be a *batch.Job, got %T", name, e.Object)
   586  	}
   587  
   588  	for _, c := range o.Status.Conditions {
   589  		if c.Type == batchinternal.JobComplete && c.Status == api.ConditionTrue {
   590  			return true, nil
   591  		} else if c.Type == batchinternal.JobFailed && c.Status == api.ConditionTrue {
   592  			return true, fmt.Errorf("Job failed: %s", c.Reason)
   593  		}
   594  	}
   595  
   596  	c.Log("%s: Jobs active: %d, jobs failed: %d, jobs succeeded: %d", name, o.Status.Active, o.Status.Failed, o.Status.Succeeded)
   597  	return false, nil
   598  }
   599  
   600  // scrubValidationError removes kubectl info from the message
   601  func scrubValidationError(err error) error {
   602  	if err == nil {
   603  		return nil
   604  	}
   605  	const stopValidateMessage = "if you choose to ignore these errors, turn validation off with --validate=false"
   606  
   607  	if strings.Contains(err.Error(), stopValidateMessage) {
   608  		return goerrors.New(strings.Replace(err.Error(), "; "+stopValidateMessage, "", -1))
   609  	}
   610  	return err
   611  }
   612  
   613  // WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase
   614  // and returns said phase (PodSucceeded or PodFailed qualify)
   615  func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (api.PodPhase, error) {
   616  	infos, err := c.Build(namespace, reader)
   617  	if err != nil {
   618  		return api.PodUnknown, err
   619  	}
   620  	info := infos[0]
   621  
   622  	kind := info.Mapping.GroupVersionKind.Kind
   623  	if kind != "Pod" {
   624  		return api.PodUnknown, fmt.Errorf("%s is not a Pod", info.Name)
   625  	}
   626  
   627  	if err := c.watchPodUntilComplete(timeout, info); err != nil {
   628  		return api.PodUnknown, err
   629  	}
   630  
   631  	if err := info.Get(); err != nil {
   632  		return api.PodUnknown, err
   633  	}
   634  	status := info.Object.(*api.Pod).Status.Phase
   635  
   636  	return status, nil
   637  }
   638  
   639  func (c *Client) watchPodUntilComplete(timeout time.Duration, info *resource.Info) error {
   640  	w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion)
   641  	if err != nil {
   642  		return err
   643  	}
   644  
   645  	c.Log("Watching pod %s for completion with timeout of %v", info.Name, timeout)
   646  	_, err = watch.Until(timeout, w, func(e watch.Event) (bool, error) {
   647  		return conditions.PodCompleted(e)
   648  	})
   649  
   650  	return err
   651  }