github.com/sdbaiguanghe/helm@v2.16.7+incompatible/pkg/kube/client.go (about)

     1  /*
     2  Copyright The Helm Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kube // import "k8s.io/helm/pkg/kube"
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	goerrors "errors"
    24  	"fmt"
    25  	"io"
    26  	"io/ioutil"
    27  	"log"
    28  	"sort"
    29  	"strings"
    30  	"sync"
    31  	"time"
    32  
    33  	"k8s.io/apimachinery/pkg/api/meta"
    34  
    35  	jsonpatch "github.com/evanphx/json-patch"
    36  	appsv1 "k8s.io/api/apps/v1"
    37  	appsv1beta1 "k8s.io/api/apps/v1beta1"
    38  	appsv1beta2 "k8s.io/api/apps/v1beta2"
    39  	batch "k8s.io/api/batch/v1"
    40  	v1 "k8s.io/api/core/v1"
    41  	extv1beta1 "k8s.io/api/extensions/v1beta1"
    42  	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
    43  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    44  	"k8s.io/apimachinery/pkg/api/errors"
    45  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    46  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    47  	metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1"
    48  	"k8s.io/apimachinery/pkg/fields"
    49  	"k8s.io/apimachinery/pkg/labels"
    50  	"k8s.io/apimachinery/pkg/runtime"
    51  	"k8s.io/apimachinery/pkg/runtime/schema"
    52  	"k8s.io/apimachinery/pkg/types"
    53  	"k8s.io/apimachinery/pkg/util/strategicpatch"
    54  	"k8s.io/apimachinery/pkg/util/wait"
    55  	"k8s.io/apimachinery/pkg/watch"
    56  	"k8s.io/cli-runtime/pkg/genericclioptions"
    57  	"k8s.io/cli-runtime/pkg/printers"
    58  	"k8s.io/cli-runtime/pkg/resource"
    59  	"k8s.io/client-go/kubernetes/scheme"
    60  	"k8s.io/client-go/rest"
    61  	cachetools "k8s.io/client-go/tools/cache"
    62  	watchtools "k8s.io/client-go/tools/watch"
    63  	cmdutil "k8s.io/kubectl/pkg/cmd/util"
    64  	"k8s.io/kubectl/pkg/validation"
    65  	"k8s.io/kubernetes/pkg/kubectl/cmd/get"
    66  )
    67  
    68  // MissingGetHeader is added to Get's output when a resource is not found.
    69  const MissingGetHeader = "==> MISSING\nKIND\t\tNAME\n"
    70  
    71  // ErrNoObjectsVisited indicates that during a visit operation, no matching objects were found.
    72  var ErrNoObjectsVisited = goerrors.New("no objects visited")
    73  
    74  var metadataAccessor = meta.NewAccessor()
    75  
    76  // Client represents a client capable of communicating with the Kubernetes API.
    77  type Client struct {
    78  	cmdutil.Factory
    79  	Log func(string, ...interface{})
    80  }
    81  
    82  // New creates a new Client.
    83  func New(getter genericclioptions.RESTClientGetter) *Client {
    84  	if getter == nil {
    85  		getter = genericclioptions.NewConfigFlags(true)
    86  	}
    87  
    88  	err := apiextv1beta1.AddToScheme(scheme.Scheme)
    89  	if err != nil {
    90  		panic(err)
    91  	}
    92  
    93  	return &Client{
    94  		Factory: cmdutil.NewFactory(getter),
    95  		Log:     nopLogger,
    96  	}
    97  }
    98  
    99  var nopLogger = func(_ string, _ ...interface{}) {}
   100  
   101  // ResourceActorFunc performs an action on a single resource.
   102  type ResourceActorFunc func(*resource.Info) error
   103  
   104  // Create creates Kubernetes resources from an io.reader.
   105  //
   106  // Namespace will set the namespace.
   107  func (c *Client) Create(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
   108  	client, err := c.KubernetesClientSet()
   109  	if err != nil {
   110  		return err
   111  	}
   112  	if err := ensureNamespace(client, namespace); err != nil {
   113  		return err
   114  	}
   115  	c.Log("building resources from manifest")
   116  	infos, buildErr := c.BuildUnstructured(namespace, reader)
   117  	if buildErr != nil {
   118  		return buildErr
   119  	}
   120  	c.Log("creating %d resource(s)", len(infos))
   121  	if err := perform(infos, createResource); err != nil {
   122  		return err
   123  	}
   124  	if shouldWait {
   125  		return c.waitForResources(time.Duration(timeout)*time.Second, infos)
   126  	}
   127  	return nil
   128  }
   129  
   130  func (c *Client) newBuilder(namespace string, reader io.Reader) *resource.Result {
   131  	return c.NewBuilder().
   132  		ContinueOnError().
   133  		WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
   134  		Schema(c.validator()).
   135  		NamespaceParam(namespace).
   136  		DefaultNamespace().
   137  		Stream(reader, "").
   138  		Flatten().
   139  		Do()
   140  }
   141  
   142  func (c *Client) validator() validation.Schema {
   143  	schema, err := c.Validator(true)
   144  	if err != nil {
   145  		c.Log("warning: failed to load schema: %s", err)
   146  	}
   147  	return schema
   148  }
   149  
   150  // BuildUnstructured reads Kubernetes objects and returns unstructured infos.
   151  func (c *Client) BuildUnstructured(namespace string, reader io.Reader) (Result, error) {
   152  	var result Result
   153  
   154  	result, err := c.NewBuilder().
   155  		Unstructured().
   156  		ContinueOnError().
   157  		NamespaceParam(namespace).
   158  		DefaultNamespace().
   159  		Stream(reader, "").
   160  		Flatten().
   161  		Do().Infos()
   162  	return result, scrubValidationError(err)
   163  }
   164  
   165  // BuildUnstructuredTable reads Kubernetes objects and returns unstructured infos
   166  // as a Table. This is meant for viewing resources and displaying them in a table.
   167  // This is similar to BuildUnstructured but transforms the request for table
   168  // display.
   169  func (c *Client) BuildUnstructuredTable(namespace string, reader io.Reader) (Result, error) {
   170  	var result Result
   171  
   172  	result, err := c.NewBuilder().
   173  		Unstructured().
   174  		ContinueOnError().
   175  		NamespaceParam(namespace).
   176  		DefaultNamespace().
   177  		Stream(reader, "").
   178  		Flatten().
   179  		TransformRequests(transformRequests).
   180  		Do().Infos()
   181  	return result, scrubValidationError(err)
   182  }
   183  
   184  // This is used to retrieve a table view of the data. A table view is how kubectl
   185  // retrieves the information Helm displays as resources in status. Note, table
   186  // data is returned as a Table type that does not conform to the runtime.Object
   187  // interface but is that type. So, you can't transform it into Go objects easily.
   188  func transformRequests(req *rest.Request) {
   189  
   190  	// The request headers are for both the v1 and v1beta1 versions of the table
   191  	// as Kubernetes 1.14 and older used the beta version.
   192  	req.SetHeader("Accept", strings.Join([]string{
   193  		fmt.Sprintf("application/json;as=Table;v=%s;g=%s", metav1.SchemeGroupVersion.Version, metav1.GroupName),
   194  		fmt.Sprintf("application/json;as=Table;v=%s;g=%s", metav1beta1.SchemeGroupVersion.Version, metav1beta1.GroupName),
   195  		"application/json",
   196  	}, ","))
   197  }
   198  
   199  // Validate reads Kubernetes manifests and validates the content.
   200  //
   201  // This function does not actually do schema validation of manifests. Adding
   202  // validation now breaks existing clients of helm: https://github.com/helm/helm/issues/5750
   203  func (c *Client) Validate(namespace string, reader io.Reader) error {
   204  	_, err := c.NewBuilder().
   205  		Unstructured().
   206  		ContinueOnError().
   207  		NamespaceParam(namespace).
   208  		DefaultNamespace().
   209  		// Schema(c.validator()). // No schema validation
   210  		Stream(reader, "").
   211  		Flatten().
   212  		Do().Infos()
   213  	return scrubValidationError(err)
   214  }
   215  
   216  // Build validates for Kubernetes objects and returns resource Infos from a io.Reader.
   217  func (c *Client) Build(namespace string, reader io.Reader) (Result, error) {
   218  	var result Result
   219  	result, err := c.newBuilder(namespace, reader).Infos()
   220  	return result, scrubValidationError(err)
   221  }
   222  
   223  // Return the resource info as internal
   224  func resourceInfoToObject(info *resource.Info, c *Client) runtime.Object {
   225  	internalObj, err := asInternal(info)
   226  	if err != nil {
   227  		// If the problem is just that the resource is not registered, don't print any
   228  		// error. This is normal for custom resources.
   229  		if !runtime.IsNotRegisteredError(err) {
   230  			c.Log("Warning: conversion to internal type failed: %v", err)
   231  		}
   232  		// Add the unstructured object in this situation. It will still get listed, just
   233  		// with less information.
   234  		return info.Object
   235  	}
   236  
   237  	return internalObj
   238  }
   239  
   240  func sortByKey(objs map[string][]runtime.Object) []string {
   241  	var keys []string
   242  	// Create a simple slice, so we can sort it
   243  	for key := range objs {
   244  		keys = append(keys, key)
   245  	}
   246  	// Sort alphabetically by version/kind keys
   247  	sort.Strings(keys)
   248  	return keys
   249  }
   250  
   251  // We have slices of tables that need to be sorted by name. In this case the
   252  // self link is used so the sorting will include namespace and name.
   253  func sortTableSlice(objs []runtime.Object) []runtime.Object {
   254  	// If there are 0 or 1 objects to sort there is nothing to sort so
   255  	// the list can be returned
   256  	if len(objs) < 2 {
   257  		return objs
   258  	}
   259  
   260  	ntbl := &metav1.Table{}
   261  	unstr, ok := objs[0].(*unstructured.Unstructured)
   262  	if !ok {
   263  		return objs
   264  	}
   265  	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr.Object, ntbl); err != nil {
   266  		return objs
   267  	}
   268  
   269  	// Sort the list of objects
   270  	var newObjs []runtime.Object
   271  	namesCache := make(map[string]runtime.Object, len(objs))
   272  	var names []string
   273  	var key string
   274  	for _, obj := range objs {
   275  		unstr, ok := obj.(*unstructured.Unstructured)
   276  		if !ok {
   277  			return objs
   278  		}
   279  		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr.Object, ntbl); err != nil {
   280  			return objs
   281  		}
   282  
   283  		// At this point we have a table. Each table has just one row. We are
   284  		// sorting the tables by the first cell (name) in the first and only
   285  		// row. If the first cell of the first row cannot be gotten as a string
   286  		// we return the original unsorted list.
   287  		if len(ntbl.Rows) == 0 { // Make sure there are rows to read from
   288  			return objs
   289  		}
   290  		if len(ntbl.Rows[0].Cells) == 0 { // Make sure there are cells to read
   291  			return objs
   292  		}
   293  		key, ok = ntbl.Rows[0].Cells[0].(string)
   294  		if !ok {
   295  			return objs
   296  		}
   297  		namesCache[key] = obj
   298  		names = append(names, key)
   299  	}
   300  
   301  	sort.Strings(names)
   302  
   303  	for _, name := range names {
   304  		newObjs = append(newObjs, namesCache[name])
   305  	}
   306  
   307  	return newObjs
   308  }
   309  
   310  // Get gets Kubernetes resources as pretty-printed string.
   311  //
   312  // Namespace will set the namespace.
   313  func (c *Client) Get(namespace string, reader io.Reader) (string, error) {
   314  	// Since we don't know what order the objects come in, let's group them by the types and then sort them, so
   315  	// that when we print them, they come out looking good (headers apply to subgroups, etc.).
   316  	objs := make(map[string][]runtime.Object)
   317  	gk := make(map[string]schema.GroupKind)
   318  	mux := &sync.Mutex{}
   319  
   320  	// The contents of the reader are used two times. The bytes are coppied out
   321  	// for use in future readers.
   322  	b, err := ioutil.ReadAll(reader)
   323  	if err != nil {
   324  		return "", err
   325  	}
   326  
   327  	// Get the table display for the objects associated with the release. This
   328  	// is done in table format so that it can be displayed in the status in
   329  	// the same way kubectl displays the resource information.
   330  	// Note, the response returns unstructured data instead of typed objects.
   331  	// These cannot be easily (i.e., via the go packages) transformed into
   332  	// Go types.
   333  	tinfos, err := c.BuildUnstructuredTable(namespace, bytes.NewBuffer(b))
   334  	if err != nil {
   335  		return "", err
   336  	}
   337  
   338  	missing := []string{}
   339  	err = perform(tinfos, func(info *resource.Info) error {
   340  		mux.Lock()
   341  		defer mux.Unlock()
   342  		c.Log("Doing get for %s: %q", info.Mapping.GroupVersionKind.Kind, info.Name)
   343  		if err := info.Get(); err != nil {
   344  			c.Log("WARNING: Failed Get for resource %q: %s", info.Name, err)
   345  			missing = append(missing, fmt.Sprintf("%v\t\t%s", info.Mapping.Resource, info.Name))
   346  			return nil
   347  		}
   348  
   349  		// Use APIVersion/Kind as grouping mechanism. I'm not sure if you can have multiple
   350  		// versions per cluster, but this certainly won't hurt anything, so let's be safe.
   351  		gvk := info.ResourceMapping().GroupVersionKind
   352  		vk := gvk.Version + "/" + gvk.Kind
   353  		gk[vk] = gvk.GroupKind()
   354  
   355  		// Initialize map. The main map groups resources based on version/kind
   356  		// The second level is a simple 'Name' to 'Object', that will help sort
   357  		// the individual resource later
   358  		if objs[vk] == nil {
   359  			objs[vk] = []runtime.Object{}
   360  		}
   361  		// Map between the resource name to the underlying info object
   362  		objs[vk] = append(objs[vk], resourceInfoToObject(info, c))
   363  
   364  		return nil
   365  	})
   366  	if err != nil {
   367  		return "", err
   368  	}
   369  
   370  	// This section finds related resources (e.g., pods). Before looking up pods
   371  	// the resources the pods are made from need to be looked up in a manner
   372  	// that can be turned into Go types and worked with.
   373  	infos, err := c.BuildUnstructured(namespace, bytes.NewBuffer(b))
   374  	if err != nil {
   375  		return "", err
   376  	}
   377  	err = perform(infos, func(info *resource.Info) error {
   378  		mux.Lock()
   379  		defer mux.Unlock()
   380  		if err := info.Get(); err != nil {
   381  			c.Log("WARNING: Failed Get for resource %q: %s", info.Name, err)
   382  			missing = append(missing, fmt.Sprintf("%v\t\t%s", info.Mapping.Resource, info.Name))
   383  			return nil
   384  		}
   385  
   386  		//Get the relation pods
   387  		objs, err = c.getSelectRelationPod(info, objs)
   388  		if err != nil {
   389  			c.Log("Warning: get the relation pod is failed, err:%s", err.Error())
   390  		}
   391  
   392  		return nil
   393  	})
   394  	if err != nil {
   395  		return "", err
   396  	}
   397  
   398  	// Ok, now we have all the objects grouped by types (say, by v1/Pod, v1/Service, etc.), so
   399  	// spin through them and print them. Printer is cool since it prints the header only when
   400  	// an object type changes, so we can just rely on that. Problem is it doesn't seem to keep
   401  	// track of tab widths.
   402  	buf := new(bytes.Buffer)
   403  
   404  	// Sort alphabetically by version/kind keys
   405  	vkKeys := sortByKey(objs)
   406  	// Iterate on sorted version/kind types
   407  	for _, t := range vkKeys {
   408  		if _, err = fmt.Fprintf(buf, "==> %s\n", t); err != nil {
   409  			return "", err
   410  		}
   411  		vk := objs[t]
   412  
   413  		// The request made for tables returns each Kubernetes object as its
   414  		// own table. The normal sorting provided by kubectl and cli-runtime
   415  		// does not handle this case. Here we sort within each of our own
   416  		// grouping.
   417  		vk = sortTableSlice(vk)
   418  
   419  		// The printer flag setup follows a simalar setup to kubectl
   420  		printFlags := get.NewHumanPrintFlags()
   421  		if lgk, ok := gk[t]; ok {
   422  			printFlags.SetKind(lgk)
   423  		}
   424  		printer, _ := printFlags.ToPrinter("")
   425  		printer, err = printers.NewTypeSetter(scheme.Scheme).WrapToPrinter(printer, nil)
   426  		if err != nil {
   427  			return "", err
   428  		}
   429  		printer = &get.TablePrinter{Delegate: printer}
   430  
   431  		for _, resource := range vk {
   432  			if err := printer.PrintObj(resource, buf); err != nil {
   433  				c.Log("failed to print object type %s: %v", t, err)
   434  				return "", err
   435  			}
   436  		}
   437  		if _, err := buf.WriteString("\n"); err != nil {
   438  			return "", err
   439  		}
   440  	}
   441  	if len(missing) > 0 {
   442  		buf.WriteString(MissingGetHeader)
   443  		for _, s := range missing {
   444  			fmt.Fprintln(buf, s)
   445  		}
   446  	}
   447  	return buf.String(), nil
   448  }
   449  
   450  // Update reads the current configuration and a target configuration from io.reader
   451  // and creates resources that don't already exist, updates resources that have been modified
   452  // in the target configuration and deletes resources from the current configuration that are
   453  // not present in the target configuration.
   454  //
   455  // Namespace will set the namespaces.
   456  //
   457  // Deprecated: use UpdateWithOptions instead.
   458  func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error {
   459  	return c.UpdateWithOptions(namespace, originalReader, targetReader, UpdateOptions{
   460  		Force:      force,
   461  		Recreate:   recreate,
   462  		Timeout:    timeout,
   463  		ShouldWait: shouldWait,
   464  	})
   465  }
   466  
   467  // UpdateOptions provides options to control update behavior
   468  type UpdateOptions struct {
   469  	Force      bool
   470  	Recreate   bool
   471  	Timeout    int64
   472  	ShouldWait bool
   473  	// Allow deletion of new resources created in this update when update failed
   474  	CleanupOnFail bool
   475  }
   476  
   477  // UpdateWithOptions reads the current configuration and a target configuration from io.reader
   478  // and creates resources that don't already exist, updates resources that have been modified
   479  // in the target configuration and deletes resources from the current configuration that are
   480  // not present in the target configuration.
   481  //
   482  // Namespace will set the namespaces. UpdateOptions provides additional parameters to control
   483  // update behavior.
   484  func (c *Client) UpdateWithOptions(namespace string, originalReader, targetReader io.Reader, opts UpdateOptions) error {
   485  	original, err := c.BuildUnstructured(namespace, originalReader)
   486  	if err != nil {
   487  		return fmt.Errorf("failed decoding reader into objects: %s", err)
   488  	}
   489  
   490  	c.Log("building resources from updated manifest")
   491  	target, err := c.BuildUnstructured(namespace, targetReader)
   492  	if err != nil {
   493  		return fmt.Errorf("failed decoding reader into objects: %s", err)
   494  	}
   495  
   496  	newlyCreatedResources := []*resource.Info{}
   497  	updateErrors := []string{}
   498  
   499  	c.Log("checking %d resources for changes", len(target))
   500  	err = target.Visit(func(info *resource.Info, err error) error {
   501  		if err != nil {
   502  			return err
   503  		}
   504  
   505  		helper := resource.NewHelper(info.Client, info.Mapping)
   506  		if _, err := helper.Get(info.Namespace, info.Name, info.Export); err != nil {
   507  			if !errors.IsNotFound(err) {
   508  				return fmt.Errorf("Could not get information about the resource: %s", err)
   509  			}
   510  
   511  			// Since the resource does not exist, create it.
   512  			if err := createResource(info); err != nil {
   513  				return fmt.Errorf("failed to create resource: %s", err)
   514  			}
   515  			newlyCreatedResources = append(newlyCreatedResources, info)
   516  
   517  			kind := info.Mapping.GroupVersionKind.Kind
   518  			c.Log("Created a new %s called %q\n", kind, info.Name)
   519  			return nil
   520  		}
   521  
   522  		originalInfo := original.Get(info)
   523  
   524  		// The resource already exists in the cluster, but it wasn't defined in the previous release.
   525  		// In this case, we consider it to be a resource that was previously un-managed by the release and error out,
   526  		// asking for the user to intervene.
   527  		//
   528  		// See https://github.com/helm/helm/issues/1193 for more info.
   529  		if originalInfo == nil {
   530  			return fmt.Errorf(
   531  				"kind %s with the name %q already exists in the cluster and wasn't defined in the previous release. Before upgrading, please either delete the resource from the cluster or remove it from the chart",
   532  				info.Mapping.GroupVersionKind.Kind,
   533  				info.Name,
   534  			)
   535  		}
   536  
   537  		if err := updateResource(c, info, originalInfo.Object, opts.Force, opts.Recreate); err != nil {
   538  			c.Log("error updating the resource %q:\n\t %v", info.Name, err)
   539  			updateErrors = append(updateErrors, err.Error())
   540  		}
   541  
   542  		return nil
   543  	})
   544  
   545  	cleanupErrors := []string{}
   546  
   547  	if opts.CleanupOnFail && (err != nil || len(updateErrors) != 0) {
   548  		c.Log("Cleanup on fail enabled: cleaning up newly created resources due to update manifests failures")
   549  		cleanupErrors = c.cleanup(newlyCreatedResources)
   550  	}
   551  
   552  	switch {
   553  	case err != nil:
   554  		return fmt.Errorf(strings.Join(append([]string{err.Error()}, cleanupErrors...), " && "))
   555  	case len(updateErrors) != 0:
   556  		return fmt.Errorf(strings.Join(append(updateErrors, cleanupErrors...), " && "))
   557  	}
   558  
   559  	for _, info := range original.Difference(target) {
   560  		c.Log("Deleting %q in %s...", info.Name, info.Namespace)
   561  
   562  		if err := info.Get(); err != nil {
   563  			c.Log("Unable to get obj %q, err: %s", info.Name, err)
   564  		}
   565  		annotations, err := metadataAccessor.Annotations(info.Object)
   566  		if err != nil {
   567  			c.Log("Unable to get annotations on %q, err: %s", info.Name, err)
   568  		}
   569  		if ResourcePolicyIsKeep(annotations) {
   570  			policy := annotations[ResourcePolicyAnno]
   571  			c.Log("Skipping delete of %q due to annotation [%s=%s]", info.Name, ResourcePolicyAnno, policy)
   572  			continue
   573  		}
   574  
   575  		if err := deleteResource(info); err != nil {
   576  			c.Log("Failed to delete %q, err: %s", info.Name, err)
   577  		}
   578  	}
   579  	if opts.ShouldWait {
   580  		err := c.waitForResources(time.Duration(opts.Timeout)*time.Second, target)
   581  
   582  		if opts.CleanupOnFail && err != nil {
   583  			c.Log("Cleanup on fail enabled: cleaning up newly created resources due to wait failure during update")
   584  			cleanupErrors = c.cleanup(newlyCreatedResources)
   585  			return fmt.Errorf(strings.Join(append([]string{err.Error()}, cleanupErrors...), " && "))
   586  		}
   587  
   588  		return err
   589  	}
   590  	return nil
   591  }
   592  
   593  func (c *Client) cleanup(newlyCreatedResources []*resource.Info) (cleanupErrors []string) {
   594  	for _, info := range newlyCreatedResources {
   595  		kind := info.Mapping.GroupVersionKind.Kind
   596  		c.Log("Deleting newly created %s with the name %q in %s...", kind, info.Name, info.Namespace)
   597  		if err := deleteResource(info); err != nil {
   598  			c.Log("Error deleting newly created %s with the name %q in %s: %s", kind, info.Name, info.Namespace, err)
   599  			cleanupErrors = append(cleanupErrors, err.Error())
   600  		}
   601  	}
   602  	return
   603  }
   604  
   605  // Delete deletes Kubernetes resources from an io.reader.
   606  //
   607  // Namespace will set the namespace.
   608  func (c *Client) Delete(namespace string, reader io.Reader) error {
   609  	return c.DeleteWithTimeout(namespace, reader, 0, false)
   610  }
   611  
   612  // DeleteWithTimeout deletes Kubernetes resources from an io.reader. If shouldWait is true, the function
   613  // will wait for all resources to be deleted from etcd before returning, or when the timeout
   614  // has expired.
   615  //
   616  // Namespace will set the namespace.
   617  func (c *Client) DeleteWithTimeout(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
   618  	infos, err := c.BuildUnstructured(namespace, reader)
   619  	if err != nil {
   620  		return err
   621  	}
   622  	err = perform(infos, func(info *resource.Info) error {
   623  		c.Log("Starting delete for %q %s", info.Name, info.Mapping.GroupVersionKind.Kind)
   624  		err := deleteResource(info)
   625  		return c.skipIfNotFound(err)
   626  	})
   627  	if err != nil {
   628  		return err
   629  	}
   630  
   631  	if shouldWait {
   632  		c.Log("Waiting for %d seconds for delete to be completed", timeout)
   633  		return waitUntilAllResourceDeleted(infos, time.Duration(timeout)*time.Second)
   634  	}
   635  
   636  	return nil
   637  }
   638  
   639  func (c *Client) skipIfNotFound(err error) error {
   640  	if errors.IsNotFound(err) {
   641  		c.Log("%v", err)
   642  		return nil
   643  	}
   644  	return err
   645  }
   646  
   647  func waitUntilAllResourceDeleted(infos Result, timeout time.Duration) error {
   648  	return wait.Poll(2*time.Second, timeout, func() (bool, error) {
   649  		allDeleted := true
   650  		err := perform(infos, func(info *resource.Info) error {
   651  			innerErr := info.Get()
   652  			if errors.IsNotFound(innerErr) {
   653  				return nil
   654  			}
   655  			if innerErr != nil {
   656  				return innerErr
   657  			}
   658  			allDeleted = false
   659  			return nil
   660  		})
   661  		if err != nil {
   662  			return false, err
   663  		}
   664  		return allDeleted, nil
   665  	})
   666  }
   667  
   668  func (c *Client) watchTimeout(t time.Duration) ResourceActorFunc {
   669  	return func(info *resource.Info) error {
   670  		return c.watchUntilReady(t, info)
   671  	}
   672  }
   673  
   674  // WatchUntilReady watches the resource given in the reader, and waits until it is ready.
   675  //
   676  // This function is mainly for hook implementations. It watches for a resource to
   677  // hit a particular milestone. The milestone depends on the Kind.
   678  //
   679  // For most kinds, it checks to see if the resource is marked as Added or Modified
   680  // by the Kubernetes event stream. For some kinds, it does more:
   681  //
   682  // - Jobs: A job is marked "Ready" when it has successfully completed. This is
   683  //   ascertained by watching the Status fields in a job's output.
   684  //
   685  // Handling for other kinds will be added as necessary.
   686  func (c *Client) WatchUntilReady(namespace string, reader io.Reader, timeout int64, shouldWait bool) error {
   687  	infos, err := c.BuildUnstructured(namespace, reader)
   688  	if err != nil {
   689  		return err
   690  	}
   691  	// For jobs, there's also the option to do poll c.Jobs(namespace).Get():
   692  	// https://github.com/adamreese/kubernetes/blob/master/test/e2e/job.go#L291-L300
   693  	return perform(infos, c.watchTimeout(time.Duration(timeout)*time.Second))
   694  }
   695  
   696  // WaitUntilCRDEstablished polls the given CRD until it reaches the established
   697  // state. A CRD needs to reach the established state before CRs can be created.
   698  //
   699  // If a naming conflict condition is found, this function will return an error.
   700  func (c *Client) WaitUntilCRDEstablished(reader io.Reader, timeout time.Duration) error {
   701  	infos, err := c.BuildUnstructured(metav1.NamespaceAll, reader)
   702  	if err != nil {
   703  		return err
   704  	}
   705  
   706  	return perform(infos, c.pollCRDEstablished(timeout))
   707  }
   708  
   709  func (c *Client) pollCRDEstablished(t time.Duration) ResourceActorFunc {
   710  	return func(info *resource.Info) error {
   711  		return c.pollCRDUntilEstablished(t, info)
   712  	}
   713  }
   714  
   715  func (c *Client) pollCRDUntilEstablished(timeout time.Duration, info *resource.Info) error {
   716  	return wait.PollImmediate(time.Second, timeout, func() (bool, error) {
   717  		err := info.Get()
   718  		if err != nil {
   719  			return false, fmt.Errorf("unable to get CRD: %v", err)
   720  		}
   721  
   722  		crd := &apiextv1beta1.CustomResourceDefinition{}
   723  		err = scheme.Scheme.Convert(info.Object, crd, nil)
   724  		if err != nil {
   725  			return false, fmt.Errorf("unable to convert to CRD type: %v", err)
   726  		}
   727  
   728  		for _, cond := range crd.Status.Conditions {
   729  			switch cond.Type {
   730  			case apiextv1beta1.Established:
   731  				if cond.Status == apiextv1beta1.ConditionTrue {
   732  					return true, nil
   733  				}
   734  			case apiextv1beta1.NamesAccepted:
   735  				if cond.Status == apiextv1beta1.ConditionFalse {
   736  					return false, fmt.Errorf("naming conflict detected for CRD %s", crd.GetName())
   737  				}
   738  			}
   739  		}
   740  
   741  		return false, nil
   742  	})
   743  }
   744  
   745  func perform(infos Result, fn ResourceActorFunc) error {
   746  	if len(infos) == 0 {
   747  		return ErrNoObjectsVisited
   748  	}
   749  
   750  	errs := make(chan error)
   751  	go batchPerform(infos, fn, errs)
   752  
   753  	for range infos {
   754  		err := <-errs
   755  		if err != nil {
   756  			return err
   757  		}
   758  	}
   759  	return nil
   760  }
   761  
   762  func batchPerform(infos Result, fn ResourceActorFunc, errs chan<- error) {
   763  	var kind string
   764  	var wg sync.WaitGroup
   765  	for _, info := range infos {
   766  		currentKind := info.Object.GetObjectKind().GroupVersionKind().Kind
   767  		if kind != currentKind {
   768  			wg.Wait()
   769  			kind = currentKind
   770  		}
   771  		wg.Add(1)
   772  		go func(i *resource.Info) {
   773  			errs <- fn(i)
   774  			wg.Done()
   775  		}(info)
   776  	}
   777  }
   778  
   779  func createResource(info *resource.Info) error {
   780  	obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object, nil)
   781  	if err != nil {
   782  		return err
   783  	}
   784  	return info.Refresh(obj, true)
   785  }
   786  
   787  func deleteResource(info *resource.Info) error {
   788  	policy := metav1.DeletePropagationBackground
   789  	opts := &metav1.DeleteOptions{PropagationPolicy: &policy}
   790  	_, err := resource.NewHelper(info.Client, info.Mapping).DeleteWithOptions(info.Namespace, info.Name, opts)
   791  	return err
   792  }
   793  
   794  func createPatch(target *resource.Info, current runtime.Object) ([]byte, types.PatchType, error) {
   795  	oldData, err := json.Marshal(current)
   796  	if err != nil {
   797  		return nil, types.StrategicMergePatchType, fmt.Errorf("serializing current configuration: %s", err)
   798  	}
   799  	newData, err := json.Marshal(target.Object)
   800  	if err != nil {
   801  		return nil, types.StrategicMergePatchType, fmt.Errorf("serializing target configuration: %s", err)
   802  	}
   803  
   804  	// While different objects need different merge types, the parent function
   805  	// that calls this does not try to create a patch when the data (first
   806  	// returned object) is nil. We can skip calculating the merge type as
   807  	// the returned merge type is ignored.
   808  	if apiequality.Semantic.DeepEqual(oldData, newData) {
   809  		return nil, types.StrategicMergePatchType, nil
   810  	}
   811  
   812  	// Get a versioned object
   813  	versionedObject, err := asVersioned(target)
   814  
   815  	// Unstructured objects, such as CRDs, may not have a not registered error
   816  	// returned from ConvertToVersion. Anything that's unstructured should
   817  	// use the jsonpatch.CreateMergePatch. Strategic Merge Patch is not supported
   818  	// on objects like CRDs.
   819  	_, isUnstructured := versionedObject.(runtime.Unstructured)
   820  
   821  	// On newer K8s versions, CRDs aren't unstructured but has this dedicated type
   822  	_, isCRD := versionedObject.(*apiextv1beta1.CustomResourceDefinition)
   823  
   824  	switch {
   825  	case runtime.IsNotRegisteredError(err), isUnstructured, isCRD:
   826  		// fall back to generic JSON merge patch
   827  		patch, err := jsonpatch.CreateMergePatch(oldData, newData)
   828  		if err != nil {
   829  			return nil, types.MergePatchType, fmt.Errorf("failed to create merge patch: %v", err)
   830  		}
   831  		return patch, types.MergePatchType, nil
   832  	case err != nil:
   833  		return nil, types.StrategicMergePatchType, fmt.Errorf("failed to get versionedObject: %s", err)
   834  	default:
   835  		patch, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, versionedObject)
   836  		if err != nil {
   837  			return nil, types.StrategicMergePatchType, fmt.Errorf("failed to create two-way merge patch: %v", err)
   838  		}
   839  		return patch, types.StrategicMergePatchType, nil
   840  	}
   841  }
   842  
   843  func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, force bool, recreate bool) error {
   844  	patch, patchType, err := createPatch(target, currentObj)
   845  	if err != nil {
   846  		return fmt.Errorf("failed to create patch: %s", err)
   847  	}
   848  	if patch == nil {
   849  		c.Log("Looks like there are no changes for %s %q", target.Mapping.GroupVersionKind.Kind, target.Name)
   850  		// This needs to happen to make sure that tiller has the latest info from the API
   851  		// Otherwise there will be no labels and other functions that use labels will panic
   852  		if err := target.Get(); err != nil {
   853  			return fmt.Errorf("error trying to refresh resource information: %v", err)
   854  		}
   855  	} else {
   856  		// send patch to server
   857  		helper := resource.NewHelper(target.Client, target.Mapping)
   858  
   859  		obj, err := helper.Patch(target.Namespace, target.Name, patchType, patch, nil)
   860  		if err != nil {
   861  			kind := target.Mapping.GroupVersionKind.Kind
   862  			log.Printf("Cannot patch %s: %q (%v)", kind, target.Name, err)
   863  
   864  			if force {
   865  				// Attempt to delete...
   866  				if err := deleteResource(target); err != nil {
   867  					return err
   868  				}
   869  				log.Printf("Deleted %s: %q", kind, target.Name)
   870  
   871  				// ... and recreate
   872  				if err := createResource(target); err != nil {
   873  					return fmt.Errorf("Failed to recreate resource: %s", err)
   874  				}
   875  				log.Printf("Created a new %s called %q\n", kind, target.Name)
   876  
   877  				// No need to refresh the target, as we recreated the resource based
   878  				// on it. In addition, it might not exist yet and a call to `Refresh`
   879  				// may fail.
   880  			} else {
   881  				log.Print("Use --force to force recreation of the resource")
   882  				return err
   883  			}
   884  		} else {
   885  			// When patch succeeds without needing to recreate, refresh target.
   886  			target.Refresh(obj, true)
   887  		}
   888  	}
   889  
   890  	if !recreate {
   891  		return nil
   892  	}
   893  
   894  	versioned := asVersionedOrUnstructured(target)
   895  	selector, ok := getSelectorFromObject(versioned)
   896  	if !ok {
   897  		return nil
   898  	}
   899  
   900  	client, err := c.KubernetesClientSet()
   901  	if err != nil {
   902  		return err
   903  	}
   904  
   905  	pods, err := client.CoreV1().Pods(target.Namespace).List(metav1.ListOptions{
   906  		LabelSelector: labels.Set(selector).AsSelector().String(),
   907  	})
   908  	if err != nil {
   909  		return err
   910  	}
   911  
   912  	// Restart pods
   913  	for _, pod := range pods.Items {
   914  		c.Log("Restarting pod: %v/%v", pod.Namespace, pod.Name)
   915  
   916  		// Delete each pod for get them restarted with changed spec.
   917  		if err := client.CoreV1().Pods(pod.Namespace).Delete(pod.Name, metav1.NewPreconditionDeleteOptions(string(pod.UID))); err != nil {
   918  			return err
   919  		}
   920  	}
   921  	return nil
   922  }
   923  
   924  func getSelectorFromObject(obj runtime.Object) (map[string]string, bool) {
   925  	switch typed := obj.(type) {
   926  
   927  	case *v1.ReplicationController:
   928  		return typed.Spec.Selector, true
   929  
   930  	case *extv1beta1.ReplicaSet:
   931  		return typed.Spec.Selector.MatchLabels, true
   932  	case *appsv1.ReplicaSet:
   933  		return typed.Spec.Selector.MatchLabels, true
   934  
   935  	case *extv1beta1.Deployment:
   936  		return typed.Spec.Selector.MatchLabels, true
   937  	case *appsv1beta1.Deployment:
   938  		return typed.Spec.Selector.MatchLabels, true
   939  	case *appsv1beta2.Deployment:
   940  		return typed.Spec.Selector.MatchLabels, true
   941  	case *appsv1.Deployment:
   942  		return typed.Spec.Selector.MatchLabels, true
   943  
   944  	case *extv1beta1.DaemonSet:
   945  		return typed.Spec.Selector.MatchLabels, true
   946  	case *appsv1beta2.DaemonSet:
   947  		return typed.Spec.Selector.MatchLabels, true
   948  	case *appsv1.DaemonSet:
   949  		return typed.Spec.Selector.MatchLabels, true
   950  
   951  	case *batch.Job:
   952  		return typed.Spec.Selector.MatchLabels, true
   953  
   954  	case *appsv1beta1.StatefulSet:
   955  		return typed.Spec.Selector.MatchLabels, true
   956  	case *appsv1beta2.StatefulSet:
   957  		return typed.Spec.Selector.MatchLabels, true
   958  	case *appsv1.StatefulSet:
   959  		return typed.Spec.Selector.MatchLabels, true
   960  
   961  	default:
   962  		return nil, false
   963  	}
   964  }
   965  
   966  func (c *Client) watchUntilReady(timeout time.Duration, info *resource.Info) error {
   967  	// Use a selector on the name of the resource. This should be unique for the
   968  	// given version and kind
   969  	selector, err := fields.ParseSelector(fmt.Sprintf("metadata.name=%s", info.Name))
   970  	if err != nil {
   971  		return err
   972  	}
   973  	lw := cachetools.NewListWatchFromClient(info.Client, info.Mapping.Resource.Resource, info.Namespace, selector)
   974  
   975  	kind := info.Mapping.GroupVersionKind.Kind
   976  	c.Log("Watching for changes to %s %s with timeout of %v", kind, info.Name, timeout)
   977  
   978  	// What we watch for depends on the Kind.
   979  	// - For a Job, we watch for completion.
   980  	// - For all else, we watch until Ready.
   981  	// In the future, we might want to add some special logic for types
   982  	// like Ingress, Volume, etc.
   983  
   984  	ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout)
   985  	defer cancel()
   986  	_, err = watchtools.ListWatchUntil(ctx, lw, func(e watch.Event) (bool, error) {
   987  		switch e.Type {
   988  		case watch.Added, watch.Modified:
   989  			// For things like a secret or a config map, this is the best indicator
   990  			// we get. We care mostly about jobs, where what we want to see is
   991  			// the status go into a good state. For other types, like ReplicaSet
   992  			// we don't really do anything to support these as hooks.
   993  			c.Log("Add/Modify event for %s: %v", info.Name, e.Type)
   994  			if kind == "Job" {
   995  				return c.waitForJob(e, info.Name)
   996  			}
   997  			return true, nil
   998  		case watch.Deleted:
   999  			c.Log("Deleted event for %s", info.Name)
  1000  			return true, nil
  1001  		case watch.Error:
  1002  			// Handle error and return with an error.
  1003  			c.Log("Error event for %s", info.Name)
  1004  			return true, fmt.Errorf("Failed to deploy %s", info.Name)
  1005  		default:
  1006  			return false, nil
  1007  		}
  1008  	})
  1009  	return err
  1010  }
  1011  
  1012  // waitForJob is a helper that waits for a job to complete.
  1013  //
  1014  // This operates on an event returned from a watcher.
  1015  func (c *Client) waitForJob(e watch.Event, name string) (bool, error) {
  1016  	job := &batch.Job{}
  1017  	err := scheme.Scheme.Convert(e.Object, job, nil)
  1018  	if err != nil {
  1019  		return true, err
  1020  	}
  1021  
  1022  	for _, c := range job.Status.Conditions {
  1023  		if c.Type == batch.JobComplete && c.Status == v1.ConditionTrue {
  1024  			return true, nil
  1025  		} else if c.Type == batch.JobFailed && c.Status == v1.ConditionTrue {
  1026  			return true, fmt.Errorf("Job failed: %s", c.Reason)
  1027  		}
  1028  	}
  1029  
  1030  	c.Log("%s: Jobs active: %d, jobs failed: %d, jobs succeeded: %d", name, job.Status.Active, job.Status.Failed, job.Status.Succeeded)
  1031  	return false, nil
  1032  }
  1033  
  1034  // scrubValidationError removes kubectl info from the message.
  1035  func scrubValidationError(err error) error {
  1036  	if err == nil {
  1037  		return nil
  1038  	}
  1039  	const stopValidateMessage = "if you choose to ignore these errors, turn validation off with --validate=false"
  1040  
  1041  	if strings.Contains(err.Error(), stopValidateMessage) {
  1042  		return goerrors.New(strings.Replace(err.Error(), "; "+stopValidateMessage, "", -1))
  1043  	}
  1044  	return err
  1045  }
  1046  
  1047  // WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase
  1048  // and returns said phase (PodSucceeded or PodFailed qualify).
  1049  func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (v1.PodPhase, error) {
  1050  	infos, err := c.Build(namespace, reader)
  1051  	if err != nil {
  1052  		return v1.PodUnknown, err
  1053  	}
  1054  	info := infos[0]
  1055  
  1056  	kind := info.Mapping.GroupVersionKind.Kind
  1057  	if kind != "Pod" {
  1058  		return v1.PodUnknown, fmt.Errorf("%s is not a Pod", info.Name)
  1059  	}
  1060  
  1061  	if err := c.watchPodUntilComplete(timeout, info); err != nil {
  1062  		return v1.PodUnknown, err
  1063  	}
  1064  
  1065  	if err := info.Get(); err != nil {
  1066  		return v1.PodUnknown, err
  1067  	}
  1068  	status := info.Object.(*v1.Pod).Status.Phase
  1069  
  1070  	return status, nil
  1071  }
  1072  
  1073  func (c *Client) watchPodUntilComplete(timeout time.Duration, info *resource.Info) error {
  1074  	lw := cachetools.NewListWatchFromClient(info.Client, info.Mapping.Resource.Resource, info.Namespace, fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", info.Name)))
  1075  
  1076  	c.Log("Watching pod %s for completion with timeout of %v", info.Name, timeout)
  1077  	ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout)
  1078  	defer cancel()
  1079  	_, err := watchtools.ListWatchUntil(ctx, lw, func(e watch.Event) (bool, error) {
  1080  		return isPodComplete(e)
  1081  	})
  1082  
  1083  	return err
  1084  }
  1085  
  1086  // GetPodLogs takes pod name and namespace and returns the current logs (streaming is NOT enabled).
  1087  func (c *Client) GetPodLogs(name, ns string) (io.ReadCloser, error) {
  1088  	client, err := c.KubernetesClientSet()
  1089  	if err != nil {
  1090  		return nil, err
  1091  	}
  1092  	req := client.CoreV1().Pods(ns).GetLogs(name, &v1.PodLogOptions{})
  1093  	logReader, err := req.Stream()
  1094  	if err != nil {
  1095  		return nil, fmt.Errorf("error in opening log stream, got: %s", err)
  1096  	}
  1097  	return logReader, nil
  1098  }
  1099  
  1100  func isPodComplete(event watch.Event) (bool, error) {
  1101  	o, ok := event.Object.(*v1.Pod)
  1102  	if !ok {
  1103  		return true, fmt.Errorf("expected a *v1.Pod, got %T", event.Object)
  1104  	}
  1105  	if event.Type == watch.Deleted {
  1106  		return false, fmt.Errorf("pod not found")
  1107  	}
  1108  	switch o.Status.Phase {
  1109  	case v1.PodFailed, v1.PodSucceeded:
  1110  		return true, nil
  1111  	}
  1112  	return false, nil
  1113  }
  1114  
  1115  // get a kubernetes resources' relation pods
  1116  // kubernetes resource used select labels to relate pods
  1117  func (c *Client) getSelectRelationPod(info *resource.Info, objs map[string][]runtime.Object) (map[string][]runtime.Object, error) {
  1118  	if info == nil {
  1119  		return objs, nil
  1120  	}
  1121  
  1122  	c.Log("get relation pod of object: %s/%s/%s", info.Namespace, info.Mapping.GroupVersionKind.Kind, info.Name)
  1123  
  1124  	versioned := asVersionedOrUnstructured(info)
  1125  	selector, ok := getSelectorFromObject(versioned)
  1126  	if !ok {
  1127  		return objs, nil
  1128  	}
  1129  
  1130  	// The related pods are looked up in Table format so that their display can
  1131  	// be printed in a manner similar to kubectl when it get pods. The response
  1132  	// can be used with a table printer.
  1133  	infos, err := c.NewBuilder().
  1134  		Unstructured().
  1135  		ContinueOnError().
  1136  		NamespaceParam(info.Namespace).
  1137  		DefaultNamespace().
  1138  		ResourceTypes("pods").
  1139  		LabelSelector(labels.Set(selector).AsSelector().String()).
  1140  		TransformRequests(transformRequests).
  1141  		Do().Infos()
  1142  	if err != nil {
  1143  		return objs, err
  1144  	}
  1145  
  1146  	for _, info := range infos {
  1147  		vk := "v1/Pod(related)"
  1148  		objs[vk] = append(objs[vk], info.Object)
  1149  	}
  1150  
  1151  	return objs, nil
  1152  }
  1153  
  1154  func asVersionedOrUnstructured(info *resource.Info) runtime.Object {
  1155  	obj, _ := asVersioned(info)
  1156  	return obj
  1157  }
  1158  
  1159  func asVersioned(info *resource.Info) (runtime.Object, error) {
  1160  	converter := runtime.ObjectConvertor(scheme.Scheme)
  1161  	groupVersioner := runtime.GroupVersioner(schema.GroupVersions(scheme.Scheme.PrioritizedVersionsAllGroups()))
  1162  	if info.Mapping != nil {
  1163  		groupVersioner = info.Mapping.GroupVersionKind.GroupVersion()
  1164  	}
  1165  
  1166  	obj, err := converter.ConvertToVersion(info.Object, groupVersioner)
  1167  	if err != nil {
  1168  		return info.Object, err
  1169  	}
  1170  	return obj, nil
  1171  }
  1172  
  1173  func asInternal(info *resource.Info) (runtime.Object, error) {
  1174  	groupVersioner := info.Mapping.GroupVersionKind.GroupKind().WithVersion(runtime.APIVersionInternal).GroupVersion()
  1175  	return scheme.Scheme.ConvertToVersion(info.Object, groupVersioner)
  1176  }