github.com/sdbaiguanghe/helm@v2.16.7+incompatible/pkg/kube/client.go (about) 1 /* 2 Copyright The Helm Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package kube // import "k8s.io/helm/pkg/kube" 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 goerrors "errors" 24 "fmt" 25 "io" 26 "io/ioutil" 27 "log" 28 "sort" 29 "strings" 30 "sync" 31 "time" 32 33 "k8s.io/apimachinery/pkg/api/meta" 34 35 jsonpatch "github.com/evanphx/json-patch" 36 appsv1 "k8s.io/api/apps/v1" 37 appsv1beta1 "k8s.io/api/apps/v1beta1" 38 appsv1beta2 "k8s.io/api/apps/v1beta2" 39 batch "k8s.io/api/batch/v1" 40 v1 "k8s.io/api/core/v1" 41 extv1beta1 "k8s.io/api/extensions/v1beta1" 42 apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" 43 apiequality "k8s.io/apimachinery/pkg/api/equality" 44 "k8s.io/apimachinery/pkg/api/errors" 45 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 46 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 47 metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" 48 "k8s.io/apimachinery/pkg/fields" 49 "k8s.io/apimachinery/pkg/labels" 50 "k8s.io/apimachinery/pkg/runtime" 51 "k8s.io/apimachinery/pkg/runtime/schema" 52 "k8s.io/apimachinery/pkg/types" 53 "k8s.io/apimachinery/pkg/util/strategicpatch" 54 "k8s.io/apimachinery/pkg/util/wait" 55 "k8s.io/apimachinery/pkg/watch" 56 "k8s.io/cli-runtime/pkg/genericclioptions" 57 "k8s.io/cli-runtime/pkg/printers" 58 "k8s.io/cli-runtime/pkg/resource" 59 "k8s.io/client-go/kubernetes/scheme" 60 "k8s.io/client-go/rest" 61 cachetools "k8s.io/client-go/tools/cache" 62 watchtools "k8s.io/client-go/tools/watch" 63 cmdutil "k8s.io/kubectl/pkg/cmd/util" 64 "k8s.io/kubectl/pkg/validation" 65 "k8s.io/kubernetes/pkg/kubectl/cmd/get" 66 ) 67 68 // MissingGetHeader is added to Get's output when a resource is not found. 69 const MissingGetHeader = "==> MISSING\nKIND\t\tNAME\n" 70 71 // ErrNoObjectsVisited indicates that during a visit operation, no matching objects were found. 72 var ErrNoObjectsVisited = goerrors.New("no objects visited") 73 74 var metadataAccessor = meta.NewAccessor() 75 76 // Client represents a client capable of communicating with the Kubernetes API. 77 type Client struct { 78 cmdutil.Factory 79 Log func(string, ...interface{}) 80 } 81 82 // New creates a new Client. 83 func New(getter genericclioptions.RESTClientGetter) *Client { 84 if getter == nil { 85 getter = genericclioptions.NewConfigFlags(true) 86 } 87 88 err := apiextv1beta1.AddToScheme(scheme.Scheme) 89 if err != nil { 90 panic(err) 91 } 92 93 return &Client{ 94 Factory: cmdutil.NewFactory(getter), 95 Log: nopLogger, 96 } 97 } 98 99 var nopLogger = func(_ string, _ ...interface{}) {} 100 101 // ResourceActorFunc performs an action on a single resource. 102 type ResourceActorFunc func(*resource.Info) error 103 104 // Create creates Kubernetes resources from an io.reader. 105 // 106 // Namespace will set the namespace. 107 func (c *Client) Create(namespace string, reader io.Reader, timeout int64, shouldWait bool) error { 108 client, err := c.KubernetesClientSet() 109 if err != nil { 110 return err 111 } 112 if err := ensureNamespace(client, namespace); err != nil { 113 return err 114 } 115 c.Log("building resources from manifest") 116 infos, buildErr := c.BuildUnstructured(namespace, reader) 117 if buildErr != nil { 118 return buildErr 119 } 120 c.Log("creating %d resource(s)", len(infos)) 121 if err := perform(infos, createResource); err != nil { 122 return err 123 } 124 if shouldWait { 125 return c.waitForResources(time.Duration(timeout)*time.Second, infos) 126 } 127 return nil 128 } 129 130 func (c *Client) newBuilder(namespace string, reader io.Reader) *resource.Result { 131 return c.NewBuilder(). 132 ContinueOnError(). 133 WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...). 134 Schema(c.validator()). 135 NamespaceParam(namespace). 136 DefaultNamespace(). 137 Stream(reader, ""). 138 Flatten(). 139 Do() 140 } 141 142 func (c *Client) validator() validation.Schema { 143 schema, err := c.Validator(true) 144 if err != nil { 145 c.Log("warning: failed to load schema: %s", err) 146 } 147 return schema 148 } 149 150 // BuildUnstructured reads Kubernetes objects and returns unstructured infos. 151 func (c *Client) BuildUnstructured(namespace string, reader io.Reader) (Result, error) { 152 var result Result 153 154 result, err := c.NewBuilder(). 155 Unstructured(). 156 ContinueOnError(). 157 NamespaceParam(namespace). 158 DefaultNamespace(). 159 Stream(reader, ""). 160 Flatten(). 161 Do().Infos() 162 return result, scrubValidationError(err) 163 } 164 165 // BuildUnstructuredTable reads Kubernetes objects and returns unstructured infos 166 // as a Table. This is meant for viewing resources and displaying them in a table. 167 // This is similar to BuildUnstructured but transforms the request for table 168 // display. 169 func (c *Client) BuildUnstructuredTable(namespace string, reader io.Reader) (Result, error) { 170 var result Result 171 172 result, err := c.NewBuilder(). 173 Unstructured(). 174 ContinueOnError(). 175 NamespaceParam(namespace). 176 DefaultNamespace(). 177 Stream(reader, ""). 178 Flatten(). 179 TransformRequests(transformRequests). 180 Do().Infos() 181 return result, scrubValidationError(err) 182 } 183 184 // This is used to retrieve a table view of the data. A table view is how kubectl 185 // retrieves the information Helm displays as resources in status. Note, table 186 // data is returned as a Table type that does not conform to the runtime.Object 187 // interface but is that type. So, you can't transform it into Go objects easily. 188 func transformRequests(req *rest.Request) { 189 190 // The request headers are for both the v1 and v1beta1 versions of the table 191 // as Kubernetes 1.14 and older used the beta version. 192 req.SetHeader("Accept", strings.Join([]string{ 193 fmt.Sprintf("application/json;as=Table;v=%s;g=%s", metav1.SchemeGroupVersion.Version, metav1.GroupName), 194 fmt.Sprintf("application/json;as=Table;v=%s;g=%s", metav1beta1.SchemeGroupVersion.Version, metav1beta1.GroupName), 195 "application/json", 196 }, ",")) 197 } 198 199 // Validate reads Kubernetes manifests and validates the content. 200 // 201 // This function does not actually do schema validation of manifests. Adding 202 // validation now breaks existing clients of helm: https://github.com/helm/helm/issues/5750 203 func (c *Client) Validate(namespace string, reader io.Reader) error { 204 _, err := c.NewBuilder(). 205 Unstructured(). 206 ContinueOnError(). 207 NamespaceParam(namespace). 208 DefaultNamespace(). 209 // Schema(c.validator()). // No schema validation 210 Stream(reader, ""). 211 Flatten(). 212 Do().Infos() 213 return scrubValidationError(err) 214 } 215 216 // Build validates for Kubernetes objects and returns resource Infos from a io.Reader. 217 func (c *Client) Build(namespace string, reader io.Reader) (Result, error) { 218 var result Result 219 result, err := c.newBuilder(namespace, reader).Infos() 220 return result, scrubValidationError(err) 221 } 222 223 // Return the resource info as internal 224 func resourceInfoToObject(info *resource.Info, c *Client) runtime.Object { 225 internalObj, err := asInternal(info) 226 if err != nil { 227 // If the problem is just that the resource is not registered, don't print any 228 // error. This is normal for custom resources. 229 if !runtime.IsNotRegisteredError(err) { 230 c.Log("Warning: conversion to internal type failed: %v", err) 231 } 232 // Add the unstructured object in this situation. It will still get listed, just 233 // with less information. 234 return info.Object 235 } 236 237 return internalObj 238 } 239 240 func sortByKey(objs map[string][]runtime.Object) []string { 241 var keys []string 242 // Create a simple slice, so we can sort it 243 for key := range objs { 244 keys = append(keys, key) 245 } 246 // Sort alphabetically by version/kind keys 247 sort.Strings(keys) 248 return keys 249 } 250 251 // We have slices of tables that need to be sorted by name. In this case the 252 // self link is used so the sorting will include namespace and name. 253 func sortTableSlice(objs []runtime.Object) []runtime.Object { 254 // If there are 0 or 1 objects to sort there is nothing to sort so 255 // the list can be returned 256 if len(objs) < 2 { 257 return objs 258 } 259 260 ntbl := &metav1.Table{} 261 unstr, ok := objs[0].(*unstructured.Unstructured) 262 if !ok { 263 return objs 264 } 265 if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr.Object, ntbl); err != nil { 266 return objs 267 } 268 269 // Sort the list of objects 270 var newObjs []runtime.Object 271 namesCache := make(map[string]runtime.Object, len(objs)) 272 var names []string 273 var key string 274 for _, obj := range objs { 275 unstr, ok := obj.(*unstructured.Unstructured) 276 if !ok { 277 return objs 278 } 279 if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr.Object, ntbl); err != nil { 280 return objs 281 } 282 283 // At this point we have a table. Each table has just one row. We are 284 // sorting the tables by the first cell (name) in the first and only 285 // row. If the first cell of the first row cannot be gotten as a string 286 // we return the original unsorted list. 287 if len(ntbl.Rows) == 0 { // Make sure there are rows to read from 288 return objs 289 } 290 if len(ntbl.Rows[0].Cells) == 0 { // Make sure there are cells to read 291 return objs 292 } 293 key, ok = ntbl.Rows[0].Cells[0].(string) 294 if !ok { 295 return objs 296 } 297 namesCache[key] = obj 298 names = append(names, key) 299 } 300 301 sort.Strings(names) 302 303 for _, name := range names { 304 newObjs = append(newObjs, namesCache[name]) 305 } 306 307 return newObjs 308 } 309 310 // Get gets Kubernetes resources as pretty-printed string. 311 // 312 // Namespace will set the namespace. 313 func (c *Client) Get(namespace string, reader io.Reader) (string, error) { 314 // Since we don't know what order the objects come in, let's group them by the types and then sort them, so 315 // that when we print them, they come out looking good (headers apply to subgroups, etc.). 316 objs := make(map[string][]runtime.Object) 317 gk := make(map[string]schema.GroupKind) 318 mux := &sync.Mutex{} 319 320 // The contents of the reader are used two times. The bytes are coppied out 321 // for use in future readers. 322 b, err := ioutil.ReadAll(reader) 323 if err != nil { 324 return "", err 325 } 326 327 // Get the table display for the objects associated with the release. This 328 // is done in table format so that it can be displayed in the status in 329 // the same way kubectl displays the resource information. 330 // Note, the response returns unstructured data instead of typed objects. 331 // These cannot be easily (i.e., via the go packages) transformed into 332 // Go types. 333 tinfos, err := c.BuildUnstructuredTable(namespace, bytes.NewBuffer(b)) 334 if err != nil { 335 return "", err 336 } 337 338 missing := []string{} 339 err = perform(tinfos, func(info *resource.Info) error { 340 mux.Lock() 341 defer mux.Unlock() 342 c.Log("Doing get for %s: %q", info.Mapping.GroupVersionKind.Kind, info.Name) 343 if err := info.Get(); err != nil { 344 c.Log("WARNING: Failed Get for resource %q: %s", info.Name, err) 345 missing = append(missing, fmt.Sprintf("%v\t\t%s", info.Mapping.Resource, info.Name)) 346 return nil 347 } 348 349 // Use APIVersion/Kind as grouping mechanism. I'm not sure if you can have multiple 350 // versions per cluster, but this certainly won't hurt anything, so let's be safe. 351 gvk := info.ResourceMapping().GroupVersionKind 352 vk := gvk.Version + "/" + gvk.Kind 353 gk[vk] = gvk.GroupKind() 354 355 // Initialize map. The main map groups resources based on version/kind 356 // The second level is a simple 'Name' to 'Object', that will help sort 357 // the individual resource later 358 if objs[vk] == nil { 359 objs[vk] = []runtime.Object{} 360 } 361 // Map between the resource name to the underlying info object 362 objs[vk] = append(objs[vk], resourceInfoToObject(info, c)) 363 364 return nil 365 }) 366 if err != nil { 367 return "", err 368 } 369 370 // This section finds related resources (e.g., pods). Before looking up pods 371 // the resources the pods are made from need to be looked up in a manner 372 // that can be turned into Go types and worked with. 373 infos, err := c.BuildUnstructured(namespace, bytes.NewBuffer(b)) 374 if err != nil { 375 return "", err 376 } 377 err = perform(infos, func(info *resource.Info) error { 378 mux.Lock() 379 defer mux.Unlock() 380 if err := info.Get(); err != nil { 381 c.Log("WARNING: Failed Get for resource %q: %s", info.Name, err) 382 missing = append(missing, fmt.Sprintf("%v\t\t%s", info.Mapping.Resource, info.Name)) 383 return nil 384 } 385 386 //Get the relation pods 387 objs, err = c.getSelectRelationPod(info, objs) 388 if err != nil { 389 c.Log("Warning: get the relation pod is failed, err:%s", err.Error()) 390 } 391 392 return nil 393 }) 394 if err != nil { 395 return "", err 396 } 397 398 // Ok, now we have all the objects grouped by types (say, by v1/Pod, v1/Service, etc.), so 399 // spin through them and print them. Printer is cool since it prints the header only when 400 // an object type changes, so we can just rely on that. Problem is it doesn't seem to keep 401 // track of tab widths. 402 buf := new(bytes.Buffer) 403 404 // Sort alphabetically by version/kind keys 405 vkKeys := sortByKey(objs) 406 // Iterate on sorted version/kind types 407 for _, t := range vkKeys { 408 if _, err = fmt.Fprintf(buf, "==> %s\n", t); err != nil { 409 return "", err 410 } 411 vk := objs[t] 412 413 // The request made for tables returns each Kubernetes object as its 414 // own table. The normal sorting provided by kubectl and cli-runtime 415 // does not handle this case. Here we sort within each of our own 416 // grouping. 417 vk = sortTableSlice(vk) 418 419 // The printer flag setup follows a simalar setup to kubectl 420 printFlags := get.NewHumanPrintFlags() 421 if lgk, ok := gk[t]; ok { 422 printFlags.SetKind(lgk) 423 } 424 printer, _ := printFlags.ToPrinter("") 425 printer, err = printers.NewTypeSetter(scheme.Scheme).WrapToPrinter(printer, nil) 426 if err != nil { 427 return "", err 428 } 429 printer = &get.TablePrinter{Delegate: printer} 430 431 for _, resource := range vk { 432 if err := printer.PrintObj(resource, buf); err != nil { 433 c.Log("failed to print object type %s: %v", t, err) 434 return "", err 435 } 436 } 437 if _, err := buf.WriteString("\n"); err != nil { 438 return "", err 439 } 440 } 441 if len(missing) > 0 { 442 buf.WriteString(MissingGetHeader) 443 for _, s := range missing { 444 fmt.Fprintln(buf, s) 445 } 446 } 447 return buf.String(), nil 448 } 449 450 // Update reads the current configuration and a target configuration from io.reader 451 // and creates resources that don't already exist, updates resources that have been modified 452 // in the target configuration and deletes resources from the current configuration that are 453 // not present in the target configuration. 454 // 455 // Namespace will set the namespaces. 456 // 457 // Deprecated: use UpdateWithOptions instead. 458 func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error { 459 return c.UpdateWithOptions(namespace, originalReader, targetReader, UpdateOptions{ 460 Force: force, 461 Recreate: recreate, 462 Timeout: timeout, 463 ShouldWait: shouldWait, 464 }) 465 } 466 467 // UpdateOptions provides options to control update behavior 468 type UpdateOptions struct { 469 Force bool 470 Recreate bool 471 Timeout int64 472 ShouldWait bool 473 // Allow deletion of new resources created in this update when update failed 474 CleanupOnFail bool 475 } 476 477 // UpdateWithOptions reads the current configuration and a target configuration from io.reader 478 // and creates resources that don't already exist, updates resources that have been modified 479 // in the target configuration and deletes resources from the current configuration that are 480 // not present in the target configuration. 481 // 482 // Namespace will set the namespaces. UpdateOptions provides additional parameters to control 483 // update behavior. 484 func (c *Client) UpdateWithOptions(namespace string, originalReader, targetReader io.Reader, opts UpdateOptions) error { 485 original, err := c.BuildUnstructured(namespace, originalReader) 486 if err != nil { 487 return fmt.Errorf("failed decoding reader into objects: %s", err) 488 } 489 490 c.Log("building resources from updated manifest") 491 target, err := c.BuildUnstructured(namespace, targetReader) 492 if err != nil { 493 return fmt.Errorf("failed decoding reader into objects: %s", err) 494 } 495 496 newlyCreatedResources := []*resource.Info{} 497 updateErrors := []string{} 498 499 c.Log("checking %d resources for changes", len(target)) 500 err = target.Visit(func(info *resource.Info, err error) error { 501 if err != nil { 502 return err 503 } 504 505 helper := resource.NewHelper(info.Client, info.Mapping) 506 if _, err := helper.Get(info.Namespace, info.Name, info.Export); err != nil { 507 if !errors.IsNotFound(err) { 508 return fmt.Errorf("Could not get information about the resource: %s", err) 509 } 510 511 // Since the resource does not exist, create it. 512 if err := createResource(info); err != nil { 513 return fmt.Errorf("failed to create resource: %s", err) 514 } 515 newlyCreatedResources = append(newlyCreatedResources, info) 516 517 kind := info.Mapping.GroupVersionKind.Kind 518 c.Log("Created a new %s called %q\n", kind, info.Name) 519 return nil 520 } 521 522 originalInfo := original.Get(info) 523 524 // The resource already exists in the cluster, but it wasn't defined in the previous release. 525 // In this case, we consider it to be a resource that was previously un-managed by the release and error out, 526 // asking for the user to intervene. 527 // 528 // See https://github.com/helm/helm/issues/1193 for more info. 529 if originalInfo == nil { 530 return fmt.Errorf( 531 "kind %s with the name %q already exists in the cluster and wasn't defined in the previous release. Before upgrading, please either delete the resource from the cluster or remove it from the chart", 532 info.Mapping.GroupVersionKind.Kind, 533 info.Name, 534 ) 535 } 536 537 if err := updateResource(c, info, originalInfo.Object, opts.Force, opts.Recreate); err != nil { 538 c.Log("error updating the resource %q:\n\t %v", info.Name, err) 539 updateErrors = append(updateErrors, err.Error()) 540 } 541 542 return nil 543 }) 544 545 cleanupErrors := []string{} 546 547 if opts.CleanupOnFail && (err != nil || len(updateErrors) != 0) { 548 c.Log("Cleanup on fail enabled: cleaning up newly created resources due to update manifests failures") 549 cleanupErrors = c.cleanup(newlyCreatedResources) 550 } 551 552 switch { 553 case err != nil: 554 return fmt.Errorf(strings.Join(append([]string{err.Error()}, cleanupErrors...), " && ")) 555 case len(updateErrors) != 0: 556 return fmt.Errorf(strings.Join(append(updateErrors, cleanupErrors...), " && ")) 557 } 558 559 for _, info := range original.Difference(target) { 560 c.Log("Deleting %q in %s...", info.Name, info.Namespace) 561 562 if err := info.Get(); err != nil { 563 c.Log("Unable to get obj %q, err: %s", info.Name, err) 564 } 565 annotations, err := metadataAccessor.Annotations(info.Object) 566 if err != nil { 567 c.Log("Unable to get annotations on %q, err: %s", info.Name, err) 568 } 569 if ResourcePolicyIsKeep(annotations) { 570 policy := annotations[ResourcePolicyAnno] 571 c.Log("Skipping delete of %q due to annotation [%s=%s]", info.Name, ResourcePolicyAnno, policy) 572 continue 573 } 574 575 if err := deleteResource(info); err != nil { 576 c.Log("Failed to delete %q, err: %s", info.Name, err) 577 } 578 } 579 if opts.ShouldWait { 580 err := c.waitForResources(time.Duration(opts.Timeout)*time.Second, target) 581 582 if opts.CleanupOnFail && err != nil { 583 c.Log("Cleanup on fail enabled: cleaning up newly created resources due to wait failure during update") 584 cleanupErrors = c.cleanup(newlyCreatedResources) 585 return fmt.Errorf(strings.Join(append([]string{err.Error()}, cleanupErrors...), " && ")) 586 } 587 588 return err 589 } 590 return nil 591 } 592 593 func (c *Client) cleanup(newlyCreatedResources []*resource.Info) (cleanupErrors []string) { 594 for _, info := range newlyCreatedResources { 595 kind := info.Mapping.GroupVersionKind.Kind 596 c.Log("Deleting newly created %s with the name %q in %s...", kind, info.Name, info.Namespace) 597 if err := deleteResource(info); err != nil { 598 c.Log("Error deleting newly created %s with the name %q in %s: %s", kind, info.Name, info.Namespace, err) 599 cleanupErrors = append(cleanupErrors, err.Error()) 600 } 601 } 602 return 603 } 604 605 // Delete deletes Kubernetes resources from an io.reader. 606 // 607 // Namespace will set the namespace. 608 func (c *Client) Delete(namespace string, reader io.Reader) error { 609 return c.DeleteWithTimeout(namespace, reader, 0, false) 610 } 611 612 // DeleteWithTimeout deletes Kubernetes resources from an io.reader. If shouldWait is true, the function 613 // will wait for all resources to be deleted from etcd before returning, or when the timeout 614 // has expired. 615 // 616 // Namespace will set the namespace. 617 func (c *Client) DeleteWithTimeout(namespace string, reader io.Reader, timeout int64, shouldWait bool) error { 618 infos, err := c.BuildUnstructured(namespace, reader) 619 if err != nil { 620 return err 621 } 622 err = perform(infos, func(info *resource.Info) error { 623 c.Log("Starting delete for %q %s", info.Name, info.Mapping.GroupVersionKind.Kind) 624 err := deleteResource(info) 625 return c.skipIfNotFound(err) 626 }) 627 if err != nil { 628 return err 629 } 630 631 if shouldWait { 632 c.Log("Waiting for %d seconds for delete to be completed", timeout) 633 return waitUntilAllResourceDeleted(infos, time.Duration(timeout)*time.Second) 634 } 635 636 return nil 637 } 638 639 func (c *Client) skipIfNotFound(err error) error { 640 if errors.IsNotFound(err) { 641 c.Log("%v", err) 642 return nil 643 } 644 return err 645 } 646 647 func waitUntilAllResourceDeleted(infos Result, timeout time.Duration) error { 648 return wait.Poll(2*time.Second, timeout, func() (bool, error) { 649 allDeleted := true 650 err := perform(infos, func(info *resource.Info) error { 651 innerErr := info.Get() 652 if errors.IsNotFound(innerErr) { 653 return nil 654 } 655 if innerErr != nil { 656 return innerErr 657 } 658 allDeleted = false 659 return nil 660 }) 661 if err != nil { 662 return false, err 663 } 664 return allDeleted, nil 665 }) 666 } 667 668 func (c *Client) watchTimeout(t time.Duration) ResourceActorFunc { 669 return func(info *resource.Info) error { 670 return c.watchUntilReady(t, info) 671 } 672 } 673 674 // WatchUntilReady watches the resource given in the reader, and waits until it is ready. 675 // 676 // This function is mainly for hook implementations. It watches for a resource to 677 // hit a particular milestone. The milestone depends on the Kind. 678 // 679 // For most kinds, it checks to see if the resource is marked as Added or Modified 680 // by the Kubernetes event stream. For some kinds, it does more: 681 // 682 // - Jobs: A job is marked "Ready" when it has successfully completed. This is 683 // ascertained by watching the Status fields in a job's output. 684 // 685 // Handling for other kinds will be added as necessary. 686 func (c *Client) WatchUntilReady(namespace string, reader io.Reader, timeout int64, shouldWait bool) error { 687 infos, err := c.BuildUnstructured(namespace, reader) 688 if err != nil { 689 return err 690 } 691 // For jobs, there's also the option to do poll c.Jobs(namespace).Get(): 692 // https://github.com/adamreese/kubernetes/blob/master/test/e2e/job.go#L291-L300 693 return perform(infos, c.watchTimeout(time.Duration(timeout)*time.Second)) 694 } 695 696 // WaitUntilCRDEstablished polls the given CRD until it reaches the established 697 // state. A CRD needs to reach the established state before CRs can be created. 698 // 699 // If a naming conflict condition is found, this function will return an error. 700 func (c *Client) WaitUntilCRDEstablished(reader io.Reader, timeout time.Duration) error { 701 infos, err := c.BuildUnstructured(metav1.NamespaceAll, reader) 702 if err != nil { 703 return err 704 } 705 706 return perform(infos, c.pollCRDEstablished(timeout)) 707 } 708 709 func (c *Client) pollCRDEstablished(t time.Duration) ResourceActorFunc { 710 return func(info *resource.Info) error { 711 return c.pollCRDUntilEstablished(t, info) 712 } 713 } 714 715 func (c *Client) pollCRDUntilEstablished(timeout time.Duration, info *resource.Info) error { 716 return wait.PollImmediate(time.Second, timeout, func() (bool, error) { 717 err := info.Get() 718 if err != nil { 719 return false, fmt.Errorf("unable to get CRD: %v", err) 720 } 721 722 crd := &apiextv1beta1.CustomResourceDefinition{} 723 err = scheme.Scheme.Convert(info.Object, crd, nil) 724 if err != nil { 725 return false, fmt.Errorf("unable to convert to CRD type: %v", err) 726 } 727 728 for _, cond := range crd.Status.Conditions { 729 switch cond.Type { 730 case apiextv1beta1.Established: 731 if cond.Status == apiextv1beta1.ConditionTrue { 732 return true, nil 733 } 734 case apiextv1beta1.NamesAccepted: 735 if cond.Status == apiextv1beta1.ConditionFalse { 736 return false, fmt.Errorf("naming conflict detected for CRD %s", crd.GetName()) 737 } 738 } 739 } 740 741 return false, nil 742 }) 743 } 744 745 func perform(infos Result, fn ResourceActorFunc) error { 746 if len(infos) == 0 { 747 return ErrNoObjectsVisited 748 } 749 750 errs := make(chan error) 751 go batchPerform(infos, fn, errs) 752 753 for range infos { 754 err := <-errs 755 if err != nil { 756 return err 757 } 758 } 759 return nil 760 } 761 762 func batchPerform(infos Result, fn ResourceActorFunc, errs chan<- error) { 763 var kind string 764 var wg sync.WaitGroup 765 for _, info := range infos { 766 currentKind := info.Object.GetObjectKind().GroupVersionKind().Kind 767 if kind != currentKind { 768 wg.Wait() 769 kind = currentKind 770 } 771 wg.Add(1) 772 go func(i *resource.Info) { 773 errs <- fn(i) 774 wg.Done() 775 }(info) 776 } 777 } 778 779 func createResource(info *resource.Info) error { 780 obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object, nil) 781 if err != nil { 782 return err 783 } 784 return info.Refresh(obj, true) 785 } 786 787 func deleteResource(info *resource.Info) error { 788 policy := metav1.DeletePropagationBackground 789 opts := &metav1.DeleteOptions{PropagationPolicy: &policy} 790 _, err := resource.NewHelper(info.Client, info.Mapping).DeleteWithOptions(info.Namespace, info.Name, opts) 791 return err 792 } 793 794 func createPatch(target *resource.Info, current runtime.Object) ([]byte, types.PatchType, error) { 795 oldData, err := json.Marshal(current) 796 if err != nil { 797 return nil, types.StrategicMergePatchType, fmt.Errorf("serializing current configuration: %s", err) 798 } 799 newData, err := json.Marshal(target.Object) 800 if err != nil { 801 return nil, types.StrategicMergePatchType, fmt.Errorf("serializing target configuration: %s", err) 802 } 803 804 // While different objects need different merge types, the parent function 805 // that calls this does not try to create a patch when the data (first 806 // returned object) is nil. We can skip calculating the merge type as 807 // the returned merge type is ignored. 808 if apiequality.Semantic.DeepEqual(oldData, newData) { 809 return nil, types.StrategicMergePatchType, nil 810 } 811 812 // Get a versioned object 813 versionedObject, err := asVersioned(target) 814 815 // Unstructured objects, such as CRDs, may not have a not registered error 816 // returned from ConvertToVersion. Anything that's unstructured should 817 // use the jsonpatch.CreateMergePatch. Strategic Merge Patch is not supported 818 // on objects like CRDs. 819 _, isUnstructured := versionedObject.(runtime.Unstructured) 820 821 // On newer K8s versions, CRDs aren't unstructured but has this dedicated type 822 _, isCRD := versionedObject.(*apiextv1beta1.CustomResourceDefinition) 823 824 switch { 825 case runtime.IsNotRegisteredError(err), isUnstructured, isCRD: 826 // fall back to generic JSON merge patch 827 patch, err := jsonpatch.CreateMergePatch(oldData, newData) 828 if err != nil { 829 return nil, types.MergePatchType, fmt.Errorf("failed to create merge patch: %v", err) 830 } 831 return patch, types.MergePatchType, nil 832 case err != nil: 833 return nil, types.StrategicMergePatchType, fmt.Errorf("failed to get versionedObject: %s", err) 834 default: 835 patch, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, versionedObject) 836 if err != nil { 837 return nil, types.StrategicMergePatchType, fmt.Errorf("failed to create two-way merge patch: %v", err) 838 } 839 return patch, types.StrategicMergePatchType, nil 840 } 841 } 842 843 func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, force bool, recreate bool) error { 844 patch, patchType, err := createPatch(target, currentObj) 845 if err != nil { 846 return fmt.Errorf("failed to create patch: %s", err) 847 } 848 if patch == nil { 849 c.Log("Looks like there are no changes for %s %q", target.Mapping.GroupVersionKind.Kind, target.Name) 850 // This needs to happen to make sure that tiller has the latest info from the API 851 // Otherwise there will be no labels and other functions that use labels will panic 852 if err := target.Get(); err != nil { 853 return fmt.Errorf("error trying to refresh resource information: %v", err) 854 } 855 } else { 856 // send patch to server 857 helper := resource.NewHelper(target.Client, target.Mapping) 858 859 obj, err := helper.Patch(target.Namespace, target.Name, patchType, patch, nil) 860 if err != nil { 861 kind := target.Mapping.GroupVersionKind.Kind 862 log.Printf("Cannot patch %s: %q (%v)", kind, target.Name, err) 863 864 if force { 865 // Attempt to delete... 866 if err := deleteResource(target); err != nil { 867 return err 868 } 869 log.Printf("Deleted %s: %q", kind, target.Name) 870 871 // ... and recreate 872 if err := createResource(target); err != nil { 873 return fmt.Errorf("Failed to recreate resource: %s", err) 874 } 875 log.Printf("Created a new %s called %q\n", kind, target.Name) 876 877 // No need to refresh the target, as we recreated the resource based 878 // on it. In addition, it might not exist yet and a call to `Refresh` 879 // may fail. 880 } else { 881 log.Print("Use --force to force recreation of the resource") 882 return err 883 } 884 } else { 885 // When patch succeeds without needing to recreate, refresh target. 886 target.Refresh(obj, true) 887 } 888 } 889 890 if !recreate { 891 return nil 892 } 893 894 versioned := asVersionedOrUnstructured(target) 895 selector, ok := getSelectorFromObject(versioned) 896 if !ok { 897 return nil 898 } 899 900 client, err := c.KubernetesClientSet() 901 if err != nil { 902 return err 903 } 904 905 pods, err := client.CoreV1().Pods(target.Namespace).List(metav1.ListOptions{ 906 LabelSelector: labels.Set(selector).AsSelector().String(), 907 }) 908 if err != nil { 909 return err 910 } 911 912 // Restart pods 913 for _, pod := range pods.Items { 914 c.Log("Restarting pod: %v/%v", pod.Namespace, pod.Name) 915 916 // Delete each pod for get them restarted with changed spec. 917 if err := client.CoreV1().Pods(pod.Namespace).Delete(pod.Name, metav1.NewPreconditionDeleteOptions(string(pod.UID))); err != nil { 918 return err 919 } 920 } 921 return nil 922 } 923 924 func getSelectorFromObject(obj runtime.Object) (map[string]string, bool) { 925 switch typed := obj.(type) { 926 927 case *v1.ReplicationController: 928 return typed.Spec.Selector, true 929 930 case *extv1beta1.ReplicaSet: 931 return typed.Spec.Selector.MatchLabels, true 932 case *appsv1.ReplicaSet: 933 return typed.Spec.Selector.MatchLabels, true 934 935 case *extv1beta1.Deployment: 936 return typed.Spec.Selector.MatchLabels, true 937 case *appsv1beta1.Deployment: 938 return typed.Spec.Selector.MatchLabels, true 939 case *appsv1beta2.Deployment: 940 return typed.Spec.Selector.MatchLabels, true 941 case *appsv1.Deployment: 942 return typed.Spec.Selector.MatchLabels, true 943 944 case *extv1beta1.DaemonSet: 945 return typed.Spec.Selector.MatchLabels, true 946 case *appsv1beta2.DaemonSet: 947 return typed.Spec.Selector.MatchLabels, true 948 case *appsv1.DaemonSet: 949 return typed.Spec.Selector.MatchLabels, true 950 951 case *batch.Job: 952 return typed.Spec.Selector.MatchLabels, true 953 954 case *appsv1beta1.StatefulSet: 955 return typed.Spec.Selector.MatchLabels, true 956 case *appsv1beta2.StatefulSet: 957 return typed.Spec.Selector.MatchLabels, true 958 case *appsv1.StatefulSet: 959 return typed.Spec.Selector.MatchLabels, true 960 961 default: 962 return nil, false 963 } 964 } 965 966 func (c *Client) watchUntilReady(timeout time.Duration, info *resource.Info) error { 967 // Use a selector on the name of the resource. This should be unique for the 968 // given version and kind 969 selector, err := fields.ParseSelector(fmt.Sprintf("metadata.name=%s", info.Name)) 970 if err != nil { 971 return err 972 } 973 lw := cachetools.NewListWatchFromClient(info.Client, info.Mapping.Resource.Resource, info.Namespace, selector) 974 975 kind := info.Mapping.GroupVersionKind.Kind 976 c.Log("Watching for changes to %s %s with timeout of %v", kind, info.Name, timeout) 977 978 // What we watch for depends on the Kind. 979 // - For a Job, we watch for completion. 980 // - For all else, we watch until Ready. 981 // In the future, we might want to add some special logic for types 982 // like Ingress, Volume, etc. 983 984 ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) 985 defer cancel() 986 _, err = watchtools.ListWatchUntil(ctx, lw, func(e watch.Event) (bool, error) { 987 switch e.Type { 988 case watch.Added, watch.Modified: 989 // For things like a secret or a config map, this is the best indicator 990 // we get. We care mostly about jobs, where what we want to see is 991 // the status go into a good state. For other types, like ReplicaSet 992 // we don't really do anything to support these as hooks. 993 c.Log("Add/Modify event for %s: %v", info.Name, e.Type) 994 if kind == "Job" { 995 return c.waitForJob(e, info.Name) 996 } 997 return true, nil 998 case watch.Deleted: 999 c.Log("Deleted event for %s", info.Name) 1000 return true, nil 1001 case watch.Error: 1002 // Handle error and return with an error. 1003 c.Log("Error event for %s", info.Name) 1004 return true, fmt.Errorf("Failed to deploy %s", info.Name) 1005 default: 1006 return false, nil 1007 } 1008 }) 1009 return err 1010 } 1011 1012 // waitForJob is a helper that waits for a job to complete. 1013 // 1014 // This operates on an event returned from a watcher. 1015 func (c *Client) waitForJob(e watch.Event, name string) (bool, error) { 1016 job := &batch.Job{} 1017 err := scheme.Scheme.Convert(e.Object, job, nil) 1018 if err != nil { 1019 return true, err 1020 } 1021 1022 for _, c := range job.Status.Conditions { 1023 if c.Type == batch.JobComplete && c.Status == v1.ConditionTrue { 1024 return true, nil 1025 } else if c.Type == batch.JobFailed && c.Status == v1.ConditionTrue { 1026 return true, fmt.Errorf("Job failed: %s", c.Reason) 1027 } 1028 } 1029 1030 c.Log("%s: Jobs active: %d, jobs failed: %d, jobs succeeded: %d", name, job.Status.Active, job.Status.Failed, job.Status.Succeeded) 1031 return false, nil 1032 } 1033 1034 // scrubValidationError removes kubectl info from the message. 1035 func scrubValidationError(err error) error { 1036 if err == nil { 1037 return nil 1038 } 1039 const stopValidateMessage = "if you choose to ignore these errors, turn validation off with --validate=false" 1040 1041 if strings.Contains(err.Error(), stopValidateMessage) { 1042 return goerrors.New(strings.Replace(err.Error(), "; "+stopValidateMessage, "", -1)) 1043 } 1044 return err 1045 } 1046 1047 // WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase 1048 // and returns said phase (PodSucceeded or PodFailed qualify). 1049 func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (v1.PodPhase, error) { 1050 infos, err := c.Build(namespace, reader) 1051 if err != nil { 1052 return v1.PodUnknown, err 1053 } 1054 info := infos[0] 1055 1056 kind := info.Mapping.GroupVersionKind.Kind 1057 if kind != "Pod" { 1058 return v1.PodUnknown, fmt.Errorf("%s is not a Pod", info.Name) 1059 } 1060 1061 if err := c.watchPodUntilComplete(timeout, info); err != nil { 1062 return v1.PodUnknown, err 1063 } 1064 1065 if err := info.Get(); err != nil { 1066 return v1.PodUnknown, err 1067 } 1068 status := info.Object.(*v1.Pod).Status.Phase 1069 1070 return status, nil 1071 } 1072 1073 func (c *Client) watchPodUntilComplete(timeout time.Duration, info *resource.Info) error { 1074 lw := cachetools.NewListWatchFromClient(info.Client, info.Mapping.Resource.Resource, info.Namespace, fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", info.Name))) 1075 1076 c.Log("Watching pod %s for completion with timeout of %v", info.Name, timeout) 1077 ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) 1078 defer cancel() 1079 _, err := watchtools.ListWatchUntil(ctx, lw, func(e watch.Event) (bool, error) { 1080 return isPodComplete(e) 1081 }) 1082 1083 return err 1084 } 1085 1086 // GetPodLogs takes pod name and namespace and returns the current logs (streaming is NOT enabled). 1087 func (c *Client) GetPodLogs(name, ns string) (io.ReadCloser, error) { 1088 client, err := c.KubernetesClientSet() 1089 if err != nil { 1090 return nil, err 1091 } 1092 req := client.CoreV1().Pods(ns).GetLogs(name, &v1.PodLogOptions{}) 1093 logReader, err := req.Stream() 1094 if err != nil { 1095 return nil, fmt.Errorf("error in opening log stream, got: %s", err) 1096 } 1097 return logReader, nil 1098 } 1099 1100 func isPodComplete(event watch.Event) (bool, error) { 1101 o, ok := event.Object.(*v1.Pod) 1102 if !ok { 1103 return true, fmt.Errorf("expected a *v1.Pod, got %T", event.Object) 1104 } 1105 if event.Type == watch.Deleted { 1106 return false, fmt.Errorf("pod not found") 1107 } 1108 switch o.Status.Phase { 1109 case v1.PodFailed, v1.PodSucceeded: 1110 return true, nil 1111 } 1112 return false, nil 1113 } 1114 1115 // get a kubernetes resources' relation pods 1116 // kubernetes resource used select labels to relate pods 1117 func (c *Client) getSelectRelationPod(info *resource.Info, objs map[string][]runtime.Object) (map[string][]runtime.Object, error) { 1118 if info == nil { 1119 return objs, nil 1120 } 1121 1122 c.Log("get relation pod of object: %s/%s/%s", info.Namespace, info.Mapping.GroupVersionKind.Kind, info.Name) 1123 1124 versioned := asVersionedOrUnstructured(info) 1125 selector, ok := getSelectorFromObject(versioned) 1126 if !ok { 1127 return objs, nil 1128 } 1129 1130 // The related pods are looked up in Table format so that their display can 1131 // be printed in a manner similar to kubectl when it get pods. The response 1132 // can be used with a table printer. 1133 infos, err := c.NewBuilder(). 1134 Unstructured(). 1135 ContinueOnError(). 1136 NamespaceParam(info.Namespace). 1137 DefaultNamespace(). 1138 ResourceTypes("pods"). 1139 LabelSelector(labels.Set(selector).AsSelector().String()). 1140 TransformRequests(transformRequests). 1141 Do().Infos() 1142 if err != nil { 1143 return objs, err 1144 } 1145 1146 for _, info := range infos { 1147 vk := "v1/Pod(related)" 1148 objs[vk] = append(objs[vk], info.Object) 1149 } 1150 1151 return objs, nil 1152 } 1153 1154 func asVersionedOrUnstructured(info *resource.Info) runtime.Object { 1155 obj, _ := asVersioned(info) 1156 return obj 1157 } 1158 1159 func asVersioned(info *resource.Info) (runtime.Object, error) { 1160 converter := runtime.ObjectConvertor(scheme.Scheme) 1161 groupVersioner := runtime.GroupVersioner(schema.GroupVersions(scheme.Scheme.PrioritizedVersionsAllGroups())) 1162 if info.Mapping != nil { 1163 groupVersioner = info.Mapping.GroupVersionKind.GroupVersion() 1164 } 1165 1166 obj, err := converter.ConvertToVersion(info.Object, groupVersioner) 1167 if err != nil { 1168 return info.Object, err 1169 } 1170 return obj, nil 1171 } 1172 1173 func asInternal(info *resource.Info) (runtime.Object, error) { 1174 groupVersioner := info.Mapping.GroupVersionKind.GroupKind().WithVersion(runtime.APIVersionInternal).GroupVersion() 1175 return scheme.Scheme.ConvertToVersion(info.Object, groupVersioner) 1176 }