github.com/caicloud/helm@v2.5.0+incompatible/pkg/kube/client.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors All rights reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package kube // import "k8s.io/helm/pkg/kube" 18 19 import ( 20 "bytes" 21 "encoding/json" 22 goerrors "errors" 23 "fmt" 24 "io" 25 "log" 26 "strings" 27 "time" 28 29 jsonpatch "github.com/evanphx/json-patch" 30 "k8s.io/apimachinery/pkg/api/errors" 31 "k8s.io/apimachinery/pkg/api/meta" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 34 "k8s.io/apimachinery/pkg/fields" 35 "k8s.io/apimachinery/pkg/labels" 36 "k8s.io/apimachinery/pkg/runtime" 37 "k8s.io/apimachinery/pkg/types" 38 "k8s.io/apimachinery/pkg/util/strategicpatch" 39 "k8s.io/apimachinery/pkg/watch" 40 "k8s.io/client-go/tools/clientcmd" 41 "k8s.io/kubernetes/pkg/api" 42 "k8s.io/kubernetes/pkg/api/v1" 43 apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1" 44 batchinternal "k8s.io/kubernetes/pkg/apis/batch" 45 batch "k8s.io/kubernetes/pkg/apis/batch/v1" 46 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" 47 conditions "k8s.io/kubernetes/pkg/client/unversioned" 48 "k8s.io/kubernetes/pkg/kubectl" 49 cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" 50 "k8s.io/kubernetes/pkg/kubectl/resource" 51 "k8s.io/kubernetes/pkg/printers" 52 53 "k8s.io/helm/pkg/releaseutil" 54 ) 55 56 // ErrNoObjectsVisited indicates that during a visit operation, no matching objects were found. 57 var ErrNoObjectsVisited = goerrors.New("no objects visited") 58 59 // Client represents a client capable of communicating with the Kubernetes API. 60 type Client struct { 61 cmdutil.Factory 62 // SchemaCacheDir is the path for loading cached schema. 63 SchemaCacheDir string 64 65 Log func(string, ...interface{}) 66 } 67 68 // New create a new Client 69 func New(config clientcmd.ClientConfig) *Client { 70 return &Client{ 71 Factory: cmdutil.NewFactory(config), 72 SchemaCacheDir: clientcmd.RecommendedSchemaFile, 73 Log: func(_ string, _ ...interface{}) {}, 74 } 75 } 76 77 // ResourceActorFunc performs an action on a single resource. 78 type ResourceActorFunc func(*resource.Info) error 79 80 // Create creates kubernetes resources from an io.reader 81 // 82 // Namespace will set the namespace 83 func (c *Client) Create(namespace string, reader io.Reader, timeout int64, shouldWait bool) error { 84 client, err := c.ClientSet() 85 if err != nil { 86 return err 87 } 88 if err := ensureNamespace(client, namespace); err != nil { 89 return err 90 } 91 c.Log("building resources from manifest") 92 infos, buildErr := c.BuildUnstructured(namespace, reader) 93 if buildErr != nil { 94 return buildErr 95 } 96 c.Log("creating %d resource(s)", len(infos)) 97 if err := perform(infos, createResource); err != nil { 98 return err 99 } 100 if shouldWait { 101 return c.waitForResources(time.Duration(timeout)*time.Second, infos) 102 } 103 return nil 104 } 105 106 func (c *Client) newBuilder(namespace string, reader io.Reader) *resource.Result { 107 schema, err := c.Validator(true, c.SchemaCacheDir) 108 if err != nil { 109 c.Log("warning: failed to load schema: %s", err) 110 } 111 return c.NewBuilder(). 112 ContinueOnError(). 113 Schema(schema). 114 NamespaceParam(namespace). 115 DefaultNamespace(). 116 Stream(reader, ""). 117 Flatten(). 118 Do() 119 } 120 121 // BuildUnstructured validates for Kubernetes objects and returns unstructured infos. 122 func (c *Client) BuildUnstructured(namespace string, reader io.Reader) (Result, error) { 123 schema, err := c.Validator(true, c.SchemaCacheDir) 124 if err != nil { 125 c.Log("warning: failed to load schema: %s", err) 126 } 127 128 mapper, typer, err := c.UnstructuredObject() 129 if err != nil { 130 c.Log("failed to load mapper: %s", err) 131 return nil, err 132 } 133 var result Result 134 result, err = resource.NewBuilder(mapper, typer, resource.ClientMapperFunc(c.UnstructuredClientForMapping), unstructured.UnstructuredJSONScheme). 135 ContinueOnError(). 136 Schema(schema). 137 NamespaceParam(namespace). 138 DefaultNamespace(). 139 Stream(reader, ""). 140 Flatten(). 141 Do().Infos() 142 return result, scrubValidationError(err) 143 } 144 145 // Build validates for Kubernetes objects and returns resource Infos from a io.Reader. 146 func (c *Client) Build(namespace string, reader io.Reader) (Result, error) { 147 var result Result 148 result, err := c.newBuilder(namespace, reader).Infos() 149 return result, scrubValidationError(err) 150 } 151 152 // Get gets kubernetes resources as pretty printed string 153 // 154 // Namespace will set the namespace 155 func (c *Client) Get(namespace string, reader io.Reader) (string, error) { 156 // Since we don't know what order the objects come in, let's group them by the types, so 157 // that when we print them, they come looking good (headers apply to subgroups, etc.) 158 objs := make(map[string][]runtime.Object) 159 infos, err := c.BuildUnstructured(namespace, reader) 160 if err != nil { 161 return "", err 162 } 163 missing := []string{} 164 err = perform(infos, func(info *resource.Info) error { 165 c.Log("Doing get for %s: %q", info.Mapping.GroupVersionKind.Kind, info.Name) 166 if err := info.Get(); err != nil { 167 c.Log("WARNING: Failed Get for resource %q: %s", info.Name, err) 168 missing = append(missing, fmt.Sprintf("%v\t\t%s", info.Mapping.Resource, info.Name)) 169 return nil 170 } 171 172 // Use APIVersion/Kind as grouping mechanism. I'm not sure if you can have multiple 173 // versions per cluster, but this certainly won't hurt anything, so let's be safe. 174 gvk := info.ResourceMapping().GroupVersionKind 175 vk := gvk.Version + "/" + gvk.Kind 176 objs[vk] = append(objs[vk], info.Object) 177 return nil 178 }) 179 if err != nil { 180 return "", err 181 } 182 183 // Ok, now we have all the objects grouped by types (say, by v1/Pod, v1/Service, etc.), so 184 // spin through them and print them. Printer is cool since it prints the header only when 185 // an object type changes, so we can just rely on that. Problem is it doesn't seem to keep 186 // track of tab widths 187 buf := new(bytes.Buffer) 188 p, _ := c.Printer(nil, printers.PrintOptions{}) 189 for t, ot := range objs { 190 if _, err = buf.WriteString("==> " + t + "\n"); err != nil { 191 return "", err 192 } 193 for _, o := range ot { 194 if err := p.PrintObj(o, buf); err != nil { 195 c.Log("failed to print object type %s, object: %q :\n %v", t, o, err) 196 return "", err 197 } 198 } 199 if _, err := buf.WriteString("\n"); err != nil { 200 return "", err 201 } 202 } 203 if len(missing) > 0 { 204 buf.WriteString("==> MISSING\nKIND\t\tNAME\n") 205 for _, s := range missing { 206 fmt.Fprintln(buf, s) 207 } 208 } 209 return buf.String(), nil 210 } 211 212 // Update reads in the current configuration and a target configuration from io.reader 213 // and creates resources that don't already exists, updates resources that have been modified 214 // in the target configuration and deletes resources from the current configuration that are 215 // not present in the target configuration 216 // 217 // Namespace will set the namespaces 218 func (c *Client) Update(namespace string, originalReader, targetReader io.Reader, force bool, recreate bool, timeout int64, shouldWait bool) error { 219 original, err := c.BuildUnstructured(namespace, originalReader) 220 if err != nil { 221 return fmt.Errorf("failed decoding reader into objects: %s", err) 222 } 223 224 c.Log("building resources from updated manifest") 225 target, err := c.BuildUnstructured(namespace, targetReader) 226 if err != nil { 227 return fmt.Errorf("failed decoding reader into objects: %s", err) 228 } 229 230 updateErrors := []string{} 231 232 c.Log("checking %d resources for changes", len(target)) 233 err = target.Visit(func(info *resource.Info, err error) error { 234 if err != nil { 235 return err 236 } 237 238 helper := resource.NewHelper(info.Client, info.Mapping) 239 if _, err := helper.Get(info.Namespace, info.Name, info.Export); err != nil { 240 if !errors.IsNotFound(err) { 241 return fmt.Errorf("Could not get information about the resource: %s", err) 242 } 243 244 // Since the resource does not exist, create it. 245 if err := createResource(info); err != nil { 246 return fmt.Errorf("failed to create resource: %s", err) 247 } 248 249 kind := info.Mapping.GroupVersionKind.Kind 250 c.Log("Created a new %s called %q\n", kind, info.Name) 251 return nil 252 } 253 254 originalInfo := original.Get(info) 255 if originalInfo == nil { 256 return fmt.Errorf("no resource with the name %q found", info.Name) 257 } 258 259 if err := updateResource(c, info, originalInfo.Object, force, recreate); err != nil { 260 c.Log("error updating the resource %q:\n\t %v", info.Name, err) 261 updateErrors = append(updateErrors, err.Error()) 262 } 263 264 return nil 265 }) 266 267 switch { 268 case err != nil: 269 return err 270 case len(updateErrors) != 0: 271 return fmt.Errorf(strings.Join(updateErrors, " && ")) 272 } 273 274 for _, info := range original.Difference(target) { 275 c.Log("Deleting %q in %s...", info.Name, info.Namespace) 276 if err := deleteResource(c, info); err != nil { 277 c.Log("Failed to delete %q, err: %s", info.Name, err) 278 } 279 } 280 if shouldWait { 281 return c.waitForResources(time.Duration(timeout)*time.Second, target) 282 } 283 return nil 284 } 285 286 // Delete deletes kubernetes resources from an io.reader 287 // 288 // Namespace will set the namespace 289 func (c *Client) Delete(namespace string, reader io.Reader) error { 290 infos, err := c.BuildUnstructured(namespace, reader) 291 if err != nil { 292 return err 293 } 294 return perform(infos, func(info *resource.Info) error { 295 c.Log("Starting delete for %q %s", info.Name, info.Mapping.GroupVersionKind.Kind) 296 err := deleteResource(c, info) 297 return c.skipIfNotFound(err) 298 }) 299 } 300 301 func (c *Client) skipIfNotFound(err error) error { 302 if errors.IsNotFound(err) { 303 c.Log("%v", err) 304 return nil 305 } 306 return err 307 } 308 309 func (c *Client) watchTimeout(t time.Duration) ResourceActorFunc { 310 return func(info *resource.Info) error { 311 return c.watchUntilReady(t, info) 312 } 313 } 314 315 // WatchUntilReady watches the resource given in the reader, and waits until it is ready. 316 // 317 // This function is mainly for hook implementations. It watches for a resource to 318 // hit a particular milestone. The milestone depends on the Kind. 319 // 320 // For most kinds, it checks to see if the resource is marked as Added or Modified 321 // by the Kubernetes event stream. For some kinds, it does more: 322 // 323 // - Jobs: A job is marked "Ready" when it has successfully completed. This is 324 // ascertained by watching the Status fields in a job's output. 325 // 326 // Handling for other kinds will be added as necessary. 327 func (c *Client) WatchUntilReady(namespace string, reader io.Reader, timeout int64, shouldWait bool) error { 328 infos, err := c.Build(namespace, reader) 329 if err != nil { 330 return err 331 } 332 // For jobs, there's also the option to do poll c.Jobs(namespace).Get(): 333 // https://github.com/adamreese/kubernetes/blob/master/test/e2e/job.go#L291-L300 334 return perform(infos, c.watchTimeout(time.Duration(timeout)*time.Second)) 335 } 336 337 func perform(infos Result, fn ResourceActorFunc) error { 338 if len(infos) == 0 { 339 return ErrNoObjectsVisited 340 } 341 342 for _, info := range infos { 343 if err := fn(info); err != nil { 344 return err 345 } 346 } 347 return nil 348 } 349 350 func createResource(info *resource.Info) error { 351 obj, err := resource.NewHelper(info.Client, info.Mapping).Create(info.Namespace, true, info.Object) 352 if err != nil { 353 return err 354 } 355 return info.Refresh(obj, true) 356 } 357 358 func deleteResource(c *Client, info *resource.Info) error { 359 helper := resource.NewHelper(info.Client, info.Mapping) 360 runningObj, err := helper.Get(info.Namespace, info.Name, info.Export) 361 if err != nil { 362 if errors.IsNotFound(err) { 363 return nil 364 } 365 return err 366 } 367 if !releaseutil.MatchRelease(info.VersionedObject, runningObj) { 368 log.Printf("Delete: Ignore unmatched object: %s/%s", info.Namespace, info.Name) 369 return nil 370 } 371 reaper, err := c.Reaper(info.Mapping) 372 if err != nil { 373 // If there is no reaper for this resources, delete it. 374 if kubectl.IsNoSuchReaperError(err) { 375 return helper.Delete(info.Namespace, info.Name) 376 } 377 return err 378 } 379 c.Log("Using reaper for deleting %q", info.Name) 380 return reaper.Stop(info.Namespace, info.Name, 0, nil) 381 } 382 383 func createPatch(mapping *meta.RESTMapping, target, current runtime.Object) ([]byte, types.PatchType, error) { 384 oldData, err := json.Marshal(current) 385 if err != nil { 386 return nil, types.StrategicMergePatchType, fmt.Errorf("serializing current configuration: %s", err) 387 } 388 newData, err := json.Marshal(target) 389 if err != nil { 390 return nil, types.StrategicMergePatchType, fmt.Errorf("serializing target configuration: %s", err) 391 } 392 393 if api.Semantic.DeepEqual(oldData, newData) { 394 return nil, types.StrategicMergePatchType, nil 395 } 396 397 // Get a versioned object 398 versionedObject, err := api.Scheme.New(mapping.GroupVersionKind) 399 switch { 400 case runtime.IsNotRegisteredError(err): 401 // fall back to generic JSON merge patch 402 patch, err := jsonpatch.CreateMergePatch(oldData, newData) 403 return patch, types.MergePatchType, err 404 case err != nil: 405 return nil, types.StrategicMergePatchType, fmt.Errorf("failed to get versionedObject: %s", err) 406 default: 407 patch, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, versionedObject) 408 return patch, types.StrategicMergePatchType, err 409 } 410 } 411 412 func updateResource(c *Client, target *resource.Info, currentObj runtime.Object, force bool, recreate bool) error { 413 helper := resource.NewHelper(target.Client, target.Mapping) 414 runningObj, err := helper.Get(target.Namespace, target.Name, target.Export) 415 if err != nil { 416 return err 417 } 418 if !releaseutil.MatchRelease(target.VersionedObject, runningObj) { 419 log.Printf("Update: Find unmatched object: %s/%s", target.Namespace, target.Name) 420 return fmt.Errorf("Update: Unmatched object: %s/%s", target.Namespace, target.Name) 421 } 422 patch, patchType, err := createPatch(target.Mapping, target.Object, currentObj) 423 if err != nil { 424 return fmt.Errorf("failed to create patch: %s", err) 425 } 426 if patch == nil { 427 c.Log("Looks like there are no changes for %s %q", target.Mapping.GroupVersionKind.Kind, target.Name) 428 // This needs to happen to make sure that tiller has the latest info from the API 429 // Otherwise there will be no labels and other functions that use labels will panic 430 if err := target.Get(); err != nil { 431 return fmt.Errorf("error trying to refresh resource information: %v", err) 432 } 433 return nil 434 } 435 436 obj, err := helper.Patch(target.Namespace, target.Name, patchType, patch) 437 if err != nil { 438 kind := target.Mapping.GroupVersionKind.Kind 439 log.Printf("Cannot patch %s: %q (%v)", kind, target.Name, err) 440 441 if force { 442 // Attempt to delete... 443 if err := deleteResource(c, target); err != nil { 444 return err 445 } 446 log.Printf("Deleted %s: %q", kind, target.Name) 447 448 // ... and recreate 449 if err := createResource(target); err != nil { 450 return fmt.Errorf("Failed to recreate resource: %s", err) 451 } 452 log.Printf("Created a new %s called %q\n", kind, target.Name) 453 454 // No need to refresh the target, as we recreated the resource based 455 // on it. In addition, it might not exist yet and a call to `Refresh` 456 // may fail. 457 } else { 458 log.Print("Use --force to force recreation of the resource") 459 return err 460 } 461 } else { 462 // When patch succeeds without needing to recreate, refresh target. 463 target.Refresh(obj, true) 464 } 465 466 if !recreate { 467 return nil 468 } 469 470 versioned, err := c.AsVersionedObject(target.Object) 471 if runtime.IsNotRegisteredError(err) { 472 return nil 473 } 474 if err != nil { 475 return err 476 } 477 478 selector, err := getSelectorFromObject(versioned) 479 if err != nil { 480 return nil 481 } 482 483 client, err := c.ClientSet() 484 if err != nil { 485 return err 486 } 487 488 pods, err := client.Core().Pods(target.Namespace).List(metav1.ListOptions{ 489 FieldSelector: fields.Everything().String(), 490 LabelSelector: labels.Set(selector).AsSelector().String(), 491 }) 492 if err != nil { 493 return err 494 } 495 496 // Restart pods 497 for _, pod := range pods.Items { 498 c.Log("Restarting pod: %v/%v", pod.Namespace, pod.Name) 499 500 // Delete each pod for get them restarted with changed spec. 501 if err := client.Core().Pods(pod.Namespace).Delete(pod.Name, metav1.NewPreconditionDeleteOptions(string(pod.UID))); err != nil { 502 return err 503 } 504 } 505 return nil 506 } 507 508 func getSelectorFromObject(obj runtime.Object) (map[string]string, error) { 509 switch typed := obj.(type) { 510 case *v1.ReplicationController: 511 return typed.Spec.Selector, nil 512 case *v1beta1.ReplicaSet: 513 return typed.Spec.Selector.MatchLabels, nil 514 case *v1beta1.Deployment: 515 return typed.Spec.Selector.MatchLabels, nil 516 case *v1beta1.DaemonSet: 517 return typed.Spec.Selector.MatchLabels, nil 518 case *batch.Job: 519 return typed.Spec.Selector.MatchLabels, nil 520 case *apps.StatefulSet: 521 return typed.Spec.Selector.MatchLabels, nil 522 default: 523 return nil, fmt.Errorf("Unsupported kind when getting selector: %v", obj) 524 } 525 } 526 527 func (c *Client) watchUntilReady(timeout time.Duration, info *resource.Info) error { 528 w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion) 529 if err != nil { 530 return err 531 } 532 533 kind := info.Mapping.GroupVersionKind.Kind 534 c.Log("Watching for changes to %s %s with timeout of %v", kind, info.Name, timeout) 535 536 // What we watch for depends on the Kind. 537 // - For a Job, we watch for completion. 538 // - For all else, we watch until Ready. 539 // In the future, we might want to add some special logic for types 540 // like Ingress, Volume, etc. 541 542 _, err = watch.Until(timeout, w, func(e watch.Event) (bool, error) { 543 switch e.Type { 544 case watch.Added, watch.Modified: 545 // For things like a secret or a config map, this is the best indicator 546 // we get. We care mostly about jobs, where what we want to see is 547 // the status go into a good state. For other types, like ReplicaSet 548 // we don't really do anything to support these as hooks. 549 c.Log("Add/Modify event for %s: %v", info.Name, e.Type) 550 if kind == "Job" { 551 return c.waitForJob(e, info.Name) 552 } 553 return true, nil 554 case watch.Deleted: 555 c.Log("Deleted event for %s", info.Name) 556 return true, nil 557 case watch.Error: 558 // Handle error and return with an error. 559 c.Log("Error event for %s", info.Name) 560 return true, fmt.Errorf("Failed to deploy %s", info.Name) 561 default: 562 return false, nil 563 } 564 }) 565 return err 566 } 567 568 // AsVersionedObject converts a runtime.object to a versioned object. 569 func (c *Client) AsVersionedObject(obj runtime.Object) (runtime.Object, error) { 570 json, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj) 571 if err != nil { 572 return nil, err 573 } 574 versions := &runtime.VersionedObjects{} 575 err = runtime.DecodeInto(c.Decoder(true), json, versions) 576 return versions.First(), err 577 } 578 579 // waitForJob is a helper that waits for a job to complete. 580 // 581 // This operates on an event returned from a watcher. 582 func (c *Client) waitForJob(e watch.Event, name string) (bool, error) { 583 o, ok := e.Object.(*batchinternal.Job) 584 if !ok { 585 return true, fmt.Errorf("Expected %s to be a *batch.Job, got %T", name, e.Object) 586 } 587 588 for _, c := range o.Status.Conditions { 589 if c.Type == batchinternal.JobComplete && c.Status == api.ConditionTrue { 590 return true, nil 591 } else if c.Type == batchinternal.JobFailed && c.Status == api.ConditionTrue { 592 return true, fmt.Errorf("Job failed: %s", c.Reason) 593 } 594 } 595 596 c.Log("%s: Jobs active: %d, jobs failed: %d, jobs succeeded: %d", name, o.Status.Active, o.Status.Failed, o.Status.Succeeded) 597 return false, nil 598 } 599 600 // scrubValidationError removes kubectl info from the message 601 func scrubValidationError(err error) error { 602 if err == nil { 603 return nil 604 } 605 const stopValidateMessage = "if you choose to ignore these errors, turn validation off with --validate=false" 606 607 if strings.Contains(err.Error(), stopValidateMessage) { 608 return goerrors.New(strings.Replace(err.Error(), "; "+stopValidateMessage, "", -1)) 609 } 610 return err 611 } 612 613 // WaitAndGetCompletedPodPhase waits up to a timeout until a pod enters a completed phase 614 // and returns said phase (PodSucceeded or PodFailed qualify) 615 func (c *Client) WaitAndGetCompletedPodPhase(namespace string, reader io.Reader, timeout time.Duration) (api.PodPhase, error) { 616 infos, err := c.Build(namespace, reader) 617 if err != nil { 618 return api.PodUnknown, err 619 } 620 info := infos[0] 621 622 kind := info.Mapping.GroupVersionKind.Kind 623 if kind != "Pod" { 624 return api.PodUnknown, fmt.Errorf("%s is not a Pod", info.Name) 625 } 626 627 if err := c.watchPodUntilComplete(timeout, info); err != nil { 628 return api.PodUnknown, err 629 } 630 631 if err := info.Get(); err != nil { 632 return api.PodUnknown, err 633 } 634 status := info.Object.(*api.Pod).Status.Phase 635 636 return status, nil 637 } 638 639 func (c *Client) watchPodUntilComplete(timeout time.Duration, info *resource.Info) error { 640 w, err := resource.NewHelper(info.Client, info.Mapping).WatchSingle(info.Namespace, info.Name, info.ResourceVersion) 641 if err != nil { 642 return err 643 } 644 645 c.Log("Watching pod %s for completion with timeout of %v", info.Name, timeout) 646 _, err = watch.Until(timeout, w, func(e watch.Event) (bool, error) { 647 return conditions.PodCompleted(e) 648 }) 649 650 return err 651 }