github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/k8s/client.go (about) 1 package k8s 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "io" 8 "net/http" 9 "net/url" 10 "regexp" 11 "strings" 12 "sync" 13 "time" 14 15 "github.com/pkg/errors" 16 "helm.sh/helm/v3/pkg/kube" 17 v1 "k8s.io/api/core/v1" 18 apierrors "k8s.io/apimachinery/pkg/api/errors" 19 "k8s.io/apimachinery/pkg/api/meta" 20 "k8s.io/apimachinery/pkg/api/validation" 21 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 "k8s.io/apimachinery/pkg/runtime/schema" 23 "k8s.io/apimachinery/pkg/util/duration" 24 utilerrors "k8s.io/apimachinery/pkg/util/errors" 25 "k8s.io/apimachinery/pkg/version" 26 "k8s.io/cli-runtime/pkg/genericclioptions" 27 "k8s.io/cli-runtime/pkg/resource" 28 "k8s.io/client-go/discovery" 29 "k8s.io/client-go/discovery/cached/memory" 30 "k8s.io/client-go/dynamic" 31 "k8s.io/client-go/kubernetes" 32 apiv1 "k8s.io/client-go/kubernetes/typed/core/v1" 33 "k8s.io/client-go/metadata" 34 "k8s.io/client-go/rest" 35 "k8s.io/client-go/restmapper" 36 "k8s.io/client-go/tools/clientcmd" 37 "k8s.io/client-go/tools/clientcmd/api" 38 clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 39 "k8s.io/kubectl/pkg/cmd/wait" 40 41 // Client auth plugins! They will auto-init if we import them. 42 _ "k8s.io/client-go/plugin/pkg/client/auth" 43 44 "github.com/tilt-dev/clusterid" 45 "github.com/tilt-dev/tilt/internal/container" 46 "github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1" 47 "github.com/tilt-dev/tilt/pkg/logger" 48 ) 49 50 // Due to the way the Kubernetes apiserver works, there's no easy way to 51 // distinguish between "server is taking a long time to respond because it's 52 // gone" and "server is taking a long time to respond because it has a slow auth 53 // plugin". 54 // 55 // So our health check timeout is a bit longer than we'd like. 56 const healthCheckTimeout = 5 * time.Second 57 58 type Namespace string 59 type NamespaceOverride string 60 type PodID string 61 type NodeID string 62 type ServiceName string 63 type KubeContext string 64 type KubeContextOverride string 65 66 // NOTE(nick): This isn't right. DefaultNamespace is a function of your kubectl context. 67 const DefaultNamespace = Namespace("default") 68 69 // Kubernetes uses "Forbidden" errors for a variety of field immutability errors. 70 // 71 // https://github.com/kubernetes/kubernetes/blob/5d6a793221370d890af6ea766d056af4e33f1118/pkg/apis/core/validation/validation.go#L4383 72 // https://github.com/kubernetes/kubernetes/blob/5d6a793221370d890af6ea766d056af4e33f1118/pkg/apis/core/validation/validation.go#L4196 73 var ForbiddenFieldsPrefix = "Forbidden:" 74 75 func (pID PodID) Empty() bool { return pID.String() == "" } 76 func (pID PodID) String() string { return string(pID) } 77 78 func (nID NodeID) String() string { return string(nID) } 79 80 func (n Namespace) Empty() bool { return n == "" } 81 82 func (n Namespace) String() string { 83 if n == "" { 84 return string(DefaultNamespace) 85 } 86 return string(n) 87 } 88 89 type ClusterHealth struct { 90 Live bool 91 LiveOutput string 92 Ready bool 93 ReadyOutput string 94 } 95 96 type Client interface { 97 InformerSet 98 99 // Updates the entities, creating them if necessary. 100 // 101 // Tries to update them in-place if possible. But for certain resource types, 102 // we might need to fallback to deleting and re-creating them. 103 // 104 // Returns entities in the order that they were applied (which may be different 105 // than they were passed in) and with UUIDs from the Kube API 106 Upsert(ctx context.Context, entities []K8sEntity, timeout time.Duration) ([]K8sEntity, error) 107 108 // Delete all given entities, optionally waiting for them to be fully deleted. 109 // 110 // Currently ignores any "not found" errors, because that seems like the correct 111 // behavior for our use cases. 112 Delete(ctx context.Context, entities []K8sEntity, wait time.Duration) error 113 114 GetMetaByReference(ctx context.Context, ref v1.ObjectReference) (metav1.Object, error) 115 ListMeta(ctx context.Context, gvk schema.GroupVersionKind, ns Namespace) ([]metav1.Object, error) 116 117 // Streams the container logs 118 ContainerLogs(ctx context.Context, podID PodID, cName container.Name, n Namespace, startTime time.Time) (io.ReadCloser, error) 119 120 // Opens a tunnel to the specified pod+port. Returns the tunnel's local port and a function that closes the tunnel 121 CreatePortForwarder(ctx context.Context, namespace Namespace, podID PodID, optionalLocalPort, remotePort int, host string) (PortForwarder, error) 122 123 WatchMeta(ctx context.Context, gvk schema.GroupVersionKind, ns Namespace) (<-chan metav1.Object, error) 124 125 ContainerRuntime(ctx context.Context) container.Runtime 126 127 // Some clusters support a local image registry that we can push to. 128 LocalRegistry(ctx context.Context) *v1alpha1.RegistryHosting 129 130 // Some clusters support a node IP where all servers are reachable. 131 NodeIP(ctx context.Context) NodeIP 132 133 Exec(ctx context.Context, podID PodID, cName container.Name, n Namespace, cmd []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error 134 135 // Returns version information about the apiserver, or an error if we're not connected. 136 CheckConnected(ctx context.Context) (*version.Info, error) 137 138 OwnerFetcher() OwnerFetcher 139 140 ClusterHealth(ctx context.Context, verbose bool) (ClusterHealth, error) 141 142 APIConfig() *api.Config 143 } 144 145 type RESTMapper interface { 146 meta.RESTMapper 147 Reset() 148 } 149 150 type K8sClient struct { 151 InformerSet 152 153 product clusterid.Product 154 core apiv1.CoreV1Interface 155 restConfig *rest.Config 156 portForwardClient PortForwardClient 157 configContext KubeContext 158 configCluster ClusterName 159 configNamespace Namespace 160 clientset kubernetes.Interface 161 discovery discovery.CachedDiscoveryInterface 162 dynamic dynamic.Interface 163 metadata metadata.Interface 164 runtimeAsync *runtimeAsync 165 registryAsync *registryAsync 166 nodeIPAsync *nodeIPAsync 167 drm RESTMapper 168 apiConfig *api.Config 169 clientLoader clientcmd.ClientConfig 170 resourceClient ResourceClient 171 ownerFetcher OwnerFetcher 172 } 173 174 var _ Client = &K8sClient{} 175 176 func ProvideK8sClient( 177 globalCtx context.Context, 178 product clusterid.Product, 179 maybeRESTConfig RESTConfigOrError, 180 maybeClientset ClientsetOrError, 181 pfClient PortForwardClient, 182 configContext KubeContext, 183 configCluster ClusterName, 184 configNamespace Namespace, 185 mkClient MinikubeClient, 186 apiConfigOrError APIConfigOrError, 187 clientLoader clientcmd.ClientConfig) Client { 188 apiConfig, err := apiConfigOrError.Config, apiConfigOrError.Error 189 if err != nil { 190 return &explodingClient{err: err} 191 } 192 193 if product == ProductNone { 194 // No k8s, so no need to get any further configs 195 return &explodingClient{err: fmt.Errorf("Kubernetes context not set in %s", clientLoader.ConfigAccess().GetLoadingPrecedence())} 196 } 197 198 restConfig, err := maybeRESTConfig.Config, maybeRESTConfig.Error 199 if err != nil { 200 return &explodingClient{err: err} 201 } 202 203 clientset, err := maybeClientset.Clientset, maybeClientset.Error 204 if err != nil { 205 return &explodingClient{err: err} 206 } 207 208 core := clientset.CoreV1() 209 runtimeAsync := newRuntimeAsync(core) 210 registryAsync := newRegistryAsync(product, core, runtimeAsync) 211 nodeIPAsync := newNodeIPAsync(product, mkClient) 212 213 di, err := dynamic.NewForConfig(restConfig) 214 if err != nil { 215 return &explodingClient{err: err} 216 } 217 218 meta, err := metadata.NewForConfig(restConfig) 219 if err != nil { 220 return &explodingClient{err: err} 221 } 222 223 discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) 224 if err != nil { 225 return &explodingClient{fmt.Errorf("unable to create discovery client: %v", err)} 226 } 227 228 discovery := memory.NewMemCacheClient(discoveryClient) 229 230 drm := restmapper.NewDeferredDiscoveryRESTMapper(discovery) 231 232 c := &K8sClient{ 233 InformerSet: newInformerSet(clientset, di), 234 235 product: product, 236 core: core, 237 restConfig: restConfig, 238 portForwardClient: pfClient, 239 discovery: discovery, 240 configContext: configContext, 241 configCluster: configCluster, 242 configNamespace: configNamespace, 243 clientset: clientset, 244 runtimeAsync: runtimeAsync, 245 registryAsync: registryAsync, 246 nodeIPAsync: nodeIPAsync, 247 dynamic: di, 248 drm: drm, 249 metadata: meta, 250 apiConfig: apiConfig, 251 clientLoader: clientLoader, 252 } 253 c.resourceClient = newResourceClient(c) 254 c.ownerFetcher = NewOwnerFetcher(globalCtx, c) 255 return c 256 } 257 258 func ServiceURL(service *v1.Service, ip NodeIP) (*url.URL, error) { 259 status := service.Status 260 261 lbStatus := status.LoadBalancer 262 263 if len(service.Spec.Ports) == 0 { 264 return nil, nil 265 } 266 267 portSpec := service.Spec.Ports[0] 268 port := portSpec.Port 269 nodePort := portSpec.NodePort 270 271 // Documentation here is helpful: 272 // https://godoc.org/k8s.io/api/core/v1#LoadBalancerIngress 273 // GKE and OpenStack typically use IP-based load balancers. 274 // AWS typically uses DNS-based load balancers. 275 for _, ingress := range lbStatus.Ingress { 276 ingressPort := port 277 if service.Spec.Type == v1.ServiceTypeNodePort { 278 ingressPort = nodePort 279 } 280 281 urlString := "" 282 if ingress.IP != "" { 283 urlString = fmt.Sprintf("http://%s:%d/", ingress.IP, ingressPort) 284 } 285 286 if ingress.Hostname != "" { 287 urlString = fmt.Sprintf("http://%s:%d/", ingress.Hostname, ingressPort) 288 } 289 290 if urlString == "" { 291 continue 292 } 293 294 url, err := url.Parse(urlString) 295 if err != nil { 296 return nil, errors.Wrap(err, "ServiceURL: malformed url") 297 } 298 return url, nil 299 } 300 301 // If the node has an IP that we can hit, we can also look 302 // at the NodePort. This is mostly useful for Minikube. 303 if ip != "" && nodePort != 0 { 304 url, err := url.Parse(fmt.Sprintf("http://%s:%d/", ip, nodePort)) 305 if err != nil { 306 return nil, errors.Wrap(err, "ServiceURL: malformed url") 307 } 308 return url, nil 309 } 310 311 return nil, nil 312 } 313 314 func timeoutError(timeout time.Duration) error { 315 return errors.New(fmt.Sprintf("Killed kubectl. Hit timeout of %v.", timeout)) 316 } 317 318 func (k *K8sClient) ToRESTConfig() (*rest.Config, error) { 319 return rest.CopyConfig(k.restConfig), nil 320 } 321 322 func (k *K8sClient) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) { 323 return k.discovery, nil 324 } 325 326 // Loosely adapted from ctlptl. 327 func (k *K8sClient) CheckConnected(ctx context.Context) (*version.Info, error) { 328 ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout) 329 defer cancel() 330 discoClient, err := k.ToDiscoveryClient() 331 if err != nil { 332 return nil, err 333 } 334 335 restClient := discoClient.RESTClient() 336 if restClient == nil { 337 return discoClient.ServerVersion() 338 } 339 340 body, err := restClient.Get().AbsPath("/version").Do(ctx).Raw() 341 if err != nil { 342 return nil, err 343 } 344 var info version.Info 345 err = json.Unmarshal(body, &info) 346 if err != nil { 347 return nil, fmt.Errorf("unable to parse the server version: %v", err) 348 } 349 return &info, nil 350 } 351 352 func (k *K8sClient) ToRESTMapper() (meta.RESTMapper, error) { 353 return k.drm, nil 354 } 355 func (k *K8sClient) ToRawKubeConfigLoader() clientcmd.ClientConfig { 356 return k.clientLoader 357 } 358 359 func (k *K8sClient) Upsert(ctx context.Context, entities []K8sEntity, timeout time.Duration) ([]K8sEntity, error) { 360 result := make([]K8sEntity, 0, len(entities)) 361 for _, e := range entities { 362 innerCtx, cancel := context.WithTimeout(ctx, timeout) 363 defer cancel() 364 365 newEntity, err := k.escalatingUpdate(innerCtx, e) 366 if err != nil { 367 if ctx.Err() == context.DeadlineExceeded { 368 return nil, timeoutError(timeout) 369 } 370 return nil, err 371 } 372 result = append(result, newEntity...) 373 } 374 375 return result, nil 376 } 377 378 func (k *K8sClient) OwnerFetcher() OwnerFetcher { 379 return k.ownerFetcher 380 } 381 382 func (k *K8sClient) APIConfig() *api.Config { 383 return k.apiConfig 384 } 385 386 // Update an entity like kubectl apply does. 387 // 388 // This is the "best" way to apply a change. 389 // It will do a 3-way merge to update the spec in the least intrusive way. 390 func (k *K8sClient) applyEntity(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) { 391 resources, err := k.prepareUpdateList(ctx, entity) 392 if err != nil { 393 return nil, errors.Wrap(err, "kubernetes apply") 394 } 395 396 result, err := k.resourceClient.Apply(resources) 397 if err != nil { 398 return nil, err 399 } 400 401 // Under rare circumstances, an apply() may result in a 3-way merge 402 // where the object has the new spec, but is simultaneously being removed. 403 // 404 // In that case, wait for the deletion to finish, then retry the apply. 405 // 406 // Discussion: 407 // https://github.com/tilt-dev/tilt/issues/6048 408 isDeleting := false 409 for _, info := range result.Updated { 410 accessor, err := meta.Accessor(info.Object) 411 if err != nil { 412 continue // handle the error later during object conversion 413 } 414 415 if accessor.GetDeletionTimestamp() != nil { 416 isDeleting = true 417 } 418 } 419 420 if isDeleting { 421 dur := 60 * time.Second 422 logger.Get(ctx).Infof("Resource %s is currently being deleted. Waiting %s for deletion before retrying...", 423 entity.Name(), duration.ShortHumanDuration(dur)) 424 err := k.waitForDelete(ctx, resources, dur) 425 if err != nil { 426 return nil, errors.Wrap(err, "kubernetes apply retry") 427 } 428 429 resources, err := k.prepareUpdateList(ctx, entity) 430 if err != nil { 431 return nil, errors.Wrap(err, "kubernetes apply retry") 432 } 433 434 result, err = k.resourceClient.Apply(resources) 435 if err != nil { 436 return nil, errors.Wrap(err, "kubernetes apply retry") 437 } 438 } 439 440 return k.kubeResultToEntities(result) 441 } 442 443 // Update an entity like kubectl create/replace does. 444 // 445 // This uses a PUT HTTP call to replace one entity with another. 446 // 447 // It's not as good as apply, because it will wipe out bookkeeping 448 // that other controllers have added. 449 // 450 // But in cases where the entity is too big to do a 3-way merge, 451 // this is the next best option. 452 func (k *K8sClient) createOrReplaceEntity(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) { 453 resources, err := k.prepareUpdateList(ctx, entity) 454 if err != nil { 455 return nil, errors.Wrap(err, "kubernetes upsert") 456 } 457 458 result, err := k.resourceClient.CreateOrReplace(resources) 459 if err != nil { 460 return nil, err 461 } 462 463 return k.kubeResultToEntities(result) 464 } 465 466 // Delete and create an entity. 467 // 468 // This is the most intrusive way to perform an update, 469 // because any children of the object will be deleted by the controller. 470 // 471 // Some objects in the Kubernetes ecosystem are immutable, so need 472 // this approach as a last resort. 473 func (k *K8sClient) deleteAndCreateEntity(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) { 474 resources, err := k.prepareUpdateList(ctx, entity) 475 if err != nil { 476 return nil, errors.Wrap(err, "kubernetes delete and re-create") 477 } 478 479 result, err := k.deleteAndCreate(ctx, resources) 480 if err != nil { 481 return nil, err 482 } 483 484 return k.kubeResultToEntities(result) 485 } 486 487 // Make sure the type exists and create a ResourceList to help update it. 488 func (k *K8sClient) prepareUpdateList(ctx context.Context, e K8sEntity) (kube.ResourceList, error) { 489 _, err := k.forceDiscovery(ctx, e.GVK()) 490 if err != nil { 491 return nil, err 492 } 493 494 return k.buildResourceList(ctx, e) 495 } 496 497 // Build a ResourceList usable by our helm client for interacting with a resource. 498 // 499 // Although the underlying API encourages you to batch these together (for 500 // better parallelization), we've found that it's more robust to handle entities 501 // individually to ensure an error in one doesn't affect the others (and the 502 // real bottleneck isn't in building). 503 func (k *K8sClient) buildResourceList(ctx context.Context, e K8sEntity) (kube.ResourceList, error) { 504 rawYAML, err := SerializeSpecYAMLToBuffer([]K8sEntity{e}) 505 if err != nil { 506 return nil, err 507 } 508 509 resources, err := k.resourceClient.Build(rawYAML, false) 510 if err != nil { 511 return nil, err 512 } 513 514 return resources, nil 515 } 516 517 func (k *K8sClient) kubeResultToEntities(result *kube.Result) ([]K8sEntity, error) { 518 entities := []K8sEntity{} 519 for _, info := range result.Created { 520 entities = append(entities, NewK8sEntity(info.Object)) 521 } 522 for _, info := range result.Updated { 523 entities = append(entities, NewK8sEntity(info.Object)) 524 } 525 526 // Helm parses the results as unstructured info, but Tilt needs them parsed with the current 527 // API scheme. The easiest way to do this is to serialize them to yaml and re-parse again. 528 buf, err := SerializeSpecYAMLToBuffer(entities) 529 if err != nil { 530 return nil, errors.Wrap(err, "reading kubernetes result") 531 } 532 533 parsed, err := ParseYAML(buf) 534 if err != nil { 535 return nil, errors.Wrap(err, "parsing kubernetes result") 536 } 537 return parsed, nil 538 } 539 540 func (k *K8sClient) deleteAndCreate(ctx context.Context, list kube.ResourceList) (*kube.Result, error) { 541 // Delete is destructive, so clone first. 542 toDelete := kube.ResourceList{} 543 for _, r := range list { 544 rClone := *r 545 rClone.Object = r.Object.DeepCopyObject() 546 toDelete = append(toDelete, &rClone) 547 } 548 549 _, errs := k.resourceClient.Delete(toDelete) 550 for _, err := range errs { 551 if isNotFoundError(err) { 552 continue 553 } 554 return nil, errors.Wrap(err, "kubernetes delete") 555 } 556 557 // ensure the delete has finished before attempting to recreate 558 err := k.waitForDelete(ctx, list, 30*time.Second) 559 if err != nil { 560 return nil, errors.Wrap(err, "kubernetes create") 561 } 562 563 result, err := k.resourceClient.Create(list) 564 if err != nil { 565 return nil, errors.Wrap(err, "kubernetes create") 566 } 567 return result, nil 568 } 569 570 // Update a resource in-place, starting with the least intrusive 571 // update strategy and escalating into the most intrusive strategy. 572 func (k *K8sClient) escalatingUpdate(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) { 573 fallback := false 574 result, err := k.applyEntity(ctx, entity) 575 if err != nil { 576 msg, match := maybeTooLargeError(err) 577 if match { 578 fallback = true 579 logger.Get(ctx).Infof("Updating %q failed: %s", entity.Name(), msg) 580 logger.Get(ctx).Infof("Attempting to create or replace") 581 result, err = k.createOrReplaceEntity(ctx, entity) 582 } 583 } 584 585 if err != nil { 586 maybeImmutable := maybeImmutableFieldStderr(err.Error()) 587 if maybeImmutable { 588 fallback = true 589 logger.Get(ctx).Infof("Updating %q failed: %s", entity.Name(), 590 truncateErrorToOneLine(err.Error())) 591 logger.Get(ctx).Infof("Attempting to delete and re-create") 592 result, err = k.deleteAndCreateEntity(ctx, entity) 593 } 594 } 595 596 if err != nil { 597 return nil, err 598 } 599 if fallback { 600 logger.Get(ctx).Infof("Updating %q succeeded!", entity.Name()) 601 } 602 return result, nil 603 } 604 605 func truncateErrorToOneLine(stderr string) string { 606 index := strings.Index(stderr, "\n") 607 if index != -1 { 608 return stderr[:index] 609 } 610 return stderr 611 } 612 613 // We're using kubectl, so we only get stderr, not structured errors. 614 // 615 // Take a wild guess if the update is failing due to immutable field errors. 616 // 617 // This should bias towards false positives (i.e., we think something is an 618 // immutable field error when it's not). 619 func maybeImmutableFieldStderr(stderr string) bool { 620 return strings.Contains(stderr, validation.FieldImmutableErrorMsg) || 621 strings.Contains(stderr, ForbiddenFieldsPrefix) 622 } 623 624 var MetadataAnnotationsTooLongRe = regexp.MustCompile(`metadata.annotations: Too long: must have at most \d+ bytes.*`) 625 626 // kubectl apply sets an annotation containing the object's previous configuration. 627 // However, annotations have a max size of 256k. Large objects such as configmaps can exceed 256k, which makes 628 // apply unusable, so we need to fall back to delete/create 629 // https://github.com/kubernetes/kubectl/issues/712 630 // 631 // We've also seen this reported differently, with a 413 HTTP error. 632 // https://github.com/tilt-dev/tilt/issues/5279 633 func maybeTooLargeError(err error) (string, bool) { 634 // We don't have an easy way to reproduce some of these problems, so we check 635 // for both the structured form of the error and the unstructured form. 636 statusErr, isStatusErr := err.(*apierrors.StatusError) 637 if isStatusErr && statusErr.ErrStatus.Code == http.StatusRequestEntityTooLarge { 638 return err.Error(), true 639 } 640 641 stderr := err.Error() 642 for _, line := range strings.Split(stderr, "\n") { 643 if MetadataAnnotationsTooLongRe.MatchString(line) { 644 return line, true 645 } 646 647 if strings.Contains(line, "the server responded with the status code 413") { 648 return line, true 649 } 650 } 651 652 return "", false 653 } 654 655 // Deletes all given entities. 656 // 657 // Currently ignores any "not found" errors, because that seems like the correct 658 // behavior for our use cases. 659 func (k *K8sClient) Delete(ctx context.Context, entities []K8sEntity, wait time.Duration) error { 660 l := logger.Get(ctx) 661 l.Infof("Deleting kubernetes objects:") 662 for _, e := range entities { 663 l.Infof("→ %s/%s", e.GVK().Kind, e.Name()) 664 } 665 666 var resources kube.ResourceList 667 for _, e := range entities { 668 resourceList, err := k.buildResourceList(ctx, e) 669 if utilerrors.FilterOut(err, isMissingKindError) != nil { 670 return errors.Wrap(err, "kubernetes delete") 671 } 672 resources = append(resources, resourceList...) 673 } 674 675 _, errs := k.resourceClient.Delete(resources) 676 for _, err := range errs { 677 if err == nil || isNotFoundError(err) { 678 continue 679 } 680 681 return errors.Wrap(err, "kubernetes delete") 682 } 683 684 if wait > 0 { 685 err := k.waitForDelete(ctx, resources, wait) 686 if err != nil { 687 return err 688 } 689 } 690 691 return nil 692 } 693 694 func (k *K8sClient) forceDiscovery(ctx context.Context, gvk schema.GroupVersionKind) (*meta.RESTMapping, error) { 695 rm, err := k.drm.RESTMapping(gvk.GroupKind(), gvk.Version) 696 if err != nil { 697 // The REST mapper doesn't have any sort of internal invalidation 698 // mechanism. So if the user applies a CRD (i.e., changing the available 699 // api resources), the REST mapper won't discover the new types. 700 // 701 // https://github.com/kubernetes/kubernetes/issues/75383 702 // 703 // But! When Tilt requests a resource by reference, we know in advance that 704 // it must exist, and therefore, its type must exist. So we can safely 705 // reset the REST mapper and retry, so that it discovers the types. 706 k.drm.Reset() 707 708 rm, err = k.drm.RESTMapping(gvk.GroupKind(), gvk.Version) 709 if err != nil { 710 return nil, errors.Wrapf(err, "error mapping %s/%s", gvk.Group, gvk.Kind) 711 } 712 } 713 return rm, nil 714 } 715 716 // Returns true if the list successfully deleted. False if we timed out. 717 func (k *K8sClient) waitForDelete(ctx context.Context, list kube.ResourceList, duration time.Duration) error { 718 results := make([]bool, len(list)) 719 var wg sync.WaitGroup 720 for i, r := range list { 721 wg.Add(1) 722 go func(i int, resourceInfo *resource.Info) { 723 waitOpt := &wait.WaitOptions{ 724 DynamicClient: k.dynamic, 725 IOStreams: genericclioptions.NewTestIOStreamsDiscard(), 726 Timeout: duration, 727 ForCondition: "delete", 728 } 729 730 _, ok, _ := wait.IsDeleted(ctx, resourceInfo, waitOpt) 731 results[i] = ok 732 wg.Done() 733 }(i, r) 734 } 735 wg.Wait() 736 737 for i, r := range results { 738 if !r { 739 return fmt.Errorf("timeout waiting for delete: %s", list[i].Name) 740 } 741 } 742 return nil 743 } 744 745 func (k *K8sClient) ListMeta(ctx context.Context, gvk schema.GroupVersionKind, ns Namespace) ([]metav1.Object, error) { 746 mapping, err := k.forceDiscovery(ctx, gvk) 747 if err != nil { 748 return nil, err 749 } 750 751 gvr := mapping.Resource 752 isRoot := mapping.Scope != nil && mapping.Scope.Name() == meta.RESTScopeNameRoot 753 var metaList *metav1.PartialObjectMetadataList 754 if isRoot { 755 metaList, err = k.metadata.Resource(gvr).List(ctx, metav1.ListOptions{}) 756 } else { 757 metaList, err = k.metadata.Resource(gvr).Namespace(ns.String()).List(ctx, metav1.ListOptions{}) 758 } 759 760 if err != nil { 761 return nil, err 762 } 763 764 // type conversion 765 result := make([]metav1.Object, len(metaList.Items)) 766 for i, meta := range metaList.Items { 767 m := meta.ObjectMeta 768 result[i] = &m 769 } 770 return result, nil 771 } 772 773 func (k *K8sClient) GetMetaByReference(ctx context.Context, ref v1.ObjectReference) (metav1.Object, error) { 774 gvk := ReferenceGVK(ref) 775 mapping, err := k.forceDiscovery(ctx, gvk) 776 if err != nil { 777 return nil, err 778 } 779 780 gvr := mapping.Resource 781 namespace := ref.Namespace 782 name := ref.Name 783 resourceVersion := ref.ResourceVersion 784 uid := ref.UID 785 786 typeAndMeta, err := k.metadata.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{ 787 ResourceVersion: resourceVersion, 788 }) 789 if err != nil { 790 return nil, err 791 } 792 meta := typeAndMeta.ObjectMeta 793 if uid != "" && meta.UID != uid { 794 return nil, apierrors.NewNotFound(v1.Resource(gvr.Resource), name) 795 } 796 return &meta, nil 797 } 798 799 func (k *K8sClient) ClusterHealth(ctx context.Context, verbose bool) (ClusterHealth, error) { 800 isLive, livezResp, err := k.apiServerHealthCheck(ctx, "/livez", verbose) 801 if err != nil { 802 return ClusterHealth{}, fmt.Errorf("cluster liveness check: %v", err) 803 } 804 805 // TODO(milas): is there any point to running the readiness check if the 806 // liveness check failed? 807 isReady, readyzResp, err := k.apiServerHealthCheck(ctx, "/readyz", verbose) 808 if err != nil { 809 return ClusterHealth{}, fmt.Errorf("cluster readiness check: %v", err) 810 } 811 812 return ClusterHealth{ 813 Live: isLive, 814 Ready: isReady, 815 LiveOutput: livezResp, 816 ReadyOutput: readyzResp, 817 }, nil 818 } 819 820 // apiServerHealthCheck issues a direct HTTP request to an apiserver health endpoint. 821 // 822 // There are not methods for this functionality exposed via client-go, so the 823 // RESTClient is used directly. 824 // 825 // See https://kubernetes.io/docs/reference/using-api/health-checks/ 826 func (k *K8sClient) apiServerHealthCheck(ctx context.Context, route string, verbose bool) (bool, string, error) { 827 req := k.discovery.RESTClient(). 828 Get(). 829 AbsPath(route). 830 // timeout will both be passed as a param to server as well as used to 831 // create a child context with a deadline 832 Timeout(10 * time.Second). 833 MaxRetries(1) 834 835 if verbose { 836 req = req.Param("verbose", "") 837 } 838 body, err := req.DoRaw(ctx) 839 if err != nil { 840 var statusErr *apierrors.StatusError 841 if errors.As(err, &statusErr) { 842 return false, statusErr.ErrStatus.Message, nil 843 } 844 return false, "", err 845 } 846 return true, string(body), nil 847 } 848 849 // Tests whether a string is a valid version for a k8s resource type. 850 // from https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#version-priority 851 // Versions start with a v followed by a number, an optional beta or alpha designation, and optional additional numeric 852 // versioning information. Broadly, a version string might look like v2 or v2beta1. 853 var versionRegex = regexp.MustCompile(`^v\d+(?:(?:alpha|beta)(?:\d+)?)?$`) 854 855 func ReferenceGVK(involvedObject v1.ObjectReference) schema.GroupVersionKind { 856 // For some types, APIVersion is incorrectly just the group w/ no version, which leads GroupVersionKind to return 857 // a value where Group is empty and Version contains the group, so we need to correct for that. 858 // An empty Group is valid, though: it's empty for apps in the core group. 859 // So, we detect this situation by checking if the version field is valid. 860 861 // this stems from group/version not necessarily being populated at other points in the API. see more info here: 862 // https://github.com/kubernetes/client-go/issues/308 863 // https://github.com/kubernetes/kubernetes/issues/3030 864 865 gvk := involvedObject.GroupVersionKind() 866 if !versionRegex.MatchString(gvk.Version) { 867 gvk.Group = involvedObject.APIVersion 868 gvk.Version = "" 869 } 870 871 return gvk 872 } 873 874 func ProvideServerVersion(maybeClientset ClientsetOrError) (*version.Info, error) { 875 if maybeClientset.Error != nil { 876 return nil, maybeClientset.Error 877 } 878 return maybeClientset.Clientset.Discovery().ServerVersion() 879 } 880 881 type ClientsetOrError struct { 882 Clientset *kubernetes.Clientset 883 Error error 884 } 885 886 func ProvideClientset(cfg RESTConfigOrError) ClientsetOrError { 887 if cfg.Error != nil { 888 return ClientsetOrError{Error: cfg.Error} 889 } 890 clientset, err := kubernetes.NewForConfig(cfg.Config) 891 return ClientsetOrError{Clientset: clientset, Error: err} 892 } 893 894 func ProvideClientConfig(contextOverride KubeContextOverride, nsFlag NamespaceOverride) clientcmd.ClientConfig { 895 rules := clientcmd.NewDefaultClientConfigLoadingRules() 896 rules.DefaultClientConfig = &clientcmd.DefaultClientConfig 897 898 overrides := &clientcmd.ConfigOverrides{ 899 CurrentContext: string(contextOverride), 900 Context: clientcmdapi.Context{ 901 Namespace: string(nsFlag), 902 }, 903 } 904 return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 905 rules, 906 overrides) 907 } 908 909 // The namespace in the kubeconfig. 910 // Used as a default namespace in some (but not all) client commands. 911 // https://godoc.org/k8s.io/client-go/tools/clientcmd/api/v1#Context 912 func ProvideConfigNamespace(clientLoader clientcmd.ClientConfig) Namespace { 913 namespace, explicit, err := clientLoader.Namespace() 914 if err != nil { 915 // If we can't get a namespace from the config, just fail gracefully to the default. 916 // If this error indicates a more serious problem, it will get handled downstream. 917 return "" 918 } 919 920 // TODO(nick): Right now, tilt doesn't provide a namespace flag. If we ever did, 921 // we would need to handle explicit namespaces different than implicit ones. 922 _ = explicit 923 924 return Namespace(namespace) 925 } 926 927 type RESTConfigOrError struct { 928 Config *rest.Config 929 Error error 930 } 931 932 func ProvideRESTConfig(clientLoader clientcmd.ClientConfig) RESTConfigOrError { 933 config, err := clientLoader.ClientConfig() 934 return RESTConfigOrError{Config: config, Error: err} 935 }