github.com/tilt-dev/tilt@v0.33.15-0.20240515162809-0a22ed45d8a0/internal/k8s/client.go (about)

     1  package k8s
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"fmt"
     7  	"io"
     8  	"net/http"
     9  	"net/url"
    10  	"regexp"
    11  	"strings"
    12  	"sync"
    13  	"time"
    14  
    15  	"github.com/pkg/errors"
    16  	"helm.sh/helm/v3/pkg/kube"
    17  	v1 "k8s.io/api/core/v1"
    18  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    19  	"k8s.io/apimachinery/pkg/api/meta"
    20  	"k8s.io/apimachinery/pkg/api/validation"
    21  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    22  	"k8s.io/apimachinery/pkg/runtime/schema"
    23  	"k8s.io/apimachinery/pkg/util/duration"
    24  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    25  	"k8s.io/apimachinery/pkg/version"
    26  	"k8s.io/cli-runtime/pkg/genericclioptions"
    27  	"k8s.io/cli-runtime/pkg/resource"
    28  	"k8s.io/client-go/discovery"
    29  	"k8s.io/client-go/discovery/cached/memory"
    30  	"k8s.io/client-go/dynamic"
    31  	"k8s.io/client-go/kubernetes"
    32  	apiv1 "k8s.io/client-go/kubernetes/typed/core/v1"
    33  	"k8s.io/client-go/metadata"
    34  	"k8s.io/client-go/rest"
    35  	"k8s.io/client-go/restmapper"
    36  	"k8s.io/client-go/tools/clientcmd"
    37  	"k8s.io/client-go/tools/clientcmd/api"
    38  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    39  	"k8s.io/kubectl/pkg/cmd/wait"
    40  
    41  	// Client auth plugins! They will auto-init if we import them.
    42  	_ "k8s.io/client-go/plugin/pkg/client/auth"
    43  
    44  	"github.com/tilt-dev/clusterid"
    45  	"github.com/tilt-dev/tilt/internal/container"
    46  	"github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1"
    47  	"github.com/tilt-dev/tilt/pkg/logger"
    48  )
    49  
    50  // Due to the way the Kubernetes apiserver works, there's no easy way to
    51  // distinguish between "server is taking a long time to respond because it's
    52  // gone" and "server is taking a long time to respond because it has a slow auth
    53  // plugin".
    54  //
    55  // So our health check timeout is a bit longer than we'd like.
    56  const healthCheckTimeout = 5 * time.Second
    57  
    58  type Namespace string
    59  type NamespaceOverride string
    60  type PodID string
    61  type NodeID string
    62  type ServiceName string
    63  type KubeContext string
    64  type KubeContextOverride string
    65  
    66  // NOTE(nick): This isn't right. DefaultNamespace is a function of your kubectl context.
    67  const DefaultNamespace = Namespace("default")
    68  
    69  // Kubernetes uses "Forbidden" errors for a variety of field immutability errors.
    70  //
    71  // https://github.com/kubernetes/kubernetes/blob/5d6a793221370d890af6ea766d056af4e33f1118/pkg/apis/core/validation/validation.go#L4383
    72  // https://github.com/kubernetes/kubernetes/blob/5d6a793221370d890af6ea766d056af4e33f1118/pkg/apis/core/validation/validation.go#L4196
    73  var ForbiddenFieldsPrefix = "Forbidden:"
    74  
    75  func (pID PodID) Empty() bool    { return pID.String() == "" }
    76  func (pID PodID) String() string { return string(pID) }
    77  
    78  func (nID NodeID) String() string { return string(nID) }
    79  
    80  func (n Namespace) Empty() bool { return n == "" }
    81  
    82  func (n Namespace) String() string {
    83  	if n == "" {
    84  		return string(DefaultNamespace)
    85  	}
    86  	return string(n)
    87  }
    88  
    89  type ClusterHealth struct {
    90  	Live        bool
    91  	LiveOutput  string
    92  	Ready       bool
    93  	ReadyOutput string
    94  }
    95  
    96  type Client interface {
    97  	InformerSet
    98  
    99  	// Updates the entities, creating them if necessary.
   100  	//
   101  	// Tries to update them in-place if possible. But for certain resource types,
   102  	// we might need to fallback to deleting and re-creating them.
   103  	//
   104  	// Returns entities in the order that they were applied (which may be different
   105  	// than they were passed in) and with UUIDs from the Kube API
   106  	Upsert(ctx context.Context, entities []K8sEntity, timeout time.Duration) ([]K8sEntity, error)
   107  
   108  	// Delete all given entities, optionally waiting for them to be fully deleted.
   109  	//
   110  	// Currently ignores any "not found" errors, because that seems like the correct
   111  	// behavior for our use cases.
   112  	Delete(ctx context.Context, entities []K8sEntity, wait time.Duration) error
   113  
   114  	GetMetaByReference(ctx context.Context, ref v1.ObjectReference) (metav1.Object, error)
   115  	ListMeta(ctx context.Context, gvk schema.GroupVersionKind, ns Namespace) ([]metav1.Object, error)
   116  
   117  	// Streams the container logs
   118  	ContainerLogs(ctx context.Context, podID PodID, cName container.Name, n Namespace, startTime time.Time) (io.ReadCloser, error)
   119  
   120  	// Opens a tunnel to the specified pod+port. Returns the tunnel's local port and a function that closes the tunnel
   121  	CreatePortForwarder(ctx context.Context, namespace Namespace, podID PodID, optionalLocalPort, remotePort int, host string) (PortForwarder, error)
   122  
   123  	WatchMeta(ctx context.Context, gvk schema.GroupVersionKind, ns Namespace) (<-chan metav1.Object, error)
   124  
   125  	ContainerRuntime(ctx context.Context) container.Runtime
   126  
   127  	// Some clusters support a local image registry that we can push to.
   128  	LocalRegistry(ctx context.Context) *v1alpha1.RegistryHosting
   129  
   130  	// Some clusters support a node IP where all servers are reachable.
   131  	NodeIP(ctx context.Context) NodeIP
   132  
   133  	Exec(ctx context.Context, podID PodID, cName container.Name, n Namespace, cmd []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error
   134  
   135  	// Returns version information about the apiserver, or an error if we're not connected.
   136  	CheckConnected(ctx context.Context) (*version.Info, error)
   137  
   138  	OwnerFetcher() OwnerFetcher
   139  
   140  	ClusterHealth(ctx context.Context, verbose bool) (ClusterHealth, error)
   141  
   142  	APIConfig() *api.Config
   143  }
   144  
   145  type RESTMapper interface {
   146  	meta.RESTMapper
   147  	Reset()
   148  }
   149  
   150  type K8sClient struct {
   151  	InformerSet
   152  
   153  	product           clusterid.Product
   154  	core              apiv1.CoreV1Interface
   155  	restConfig        *rest.Config
   156  	portForwardClient PortForwardClient
   157  	configContext     KubeContext
   158  	configCluster     ClusterName
   159  	configNamespace   Namespace
   160  	clientset         kubernetes.Interface
   161  	discovery         discovery.CachedDiscoveryInterface
   162  	dynamic           dynamic.Interface
   163  	metadata          metadata.Interface
   164  	runtimeAsync      *runtimeAsync
   165  	registryAsync     *registryAsync
   166  	nodeIPAsync       *nodeIPAsync
   167  	drm               RESTMapper
   168  	apiConfig         *api.Config
   169  	clientLoader      clientcmd.ClientConfig
   170  	resourceClient    ResourceClient
   171  	ownerFetcher      OwnerFetcher
   172  }
   173  
   174  var _ Client = &K8sClient{}
   175  
   176  func ProvideK8sClient(
   177  	globalCtx context.Context,
   178  	product clusterid.Product,
   179  	maybeRESTConfig RESTConfigOrError,
   180  	maybeClientset ClientsetOrError,
   181  	pfClient PortForwardClient,
   182  	configContext KubeContext,
   183  	configCluster ClusterName,
   184  	configNamespace Namespace,
   185  	mkClient MinikubeClient,
   186  	apiConfigOrError APIConfigOrError,
   187  	clientLoader clientcmd.ClientConfig) Client {
   188  	apiConfig, err := apiConfigOrError.Config, apiConfigOrError.Error
   189  	if err != nil {
   190  		return &explodingClient{err: err}
   191  	}
   192  
   193  	if product == ProductNone {
   194  		// No k8s, so no need to get any further configs
   195  		return &explodingClient{err: fmt.Errorf("Kubernetes context not set in %s", clientLoader.ConfigAccess().GetLoadingPrecedence())}
   196  	}
   197  
   198  	restConfig, err := maybeRESTConfig.Config, maybeRESTConfig.Error
   199  	if err != nil {
   200  		return &explodingClient{err: err}
   201  	}
   202  
   203  	clientset, err := maybeClientset.Clientset, maybeClientset.Error
   204  	if err != nil {
   205  		return &explodingClient{err: err}
   206  	}
   207  
   208  	core := clientset.CoreV1()
   209  	runtimeAsync := newRuntimeAsync(core)
   210  	registryAsync := newRegistryAsync(product, core, runtimeAsync)
   211  	nodeIPAsync := newNodeIPAsync(product, mkClient)
   212  
   213  	di, err := dynamic.NewForConfig(restConfig)
   214  	if err != nil {
   215  		return &explodingClient{err: err}
   216  	}
   217  
   218  	meta, err := metadata.NewForConfig(restConfig)
   219  	if err != nil {
   220  		return &explodingClient{err: err}
   221  	}
   222  
   223  	discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig)
   224  	if err != nil {
   225  		return &explodingClient{fmt.Errorf("unable to create discovery client: %v", err)}
   226  	}
   227  
   228  	discovery := memory.NewMemCacheClient(discoveryClient)
   229  
   230  	drm := restmapper.NewDeferredDiscoveryRESTMapper(discovery)
   231  
   232  	c := &K8sClient{
   233  		InformerSet: newInformerSet(clientset, di),
   234  
   235  		product:           product,
   236  		core:              core,
   237  		restConfig:        restConfig,
   238  		portForwardClient: pfClient,
   239  		discovery:         discovery,
   240  		configContext:     configContext,
   241  		configCluster:     configCluster,
   242  		configNamespace:   configNamespace,
   243  		clientset:         clientset,
   244  		runtimeAsync:      runtimeAsync,
   245  		registryAsync:     registryAsync,
   246  		nodeIPAsync:       nodeIPAsync,
   247  		dynamic:           di,
   248  		drm:               drm,
   249  		metadata:          meta,
   250  		apiConfig:         apiConfig,
   251  		clientLoader:      clientLoader,
   252  	}
   253  	c.resourceClient = newResourceClient(c)
   254  	c.ownerFetcher = NewOwnerFetcher(globalCtx, c)
   255  	return c
   256  }
   257  
   258  func ServiceURL(service *v1.Service, ip NodeIP) (*url.URL, error) {
   259  	status := service.Status
   260  
   261  	lbStatus := status.LoadBalancer
   262  
   263  	if len(service.Spec.Ports) == 0 {
   264  		return nil, nil
   265  	}
   266  
   267  	portSpec := service.Spec.Ports[0]
   268  	port := portSpec.Port
   269  	nodePort := portSpec.NodePort
   270  
   271  	// Documentation here is helpful:
   272  	// https://godoc.org/k8s.io/api/core/v1#LoadBalancerIngress
   273  	// GKE and OpenStack typically use IP-based load balancers.
   274  	// AWS typically uses DNS-based load balancers.
   275  	for _, ingress := range lbStatus.Ingress {
   276  		ingressPort := port
   277  		if service.Spec.Type == v1.ServiceTypeNodePort {
   278  			ingressPort = nodePort
   279  		}
   280  
   281  		urlString := ""
   282  		if ingress.IP != "" {
   283  			urlString = fmt.Sprintf("http://%s:%d/", ingress.IP, ingressPort)
   284  		}
   285  
   286  		if ingress.Hostname != "" {
   287  			urlString = fmt.Sprintf("http://%s:%d/", ingress.Hostname, ingressPort)
   288  		}
   289  
   290  		if urlString == "" {
   291  			continue
   292  		}
   293  
   294  		url, err := url.Parse(urlString)
   295  		if err != nil {
   296  			return nil, errors.Wrap(err, "ServiceURL: malformed url")
   297  		}
   298  		return url, nil
   299  	}
   300  
   301  	// If the node has an IP that we can hit, we can also look
   302  	// at the NodePort. This is mostly useful for Minikube.
   303  	if ip != "" && nodePort != 0 {
   304  		url, err := url.Parse(fmt.Sprintf("http://%s:%d/", ip, nodePort))
   305  		if err != nil {
   306  			return nil, errors.Wrap(err, "ServiceURL: malformed url")
   307  		}
   308  		return url, nil
   309  	}
   310  
   311  	return nil, nil
   312  }
   313  
   314  func timeoutError(timeout time.Duration) error {
   315  	return errors.New(fmt.Sprintf("Killed kubectl. Hit timeout of %v.", timeout))
   316  }
   317  
   318  func (k *K8sClient) ToRESTConfig() (*rest.Config, error) {
   319  	return rest.CopyConfig(k.restConfig), nil
   320  }
   321  
   322  func (k *K8sClient) ToDiscoveryClient() (discovery.CachedDiscoveryInterface, error) {
   323  	return k.discovery, nil
   324  }
   325  
   326  // Loosely adapted from ctlptl.
   327  func (k *K8sClient) CheckConnected(ctx context.Context) (*version.Info, error) {
   328  	ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
   329  	defer cancel()
   330  	discoClient, err := k.ToDiscoveryClient()
   331  	if err != nil {
   332  		return nil, err
   333  	}
   334  
   335  	restClient := discoClient.RESTClient()
   336  	if restClient == nil {
   337  		return discoClient.ServerVersion()
   338  	}
   339  
   340  	body, err := restClient.Get().AbsPath("/version").Do(ctx).Raw()
   341  	if err != nil {
   342  		return nil, err
   343  	}
   344  	var info version.Info
   345  	err = json.Unmarshal(body, &info)
   346  	if err != nil {
   347  		return nil, fmt.Errorf("unable to parse the server version: %v", err)
   348  	}
   349  	return &info, nil
   350  }
   351  
   352  func (k *K8sClient) ToRESTMapper() (meta.RESTMapper, error) {
   353  	return k.drm, nil
   354  }
   355  func (k *K8sClient) ToRawKubeConfigLoader() clientcmd.ClientConfig {
   356  	return k.clientLoader
   357  }
   358  
   359  func (k *K8sClient) Upsert(ctx context.Context, entities []K8sEntity, timeout time.Duration) ([]K8sEntity, error) {
   360  	result := make([]K8sEntity, 0, len(entities))
   361  	for _, e := range entities {
   362  		innerCtx, cancel := context.WithTimeout(ctx, timeout)
   363  		defer cancel()
   364  
   365  		newEntity, err := k.escalatingUpdate(innerCtx, e)
   366  		if err != nil {
   367  			if ctx.Err() == context.DeadlineExceeded {
   368  				return nil, timeoutError(timeout)
   369  			}
   370  			return nil, err
   371  		}
   372  		result = append(result, newEntity...)
   373  	}
   374  
   375  	return result, nil
   376  }
   377  
   378  func (k *K8sClient) OwnerFetcher() OwnerFetcher {
   379  	return k.ownerFetcher
   380  }
   381  
   382  func (k *K8sClient) APIConfig() *api.Config {
   383  	return k.apiConfig
   384  }
   385  
   386  // Update an entity like kubectl apply does.
   387  //
   388  // This is the "best" way to apply a change.
   389  // It will do a 3-way merge to update the spec in the least intrusive way.
   390  func (k *K8sClient) applyEntity(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) {
   391  	resources, err := k.prepareUpdateList(ctx, entity)
   392  	if err != nil {
   393  		return nil, errors.Wrap(err, "kubernetes apply")
   394  	}
   395  
   396  	result, err := k.resourceClient.Apply(resources)
   397  	if err != nil {
   398  		return nil, err
   399  	}
   400  
   401  	// Under rare circumstances, an apply() may result in a 3-way merge
   402  	// where the object has the new spec, but is simultaneously being removed.
   403  	//
   404  	// In that case, wait for the deletion to finish, then retry the apply.
   405  	//
   406  	// Discussion:
   407  	// https://github.com/tilt-dev/tilt/issues/6048
   408  	isDeleting := false
   409  	for _, info := range result.Updated {
   410  		accessor, err := meta.Accessor(info.Object)
   411  		if err != nil {
   412  			continue // handle the error later during object conversion
   413  		}
   414  
   415  		if accessor.GetDeletionTimestamp() != nil {
   416  			isDeleting = true
   417  		}
   418  	}
   419  
   420  	if isDeleting {
   421  		dur := 60 * time.Second
   422  		logger.Get(ctx).Infof("Resource %s is currently being deleted. Waiting %s for deletion before retrying...",
   423  			entity.Name(), duration.ShortHumanDuration(dur))
   424  		err := k.waitForDelete(ctx, resources, dur)
   425  		if err != nil {
   426  			return nil, errors.Wrap(err, "kubernetes apply retry")
   427  		}
   428  
   429  		resources, err := k.prepareUpdateList(ctx, entity)
   430  		if err != nil {
   431  			return nil, errors.Wrap(err, "kubernetes apply retry")
   432  		}
   433  
   434  		result, err = k.resourceClient.Apply(resources)
   435  		if err != nil {
   436  			return nil, errors.Wrap(err, "kubernetes apply retry")
   437  		}
   438  	}
   439  
   440  	return k.kubeResultToEntities(result)
   441  }
   442  
   443  // Update an entity like kubectl create/replace does.
   444  //
   445  // This uses a PUT HTTP call to replace one entity with another.
   446  //
   447  // It's not as good as apply, because it will wipe out bookkeeping
   448  // that other controllers have added.
   449  //
   450  // But in cases where the entity is too big to do a 3-way merge,
   451  // this is the next best option.
   452  func (k *K8sClient) createOrReplaceEntity(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) {
   453  	resources, err := k.prepareUpdateList(ctx, entity)
   454  	if err != nil {
   455  		return nil, errors.Wrap(err, "kubernetes upsert")
   456  	}
   457  
   458  	result, err := k.resourceClient.CreateOrReplace(resources)
   459  	if err != nil {
   460  		return nil, err
   461  	}
   462  
   463  	return k.kubeResultToEntities(result)
   464  }
   465  
   466  // Delete and create an entity.
   467  //
   468  // This is the most intrusive way to perform an update,
   469  // because any children of the object will be deleted by the controller.
   470  //
   471  // Some objects in the Kubernetes ecosystem are immutable, so need
   472  // this approach as a last resort.
   473  func (k *K8sClient) deleteAndCreateEntity(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) {
   474  	resources, err := k.prepareUpdateList(ctx, entity)
   475  	if err != nil {
   476  		return nil, errors.Wrap(err, "kubernetes delete and re-create")
   477  	}
   478  
   479  	result, err := k.deleteAndCreate(ctx, resources)
   480  	if err != nil {
   481  		return nil, err
   482  	}
   483  
   484  	return k.kubeResultToEntities(result)
   485  }
   486  
   487  // Make sure the type exists and create a ResourceList to help update it.
   488  func (k *K8sClient) prepareUpdateList(ctx context.Context, e K8sEntity) (kube.ResourceList, error) {
   489  	_, err := k.forceDiscovery(ctx, e.GVK())
   490  	if err != nil {
   491  		return nil, err
   492  	}
   493  
   494  	return k.buildResourceList(ctx, e)
   495  }
   496  
   497  // Build a ResourceList usable by our helm client for interacting with a resource.
   498  //
   499  // Although the underlying API encourages you to batch these together (for
   500  // better parallelization), we've found that it's more robust to handle entities
   501  // individually to ensure an error in one doesn't affect the others (and the
   502  // real bottleneck isn't in building).
   503  func (k *K8sClient) buildResourceList(ctx context.Context, e K8sEntity) (kube.ResourceList, error) {
   504  	rawYAML, err := SerializeSpecYAMLToBuffer([]K8sEntity{e})
   505  	if err != nil {
   506  		return nil, err
   507  	}
   508  
   509  	resources, err := k.resourceClient.Build(rawYAML, false)
   510  	if err != nil {
   511  		return nil, err
   512  	}
   513  
   514  	return resources, nil
   515  }
   516  
   517  func (k *K8sClient) kubeResultToEntities(result *kube.Result) ([]K8sEntity, error) {
   518  	entities := []K8sEntity{}
   519  	for _, info := range result.Created {
   520  		entities = append(entities, NewK8sEntity(info.Object))
   521  	}
   522  	for _, info := range result.Updated {
   523  		entities = append(entities, NewK8sEntity(info.Object))
   524  	}
   525  
   526  	// Helm parses the results as unstructured info, but Tilt needs them parsed with the current
   527  	// API scheme. The easiest way to do this is to serialize them to yaml and re-parse again.
   528  	buf, err := SerializeSpecYAMLToBuffer(entities)
   529  	if err != nil {
   530  		return nil, errors.Wrap(err, "reading kubernetes result")
   531  	}
   532  
   533  	parsed, err := ParseYAML(buf)
   534  	if err != nil {
   535  		return nil, errors.Wrap(err, "parsing kubernetes result")
   536  	}
   537  	return parsed, nil
   538  }
   539  
   540  func (k *K8sClient) deleteAndCreate(ctx context.Context, list kube.ResourceList) (*kube.Result, error) {
   541  	// Delete is destructive, so clone first.
   542  	toDelete := kube.ResourceList{}
   543  	for _, r := range list {
   544  		rClone := *r
   545  		rClone.Object = r.Object.DeepCopyObject()
   546  		toDelete = append(toDelete, &rClone)
   547  	}
   548  
   549  	_, errs := k.resourceClient.Delete(toDelete)
   550  	for _, err := range errs {
   551  		if isNotFoundError(err) {
   552  			continue
   553  		}
   554  		return nil, errors.Wrap(err, "kubernetes delete")
   555  	}
   556  
   557  	// ensure the delete has finished before attempting to recreate
   558  	err := k.waitForDelete(ctx, list, 30*time.Second)
   559  	if err != nil {
   560  		return nil, errors.Wrap(err, "kubernetes create")
   561  	}
   562  
   563  	result, err := k.resourceClient.Create(list)
   564  	if err != nil {
   565  		return nil, errors.Wrap(err, "kubernetes create")
   566  	}
   567  	return result, nil
   568  }
   569  
   570  // Update a resource in-place, starting with the least intrusive
   571  // update strategy and escalating into the most intrusive strategy.
   572  func (k *K8sClient) escalatingUpdate(ctx context.Context, entity K8sEntity) ([]K8sEntity, error) {
   573  	fallback := false
   574  	result, err := k.applyEntity(ctx, entity)
   575  	if err != nil {
   576  		msg, match := maybeTooLargeError(err)
   577  		if match {
   578  			fallback = true
   579  			logger.Get(ctx).Infof("Updating %q failed: %s", entity.Name(), msg)
   580  			logger.Get(ctx).Infof("Attempting to create or replace")
   581  			result, err = k.createOrReplaceEntity(ctx, entity)
   582  		}
   583  	}
   584  
   585  	if err != nil {
   586  		maybeImmutable := maybeImmutableFieldStderr(err.Error())
   587  		if maybeImmutable {
   588  			fallback = true
   589  			logger.Get(ctx).Infof("Updating %q failed: %s", entity.Name(),
   590  				truncateErrorToOneLine(err.Error()))
   591  			logger.Get(ctx).Infof("Attempting to delete and re-create")
   592  			result, err = k.deleteAndCreateEntity(ctx, entity)
   593  		}
   594  	}
   595  
   596  	if err != nil {
   597  		return nil, err
   598  	}
   599  	if fallback {
   600  		logger.Get(ctx).Infof("Updating %q succeeded!", entity.Name())
   601  	}
   602  	return result, nil
   603  }
   604  
   605  func truncateErrorToOneLine(stderr string) string {
   606  	index := strings.Index(stderr, "\n")
   607  	if index != -1 {
   608  		return stderr[:index]
   609  	}
   610  	return stderr
   611  }
   612  
   613  // We're using kubectl, so we only get stderr, not structured errors.
   614  //
   615  // Take a wild guess if the update is failing due to immutable field errors.
   616  //
   617  // This should bias towards false positives (i.e., we think something is an
   618  // immutable field error when it's not).
   619  func maybeImmutableFieldStderr(stderr string) bool {
   620  	return strings.Contains(stderr, validation.FieldImmutableErrorMsg) ||
   621  		strings.Contains(stderr, ForbiddenFieldsPrefix)
   622  }
   623  
   624  var MetadataAnnotationsTooLongRe = regexp.MustCompile(`metadata.annotations: Too long: must have at most \d+ bytes.*`)
   625  
   626  // kubectl apply sets an annotation containing the object's previous configuration.
   627  // However, annotations have a max size of 256k. Large objects such as configmaps can exceed 256k, which makes
   628  // apply unusable, so we need to fall back to delete/create
   629  // https://github.com/kubernetes/kubectl/issues/712
   630  //
   631  // We've also seen this reported differently, with a 413 HTTP error.
   632  // https://github.com/tilt-dev/tilt/issues/5279
   633  func maybeTooLargeError(err error) (string, bool) {
   634  	// We don't have an easy way to reproduce some of these problems, so we check
   635  	// for both the structured form of the error and the unstructured form.
   636  	statusErr, isStatusErr := err.(*apierrors.StatusError)
   637  	if isStatusErr && statusErr.ErrStatus.Code == http.StatusRequestEntityTooLarge {
   638  		return err.Error(), true
   639  	}
   640  
   641  	stderr := err.Error()
   642  	for _, line := range strings.Split(stderr, "\n") {
   643  		if MetadataAnnotationsTooLongRe.MatchString(line) {
   644  			return line, true
   645  		}
   646  
   647  		if strings.Contains(line, "the server responded with the status code 413") {
   648  			return line, true
   649  		}
   650  	}
   651  
   652  	return "", false
   653  }
   654  
   655  // Deletes all given entities.
   656  //
   657  // Currently ignores any "not found" errors, because that seems like the correct
   658  // behavior for our use cases.
   659  func (k *K8sClient) Delete(ctx context.Context, entities []K8sEntity, wait time.Duration) error {
   660  	l := logger.Get(ctx)
   661  	l.Infof("Deleting kubernetes objects:")
   662  	for _, e := range entities {
   663  		l.Infof("→ %s/%s", e.GVK().Kind, e.Name())
   664  	}
   665  
   666  	var resources kube.ResourceList
   667  	for _, e := range entities {
   668  		resourceList, err := k.buildResourceList(ctx, e)
   669  		if utilerrors.FilterOut(err, isMissingKindError) != nil {
   670  			return errors.Wrap(err, "kubernetes delete")
   671  		}
   672  		resources = append(resources, resourceList...)
   673  	}
   674  
   675  	_, errs := k.resourceClient.Delete(resources)
   676  	for _, err := range errs {
   677  		if err == nil || isNotFoundError(err) {
   678  			continue
   679  		}
   680  
   681  		return errors.Wrap(err, "kubernetes delete")
   682  	}
   683  
   684  	if wait > 0 {
   685  		err := k.waitForDelete(ctx, resources, wait)
   686  		if err != nil {
   687  			return err
   688  		}
   689  	}
   690  
   691  	return nil
   692  }
   693  
   694  func (k *K8sClient) forceDiscovery(ctx context.Context, gvk schema.GroupVersionKind) (*meta.RESTMapping, error) {
   695  	rm, err := k.drm.RESTMapping(gvk.GroupKind(), gvk.Version)
   696  	if err != nil {
   697  		// The REST mapper doesn't have any sort of internal invalidation
   698  		// mechanism. So if the user applies a CRD (i.e., changing the available
   699  		// api resources), the REST mapper won't discover the new types.
   700  		//
   701  		// https://github.com/kubernetes/kubernetes/issues/75383
   702  		//
   703  		// But! When Tilt requests a resource by reference, we know in advance that
   704  		// it must exist, and therefore, its type must exist.  So we can safely
   705  		// reset the REST mapper and retry, so that it discovers the types.
   706  		k.drm.Reset()
   707  
   708  		rm, err = k.drm.RESTMapping(gvk.GroupKind(), gvk.Version)
   709  		if err != nil {
   710  			return nil, errors.Wrapf(err, "error mapping %s/%s", gvk.Group, gvk.Kind)
   711  		}
   712  	}
   713  	return rm, nil
   714  }
   715  
   716  // Returns true if the list successfully deleted. False if we timed out.
   717  func (k *K8sClient) waitForDelete(ctx context.Context, list kube.ResourceList, duration time.Duration) error {
   718  	results := make([]bool, len(list))
   719  	var wg sync.WaitGroup
   720  	for i, r := range list {
   721  		wg.Add(1)
   722  		go func(i int, resourceInfo *resource.Info) {
   723  			waitOpt := &wait.WaitOptions{
   724  				DynamicClient: k.dynamic,
   725  				IOStreams:     genericclioptions.NewTestIOStreamsDiscard(),
   726  				Timeout:       duration,
   727  				ForCondition:  "delete",
   728  			}
   729  
   730  			_, ok, _ := wait.IsDeleted(ctx, resourceInfo, waitOpt)
   731  			results[i] = ok
   732  			wg.Done()
   733  		}(i, r)
   734  	}
   735  	wg.Wait()
   736  
   737  	for i, r := range results {
   738  		if !r {
   739  			return fmt.Errorf("timeout waiting for delete: %s", list[i].Name)
   740  		}
   741  	}
   742  	return nil
   743  }
   744  
   745  func (k *K8sClient) ListMeta(ctx context.Context, gvk schema.GroupVersionKind, ns Namespace) ([]metav1.Object, error) {
   746  	mapping, err := k.forceDiscovery(ctx, gvk)
   747  	if err != nil {
   748  		return nil, err
   749  	}
   750  
   751  	gvr := mapping.Resource
   752  	isRoot := mapping.Scope != nil && mapping.Scope.Name() == meta.RESTScopeNameRoot
   753  	var metaList *metav1.PartialObjectMetadataList
   754  	if isRoot {
   755  		metaList, err = k.metadata.Resource(gvr).List(ctx, metav1.ListOptions{})
   756  	} else {
   757  		metaList, err = k.metadata.Resource(gvr).Namespace(ns.String()).List(ctx, metav1.ListOptions{})
   758  	}
   759  
   760  	if err != nil {
   761  		return nil, err
   762  	}
   763  
   764  	// type conversion
   765  	result := make([]metav1.Object, len(metaList.Items))
   766  	for i, meta := range metaList.Items {
   767  		m := meta.ObjectMeta
   768  		result[i] = &m
   769  	}
   770  	return result, nil
   771  }
   772  
   773  func (k *K8sClient) GetMetaByReference(ctx context.Context, ref v1.ObjectReference) (metav1.Object, error) {
   774  	gvk := ReferenceGVK(ref)
   775  	mapping, err := k.forceDiscovery(ctx, gvk)
   776  	if err != nil {
   777  		return nil, err
   778  	}
   779  
   780  	gvr := mapping.Resource
   781  	namespace := ref.Namespace
   782  	name := ref.Name
   783  	resourceVersion := ref.ResourceVersion
   784  	uid := ref.UID
   785  
   786  	typeAndMeta, err := k.metadata.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{
   787  		ResourceVersion: resourceVersion,
   788  	})
   789  	if err != nil {
   790  		return nil, err
   791  	}
   792  	meta := typeAndMeta.ObjectMeta
   793  	if uid != "" && meta.UID != uid {
   794  		return nil, apierrors.NewNotFound(v1.Resource(gvr.Resource), name)
   795  	}
   796  	return &meta, nil
   797  }
   798  
   799  func (k *K8sClient) ClusterHealth(ctx context.Context, verbose bool) (ClusterHealth, error) {
   800  	isLive, livezResp, err := k.apiServerHealthCheck(ctx, "/livez", verbose)
   801  	if err != nil {
   802  		return ClusterHealth{}, fmt.Errorf("cluster liveness check: %v", err)
   803  	}
   804  
   805  	// TODO(milas): is there any point to running the readiness check if the
   806  	// 	liveness check failed?
   807  	isReady, readyzResp, err := k.apiServerHealthCheck(ctx, "/readyz", verbose)
   808  	if err != nil {
   809  		return ClusterHealth{}, fmt.Errorf("cluster readiness check: %v", err)
   810  	}
   811  
   812  	return ClusterHealth{
   813  		Live:        isLive,
   814  		Ready:       isReady,
   815  		LiveOutput:  livezResp,
   816  		ReadyOutput: readyzResp,
   817  	}, nil
   818  }
   819  
   820  // apiServerHealthCheck issues a direct HTTP request to an apiserver health endpoint.
   821  //
   822  // There are not methods for this functionality exposed via client-go, so the
   823  // RESTClient is used directly.
   824  //
   825  // See https://kubernetes.io/docs/reference/using-api/health-checks/
   826  func (k *K8sClient) apiServerHealthCheck(ctx context.Context, route string, verbose bool) (bool, string, error) {
   827  	req := k.discovery.RESTClient().
   828  		Get().
   829  		AbsPath(route).
   830  		// timeout will both be passed as a param to server as well as used to
   831  		// create a child context with a deadline
   832  		Timeout(10 * time.Second).
   833  		MaxRetries(1)
   834  
   835  	if verbose {
   836  		req = req.Param("verbose", "")
   837  	}
   838  	body, err := req.DoRaw(ctx)
   839  	if err != nil {
   840  		var statusErr *apierrors.StatusError
   841  		if errors.As(err, &statusErr) {
   842  			return false, statusErr.ErrStatus.Message, nil
   843  		}
   844  		return false, "", err
   845  	}
   846  	return true, string(body), nil
   847  }
   848  
   849  // Tests whether a string is a valid version for a k8s resource type.
   850  // from https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#version-priority
   851  // Versions start with a v followed by a number, an optional beta or alpha designation, and optional additional numeric
   852  // versioning information. Broadly, a version string might look like v2 or v2beta1.
   853  var versionRegex = regexp.MustCompile(`^v\d+(?:(?:alpha|beta)(?:\d+)?)?$`)
   854  
   855  func ReferenceGVK(involvedObject v1.ObjectReference) schema.GroupVersionKind {
   856  	// For some types, APIVersion is incorrectly just the group w/ no version, which leads GroupVersionKind to return
   857  	// a value where Group is empty and Version contains the group, so we need to correct for that.
   858  	// An empty Group is valid, though: it's empty for apps in the core group.
   859  	// So, we detect this situation by checking if the version field is valid.
   860  
   861  	// this stems from group/version not necessarily being populated at other points in the API. see more info here:
   862  	// https://github.com/kubernetes/client-go/issues/308
   863  	// https://github.com/kubernetes/kubernetes/issues/3030
   864  
   865  	gvk := involvedObject.GroupVersionKind()
   866  	if !versionRegex.MatchString(gvk.Version) {
   867  		gvk.Group = involvedObject.APIVersion
   868  		gvk.Version = ""
   869  	}
   870  
   871  	return gvk
   872  }
   873  
   874  func ProvideServerVersion(maybeClientset ClientsetOrError) (*version.Info, error) {
   875  	if maybeClientset.Error != nil {
   876  		return nil, maybeClientset.Error
   877  	}
   878  	return maybeClientset.Clientset.Discovery().ServerVersion()
   879  }
   880  
   881  type ClientsetOrError struct {
   882  	Clientset *kubernetes.Clientset
   883  	Error     error
   884  }
   885  
   886  func ProvideClientset(cfg RESTConfigOrError) ClientsetOrError {
   887  	if cfg.Error != nil {
   888  		return ClientsetOrError{Error: cfg.Error}
   889  	}
   890  	clientset, err := kubernetes.NewForConfig(cfg.Config)
   891  	return ClientsetOrError{Clientset: clientset, Error: err}
   892  }
   893  
   894  func ProvideClientConfig(contextOverride KubeContextOverride, nsFlag NamespaceOverride) clientcmd.ClientConfig {
   895  	rules := clientcmd.NewDefaultClientConfigLoadingRules()
   896  	rules.DefaultClientConfig = &clientcmd.DefaultClientConfig
   897  
   898  	overrides := &clientcmd.ConfigOverrides{
   899  		CurrentContext: string(contextOverride),
   900  		Context: clientcmdapi.Context{
   901  			Namespace: string(nsFlag),
   902  		},
   903  	}
   904  	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
   905  		rules,
   906  		overrides)
   907  }
   908  
   909  // The namespace in the kubeconfig.
   910  // Used as a default namespace in some (but not all) client commands.
   911  // https://godoc.org/k8s.io/client-go/tools/clientcmd/api/v1#Context
   912  func ProvideConfigNamespace(clientLoader clientcmd.ClientConfig) Namespace {
   913  	namespace, explicit, err := clientLoader.Namespace()
   914  	if err != nil {
   915  		// If we can't get a namespace from the config, just fail gracefully to the default.
   916  		// If this error indicates a more serious problem, it will get handled downstream.
   917  		return ""
   918  	}
   919  
   920  	// TODO(nick): Right now, tilt doesn't provide a namespace flag. If we ever did,
   921  	// we would need to handle explicit namespaces different than implicit ones.
   922  	_ = explicit
   923  
   924  	return Namespace(namespace)
   925  }
   926  
   927  type RESTConfigOrError struct {
   928  	Config *rest.Config
   929  	Error  error
   930  }
   931  
   932  func ProvideRESTConfig(clientLoader clientcmd.ClientConfig) RESTConfigOrError {
   933  	config, err := clientLoader.ClientConfig()
   934  	return RESTConfigOrError{Config: config, Error: err}
   935  }