github.com/GoogleContainerTools/skaffold@v1.39.18/pkg/skaffold/kubernetes/debugging/transform.go (about)

     1  /*
     2  Copyright 2021 The Skaffold Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package debugging
    18  
    19  import (
    20  	"bufio"
    21  	"bytes"
    22  	"context"
    23  	"encoding/json"
    24  	"fmt"
    25  	"strings"
    26  	"time"
    27  
    28  	appsv1 "k8s.io/api/apps/v1"
    29  	batchv1 "k8s.io/api/batch/v1"
    30  	v1 "k8s.io/api/core/v1"
    31  	"k8s.io/apimachinery/pkg/api/meta"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json"
    35  	"k8s.io/kubectl/pkg/scheme"
    36  
    37  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/debug"
    38  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/debug/types"
    39  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
    40  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/debugging/adapter"
    41  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/manifest"
    42  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
    43  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
    44  )
    45  
    46  var (
    47  	decodeFromYaml = scheme.Codecs.UniversalDeserializer().Decode
    48  	encodeAsYaml   = func(o runtime.Object) ([]byte, error) {
    49  		s := k8sjson.NewYAMLSerializer(k8sjson.DefaultMetaFactory, scheme.Scheme, scheme.Scheme)
    50  		var b bytes.Buffer
    51  		w := bufio.NewWriter(&b)
    52  		if err := s.Encode(o, w); err != nil {
    53  			return nil, err
    54  		}
    55  		w.Flush()
    56  		return b.Bytes(), nil
    57  	}
    58  )
    59  
    60  // ApplyDebuggingTransforms applies language-platform-specific transforms to a list of manifests.
    61  func ApplyDebuggingTransforms(l manifest.ManifestList, builds []graph.Artifact, registries manifest.Registries) (manifest.ManifestList, error) {
    62  	ctx, cancel := context.WithCancel(context.Background())
    63  	defer cancel()
    64  
    65  	retriever := func(image string) (debug.ImageConfiguration, error) {
    66  		return debug.ConfigRetriever(ctx, image, builds, registries.InsecureRegistries)
    67  	}
    68  
    69  	return applyDebuggingTransforms(l, retriever, registries.DebugHelpersRegistry)
    70  }
    71  
    72  func Describe(obj runtime.Object) (group, version, kind, description string) {
    73  	// get metadata/name; shamelessly stolen from from k8s.io/cli-runtime/pkg/printers/name.go
    74  	name := "<unknown>"
    75  	if acc, err := meta.Accessor(obj); err == nil {
    76  		if n := acc.GetName(); len(n) > 0 {
    77  			name = n
    78  		}
    79  	}
    80  
    81  	gvk := obj.GetObjectKind().GroupVersionKind()
    82  	group = gvk.Group
    83  	version = gvk.Version
    84  	kind = gvk.Kind
    85  	if group == "" {
    86  		description = fmt.Sprintf("%s/%s", strings.ToLower(kind), name)
    87  	} else {
    88  		description = fmt.Sprintf("%s.%s/%s", strings.ToLower(kind), group, name)
    89  	}
    90  	return
    91  }
    92  
    93  func applyDebuggingTransforms(l manifest.ManifestList, retriever debug.ConfigurationRetriever, debugHelpersRegistry string) (manifest.ManifestList, error) {
    94  	var updated manifest.ManifestList
    95  	for _, manifest := range l {
    96  		obj, _, err := decodeFromYaml(manifest, nil, nil)
    97  		if err != nil {
    98  			log.Entry(context.Background()).Debugf("Unable to interpret manifest for debugging: %v\n", err)
    99  		} else if transformManifest(obj, retriever, debugHelpersRegistry) {
   100  			manifest, err = encodeAsYaml(obj)
   101  			if err != nil {
   102  				return nil, fmt.Errorf("marshalling yaml: %w", err)
   103  			}
   104  			if log.IsDebugLevelEnabled() {
   105  				log.Entry(context.Background()).Debugln("Applied debugging transform:\n", string(manifest))
   106  			}
   107  		}
   108  		updated = append(updated, manifest)
   109  	}
   110  
   111  	return updated, nil
   112  }
   113  
   114  // isPortAvailable returns true if none of the pod's containers specify the given port.
   115  func isPortAvailable(podSpec *v1.PodSpec, port int32) bool {
   116  	for _, container := range podSpec.Containers {
   117  		for _, portSpec := range container.Ports {
   118  			if portSpec.ContainerPort == port {
   119  				return false
   120  			}
   121  		}
   122  	}
   123  	return true
   124  }
   125  
   126  // rewriteProbes rewrites k8s probes to expand timeouts to 10 minutes to allow debugging local probes.
   127  func rewriteProbes(metadata *metav1.ObjectMeta, podSpec *v1.PodSpec) bool {
   128  	minTimeout := 10 * time.Minute // make it configurable?
   129  	if annotation, found := metadata.Annotations[types.DebugProbeTimeouts]; found {
   130  		if annotation == "skip" {
   131  			log.Entry(context.Background()).Debugf("skipping probe rewrite on %q by request", metadata.Name)
   132  			return false
   133  		}
   134  		if d, err := time.ParseDuration(annotation); err != nil {
   135  			log.Entry(context.Background()).Warnf("invalid probe timeout value for %q: %q: %v", metadata.Name, annotation, err)
   136  		} else {
   137  			minTimeout = d
   138  		}
   139  	}
   140  	annotation, found := metadata.Annotations[types.DebugConfig]
   141  	if !found {
   142  		log.Entry(context.Background()).Debugf("skipping probe rewrite on %q: not configured for debugging", metadata.Name)
   143  		return false
   144  	}
   145  	var config map[string]types.ContainerDebugConfiguration
   146  	if err := json.Unmarshal([]byte(annotation), &config); err != nil {
   147  		log.Entry(context.Background()).Warnf("error unmarshalling debugging configuration for %q: %v", metadata.Name, err)
   148  		return false
   149  	}
   150  
   151  	changed := false
   152  	for i := range podSpec.Containers {
   153  		c := &podSpec.Containers[i]
   154  		// only affect containers listed in debug-config
   155  		if _, found := config[c.Name]; found {
   156  			lp := rewriteHTTPGetProbe(c.LivenessProbe, minTimeout)
   157  			rp := rewriteHTTPGetProbe(c.ReadinessProbe, minTimeout)
   158  			sp := rewriteHTTPGetProbe(c.StartupProbe, minTimeout)
   159  			if lp || rp || sp {
   160  				log.Entry(context.Background()).Infof("Updated probe timeouts for %s/%s", metadata.Name, c.Name)
   161  			}
   162  			changed = changed || lp || rp || sp
   163  		}
   164  	}
   165  	return changed
   166  }
   167  
   168  func rewriteHTTPGetProbe(probe *v1.Probe, minTimeout time.Duration) bool {
   169  	if probe == nil || probe.HTTPGet == nil || int32(minTimeout.Seconds()) < probe.TimeoutSeconds {
   170  		return false
   171  	}
   172  	probe.TimeoutSeconds = int32(minTimeout.Seconds())
   173  	return true
   174  }
   175  
   176  // transformManifest attempts to configure a manifest for debugging.
   177  // Returns true if changed, false otherwise.
   178  func transformManifest(obj runtime.Object, retrieveImageConfiguration debug.ConfigurationRetriever, debugHelpersRegistry string) bool {
   179  	one := int32(1)
   180  	switch o := obj.(type) {
   181  	case *v1.Pod:
   182  		return transformPodSpec(&o.ObjectMeta, &o.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   183  	case *v1.PodList:
   184  		changed := false
   185  		for i := range o.Items {
   186  			if transformPodSpec(&o.Items[i].ObjectMeta, &o.Items[i].Spec, retrieveImageConfiguration, debugHelpersRegistry) {
   187  				changed = true
   188  			}
   189  		}
   190  		return changed
   191  	case *v1.ReplicationController:
   192  		if o.Spec.Replicas != nil {
   193  			o.Spec.Replicas = &one
   194  		}
   195  		return transformPodSpec(&o.Spec.Template.ObjectMeta, &o.Spec.Template.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   196  	case *appsv1.Deployment:
   197  		if o.Spec.Replicas != nil {
   198  			o.Spec.Replicas = &one
   199  		}
   200  		return transformPodSpec(&o.Spec.Template.ObjectMeta, &o.Spec.Template.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   201  	case *appsv1.DaemonSet:
   202  		return transformPodSpec(&o.Spec.Template.ObjectMeta, &o.Spec.Template.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   203  	case *appsv1.ReplicaSet:
   204  		if o.Spec.Replicas != nil {
   205  			o.Spec.Replicas = &one
   206  		}
   207  		return transformPodSpec(&o.Spec.Template.ObjectMeta, &o.Spec.Template.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   208  	case *appsv1.StatefulSet:
   209  		if o.Spec.Replicas != nil {
   210  			o.Spec.Replicas = &one
   211  		}
   212  		return transformPodSpec(&o.Spec.Template.ObjectMeta, &o.Spec.Template.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   213  	case *batchv1.Job:
   214  		return transformPodSpec(&o.Spec.Template.ObjectMeta, &o.Spec.Template.Spec, retrieveImageConfiguration, debugHelpersRegistry)
   215  
   216  	default:
   217  		group, version, _, description := Describe(obj)
   218  		if group == "apps" || group == "batch" {
   219  			if version != "v1" {
   220  				// treat deprecated objects as errors
   221  				log.Entry(context.Background()).Errorf("deprecated versions not supported by debug: %s (%s)", description, version)
   222  			} else {
   223  				log.Entry(context.Background()).Warnf("no debug transformation for: %s", description)
   224  			}
   225  		} else {
   226  			log.Entry(context.Background()).Debugf("no debug transformation for: %s", description)
   227  		}
   228  		return false
   229  	}
   230  }
   231  
   232  // transformPodSpec attempts to configure a podspec for debugging.
   233  // Returns true if changed, false otherwise.
   234  func transformPodSpec(metadata *metav1.ObjectMeta, podSpec *v1.PodSpec, retrieveImageConfiguration debug.ConfigurationRetriever, debugHelpersRegistry string) bool {
   235  	// order matters as rewriteProbes only affects containers marked for debugging
   236  	containers := rewriteContainers(metadata, podSpec, retrieveImageConfiguration, debugHelpersRegistry)
   237  	timeouts := rewriteProbes(metadata, podSpec)
   238  	return containers || timeouts
   239  }
   240  
   241  func rewriteContainers(metadata *metav1.ObjectMeta, podSpec *v1.PodSpec, retrieveImageConfiguration debug.ConfigurationRetriever, debugHelpersRegistry string) bool {
   242  	// skip annotated podspecs — allows users to customize their own image
   243  	if _, found := metadata.Annotations[types.DebugConfig]; found {
   244  		return false
   245  	}
   246  
   247  	portAvailable := func(port int32) bool {
   248  		return isPortAvailable(podSpec, port)
   249  	}
   250  	portAlloc := func(desiredPort int32) int32 {
   251  		return util.AllocatePort(portAvailable, desiredPort)
   252  	}
   253  	// map of containers -> debugging configuration maps; k8s ensures that a pod's containers are uniquely named
   254  	configurations := make(map[string]types.ContainerDebugConfiguration)
   255  	// the container images that require debugging support files
   256  	var containersRequiringSupport []*v1.Container
   257  	// the set of image IDs required to provide debugging support files
   258  	requiredSupportImages := make(map[string]bool)
   259  	for i := range podSpec.Containers {
   260  		container := podSpec.Containers[i] // make a copy and only apply changes on successful transform
   261  
   262  		// the usual retriever returns an error for non-build artifacts
   263  		imageConfig, err := retrieveImageConfiguration(container.Image)
   264  		if err != nil {
   265  			continue
   266  		}
   267  		a := adapter.NewAdapter(&container)
   268  		// requiredImage, if not empty, is the image ID providing the debugging support files
   269  		// `err != nil` means that the container did not or could not be transformed
   270  		if configuration, requiredImage, err := debug.TransformContainer(a, imageConfig, portAlloc); err == nil {
   271  			configurations[container.Name] = configuration
   272  			podSpec.Containers[i] = container // apply any configuration changes
   273  			if len(requiredImage) > 0 {
   274  				log.Entry(context.Background()).Infof("%q requires debugging support image %q", container.Name, requiredImage)
   275  				containersRequiringSupport = append(containersRequiringSupport, &podSpec.Containers[i])
   276  				requiredSupportImages[requiredImage] = true
   277  			}
   278  		} else {
   279  			log.Entry(context.Background()).Warnf("Image %q not configured for debugging: %v", container.Name, err)
   280  		}
   281  	}
   282  
   283  	// check if we have any images requiring additional debugging support files
   284  	if len(containersRequiringSupport) > 0 {
   285  		log.Entry(context.Background()).Infof("Configuring installation of debugging support files")
   286  		// we create the volume that will hold the debugging support files
   287  		supportVolume := v1.Volume{Name: debug.DebuggingSupportFilesVolume, VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}
   288  		podSpec.Volumes = append(podSpec.Volumes, supportVolume)
   289  
   290  		// this volume is mounted in the containers at `/dbg`
   291  		supportVolumeMount := v1.VolumeMount{Name: debug.DebuggingSupportFilesVolume, MountPath: "/dbg"}
   292  		// the initContainers are responsible for populating the contents of `/dbg`
   293  		for imageID := range requiredSupportImages {
   294  			supportFilesInitContainer := v1.Container{
   295  				Name:         fmt.Sprintf("install-%s-debug-support", imageID),
   296  				Image:        fmt.Sprintf("%s/%s", debugHelpersRegistry, imageID),
   297  				VolumeMounts: []v1.VolumeMount{supportVolumeMount},
   298  			}
   299  			podSpec.InitContainers = append(podSpec.InitContainers, supportFilesInitContainer)
   300  		}
   301  		// the populated volume is then mounted in the containers at `/dbg` too
   302  		for _, container := range containersRequiringSupport {
   303  			container.VolumeMounts = append(container.VolumeMounts, supportVolumeMount)
   304  		}
   305  	}
   306  
   307  	if len(configurations) > 0 {
   308  		if metadata.Annotations == nil {
   309  			metadata.Annotations = make(map[string]string)
   310  		}
   311  		metadata.Annotations[types.DebugConfig] = debug.EncodeConfigurations(configurations)
   312  		return true
   313  	}
   314  	return false
   315  }