github.com/Datadog/cnab-go@v0.3.3-beta1.0.20191007143216-bba4b7e723d0/driver/kubernetes/kubernetes.go (about)

     1  package kubernetes
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  	"time"
    10  
    11  	// load credential helpers
    12  	_ "k8s.io/client-go/plugin/pkg/client/auth"
    13  	// Convert transitive deps to direct deps so that we can use constraints in our Gopkg.toml
    14  	_ "github.com/Azure/go-autorest/autorest"
    15  
    16  	"github.com/deislabs/cnab-go/bundle"
    17  	"github.com/deislabs/cnab-go/driver"
    18  	batchv1 "k8s.io/api/batch/v1"
    19  	v1 "k8s.io/api/core/v1"
    20  	"k8s.io/apimachinery/pkg/api/resource"
    21  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    22  	"k8s.io/apimachinery/pkg/labels"
    23  	batchclientv1 "k8s.io/client-go/kubernetes/typed/batch/v1"
    24  	coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
    25  	"k8s.io/client-go/rest"
    26  	"k8s.io/client-go/tools/clientcmd"
    27  )
    28  
    29  const (
    30  	k8sContainerName    = "invocation"
    31  	k8sFileSecretVolume = "files"
    32  	numBackoffLoops     = 6
    33  )
    34  
    35  // Driver runs an invocation image in a Kubernetes cluster.
    36  type Driver struct {
    37  	Namespace             string
    38  	ServiceAccountName    string
    39  	Annotations           map[string]string
    40  	LimitCPU              resource.Quantity
    41  	LimitMemory           resource.Quantity
    42  	Tolerations           []v1.Toleration
    43  	ActiveDeadlineSeconds int64
    44  	BackoffLimit          int32
    45  	SkipCleanup           bool
    46  	skipJobStatusCheck    bool
    47  	jobs                  batchclientv1.JobInterface
    48  	secrets               coreclientv1.SecretInterface
    49  	pods                  coreclientv1.PodInterface
    50  	deletionPolicy        metav1.DeletionPropagation
    51  	requiredCompletions   int32
    52  }
    53  
    54  // New initializes a Kubernetes driver.
    55  func New(namespace, serviceAccount string, conf *rest.Config) (*Driver, error) {
    56  	driver := &Driver{
    57  		Namespace:          namespace,
    58  		ServiceAccountName: serviceAccount,
    59  	}
    60  	driver.setDefaults()
    61  	err := driver.setClient(conf)
    62  	return driver, err
    63  }
    64  
    65  // Handles receives an ImageType* and answers whether this driver supports that type.
    66  func (k *Driver) Handles(imagetype string) bool {
    67  	return imagetype == driver.ImageTypeDocker || imagetype == driver.ImageTypeOCI
    68  }
    69  
    70  // Config returns the Kubernetes driver configuration options.
    71  func (k *Driver) Config() map[string]string {
    72  	return map[string]string{
    73  		"KUBE_NAMESPACE":  "Kubernetes namespace in which to run the invocation image",
    74  		"SERVICE_ACCOUNT": "Kubernetes service account to be mounted by the invocation image (if empty, no service account token will be mounted)",
    75  		"KUBE_CONFIG":     "Absolute path to the kubeconfig file",
    76  		"MASTER_URL":      "Kubernetes master endpoint",
    77  	}
    78  }
    79  
    80  // SetConfig sets Kubernetes driver configuration.
    81  func (k *Driver) SetConfig(settings map[string]string) {
    82  	k.setDefaults()
    83  	k.Namespace = settings["KUBE_NAMESPACE"]
    84  	k.ServiceAccountName = settings["SERVICE_ACCOUNT"]
    85  
    86  	var kubeconfig string
    87  	if kpath := settings["KUBE_CONFIG"]; kpath != "" {
    88  		kubeconfig = kpath
    89  	} else if home := homeDir(); home != "" {
    90  		kubeconfig = filepath.Join(home, ".kube", "config")
    91  	}
    92  
    93  	conf, err := clientcmd.BuildConfigFromFlags(settings["MASTER_URL"], kubeconfig)
    94  	if err != nil {
    95  		panic(err)
    96  	}
    97  	err = k.setClient(conf)
    98  	if err != nil {
    99  		panic(err)
   100  	}
   101  }
   102  
   103  func (k *Driver) setDefaults() {
   104  	k.SkipCleanup = false
   105  	k.BackoffLimit = 0
   106  	k.ActiveDeadlineSeconds = 300
   107  	k.requiredCompletions = 1
   108  	k.deletionPolicy = metav1.DeletePropagationBackground
   109  }
   110  
   111  func (k *Driver) setClient(conf *rest.Config) error {
   112  	coreClient, err := coreclientv1.NewForConfig(conf)
   113  	if err != nil {
   114  		return err
   115  	}
   116  	batchClient, err := batchclientv1.NewForConfig(conf)
   117  	if err != nil {
   118  		return err
   119  	}
   120  	k.jobs = batchClient.Jobs(k.Namespace)
   121  	k.secrets = coreClient.Secrets(k.Namespace)
   122  	k.pods = coreClient.Pods(k.Namespace)
   123  
   124  	return nil
   125  }
   126  
   127  // Run executes the operation inside of the invocation image.
   128  func (k *Driver) Run(op *driver.Operation) (driver.OperationResult, error) {
   129  	if k.Namespace == "" {
   130  		return driver.OperationResult{}, fmt.Errorf("KUBE_NAMESPACE is required")
   131  	}
   132  	labelMap := generateLabels(op)
   133  	meta := metav1.ObjectMeta{
   134  		Namespace:    k.Namespace,
   135  		GenerateName: generateNameTemplate(op),
   136  		Labels:       labelMap,
   137  	}
   138  	// Mount SA token if a non-zero value for ServiceAccountName has been specified
   139  	mountServiceAccountToken := k.ServiceAccountName != ""
   140  	job := &batchv1.Job{
   141  		ObjectMeta: meta,
   142  		Spec: batchv1.JobSpec{
   143  			ActiveDeadlineSeconds: &k.ActiveDeadlineSeconds,
   144  			Completions:           &k.requiredCompletions,
   145  			BackoffLimit:          &k.BackoffLimit,
   146  			Template: v1.PodTemplateSpec{
   147  				ObjectMeta: metav1.ObjectMeta{
   148  					Labels:      labelMap,
   149  					Annotations: k.Annotations,
   150  				},
   151  				Spec: v1.PodSpec{
   152  					ServiceAccountName:           k.ServiceAccountName,
   153  					AutomountServiceAccountToken: &mountServiceAccountToken,
   154  					RestartPolicy:                v1.RestartPolicyNever,
   155  					Tolerations:                  k.Tolerations,
   156  				},
   157  			},
   158  		},
   159  	}
   160  	container := v1.Container{
   161  		Name:    k8sContainerName,
   162  		Image:   imageWithDigest(op.Image),
   163  		Command: []string{"/cnab/app/run"},
   164  		Resources: v1.ResourceRequirements{
   165  			Limits: v1.ResourceList{
   166  				v1.ResourceCPU:    k.LimitCPU,
   167  				v1.ResourceMemory: k.LimitMemory,
   168  			},
   169  		},
   170  		ImagePullPolicy: v1.PullIfNotPresent,
   171  	}
   172  
   173  	if len(op.Environment) > 0 {
   174  		secret := &v1.Secret{
   175  			ObjectMeta: meta,
   176  			StringData: op.Environment,
   177  		}
   178  		secret.ObjectMeta.GenerateName += "env-"
   179  		envsecret, err := k.secrets.Create(secret)
   180  		if err != nil {
   181  			return driver.OperationResult{}, err
   182  		}
   183  		if !k.SkipCleanup {
   184  			defer k.deleteSecret(envsecret.ObjectMeta.Name)
   185  		}
   186  
   187  		container.EnvFrom = []v1.EnvFromSource{
   188  			{
   189  				SecretRef: &v1.SecretEnvSource{
   190  					LocalObjectReference: v1.LocalObjectReference{
   191  						Name: envsecret.ObjectMeta.Name,
   192  					},
   193  				},
   194  			},
   195  		}
   196  	}
   197  
   198  	if len(op.Files) > 0 {
   199  		secret, mounts := generateFileSecret(op.Files)
   200  		secret.ObjectMeta = metav1.ObjectMeta{
   201  			Namespace:    k.Namespace,
   202  			GenerateName: generateNameTemplate(op) + "files-",
   203  			Labels:       labelMap,
   204  		}
   205  		secret, err := k.secrets.Create(secret)
   206  		if err != nil {
   207  			return driver.OperationResult{}, err
   208  		}
   209  		if !k.SkipCleanup {
   210  			defer k.deleteSecret(secret.ObjectMeta.Name)
   211  		}
   212  
   213  		job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, v1.Volume{
   214  			Name: k8sFileSecretVolume,
   215  			VolumeSource: v1.VolumeSource{
   216  				Secret: &v1.SecretVolumeSource{
   217  					SecretName: secret.ObjectMeta.Name,
   218  				},
   219  			},
   220  		})
   221  		container.VolumeMounts = mounts
   222  	}
   223  
   224  	job.Spec.Template.Spec.Containers = []v1.Container{container}
   225  	job, err := k.jobs.Create(job)
   226  	if err != nil {
   227  		return driver.OperationResult{}, err
   228  	}
   229  	if !k.SkipCleanup {
   230  		defer k.deleteJob(job.ObjectMeta.Name)
   231  	}
   232  
   233  	// Return early for unit testing purposes (the fake k8s client implementation just
   234  	// hangs during watch because no events are ever created on the Job)
   235  	if k.skipJobStatusCheck {
   236  		return driver.OperationResult{}, nil
   237  	}
   238  
   239  	// Create a selector to detect the job just created
   240  	jobSelector := metav1.ListOptions{
   241  		LabelSelector: labels.Set(job.ObjectMeta.Labels).String(),
   242  		FieldSelector: newSingleFieldSelector("metadata.name", job.ObjectMeta.Name),
   243  	}
   244  
   245  	// Prevent detecting pods from prior jobs by adding the job name to the labels
   246  	podSelector := metav1.ListOptions{
   247  		LabelSelector: newSingleFieldSelector("job-name", job.ObjectMeta.Name),
   248  	}
   249  
   250  	return driver.OperationResult{}, k.watchJobStatusAndLogs(podSelector, jobSelector, op.Out)
   251  }
   252  
   253  func (k *Driver) watchJobStatusAndLogs(podSelector metav1.ListOptions, jobSelector metav1.ListOptions, out io.Writer) error {
   254  	// Stream Pod logs in the background
   255  	logsStreamingComplete := make(chan bool)
   256  	err := k.streamPodLogs(podSelector, out, logsStreamingComplete)
   257  	if err != nil {
   258  		return err
   259  	}
   260  	// Watch job events and exit on failure/success
   261  	watch, err := k.jobs.Watch(jobSelector)
   262  	if err != nil {
   263  		return err
   264  	}
   265  	for event := range watch.ResultChan() {
   266  		job, ok := event.Object.(*batchv1.Job)
   267  		if !ok {
   268  			return fmt.Errorf("unexpected type")
   269  		}
   270  		complete := false
   271  		for _, cond := range job.Status.Conditions {
   272  			if cond.Type == batchv1.JobFailed {
   273  				err = fmt.Errorf(cond.Message)
   274  				complete = true
   275  				break
   276  			}
   277  			if cond.Type == batchv1.JobComplete {
   278  				complete = true
   279  				break
   280  			}
   281  		}
   282  		if complete {
   283  			break
   284  		}
   285  	}
   286  	if err != nil {
   287  		return err
   288  	}
   289  
   290  	// Wait for pod logs to finish printing
   291  	for i := 0; i < int(k.requiredCompletions); i++ {
   292  		<-logsStreamingComplete
   293  	}
   294  
   295  	return nil
   296  }
   297  
   298  func (k *Driver) streamPodLogs(options metav1.ListOptions, out io.Writer, done chan bool) error {
   299  	watcher, err := k.pods.Watch(options)
   300  	if err != nil {
   301  		return err
   302  	}
   303  
   304  	go func() {
   305  		// Track pods whose logs have been streamed by pod name. We need to know when we've already
   306  		// processed logs for a given pod, since multiple lifecycle events are received per pod.
   307  		streamedLogs := map[string]bool{}
   308  		for event := range watcher.ResultChan() {
   309  			pod, ok := event.Object.(*v1.Pod)
   310  			if !ok {
   311  				continue
   312  			}
   313  			podName := pod.GetName()
   314  			if streamedLogs[podName] {
   315  				// The event was for a pod whose logs have already been streamed, so do nothing.
   316  				continue
   317  			}
   318  
   319  			for i := 0; i < numBackoffLoops; i++ {
   320  				time.Sleep(time.Duration(i*i/2) * time.Second)
   321  				req := k.pods.GetLogs(podName, &v1.PodLogOptions{
   322  					Container: k8sContainerName,
   323  					Follow:    true,
   324  				})
   325  				reader, err := req.Stream()
   326  				if err != nil {
   327  					// There was an error connecting to the pod, so continue the loop and attempt streaming
   328  					// the logs again.
   329  					continue
   330  				}
   331  
   332  				// Block the loop until all logs from the pod have been processed.
   333  				bytesRead, err := io.Copy(out, reader)
   334  				reader.Close()
   335  				if err != nil {
   336  					continue
   337  				}
   338  				if bytesRead == 0 {
   339  					// There is a chance where we have connected to the pod, but it has yet to write something.
   340  					// In that case, we continue to to keep streaming until it does.
   341  					continue
   342  				}
   343  				// Set the pod to have successfully streamed data.
   344  				streamedLogs[podName] = true
   345  				break
   346  			}
   347  
   348  			done <- true
   349  		}
   350  	}()
   351  
   352  	return nil
   353  }
   354  
   355  func (k *Driver) deleteSecret(name string) error {
   356  	return k.secrets.Delete(name, &metav1.DeleteOptions{
   357  		PropagationPolicy: &k.deletionPolicy,
   358  	})
   359  }
   360  
   361  func (k *Driver) deleteJob(name string) error {
   362  	return k.jobs.Delete(name, &metav1.DeleteOptions{
   363  		PropagationPolicy: &k.deletionPolicy,
   364  	})
   365  }
   366  
   367  func generateNameTemplate(op *driver.Operation) string {
   368  	return fmt.Sprintf("%s-%s-", op.Installation, op.Action)
   369  }
   370  
   371  func generateLabels(op *driver.Operation) map[string]string {
   372  	return map[string]string{
   373  		"cnab.io/installation": op.Installation,
   374  		"cnab.io/action":       op.Action,
   375  		"cnab.io/revision":     op.Revision,
   376  	}
   377  }
   378  
   379  func generateFileSecret(files map[string]string) (*v1.Secret, []v1.VolumeMount) {
   380  	size := len(files)
   381  	data := make(map[string]string, size)
   382  	mounts := make([]v1.VolumeMount, size)
   383  
   384  	i := 0
   385  	for path, contents := range files {
   386  		key := strings.Replace(filepath.ToSlash(path), "/", "_", -1)
   387  		data[key] = contents
   388  		mounts[i] = v1.VolumeMount{
   389  			Name:      k8sFileSecretVolume,
   390  			MountPath: path,
   391  			SubPath:   key,
   392  		}
   393  		i++
   394  	}
   395  
   396  	secret := &v1.Secret{
   397  		StringData: data,
   398  	}
   399  
   400  	return secret, mounts
   401  }
   402  
   403  func newSingleFieldSelector(k, v string) string {
   404  	return labels.Set(map[string]string{
   405  		k: v,
   406  	}).String()
   407  }
   408  
   409  func homeDir() string {
   410  	if h := os.Getenv("HOME"); h != "" {
   411  		return h
   412  	}
   413  	return os.Getenv("USERPROFILE") // windows
   414  }
   415  
   416  func imageWithDigest(img bundle.InvocationImage) string {
   417  	if img.Digest == "" {
   418  		return img.Image
   419  	}
   420  	return img.Image + "@" + img.Digest
   421  }