github.com/Datadog/cnab-go@v0.3.3-beta1.0.20191007143216-bba4b7e723d0/driver/kubernetes/kubernetes.go (about) 1 package kubernetes 2 3 import ( 4 "fmt" 5 "io" 6 "os" 7 "path/filepath" 8 "strings" 9 "time" 10 11 // load credential helpers 12 _ "k8s.io/client-go/plugin/pkg/client/auth" 13 // Convert transitive deps to direct deps so that we can use constraints in our Gopkg.toml 14 _ "github.com/Azure/go-autorest/autorest" 15 16 "github.com/deislabs/cnab-go/bundle" 17 "github.com/deislabs/cnab-go/driver" 18 batchv1 "k8s.io/api/batch/v1" 19 v1 "k8s.io/api/core/v1" 20 "k8s.io/apimachinery/pkg/api/resource" 21 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 "k8s.io/apimachinery/pkg/labels" 23 batchclientv1 "k8s.io/client-go/kubernetes/typed/batch/v1" 24 coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" 25 "k8s.io/client-go/rest" 26 "k8s.io/client-go/tools/clientcmd" 27 ) 28 29 const ( 30 k8sContainerName = "invocation" 31 k8sFileSecretVolume = "files" 32 numBackoffLoops = 6 33 ) 34 35 // Driver runs an invocation image in a Kubernetes cluster. 36 type Driver struct { 37 Namespace string 38 ServiceAccountName string 39 Annotations map[string]string 40 LimitCPU resource.Quantity 41 LimitMemory resource.Quantity 42 Tolerations []v1.Toleration 43 ActiveDeadlineSeconds int64 44 BackoffLimit int32 45 SkipCleanup bool 46 skipJobStatusCheck bool 47 jobs batchclientv1.JobInterface 48 secrets coreclientv1.SecretInterface 49 pods coreclientv1.PodInterface 50 deletionPolicy metav1.DeletionPropagation 51 requiredCompletions int32 52 } 53 54 // New initializes a Kubernetes driver. 55 func New(namespace, serviceAccount string, conf *rest.Config) (*Driver, error) { 56 driver := &Driver{ 57 Namespace: namespace, 58 ServiceAccountName: serviceAccount, 59 } 60 driver.setDefaults() 61 err := driver.setClient(conf) 62 return driver, err 63 } 64 65 // Handles receives an ImageType* and answers whether this driver supports that type. 66 func (k *Driver) Handles(imagetype string) bool { 67 return imagetype == driver.ImageTypeDocker || imagetype == driver.ImageTypeOCI 68 } 69 70 // Config returns the Kubernetes driver configuration options. 71 func (k *Driver) Config() map[string]string { 72 return map[string]string{ 73 "KUBE_NAMESPACE": "Kubernetes namespace in which to run the invocation image", 74 "SERVICE_ACCOUNT": "Kubernetes service account to be mounted by the invocation image (if empty, no service account token will be mounted)", 75 "KUBE_CONFIG": "Absolute path to the kubeconfig file", 76 "MASTER_URL": "Kubernetes master endpoint", 77 } 78 } 79 80 // SetConfig sets Kubernetes driver configuration. 81 func (k *Driver) SetConfig(settings map[string]string) { 82 k.setDefaults() 83 k.Namespace = settings["KUBE_NAMESPACE"] 84 k.ServiceAccountName = settings["SERVICE_ACCOUNT"] 85 86 var kubeconfig string 87 if kpath := settings["KUBE_CONFIG"]; kpath != "" { 88 kubeconfig = kpath 89 } else if home := homeDir(); home != "" { 90 kubeconfig = filepath.Join(home, ".kube", "config") 91 } 92 93 conf, err := clientcmd.BuildConfigFromFlags(settings["MASTER_URL"], kubeconfig) 94 if err != nil { 95 panic(err) 96 } 97 err = k.setClient(conf) 98 if err != nil { 99 panic(err) 100 } 101 } 102 103 func (k *Driver) setDefaults() { 104 k.SkipCleanup = false 105 k.BackoffLimit = 0 106 k.ActiveDeadlineSeconds = 300 107 k.requiredCompletions = 1 108 k.deletionPolicy = metav1.DeletePropagationBackground 109 } 110 111 func (k *Driver) setClient(conf *rest.Config) error { 112 coreClient, err := coreclientv1.NewForConfig(conf) 113 if err != nil { 114 return err 115 } 116 batchClient, err := batchclientv1.NewForConfig(conf) 117 if err != nil { 118 return err 119 } 120 k.jobs = batchClient.Jobs(k.Namespace) 121 k.secrets = coreClient.Secrets(k.Namespace) 122 k.pods = coreClient.Pods(k.Namespace) 123 124 return nil 125 } 126 127 // Run executes the operation inside of the invocation image. 128 func (k *Driver) Run(op *driver.Operation) (driver.OperationResult, error) { 129 if k.Namespace == "" { 130 return driver.OperationResult{}, fmt.Errorf("KUBE_NAMESPACE is required") 131 } 132 labelMap := generateLabels(op) 133 meta := metav1.ObjectMeta{ 134 Namespace: k.Namespace, 135 GenerateName: generateNameTemplate(op), 136 Labels: labelMap, 137 } 138 // Mount SA token if a non-zero value for ServiceAccountName has been specified 139 mountServiceAccountToken := k.ServiceAccountName != "" 140 job := &batchv1.Job{ 141 ObjectMeta: meta, 142 Spec: batchv1.JobSpec{ 143 ActiveDeadlineSeconds: &k.ActiveDeadlineSeconds, 144 Completions: &k.requiredCompletions, 145 BackoffLimit: &k.BackoffLimit, 146 Template: v1.PodTemplateSpec{ 147 ObjectMeta: metav1.ObjectMeta{ 148 Labels: labelMap, 149 Annotations: k.Annotations, 150 }, 151 Spec: v1.PodSpec{ 152 ServiceAccountName: k.ServiceAccountName, 153 AutomountServiceAccountToken: &mountServiceAccountToken, 154 RestartPolicy: v1.RestartPolicyNever, 155 Tolerations: k.Tolerations, 156 }, 157 }, 158 }, 159 } 160 container := v1.Container{ 161 Name: k8sContainerName, 162 Image: imageWithDigest(op.Image), 163 Command: []string{"/cnab/app/run"}, 164 Resources: v1.ResourceRequirements{ 165 Limits: v1.ResourceList{ 166 v1.ResourceCPU: k.LimitCPU, 167 v1.ResourceMemory: k.LimitMemory, 168 }, 169 }, 170 ImagePullPolicy: v1.PullIfNotPresent, 171 } 172 173 if len(op.Environment) > 0 { 174 secret := &v1.Secret{ 175 ObjectMeta: meta, 176 StringData: op.Environment, 177 } 178 secret.ObjectMeta.GenerateName += "env-" 179 envsecret, err := k.secrets.Create(secret) 180 if err != nil { 181 return driver.OperationResult{}, err 182 } 183 if !k.SkipCleanup { 184 defer k.deleteSecret(envsecret.ObjectMeta.Name) 185 } 186 187 container.EnvFrom = []v1.EnvFromSource{ 188 { 189 SecretRef: &v1.SecretEnvSource{ 190 LocalObjectReference: v1.LocalObjectReference{ 191 Name: envsecret.ObjectMeta.Name, 192 }, 193 }, 194 }, 195 } 196 } 197 198 if len(op.Files) > 0 { 199 secret, mounts := generateFileSecret(op.Files) 200 secret.ObjectMeta = metav1.ObjectMeta{ 201 Namespace: k.Namespace, 202 GenerateName: generateNameTemplate(op) + "files-", 203 Labels: labelMap, 204 } 205 secret, err := k.secrets.Create(secret) 206 if err != nil { 207 return driver.OperationResult{}, err 208 } 209 if !k.SkipCleanup { 210 defer k.deleteSecret(secret.ObjectMeta.Name) 211 } 212 213 job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, v1.Volume{ 214 Name: k8sFileSecretVolume, 215 VolumeSource: v1.VolumeSource{ 216 Secret: &v1.SecretVolumeSource{ 217 SecretName: secret.ObjectMeta.Name, 218 }, 219 }, 220 }) 221 container.VolumeMounts = mounts 222 } 223 224 job.Spec.Template.Spec.Containers = []v1.Container{container} 225 job, err := k.jobs.Create(job) 226 if err != nil { 227 return driver.OperationResult{}, err 228 } 229 if !k.SkipCleanup { 230 defer k.deleteJob(job.ObjectMeta.Name) 231 } 232 233 // Return early for unit testing purposes (the fake k8s client implementation just 234 // hangs during watch because no events are ever created on the Job) 235 if k.skipJobStatusCheck { 236 return driver.OperationResult{}, nil 237 } 238 239 // Create a selector to detect the job just created 240 jobSelector := metav1.ListOptions{ 241 LabelSelector: labels.Set(job.ObjectMeta.Labels).String(), 242 FieldSelector: newSingleFieldSelector("metadata.name", job.ObjectMeta.Name), 243 } 244 245 // Prevent detecting pods from prior jobs by adding the job name to the labels 246 podSelector := metav1.ListOptions{ 247 LabelSelector: newSingleFieldSelector("job-name", job.ObjectMeta.Name), 248 } 249 250 return driver.OperationResult{}, k.watchJobStatusAndLogs(podSelector, jobSelector, op.Out) 251 } 252 253 func (k *Driver) watchJobStatusAndLogs(podSelector metav1.ListOptions, jobSelector metav1.ListOptions, out io.Writer) error { 254 // Stream Pod logs in the background 255 logsStreamingComplete := make(chan bool) 256 err := k.streamPodLogs(podSelector, out, logsStreamingComplete) 257 if err != nil { 258 return err 259 } 260 // Watch job events and exit on failure/success 261 watch, err := k.jobs.Watch(jobSelector) 262 if err != nil { 263 return err 264 } 265 for event := range watch.ResultChan() { 266 job, ok := event.Object.(*batchv1.Job) 267 if !ok { 268 return fmt.Errorf("unexpected type") 269 } 270 complete := false 271 for _, cond := range job.Status.Conditions { 272 if cond.Type == batchv1.JobFailed { 273 err = fmt.Errorf(cond.Message) 274 complete = true 275 break 276 } 277 if cond.Type == batchv1.JobComplete { 278 complete = true 279 break 280 } 281 } 282 if complete { 283 break 284 } 285 } 286 if err != nil { 287 return err 288 } 289 290 // Wait for pod logs to finish printing 291 for i := 0; i < int(k.requiredCompletions); i++ { 292 <-logsStreamingComplete 293 } 294 295 return nil 296 } 297 298 func (k *Driver) streamPodLogs(options metav1.ListOptions, out io.Writer, done chan bool) error { 299 watcher, err := k.pods.Watch(options) 300 if err != nil { 301 return err 302 } 303 304 go func() { 305 // Track pods whose logs have been streamed by pod name. We need to know when we've already 306 // processed logs for a given pod, since multiple lifecycle events are received per pod. 307 streamedLogs := map[string]bool{} 308 for event := range watcher.ResultChan() { 309 pod, ok := event.Object.(*v1.Pod) 310 if !ok { 311 continue 312 } 313 podName := pod.GetName() 314 if streamedLogs[podName] { 315 // The event was for a pod whose logs have already been streamed, so do nothing. 316 continue 317 } 318 319 for i := 0; i < numBackoffLoops; i++ { 320 time.Sleep(time.Duration(i*i/2) * time.Second) 321 req := k.pods.GetLogs(podName, &v1.PodLogOptions{ 322 Container: k8sContainerName, 323 Follow: true, 324 }) 325 reader, err := req.Stream() 326 if err != nil { 327 // There was an error connecting to the pod, so continue the loop and attempt streaming 328 // the logs again. 329 continue 330 } 331 332 // Block the loop until all logs from the pod have been processed. 333 bytesRead, err := io.Copy(out, reader) 334 reader.Close() 335 if err != nil { 336 continue 337 } 338 if bytesRead == 0 { 339 // There is a chance where we have connected to the pod, but it has yet to write something. 340 // In that case, we continue to to keep streaming until it does. 341 continue 342 } 343 // Set the pod to have successfully streamed data. 344 streamedLogs[podName] = true 345 break 346 } 347 348 done <- true 349 } 350 }() 351 352 return nil 353 } 354 355 func (k *Driver) deleteSecret(name string) error { 356 return k.secrets.Delete(name, &metav1.DeleteOptions{ 357 PropagationPolicy: &k.deletionPolicy, 358 }) 359 } 360 361 func (k *Driver) deleteJob(name string) error { 362 return k.jobs.Delete(name, &metav1.DeleteOptions{ 363 PropagationPolicy: &k.deletionPolicy, 364 }) 365 } 366 367 func generateNameTemplate(op *driver.Operation) string { 368 return fmt.Sprintf("%s-%s-", op.Installation, op.Action) 369 } 370 371 func generateLabels(op *driver.Operation) map[string]string { 372 return map[string]string{ 373 "cnab.io/installation": op.Installation, 374 "cnab.io/action": op.Action, 375 "cnab.io/revision": op.Revision, 376 } 377 } 378 379 func generateFileSecret(files map[string]string) (*v1.Secret, []v1.VolumeMount) { 380 size := len(files) 381 data := make(map[string]string, size) 382 mounts := make([]v1.VolumeMount, size) 383 384 i := 0 385 for path, contents := range files { 386 key := strings.Replace(filepath.ToSlash(path), "/", "_", -1) 387 data[key] = contents 388 mounts[i] = v1.VolumeMount{ 389 Name: k8sFileSecretVolume, 390 MountPath: path, 391 SubPath: key, 392 } 393 i++ 394 } 395 396 secret := &v1.Secret{ 397 StringData: data, 398 } 399 400 return secret, mounts 401 } 402 403 func newSingleFieldSelector(k, v string) string { 404 return labels.Set(map[string]string{ 405 k: v, 406 }).String() 407 } 408 409 func homeDir() string { 410 if h := os.Getenv("HOME"); h != "" { 411 return h 412 } 413 return os.Getenv("USERPROFILE") // windows 414 } 415 416 func imageWithDigest(img bundle.InvocationImage) string { 417 if img.Digest == "" { 418 return img.Image 419 } 420 return img.Image + "@" + img.Digest 421 }