github.com/mirantis/virtlet@v1.5.2-0.20191204181327-1659b8a48e9b/tests/e2e/framework/controller.go (about) 1 /* 2 Copyright 2017 Mirantis 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Some parts of this file are borrowed from Kubernetes source 18 // (test/utils/density_utils.go). The original copyright notice 19 // follows. 20 21 /* 22 Copyright 2016 The Kubernetes Authors. 23 Licensed under the Apache License, Version 2.0 (the "License"); 24 you may not use this file except in compliance with the License. 25 You may obtain a copy of the License at 26 http://www.apache.org/licenses/LICENSE-2.0 27 Unless required by applicable law or agreed to in writing, software 28 distributed under the License is distributed on an "AS IS" BASIS, 29 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 30 See the License for the specific language governing permissions and 31 limitations under the License. 32 */ 33 34 package framework 35 36 import ( 37 "errors" 38 "flag" 39 "fmt" 40 "strings" 41 "time" 42 43 "k8s.io/api/core/v1" 44 apierrs "k8s.io/apimachinery/pkg/api/errors" 45 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 46 "k8s.io/apimachinery/pkg/labels" 47 "k8s.io/apimachinery/pkg/types" 48 typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" 49 restclient "k8s.io/client-go/rest" 50 "k8s.io/client-go/tools/clientcmd" 51 52 virtlet_v1 "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1" 53 virtletclientv1 "github.com/Mirantis/virtlet/pkg/client/clientset/versioned/typed/virtlet.k8s/v1" 54 ) 55 56 const ( 57 retries = 5 58 defaultRunPodTimeout = 4 * time.Minute 59 ) 60 61 var ClusterURL = flag.String("cluster-url", "http://127.0.0.1:8080", "apiserver URL") 62 63 // HostPathMount specifies a host path to mount into a pod sandbox. 64 type HostPathMount struct { 65 // The path on the host. 66 HostPath string 67 // The path inside the container. 68 ContainerPath string 69 } 70 71 // RunPodOptions specifies the options for RunPod 72 type RunPodOptions struct { 73 // The command to run (optional). 74 Command []string 75 // Timeout. Defaults to 4 minutes. 76 Timeout time.Duration 77 // The list of ports to expose. 78 ExposePorts []int32 79 // The list of host paths to mount. 80 HostPathMounts []HostPathMount 81 // Node name to run this pod on. 82 NodeName string 83 } 84 85 // Controller is the entry point for various operations on k8s+virtlet entities 86 type Controller struct { 87 fixedNs bool 88 89 client typedv1.CoreV1Interface 90 virtletClient virtletclientv1.VirtletV1Interface 91 namespace *v1.Namespace 92 restConfig *restclient.Config 93 } 94 95 // NewController creates instance of controller for specified k8s namespace. 96 // If namespace is empty string then namespace with random name is going to be created 97 func NewController(namespace string) (*Controller, error) { 98 config, err := clientcmd.BuildConfigFromFlags(*ClusterURL, "") 99 if err != nil { 100 return nil, err 101 } 102 103 clientset, err := typedv1.NewForConfig(config) 104 if err != nil { 105 return nil, err 106 } 107 108 virtletClient, err := virtletclientv1.NewForConfig(config) 109 if err != nil { 110 return nil, err 111 } 112 113 var ns *v1.Namespace 114 if namespace != "" { 115 ns, err = clientset.Namespaces().Get(namespace, metav1.GetOptions{}) 116 } else { 117 ns, err = createNamespace(clientset) 118 } 119 if err != nil { 120 return nil, err 121 } 122 123 return &Controller{ 124 client: clientset, 125 virtletClient: virtletClient, 126 namespace: ns, 127 restConfig: config, 128 fixedNs: namespace != "", 129 }, nil 130 } 131 132 func createNamespace(client *typedv1.CoreV1Client) (*v1.Namespace, error) { 133 namespaceObj := &v1.Namespace{ 134 ObjectMeta: metav1.ObjectMeta{ 135 GenerateName: "virtlet-tests-", 136 }, 137 Status: v1.NamespaceStatus{}, 138 } 139 return client.Namespaces().Create(namespaceObj) 140 } 141 142 // Finalize deletes random namespace that might has been created by NewController 143 func (c *Controller) Finalize() error { 144 if c.fixedNs { 145 return nil 146 } 147 return c.client.Namespaces().Delete(c.namespace.Name, nil) 148 } 149 150 func (c *Controller) CreateVirtletImageMapping(mapping virtlet_v1.VirtletImageMapping) (*virtlet_v1.VirtletImageMapping, error) { 151 return c.virtletClient.VirtletImageMappings("kube-system").Create(&mapping) 152 } 153 154 func (c *Controller) DeleteVirtletImageMapping(name string) error { 155 return c.virtletClient.VirtletImageMappings("kube-system").Delete(name, &metav1.DeleteOptions{}) 156 } 157 158 func (c *Controller) CreateVirtletConfigMapping(configMapping virtlet_v1.VirtletConfigMapping) (*virtlet_v1.VirtletConfigMapping, error) { 159 return c.virtletClient.VirtletConfigMappings("kube-system").Create(&configMapping) 160 } 161 162 func (c *Controller) DeleteVirtletConfigMapping(name string) error { 163 return c.virtletClient.VirtletConfigMappings("kube-system").Delete(name, &metav1.DeleteOptions{}) 164 } 165 166 // PersistentVolumesClient returns interface for PVs 167 func (c *Controller) PersistentVolumesClient() typedv1.PersistentVolumeInterface { 168 return c.client.PersistentVolumes() 169 } 170 171 // PersistentVolumeClaimsClient returns interface for PVCs 172 func (c *Controller) PersistentVolumeClaimsClient() typedv1.PersistentVolumeClaimInterface { 173 return c.client.PersistentVolumeClaims(c.namespace.Name) 174 } 175 176 // ConfigMaps returns interface for ConfigMap objects 177 func (c *Controller) ConfigMaps() typedv1.ConfigMapInterface { 178 return c.client.ConfigMaps(c.namespace.Name) 179 } 180 181 // Secrets returns interface for Secret objects 182 func (c *Controller) Secrets() typedv1.SecretInterface { 183 return c.client.Secrets(c.namespace.Name) 184 } 185 186 // VM returns interface for operations on virtlet VM pods 187 func (c *Controller) VM(name string) *VMInterface { 188 return newVMInterface(c, name) 189 } 190 191 // Pod returns interface for operations on k8s pod in a given namespace. 192 // If namespace is an empty string then default controller namespace is used 193 func (c *Controller) Pod(name, namespace string) (*PodInterface, error) { 194 if namespace == "" { 195 namespace = c.namespace.Name 196 } 197 pod, err := c.client.Pods(namespace).Get(name, metav1.GetOptions{}) 198 if err != nil { 199 return nil, err 200 } 201 return newPodInterface(c, pod), nil 202 } 203 204 // FindPod looks for a pod in a given namespace having specified labels and matching optional predicate function 205 func (c *Controller) FindPod(namespace string, labelMap map[string]string, 206 predicate func(podInterface *PodInterface) bool) (*PodInterface, error) { 207 208 if namespace == "" { 209 namespace = c.namespace.Name 210 } 211 lst, err := c.client.Pods(namespace).List(metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labelMap).String()}) 212 if err != nil { 213 return nil, err 214 } 215 for _, pod := range lst.Items { 216 pi := newPodInterface(c, &pod) 217 if predicate == nil || predicate(pi) { 218 return pi, nil 219 } 220 } 221 return nil, nil 222 } 223 224 // VirtletPod returns one of the active virtlet pods 225 func (c *Controller) VirtletPod() (*PodInterface, error) { 226 pod, err := c.FindPod("kube-system", map[string]string{"runtime": "virtlet"}, nil) 227 if err != nil { 228 return nil, err 229 } else if pod == nil { 230 return nil, fmt.Errorf("cannot find virtlet pod") 231 } 232 return pod, nil 233 } 234 235 // VirtletNodeName returns the name of one of the nodes that run Virtlet 236 func (c *Controller) VirtletNodeName() (string, error) { 237 virtletPod, err := c.VirtletPod() 238 if err != nil { 239 return "", err 240 } 241 return virtletPod.Pod.Spec.NodeName, nil 242 } 243 244 // AvailableNodeName returns the name of a node that doesn't run 245 // Virtlet after the standard test setup is done but which can be 246 // labelled to run Virtlet. 247 func (c *Controller) AvailableNodeName() (string, error) { 248 virtletNodeName, err := c.VirtletNodeName() 249 if err != nil { 250 return "", err 251 } 252 253 nodeList, err := c.client.Nodes().List(metav1.ListOptions{}) 254 if err != nil { 255 return "", err 256 } 257 258 for _, node := range nodeList.Items { 259 if node.Name == virtletNodeName { 260 continue 261 } 262 if len(node.Spec.Taints) == 0 { 263 return node.Name, nil 264 } 265 } 266 267 return "", errors.New("couldn't find an available node") 268 } 269 270 // AddLabelsToNode adds the specified labels to the node. 271 // Based on test/utils/density_utils.go in the Kubernetes source. 272 func (c *Controller) AddLabelsToNode(nodeName string, labels map[string]string) error { 273 tokens := make([]string, 0, len(labels)) 274 for k, v := range labels { 275 tokens = append(tokens, "\""+k+"\":\""+v+"\"") 276 } 277 labelString := "{" + strings.Join(tokens, ",") + "}" 278 patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString) 279 var err error 280 for attempt := 0; attempt < retries; attempt++ { 281 _, err = c.client.Nodes().Patch(nodeName, types.MergePatchType, []byte(patch)) 282 if err != nil { 283 if !apierrs.IsConflict(err) { 284 return err 285 } 286 } else { 287 break 288 } 289 time.Sleep(100 * time.Millisecond) 290 } 291 return err 292 } 293 294 // RemoveLabelOffNode is for cleaning up labels temporarily added to node, 295 // won't fail if target label doesn't exist or has been removed. 296 // Based on test/utils/density_utils.go in the Kubernetes source. 297 func (c *Controller) RemoveLabelOffNode(nodeName string, labelKeys []string) error { 298 var node *v1.Node 299 var err error 300 for attempt := 0; attempt < retries; attempt++ { 301 node, err = c.client.Nodes().Get(nodeName, metav1.GetOptions{}) 302 if err != nil { 303 return err 304 } 305 if node.Labels == nil { 306 return nil 307 } 308 for _, labelKey := range labelKeys { 309 if node.Labels == nil || len(node.Labels[labelKey]) == 0 { 310 break 311 } 312 delete(node.Labels, labelKey) 313 } 314 _, err = c.client.Nodes().Update(node) 315 if err != nil { 316 if !apierrs.IsConflict(err) { 317 return err 318 } 319 } else { 320 break 321 } 322 time.Sleep(100 * time.Millisecond) 323 } 324 return err 325 } 326 327 func (c *Controller) WaitForVirtletPodOnTheNode(name string) (*PodInterface, error) { 328 var virtletPod *PodInterface 329 if err := waitFor(func() error { 330 var err error 331 virtletPod, err = c.FindPod("kube-system", map[string]string{"runtime": "virtlet"}, func(podInterface *PodInterface) bool { 332 return podInterface.Pod.Spec.NodeName == name 333 }) 334 switch { 335 case err != nil: 336 return err 337 case virtletPod != nil: 338 return nil 339 default: 340 return fmt.Errorf("no Virtlet pod on the node %q", name) 341 } 342 }, 5*time.Minute, 5*time.Second, false); err != nil { 343 return nil, err 344 } 345 346 if err := virtletPod.Wait(5 * time.Minute); err != nil { 347 return nil, err 348 } 349 350 return virtletPod, nil 351 } 352 353 func (c *Controller) WaitForVirtletPodToDisappearFromTheNode(name string) error { 354 return waitFor(func() error { 355 virtletPod, err := c.FindPod("kube-system", map[string]string{"runtime": "virtlet"}, func(podInterface *PodInterface) bool { 356 return podInterface.Pod.Spec.NodeName == name 357 }) 358 switch { 359 case err != nil: 360 return err 361 case virtletPod == nil: 362 return nil 363 default: 364 return fmt.Errorf("Virtlet pod still present on the node %q", name) 365 } 366 }, 5*time.Minute, 5*time.Second, false) 367 } 368 369 // DinDNodeExecutor returns executor in DinD container for one of k8s nodes 370 func (c *Controller) DinDNodeExecutor(name string) (Executor, error) { 371 dockerInterface, err := newDockerContainerInterface(name) 372 if err != nil { 373 return nil, err 374 } 375 return dockerInterface.Executor(false, ""), nil 376 } 377 378 // DockerContainer returns interface for operations on a docker container with a given name 379 func (c *Controller) DockerContainer(name string) (*DockerContainerInterface, error) { 380 return newDockerContainerInterface(name) 381 } 382 383 // Namespace returns default controller namespace name 384 func (c *Controller) Namespace() string { 385 return c.namespace.Name 386 } 387 388 // RunPod is a helper method to create a pod in a simple configuration (similar to `kubectl run`) 389 func (c *Controller) RunPod(name, image string, opts RunPodOptions) (*PodInterface, error) { 390 if opts.Timeout == 0 { 391 opts.Timeout = defaultRunPodTimeout 392 } 393 pod := generatePodSpec(name, image, opts) 394 podInterface := newPodInterface(c, pod) 395 if err := podInterface.Create(); err != nil { 396 return nil, err 397 } 398 if err := podInterface.Wait(opts.Timeout); err != nil { 399 return nil, err 400 } 401 if len(opts.ExposePorts) > 0 { 402 svc := &v1.Service{ 403 ObjectMeta: metav1.ObjectMeta{ 404 Name: name, 405 }, 406 Spec: v1.ServiceSpec{ 407 Selector: map[string]string{"id": name}, 408 }, 409 } 410 for _, port := range opts.ExposePorts { 411 svc.Spec.Ports = append(svc.Spec.Ports, v1.ServicePort{ 412 Name: fmt.Sprintf("port%d", port), 413 Port: port, 414 }) 415 } 416 _, err := c.client.Services(c.namespace.Name).Create(svc) 417 if err != nil { 418 return nil, err 419 } 420 podInterface.hasService = true 421 } 422 return podInterface, nil 423 } 424 425 func generatePodSpec(name, image string, opts RunPodOptions) *v1.Pod { 426 pod := &v1.Pod{ 427 ObjectMeta: metav1.ObjectMeta{ 428 Name: name, 429 Labels: map[string]string{"id": name}, 430 }, 431 Spec: v1.PodSpec{ 432 Containers: []v1.Container{ 433 { 434 Name: name, 435 Image: image, 436 ImagePullPolicy: v1.PullIfNotPresent, 437 Command: opts.Command, 438 }, 439 }, 440 }, 441 } 442 443 if opts.NodeName != "" { 444 pod.Spec.NodeSelector = map[string]string{ 445 "kubernetes.io/hostname": opts.NodeName, 446 } 447 } 448 449 for n, hpm := range opts.HostPathMounts { 450 name := fmt.Sprintf("vol%d", n) 451 pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ 452 Name: name, 453 VolumeSource: v1.VolumeSource{ 454 HostPath: &v1.HostPathVolumeSource{ 455 Path: hpm.HostPath, 456 }, 457 }, 458 }) 459 pod.Spec.Containers[0].VolumeMounts = append( 460 pod.Spec.Containers[0].VolumeMounts, 461 v1.VolumeMount{ 462 Name: name, 463 MountPath: hpm.ContainerPath, 464 }) 465 } 466 467 return pod 468 }