github.com/AliyunContainerService/heapster@v1.6.0-beta.1.0.20190403101729-c280e398e293/integration/framework.go (about) 1 // Copyright 2014 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "flag" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "strings" 25 "time" 26 27 "github.com/golang/glog" 28 "k8s.io/api/core/v1" 29 rbacv1 "k8s.io/api/rbac/v1" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/labels" 32 "k8s.io/apimachinery/pkg/runtime" 33 kclient "k8s.io/client-go/kubernetes" 34 kclientcmd "k8s.io/client-go/tools/clientcmd" 35 kclientcmdapi "k8s.io/client-go/tools/clientcmd/api" 36 "k8s.io/kubernetes/pkg/api/legacyscheme" 37 // ensure the core apis are installed 38 _ "k8s.io/kubernetes/pkg/apis/core/install" 39 // ensure the rbac apis are installed 40 _ "k8s.io/kubernetes/pkg/apis/rbac/install" 41 ) 42 43 type kubeFramework interface { 44 // Kube client 45 Client() *kclient.Clientset 46 47 // Parses and Returns a replication Controller object contained in 'filePath' 48 ParseRC(filePath string) (*v1.ReplicationController, error) 49 50 // Parses and Returns a service object contained in 'filePath' 51 ParseService(filePath string) (*v1.Service, error) 52 53 // Parses and Returns a RBAC object contained in 'filePath' 54 ParseRBAC(filePath string) (*rbacv1.ClusterRoleBinding, error) 55 56 // Parses and Returns a ServiceAccount object contained in 'filePath' 57 ParseServiceAccount(filePath string) (*v1.ServiceAccount, error) 58 59 // Creates a kube service. 60 CreateService(ns string, service *v1.Service) (*v1.Service, error) 61 62 // Creates a namespace. 63 CreateNs(ns *v1.Namespace) (*v1.Namespace, error) 64 65 // Creates a RBAC. 66 CreateRBAC(crb *rbacv1.ClusterRoleBinding) error 67 68 // Creates a ServiceAccount. 69 CreateServiceAccount(sa *v1.ServiceAccount) error 70 71 // Creates a kube replication controller. 72 CreateRC(ns string, rc *v1.ReplicationController) (*v1.ReplicationController, error) 73 74 // Deletes a namespace 75 DeleteNs(ns string) error 76 77 // Destroy cluster 78 DestroyCluster() 79 80 // Returns a url that provides access to a kubernetes service via the proxy on the apiserver. 81 // This url requires master auth. 82 GetProxyUrlForService(service *v1.Service) string 83 84 // Returns the node hostnames. 85 GetNodeNames() ([]string, error) 86 87 // Returns the nodes. 88 GetNodes() (*v1.NodeList, error) 89 90 // Returns pod names in the cluster. 91 // TODO: Remove, or mix with namespace 92 GetRunningPodNames() ([]string, error) 93 94 // Returns pods in the cluster running outside kubernetes-master. 95 GetPodsRunningOnNodes() ([]v1.Pod, error) 96 97 // Returns pods in the cluster. 98 GetAllRunningPods() ([]v1.Pod, error) 99 100 WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error 101 WaitUntilServiceActive(svc *v1.Service, timeout time.Duration) error 102 } 103 104 type realKubeFramework struct { 105 // Kube client. 106 kubeClient *kclient.Clientset 107 108 // The version of the kube cluster 109 version string 110 111 // Master IP for this framework 112 masterIP string 113 114 // The base directory of current kubernetes release. 115 baseDir string 116 } 117 118 const imageUrlTemplate = "https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz" 119 120 var ( 121 kubeConfig = flag.String("kube_config", os.Getenv("HOME")+"/.kube/config", "Path to cluster info file.") 122 workDir = flag.String("work_dir", "/tmp/heapster_test", "Filesystem path where test files will be stored. Files will persist across runs to speed up tests.") 123 ) 124 125 func exists(path string) bool { 126 if _, err := os.Stat(path); err != nil { 127 glog.V(2).Infof("%q does not exist", path) 128 return false 129 } 130 return true 131 } 132 133 const pathToGCEConfig = "cluster/gce/config-default.sh" 134 135 func disableClusterMonitoring(kubeBaseDir string) error { 136 kubeConfigFilePath := filepath.Join(kubeBaseDir, pathToGCEConfig) 137 input, err := ioutil.ReadFile(kubeConfigFilePath) 138 if err != nil { 139 return err 140 } 141 142 lines := strings.Split(string(input), "\n") 143 144 for i, line := range lines { 145 if strings.Contains(line, "ENABLE_CLUSTER_MONITORING") { 146 lines[i] = "ENABLE_CLUSTER_MONITORING=false" 147 } else if strings.Contains(line, "NUM_MINIONS=") { 148 lines[i] = "NUM_MINIONS=2" 149 } else if strings.Contains(line, "MASTER_SIZE=") { 150 // TODO(piosz): remove this once everything fits onto master 151 lines[i] = "MASTER_SIZE=n1-standard-2" 152 } 153 } 154 output := strings.Join(lines, "\n") 155 return ioutil.WriteFile(kubeConfigFilePath, []byte(output), 0644) 156 } 157 158 func runKubeClusterCommand(kubeBaseDir, command string) ([]byte, error) { 159 cmd := exec.Command(filepath.Join(kubeBaseDir, "cluster", command)) 160 env := os.Environ() 161 env = append(env, "KUBE_RUNTIME_CONFIG=--runtime-config=metrics.k8s.io/v1alpha1=true") 162 cmd.Env = env 163 glog.V(2).Infof("about to run %v", cmd) 164 return cmd.CombinedOutput() 165 } 166 167 func setupNewCluster(kubeBaseDir string) error { 168 cmd := "kube-up.sh" 169 destroyCluster(kubeBaseDir) 170 out, err := runKubeClusterCommand(kubeBaseDir, cmd) 171 if err != nil { 172 glog.Errorf("failed to bring up cluster - %q\n%s", err, out) 173 return fmt.Errorf("failed to bring up cluster - %q", err) 174 } 175 glog.V(2).Info(string(out)) 176 glog.V(2).Infof("Giving the cluster 30 sec to stabilize") 177 time.Sleep(30 * time.Second) 178 return nil 179 } 180 181 func destroyCluster(kubeBaseDir string) error { 182 if kubeBaseDir == "" { 183 glog.Infof("Skipping cluster tear down since kubernetes repo base path is not set.") 184 return nil 185 } 186 glog.V(1).Info("Bringing down any existing kube cluster") 187 out, err := runKubeClusterCommand(kubeBaseDir, "kube-down.sh") 188 if err != nil { 189 glog.Errorf("failed to tear down cluster - %q\n%s", err, out) 190 return fmt.Errorf("failed to tear down kube cluster - %q", err) 191 } 192 193 return nil 194 } 195 196 func downloadRelease(workDir, version string) error { 197 // Temporary download path. 198 downloadPath := filepath.Join(workDir, "kube") 199 // Format url. 200 downloadUrl := fmt.Sprintf(imageUrlTemplate, version) 201 glog.V(1).Infof("About to download kube release using url: %q", downloadUrl) 202 203 // Download kube code and store it in a temp dir. 204 if err := exec.Command("wget", downloadUrl, "-O", downloadPath).Run(); err != nil { 205 return fmt.Errorf("failed to wget kubernetes release @ %q - %v", downloadUrl, err) 206 } 207 208 // Un-tar kube release. 209 if err := exec.Command("tar", "-xf", downloadPath, "-C", workDir).Run(); err != nil { 210 return fmt.Errorf("failed to un-tar kubernetes release at %q - %v", downloadPath, err) 211 } 212 return nil 213 } 214 215 func getKubeClient() (string, *kclient.Clientset, error) { 216 c, err := kclientcmd.LoadFromFile(*kubeConfig) 217 if err != nil { 218 return "", nil, fmt.Errorf("error loading kubeConfig: %v", err.Error()) 219 } 220 if c.CurrentContext == "" || len(c.Clusters) == 0 { 221 return "", nil, fmt.Errorf("invalid kubeConfig: %+v", *c) 222 } 223 config, err := kclientcmd.NewDefaultClientConfig( 224 *c, 225 &kclientcmd.ConfigOverrides{ 226 ClusterInfo: kclientcmdapi.Cluster{}, 227 }).ClientConfig() 228 if err != nil { 229 return "", nil, fmt.Errorf("error parsing kubeConfig: %v", err.Error()) 230 } 231 kubeClient, err := kclient.NewForConfig(config) 232 if err != nil { 233 return "", nil, fmt.Errorf("error creating client - %q", err) 234 } 235 236 return c.Clusters[c.CurrentContext].Server, kubeClient, nil 237 } 238 239 func validateCluster(baseDir string) bool { 240 glog.V(1).Info("validating existing cluster") 241 out, err := runKubeClusterCommand(baseDir, "validate-cluster.sh") 242 if err != nil { 243 glog.V(1).Infof("cluster validation failed - %q\n %s", err, out) 244 return false 245 } 246 return true 247 } 248 249 func requireNewCluster(baseDir, version string) bool { 250 // Setup kube client 251 _, kubeClient, err := getKubeClient() 252 if err != nil { 253 glog.V(1).Infof("kube client creation failed - %q", err) 254 return true 255 } 256 glog.V(1).Infof("checking if existing cluster can be used") 257 versionInfo, err := kubeClient.ServerVersion() 258 if err != nil { 259 glog.V(1).Infof("failed to get kube version info - %q", err) 260 return true 261 } 262 return !strings.Contains(versionInfo.GitVersion, version) 263 } 264 265 func requireDownload(baseDir string) bool { 266 // Check that cluster scripts are present. 267 return !exists(filepath.Join(baseDir, "cluster", "kube-up.sh")) || 268 !exists(filepath.Join(baseDir, "cluster", "kube-down.sh")) || 269 !exists(filepath.Join(baseDir, "cluster", "validate-cluster.sh")) 270 } 271 272 func downloadAndSetupCluster(version string) (baseDir string, err error) { 273 // Create a temp dir to store the kube release files. 274 tempDir := filepath.Join(*workDir, version) 275 if !exists(tempDir) { 276 if err := os.MkdirAll(tempDir, 0700); err != nil { 277 return "", fmt.Errorf("failed to create a temp dir at %s - %q", tempDir, err) 278 } 279 glog.V(1).Infof("Successfully setup work dir at %s", tempDir) 280 } 281 282 kubeBaseDir := filepath.Join(tempDir, "kubernetes") 283 284 if requireDownload(kubeBaseDir) { 285 if exists(kubeBaseDir) { 286 os.RemoveAll(kubeBaseDir) 287 } 288 if err := downloadRelease(tempDir, version); err != nil { 289 return "", err 290 } 291 glog.V(1).Infof("Successfully downloaded kubernetes release at %s", tempDir) 292 } 293 294 // Disable monitoring 295 if err := disableClusterMonitoring(kubeBaseDir); err != nil { 296 return "", fmt.Errorf("failed to disable cluster monitoring in kube cluster config - %q", err) 297 } 298 glog.V(1).Info("Disabled cluster monitoring") 299 if !requireNewCluster(kubeBaseDir, version) { 300 glog.V(1).Infof("skipping cluster setup since a cluster with required version already exists") 301 return kubeBaseDir, nil 302 } 303 304 // Setup kube cluster 305 glog.V(1).Infof("Setting up new kubernetes cluster version: %s", version) 306 if err := os.Setenv("KUBERNETES_SKIP_CONFIRM", "y"); err != nil { 307 return "", err 308 } 309 if err := setupNewCluster(kubeBaseDir); err != nil { 310 // Cluster setup failed for some reason. 311 // Attempting to validate the cluster to see if it failed in the validate phase. 312 sleepDuration := 10 * time.Second 313 clusterReady := false 314 for i := 0; i < int(time.Minute/sleepDuration); i++ { 315 if !validateCluster(kubeBaseDir) { 316 glog.Infof("Retry validation after %v seconds.", sleepDuration/time.Second) 317 time.Sleep(sleepDuration) 318 } else { 319 clusterReady = true 320 break 321 } 322 } 323 if !clusterReady { 324 return "", fmt.Errorf("failed to setup cluster - %q", err) 325 } 326 } 327 glog.V(1).Infof("Successfully setup new kubernetes cluster version %s", version) 328 329 return kubeBaseDir, nil 330 } 331 332 func newKubeFramework(version string) (kubeFramework, error) { 333 // Install gcloud components. 334 // TODO(piosz): move this to the image creation 335 cmd := exec.Command("gcloud", "components", "install", "alpha", "beta", "kubectl", "--quiet") 336 glog.V(2).Infof("about to install gcloud components") 337 if o, err := cmd.CombinedOutput(); err != nil { 338 return nil, fmt.Errorf("Error while installing gcloud components: %v\n%s", err, o) 339 } 340 341 var err error 342 kubeBaseDir := "" 343 if version != "" { 344 if len(strings.Split(version, ".")) != 3 { 345 glog.Warningf("Using not stable version - %q", version) 346 } 347 kubeBaseDir, err = downloadAndSetupCluster(version) 348 if err != nil { 349 return nil, err 350 } 351 } 352 353 // Setup kube client 354 masterIP, kubeClient, err := getKubeClient() 355 if err != nil { 356 return nil, err 357 } 358 return &realKubeFramework{ 359 kubeClient: kubeClient, 360 baseDir: kubeBaseDir, 361 version: version, 362 masterIP: masterIP, 363 }, nil 364 } 365 366 func (self *realKubeFramework) Client() *kclient.Clientset { 367 return self.kubeClient 368 } 369 370 func (self *realKubeFramework) loadObject(filePath string) (runtime.Object, error) { 371 data, err := ioutil.ReadFile(filePath) 372 if err != nil { 373 return nil, fmt.Errorf("failed to read object: %v", err) 374 } 375 obj, _, err := legacyscheme.Codecs.UniversalDecoder(v1.SchemeGroupVersion).Decode(data, nil, nil) 376 return obj, err 377 } 378 379 func (self *realKubeFramework) loadRBACObject(filePath string) (runtime.Object, error) { 380 data, err := ioutil.ReadFile(filePath) 381 if err != nil { 382 return nil, fmt.Errorf("failed to read object: %v", err) 383 } 384 obj, _, err := legacyscheme.Codecs.UniversalDecoder(rbacv1.SchemeGroupVersion).Decode(data, nil, nil) 385 return obj, err 386 } 387 388 func (self *realKubeFramework) ParseRC(filePath string) (*v1.ReplicationController, error) { 389 obj, err := self.loadObject(filePath) 390 if err != nil { 391 return nil, err 392 } 393 394 rc, ok := obj.(*v1.ReplicationController) 395 if !ok { 396 return nil, fmt.Errorf("Failed to cast replicationController: %#v", obj) 397 } 398 return rc, nil 399 } 400 401 // Parses and Returns a RBAC object contained in 'filePath' 402 func (self *realKubeFramework) ParseRBAC(filePath string) (*rbacv1.ClusterRoleBinding, error) { 403 obj, err := self.loadRBACObject(filePath) 404 if err != nil { 405 return nil, err 406 } 407 408 rbac, ok := obj.(*rbacv1.ClusterRoleBinding) 409 if !ok { 410 return nil, fmt.Errorf("Failed to cast clusterrolebinding: %v", obj) 411 } 412 return rbac, nil 413 } 414 415 // CreateRBAC creates the RBAC object 416 func (self *realKubeFramework) CreateRBAC(rbac *rbacv1.ClusterRoleBinding) error { 417 _, err := self.kubeClient.RbacV1().ClusterRoleBindings().Create(rbac) 418 return err 419 } 420 421 // Parses and Returns a ServiceAccount object contained in 'filePath' 422 func (self *realKubeFramework) ParseServiceAccount(filePath string) (*v1.ServiceAccount, error) { 423 obj, err := self.loadObject(filePath) 424 if err != nil { 425 return nil, err 426 } 427 428 sa, ok := obj.(*v1.ServiceAccount) 429 if !ok { 430 return nil, fmt.Errorf("Failed to cast serviceaccount: %v", obj) 431 } 432 return sa, nil 433 } 434 435 // CreateServiceAccount creates the ServiceAccount object 436 func (self *realKubeFramework) CreateServiceAccount(sa *v1.ServiceAccount) error { 437 _, err := self.kubeClient.CoreV1().ServiceAccounts(sa.Namespace).Create(sa) 438 return err 439 } 440 441 func (self *realKubeFramework) ParseService(filePath string) (*v1.Service, error) { 442 obj, err := self.loadObject(filePath) 443 if err != nil { 444 return nil, err 445 } 446 service, ok := obj.(*v1.Service) 447 if !ok { 448 return nil, fmt.Errorf("Failed to cast service: %v", obj) 449 } 450 return service, nil 451 } 452 453 func (self *realKubeFramework) CreateService(ns string, service *v1.Service) (*v1.Service, error) { 454 service.Namespace = ns 455 newSvc, err := self.kubeClient.CoreV1().Services(ns).Create(service) 456 return newSvc, err 457 } 458 459 func (self *realKubeFramework) DeleteNs(ns string) error { 460 461 _, err := self.kubeClient.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) 462 if err != nil { 463 glog.V(0).Infof("Cannot get namespace %q. Skipping deletion: %s", ns, err) 464 return nil 465 } 466 glog.V(0).Infof("Deleting namespace %s", ns) 467 self.kubeClient.CoreV1().Namespaces().Delete(ns, nil) 468 469 for i := 0; i < 5; i++ { 470 glog.V(0).Infof("Checking for namespace %s", ns) 471 _, err := self.kubeClient.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) 472 if err != nil { 473 glog.V(0).Infof("%s doesn't exist", ns) 474 return nil 475 } 476 time.Sleep(10 * time.Second) 477 } 478 return fmt.Errorf("Namespace %s still exists", ns) 479 } 480 481 func (self *realKubeFramework) CreateNs(ns *v1.Namespace) (*v1.Namespace, error) { 482 return self.kubeClient.CoreV1().Namespaces().Create(ns) 483 } 484 485 func (self *realKubeFramework) CreateRC(ns string, rc *v1.ReplicationController) (*v1.ReplicationController, error) { 486 rc.Namespace = ns 487 return self.kubeClient.CoreV1().ReplicationControllers(ns).Create(rc) 488 } 489 490 func (self *realKubeFramework) DestroyCluster() { 491 destroyCluster(self.baseDir) 492 } 493 494 func (self *realKubeFramework) GetProxyUrlForService(service *v1.Service) string { 495 return fmt.Sprintf("%s/api/v1/proxy/namespaces/default/services/%s/", self.masterIP, service.Name) 496 } 497 498 func (self *realKubeFramework) GetNodeNames() ([]string, error) { 499 var nodes []string 500 nodeList, err := self.GetNodes() 501 if err != nil { 502 return nodes, err 503 } 504 for _, node := range nodeList.Items { 505 nodes = append(nodes, node.Name) 506 } 507 return nodes, nil 508 } 509 510 func (self *realKubeFramework) GetNodes() (*v1.NodeList, error) { 511 return self.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{}) 512 } 513 514 func (self *realKubeFramework) GetAllRunningPods() ([]v1.Pod, error) { 515 return getRunningPods(true, self.kubeClient) 516 } 517 518 func (self *realKubeFramework) GetPodsRunningOnNodes() ([]v1.Pod, error) { 519 return getRunningPods(false, self.kubeClient) 520 } 521 522 func getRunningPods(includeMaster bool, kubeClient *kclient.Clientset) ([]v1.Pod, error) { 523 glog.V(0).Infof("Getting running pods") 524 podList, err := kubeClient.CoreV1().Pods(v1.NamespaceAll).List(metav1.ListOptions{}) 525 if err != nil { 526 return nil, err 527 } 528 pods := []v1.Pod{} 529 for _, pod := range podList.Items { 530 if pod.Status.Phase == v1.PodRunning { 531 if includeMaster || !isMasterNode(pod.Spec.NodeName) { 532 pods = append(pods, pod) 533 } 534 } 535 } 536 return pods, nil 537 } 538 539 func isMasterNode(nodeName string) bool { 540 return strings.Contains(nodeName, "kubernetes-master") 541 } 542 543 func (self *realKubeFramework) GetRunningPodNames() ([]string, error) { 544 var pods []string 545 podList, err := self.GetAllRunningPods() 546 if err != nil { 547 return pods, err 548 } 549 for _, pod := range podList { 550 pods = append(pods, string(pod.Name)) 551 } 552 return pods, nil 553 } 554 555 func (rkf *realKubeFramework) WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error { 556 glog.V(2).Infof("Waiting for pod %v in %s...", podLabels, ns) 557 podsInterface := rkf.Client().CoreV1().Pods(ns) 558 for i := 0; i < int(timeout/time.Second); i++ { 559 podList, err := podsInterface.List(metav1.ListOptions{ 560 LabelSelector: labels.Set(podLabels).AsSelector().String(), 561 }) 562 if err != nil { 563 glog.V(1).Info(err) 564 return err 565 } 566 if len(podList.Items) > 0 { 567 podSpec := podList.Items[0] 568 if podSpec.Status.Phase == v1.PodRunning { 569 return nil 570 } 571 } 572 time.Sleep(time.Second) 573 } 574 return fmt.Errorf("pod not in running state after %d", timeout/time.Second) 575 } 576 577 func (rkf *realKubeFramework) WaitUntilServiceActive(svc *v1.Service, timeout time.Duration) error { 578 glog.V(2).Infof("Waiting for endpoints in service %s/%s", svc.Namespace, svc.Name) 579 for i := 0; i < int(timeout/time.Second); i++ { 580 e, err := rkf.Client().CoreV1().Endpoints(svc.Namespace).Get(svc.Name, metav1.GetOptions{}) 581 if err != nil { 582 return err 583 } 584 if len(e.Subsets) > 0 { 585 return nil 586 } 587 time.Sleep(time.Second) 588 } 589 590 return fmt.Errorf("Service %q not active after %d seconds - no endpoints found", svc.Name, timeout/time.Second) 591 592 }