github.com/intelsdi-x/heapster@v1.2.1-0.20221107230227-7f05c7c5dbad/integration/framework.go (about) 1 // Copyright 2014 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "flag" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "strings" 25 "time" 26 27 "github.com/golang/glog" 28 "k8s.io/kubernetes/pkg/api" 29 kclient "k8s.io/kubernetes/pkg/client/unversioned" 30 kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" 31 kclientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" 32 "k8s.io/kubernetes/pkg/fields" 33 "k8s.io/kubernetes/pkg/labels" 34 "k8s.io/kubernetes/pkg/runtime" 35 ) 36 37 type kubeFramework interface { 38 // Kube client 39 Client() *kclient.Client 40 41 // Parses and Returns a replication Controller object contained in 'filePath' 42 ParseRC(filePath string) (*api.ReplicationController, error) 43 44 // Parses and Returns a service object contained in 'filePath' 45 ParseService(filePath string) (*api.Service, error) 46 47 // Creates a kube service. 48 CreateService(ns string, service *api.Service) (*api.Service, error) 49 50 // Creates a namespace. 51 CreateNs(ns *api.Namespace) (*api.Namespace, error) 52 53 // Creates a kube replication controller. 54 CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) 55 56 // Deletes a namespace 57 DeleteNs(ns string) error 58 59 // Destroy cluster 60 DestroyCluster() 61 62 // Returns a url that provides access to a kubernetes service via the proxy on the apiserver. 63 // This url requires master auth. 64 GetProxyUrlForService(service *api.Service) string 65 66 // Returns the node hostnames. 67 GetNodeNames() ([]string, error) 68 69 // Returns the nodes. 70 GetNodes() (*api.NodeList, error) 71 72 // Returns pod names in the cluster. 73 // TODO: Remove, or mix with namespace 74 GetRunningPodNames() ([]string, error) 75 76 // Returns pods in the cluster running outside kubernetes-master. 77 GetPodsRunningOnNodes() ([]api.Pod, error) 78 79 // Returns pods in the cluster. 80 GetAllRunningPods() ([]api.Pod, error) 81 82 WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error 83 WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error 84 } 85 86 type realKubeFramework struct { 87 // Kube client. 88 kubeClient *kclient.Client 89 90 // The version of the kube cluster 91 version string 92 93 // Master IP for this framework 94 masterIP string 95 96 // The base directory of current kubernetes release. 97 baseDir string 98 } 99 100 const imageUrlTemplate = "https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz" 101 102 var ( 103 kubeConfig = flag.String("kube_config", os.Getenv("HOME")+"/.kube/config", "Path to cluster info file.") 104 workDir = flag.String("work_dir", "/tmp/heapster_test", "Filesystem path where test files will be stored. Files will persist across runs to speed up tests.") 105 ) 106 107 func exists(path string) bool { 108 if _, err := os.Stat(path); err != nil { 109 glog.V(2).Infof("%q does not exist", path) 110 return false 111 } 112 return true 113 } 114 115 const pathToGCEConfig = "cluster/gce/config-default.sh" 116 117 func disableClusterMonitoring(kubeBaseDir string) error { 118 kubeConfigFilePath := filepath.Join(kubeBaseDir, pathToGCEConfig) 119 input, err := ioutil.ReadFile(kubeConfigFilePath) 120 if err != nil { 121 return err 122 } 123 124 lines := strings.Split(string(input), "\n") 125 126 for i, line := range lines { 127 if strings.Contains(line, "ENABLE_CLUSTER_MONITORING") { 128 lines[i] = "ENABLE_CLUSTER_MONITORING=false" 129 } else if strings.Contains(line, "NUM_MINIONS=") { 130 lines[i] = "NUM_MINIONS=2" 131 } 132 } 133 output := strings.Join(lines, "\n") 134 return ioutil.WriteFile(kubeConfigFilePath, []byte(output), 0644) 135 } 136 137 func runKubeClusterCommand(kubeBaseDir, command string) ([]byte, error) { 138 cmd := exec.Command(filepath.Join(kubeBaseDir, "cluster", command)) 139 glog.V(2).Infof("about to run %v", cmd) 140 return cmd.CombinedOutput() 141 } 142 143 func setupNewCluster(kubeBaseDir string) error { 144 cmd := "kube-up.sh" 145 destroyCluster(kubeBaseDir) 146 out, err := runKubeClusterCommand(kubeBaseDir, cmd) 147 if err != nil { 148 glog.Errorf("failed to bring up cluster - %q\n%s", err, out) 149 return fmt.Errorf("failed to bring up cluster - %q", err) 150 } 151 glog.V(2).Info(string(out)) 152 glog.V(2).Infof("Giving the cluster 30 sec to stabilize") 153 time.Sleep(30 * time.Second) 154 return nil 155 } 156 157 func destroyCluster(kubeBaseDir string) error { 158 if kubeBaseDir == "" { 159 glog.Infof("Skipping cluster tear down since kubernetes repo base path is not set.") 160 return nil 161 } 162 glog.V(1).Info("Bringing down any existing kube cluster") 163 out, err := runKubeClusterCommand(kubeBaseDir, "kube-down.sh") 164 if err != nil { 165 glog.Errorf("failed to tear down cluster - %q\n%s", err, out) 166 return fmt.Errorf("failed to tear down kube cluster - %q", err) 167 } 168 169 return nil 170 } 171 172 func downloadRelease(workDir, version string) error { 173 // Temporary download path. 174 downloadPath := filepath.Join(workDir, "kube") 175 // Format url. 176 downloadUrl := fmt.Sprintf(imageUrlTemplate, version) 177 glog.V(1).Infof("About to download kube release using url: %q", downloadUrl) 178 179 // Download kube code and store it in a temp dir. 180 if err := exec.Command("wget", downloadUrl, "-O", downloadPath).Run(); err != nil { 181 return fmt.Errorf("failed to wget kubernetes release @ %q - %v", downloadUrl, err) 182 } 183 184 // Un-tar kube release. 185 if err := exec.Command("tar", "-xf", downloadPath, "-C", workDir).Run(); err != nil { 186 return fmt.Errorf("failed to un-tar kubernetes release at %q - %v", downloadPath, err) 187 } 188 return nil 189 } 190 191 func getKubeClient() (string, *kclient.Client, error) { 192 c, err := kclientcmd.LoadFromFile(*kubeConfig) 193 if err != nil { 194 return "", nil, fmt.Errorf("error loading kubeConfig: %v", err.Error()) 195 } 196 if c.CurrentContext == "" || len(c.Clusters) == 0 { 197 return "", nil, fmt.Errorf("invalid kubeConfig: %+v", *c) 198 } 199 config, err := kclientcmd.NewDefaultClientConfig( 200 *c, 201 &kclientcmd.ConfigOverrides{ 202 ClusterInfo: kclientcmdapi.Cluster{ 203 APIVersion: "v1", 204 }, 205 }).ClientConfig() 206 if err != nil { 207 return "", nil, fmt.Errorf("error parsing kubeConfig: %v", err.Error()) 208 } 209 kubeClient, err := kclient.New(config) 210 if err != nil { 211 return "", nil, fmt.Errorf("error creating client - %q", err) 212 } 213 214 return c.Clusters[c.CurrentContext].Server, kubeClient, nil 215 } 216 217 func validateCluster(baseDir string) bool { 218 glog.V(1).Info("validating existing cluster") 219 out, err := runKubeClusterCommand(baseDir, "validate-cluster.sh") 220 if err != nil { 221 glog.V(1).Infof("cluster validation failed - %q\n %s", err, out) 222 return false 223 } 224 return true 225 } 226 227 func requireNewCluster(baseDir, version string) bool { 228 // Setup kube client 229 _, kubeClient, err := getKubeClient() 230 if err != nil { 231 glog.V(1).Infof("kube client creation failed - %q", err) 232 return true 233 } 234 glog.V(1).Infof("checking if existing cluster can be used") 235 versionInfo, err := kubeClient.ServerVersion() 236 if err != nil { 237 glog.V(1).Infof("failed to get kube version info - %q", err) 238 return true 239 } 240 return !strings.Contains(versionInfo.GitVersion, version) 241 } 242 243 func requireDownload(baseDir string) bool { 244 // Check that cluster scripts are present. 245 return !exists(filepath.Join(baseDir, "cluster", "kube-up.sh")) || 246 !exists(filepath.Join(baseDir, "cluster", "kube-down.sh")) || 247 !exists(filepath.Join(baseDir, "cluster", "validate-cluster.sh")) 248 } 249 250 func downloadAndSetupCluster(version string) (baseDir string, err error) { 251 // Create a temp dir to store the kube release files. 252 tempDir := filepath.Join(*workDir, version) 253 if !exists(tempDir) { 254 if err := os.MkdirAll(tempDir, 0700); err != nil { 255 return "", fmt.Errorf("failed to create a temp dir at %s - %q", tempDir, err) 256 } 257 glog.V(1).Infof("Successfully setup work dir at %s", tempDir) 258 } 259 260 kubeBaseDir := filepath.Join(tempDir, "kubernetes") 261 262 if requireDownload(kubeBaseDir) { 263 if exists(kubeBaseDir) { 264 os.RemoveAll(kubeBaseDir) 265 } 266 if err := downloadRelease(tempDir, version); err != nil { 267 return "", err 268 } 269 glog.V(1).Infof("Successfully downloaded kubernetes release at %s", tempDir) 270 } 271 272 // Disable monitoring 273 if err := disableClusterMonitoring(kubeBaseDir); err != nil { 274 return "", fmt.Errorf("failed to disable cluster monitoring in kube cluster config - %q", err) 275 } 276 glog.V(1).Info("Disabled cluster monitoring") 277 if !requireNewCluster(kubeBaseDir, version) { 278 glog.V(1).Infof("skipping cluster setup since a cluster with required version already exists") 279 return kubeBaseDir, nil 280 } 281 282 // Setup kube cluster 283 glog.V(1).Infof("Setting up new kubernetes cluster version: %s", version) 284 if err := setupNewCluster(kubeBaseDir); err != nil { 285 // Cluster setup failed for some reason. 286 // Attempting to validate the cluster to see if it failed in the validate phase. 287 sleepDuration := 10 * time.Second 288 clusterReady := false 289 for i := 0; i < int(time.Minute/sleepDuration); i++ { 290 if !validateCluster(kubeBaseDir) { 291 glog.Infof("Retry validation after %v seconds.", sleepDuration/time.Second) 292 time.Sleep(sleepDuration) 293 } else { 294 clusterReady = true 295 break 296 } 297 } 298 if !clusterReady { 299 return "", fmt.Errorf("failed to setup cluster - %q", err) 300 } 301 } 302 glog.V(1).Infof("Successfully setup new kubernetes cluster version %s", version) 303 304 return kubeBaseDir, nil 305 } 306 307 func newKubeFramework(version string) (kubeFramework, error) { 308 var err error 309 kubeBaseDir := "" 310 if version != "" { 311 if len(strings.Split(version, ".")) != 3 { 312 return nil, fmt.Errorf("invalid kubernetes version specified - %q", version) 313 } 314 kubeBaseDir, err = downloadAndSetupCluster(version) 315 if err != nil { 316 return nil, err 317 } 318 } 319 320 // Setup kube client 321 masterIP, kubeClient, err := getKubeClient() 322 if err != nil { 323 return nil, err 324 } 325 return &realKubeFramework{ 326 kubeClient: kubeClient, 327 baseDir: kubeBaseDir, 328 version: version, 329 masterIP: masterIP, 330 }, nil 331 } 332 333 func (self *realKubeFramework) Client() *kclient.Client { 334 return self.kubeClient 335 } 336 337 func (self *realKubeFramework) loadObject(filePath string) (runtime.Object, error) { 338 data, err := ioutil.ReadFile(filePath) 339 if err != nil { 340 return nil, fmt.Errorf("failed to read object: %v", err) 341 } 342 obj, _, err := api.Codecs.UniversalDecoder().Decode(data, nil, nil) 343 return obj, err 344 } 345 346 func (self *realKubeFramework) ParseRC(filePath string) (*api.ReplicationController, error) { 347 obj, err := self.loadObject(filePath) 348 if err != nil { 349 return nil, err 350 } 351 352 rc, ok := obj.(*api.ReplicationController) 353 if !ok { 354 return nil, fmt.Errorf("Failed to cast replicationController: %v", obj) 355 } 356 return rc, nil 357 } 358 359 func (self *realKubeFramework) ParseService(filePath string) (*api.Service, error) { 360 obj, err := self.loadObject(filePath) 361 if err != nil { 362 return nil, err 363 } 364 service, ok := obj.(*api.Service) 365 if !ok { 366 return nil, fmt.Errorf("Failed to cast service: %v", obj) 367 } 368 return service, nil 369 } 370 371 func (self *realKubeFramework) CreateService(ns string, service *api.Service) (*api.Service, error) { 372 service.Namespace = ns 373 newSvc, err := self.kubeClient.Services(ns).Create(service) 374 return newSvc, err 375 } 376 377 func (self *realKubeFramework) DeleteNs(ns string) error { 378 379 _, err := self.kubeClient.Namespaces().Get(ns) 380 if err != nil { 381 glog.V(0).Infof("Cannot get namespace %q. Skipping deletion: %s", ns, err) 382 return nil 383 } 384 glog.V(0).Infof("Deleting namespace %s", ns) 385 self.kubeClient.Namespaces().Delete(ns) 386 387 for i := 0; i < 5; i++ { 388 glog.V(0).Infof("Checking for namespace %s", ns) 389 _, err := self.kubeClient.Namespaces().Get(ns) 390 if err != nil { 391 glog.V(0).Infof("%s doesn't exist", ns) 392 return nil 393 } 394 time.Sleep(10 * time.Second) 395 } 396 return fmt.Errorf("Namespace %s still exists", ns) 397 } 398 399 func (self *realKubeFramework) CreateNs(ns *api.Namespace) (*api.Namespace, error) { 400 return self.kubeClient.Namespaces().Create(ns) 401 } 402 403 func (self *realKubeFramework) CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) { 404 rc.Namespace = ns 405 return self.kubeClient.ReplicationControllers(ns).Create(rc) 406 } 407 408 func (self *realKubeFramework) DestroyCluster() { 409 destroyCluster(self.baseDir) 410 } 411 412 func (self *realKubeFramework) GetProxyUrlForService(service *api.Service) string { 413 return fmt.Sprintf("%s/api/v1/proxy/namespaces/default/services/%s/", self.masterIP, service.Name) 414 } 415 416 func (self *realKubeFramework) GetNodeNames() ([]string, error) { 417 var nodes []string 418 nodeList, err := self.GetNodes() 419 if err != nil { 420 return nodes, err 421 } 422 for _, node := range nodeList.Items { 423 nodes = append(nodes, node.Name) 424 } 425 return nodes, nil 426 } 427 428 func (self *realKubeFramework) GetNodes() (*api.NodeList, error) { 429 return self.kubeClient.Nodes().List(api.ListOptions{ 430 LabelSelector: labels.Everything(), 431 FieldSelector: fields.Everything(), 432 }) 433 } 434 435 func (self *realKubeFramework) GetAllRunningPods() ([]api.Pod, error) { 436 return getRunningPods(true, self.kubeClient) 437 } 438 439 func (self *realKubeFramework) GetPodsRunningOnNodes() ([]api.Pod, error) { 440 return getRunningPods(false, self.kubeClient) 441 } 442 443 func getRunningPods(includeMaster bool, kubeClient *kclient.Client) ([]api.Pod, error) { 444 glog.V(0).Infof("Getting running pods") 445 podList, err := kubeClient.Pods(api.NamespaceAll).List(api.ListOptions{ 446 LabelSelector: labels.Everything(), 447 FieldSelector: fields.Everything(), 448 }) 449 if err != nil { 450 return nil, err 451 } 452 pods := []api.Pod{} 453 for _, pod := range podList.Items { 454 if pod.Status.Phase == api.PodRunning { 455 if includeMaster || !isMasterNode(pod.Spec.NodeName) { 456 pods = append(pods, pod) 457 } 458 } 459 } 460 return pods, nil 461 } 462 463 func isMasterNode(nodeName string) bool { 464 return strings.Contains(nodeName, "kubernetes-master") 465 } 466 467 func (self *realKubeFramework) GetRunningPodNames() ([]string, error) { 468 var pods []string 469 podList, err := self.GetAllRunningPods() 470 if err != nil { 471 return pods, err 472 } 473 for _, pod := range podList { 474 pods = append(pods, string(pod.Name)) 475 } 476 return pods, nil 477 } 478 479 func (rkf *realKubeFramework) WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error { 480 glog.V(2).Infof("Waiting for pod %v in %s...", podLabels, ns) 481 podsInterface := rkf.Client().Pods(ns) 482 for i := 0; i < int(timeout/time.Second); i++ { 483 podList, err := podsInterface.List(api.ListOptions{ 484 LabelSelector: labels.Set(podLabels).AsSelector(), 485 FieldSelector: fields.Everything(), 486 }) 487 if err != nil { 488 glog.V(1).Info(err) 489 return err 490 } 491 if len(podList.Items) > 0 { 492 podSpec := podList.Items[0] 493 if podSpec.Status.Phase == api.PodRunning { 494 return nil 495 } 496 } 497 time.Sleep(time.Second) 498 } 499 return fmt.Errorf("pod not in running state after %d", timeout/time.Second) 500 } 501 502 func (rkf *realKubeFramework) WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error { 503 glog.V(2).Infof("Waiting for endpoints in service %s/%s", svc.Namespace, svc.Name) 504 for i := 0; i < int(timeout/time.Second); i++ { 505 e, err := rkf.Client().Endpoints(svc.Namespace).Get(svc.Name) 506 if err != nil { 507 return err 508 } 509 if len(e.Subsets) > 0 { 510 return nil 511 } 512 time.Sleep(time.Second) 513 } 514 515 return fmt.Errorf("Service %q not active after %d seconds - no endpoints found", svc.Name, timeout/time.Second) 516 517 }