github.com/mrgossett/heapster@v0.18.2/integration/framework.go (about) 1 // Copyright 2014 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "flag" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "strings" 25 "time" 26 27 "github.com/golang/glog" 28 "k8s.io/kubernetes/pkg/api" 29 "k8s.io/kubernetes/pkg/api/v1" 30 kclient "k8s.io/kubernetes/pkg/client" 31 kclientcmd "k8s.io/kubernetes/pkg/client/clientcmd" 32 kclientcmdapi "k8s.io/kubernetes/pkg/client/clientcmd/api" 33 "k8s.io/kubernetes/pkg/fields" 34 "k8s.io/kubernetes/pkg/labels" 35 "k8s.io/kubernetes/pkg/runtime" 36 ) 37 38 type kubeFramework interface { 39 // Kube client 40 Client() *kclient.Client 41 42 // Parses and Returns a replication Controller object contained in 'filePath' 43 ParseRC(filePath string) (*api.ReplicationController, error) 44 45 // Parses and Returns a service object contained in 'filePath' 46 ParseService(filePath string) (*api.Service, error) 47 48 // Creates a kube service. 49 CreateService(ns string, service *api.Service) (*api.Service, error) 50 51 // Deletes a kube service. 52 DeleteService(ns string, service *api.Service) error 53 54 // Creates a kube replication controller. 55 CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) 56 57 // Deletes a kube replication controller. 58 DeleteRC(ns string, rc *api.ReplicationController) error 59 60 // Destroy cluster 61 DestroyCluster() 62 63 // Returns a url that provides access to a kubernetes service via the proxy on the apiserver. 64 // This url requires master auth. 65 GetProxyUrlForService(service *api.Service) string 66 67 // Returns the node hostnames. 68 GetNodes() ([]string, error) 69 70 // Returns pod names in the cluster. 71 // TODO: Remove, or mix with namespace 72 GetRunningPodNames() ([]string, error) 73 74 // Returns pods in the cluster. 75 GetRunningPods() ([]*api.Pod, error) 76 77 WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error 78 WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error 79 } 80 81 type realKubeFramework struct { 82 // Kube client. 83 kubeClient *kclient.Client 84 85 // The version of the kube cluster 86 version string 87 88 // Master IP for this framework 89 masterIP string 90 91 // The base directory of current kubernetes release. 92 baseDir string 93 } 94 95 const ( 96 imageUrlTemplate = "https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz" 97 kubectl = "kubectl.sh" 98 ) 99 100 var ( 101 kubeConfig = flag.String("kube_config", os.Getenv("HOME")+"/.kube/config", "Path to cluster info file.") 102 workDir = flag.String("work_dir", "/tmp/heapster_test", "Filesystem path where test files will be stored. Files will persist across runs to speed up tests.") 103 ) 104 105 func exists(path string) bool { 106 if _, err := os.Stat(path); err != nil { 107 return false 108 } 109 return true 110 } 111 112 const pathToGCEConfig = "cluster/gce/config-default.sh" 113 114 func disableClusterMonitoring(kubeBaseDir string) error { 115 kubeConfigFilePath := filepath.Join(kubeBaseDir, pathToGCEConfig) 116 input, err := ioutil.ReadFile(kubeConfigFilePath) 117 if err != nil { 118 return err 119 } 120 121 lines := strings.Split(string(input), "\n") 122 123 for i, line := range lines { 124 if strings.Contains(line, "ENABLE_CLUSTER_MONITORING") { 125 lines[i] = "ENABLE_CLUSTER_MONITORING=false" 126 } else if strings.Contains(line, "NUM_MINIONS=") { 127 lines[i] = "NUM_MINIONS=2" 128 } 129 } 130 output := strings.Join(lines, "\n") 131 return ioutil.WriteFile(kubeConfigFilePath, []byte(output), 0644) 132 } 133 134 func runKubeClusterCommand(kubeBaseDir, command string) ([]byte, error) { 135 cmd := exec.Command(filepath.Join(kubeBaseDir, "cluster", command)) 136 glog.V(2).Infof("about to run %v", cmd) 137 return cmd.CombinedOutput() 138 } 139 140 func setupNewCluster(kubeBaseDir string) error { 141 cmd := "kube-up.sh" 142 destroyCluster(kubeBaseDir) 143 out, err := runKubeClusterCommand(kubeBaseDir, cmd) 144 if err != nil { 145 glog.Errorf("failed to bring up cluster - %q\n%s", err, out) 146 return fmt.Errorf("failed to bring up cluster - %q", err) 147 } 148 149 return nil 150 } 151 152 func destroyCluster(kubeBaseDir string) error { 153 if kubeBaseDir == "" { 154 glog.Infof("Skipping cluster tear down since kubernetes repo base path is not set.") 155 return nil 156 } 157 glog.V(1).Info("Bringing down any existing kube cluster") 158 out, err := runKubeClusterCommand(kubeBaseDir, "kube-down.sh") 159 if err != nil { 160 glog.Errorf("failed to tear down cluster - %q\n%s", err, out) 161 return fmt.Errorf("failed to tear down kube cluster - %q", err) 162 } 163 164 return nil 165 } 166 167 func downloadRelease(workDir, version string) error { 168 // Temporary download path. 169 downloadPath := filepath.Join(workDir, "kube") 170 // Format url. 171 downloadUrl := fmt.Sprintf(imageUrlTemplate, version) 172 glog.V(1).Infof("About to download kube release using url: %q", downloadUrl) 173 174 // Download kube code and store it in a temp dir. 175 if err := exec.Command("wget", downloadUrl, "-O", downloadPath).Run(); err != nil { 176 return fmt.Errorf("failed to wget kubernetes release @ %q - %v", downloadUrl, err) 177 } 178 179 // Un-tar kube release. 180 if err := exec.Command("tar", "-xf", downloadPath, "-C", workDir).Run(); err != nil { 181 return fmt.Errorf("failed to un-tar kubernetes release at %q - %v", downloadPath, err) 182 } 183 return nil 184 } 185 186 func getKubeClient() (string, *kclient.Client, error) { 187 c, err := kclientcmd.LoadFromFile(*kubeConfig) 188 if err != nil { 189 return "", nil, fmt.Errorf("error loading kubeConfig: %v", err.Error()) 190 } 191 config, err := kclientcmd.NewDefaultClientConfig( 192 *c, 193 &kclientcmd.ConfigOverrides{ 194 ClusterInfo: kclientcmdapi.Cluster{ 195 APIVersion: "v1", 196 }, 197 }).ClientConfig() 198 if err != nil { 199 return "", nil, fmt.Errorf("error parsing kubeConfig: %v", err.Error()) 200 } 201 kubeClient, err := kclient.New(config) 202 if err != nil { 203 return "", nil, fmt.Errorf("error creating client - %q", err) 204 } 205 206 return c.Clusters[c.CurrentContext].Server, kubeClient, nil 207 } 208 209 func validateCluster(baseDir string) bool { 210 glog.V(1).Info("validating existing cluster") 211 out, err := runKubeClusterCommand(baseDir, "validate-cluster.sh") 212 if err != nil { 213 glog.V(1).Infof("cluster validation failed - %q\n %s", err, out) 214 return false 215 } 216 return true 217 } 218 219 func requireNewCluster(baseDir, version string) bool { 220 // Setup kube client 221 _, kubeClient, err := getKubeClient() 222 if err != nil { 223 glog.V(1).Infof("kube client creation failed - %q", err) 224 return true 225 } 226 glog.V(1).Infof("checking if existing cluster can be used") 227 versionInfo, err := kubeClient.ServerVersion() 228 if err != nil { 229 glog.V(1).Infof("failed to get kube version info - %q", err) 230 return true 231 } 232 return !strings.Contains(versionInfo.GitVersion, version) 233 } 234 235 func downloadAndSetupCluster(version string) (baseDir string, err error) { 236 // Create a temp dir to store the kube release files. 237 tempDir := filepath.Join(*workDir, version) 238 if !exists(tempDir) { 239 if err := os.MkdirAll(tempDir, 0700); err != nil { 240 return "", fmt.Errorf("failed to create a temp dir at %s - %q", tempDir, err) 241 } 242 glog.V(1).Infof("Successfully setup work dir at %s", tempDir) 243 } 244 245 kubeBaseDir := filepath.Join(tempDir, "kubernetes") 246 247 if !exists(kubeBaseDir) { 248 if err := downloadRelease(tempDir, version); err != nil { 249 return "", err 250 } 251 glog.V(1).Infof("Successfully downloaded kubernetes release at %s", tempDir) 252 } 253 254 // Disable monitoring 255 if err := disableClusterMonitoring(kubeBaseDir); err != nil { 256 return "", fmt.Errorf("failed to disable cluster monitoring in kube cluster config - %q", err) 257 } 258 glog.V(1).Info("Disabled cluster monitoring") 259 if !requireNewCluster(kubeBaseDir, version) { 260 glog.V(1).Infof("skipping cluster setup since a cluster with required version already exists") 261 return kubeBaseDir, nil 262 } 263 264 // Setup kube cluster 265 glog.V(1).Infof("Setting up new kubernetes cluster version: %s", version) 266 if err := setupNewCluster(kubeBaseDir); err != nil { 267 // Cluster setup failed for some reason. 268 // Attempting to validate the cluster to see if it failed in the validate phase. 269 sleepDuration := 10 * time.Second 270 var clusterReady bool = false 271 for i := 0; i < int(time.Minute/sleepDuration); i++ { 272 if !validateCluster(kubeBaseDir) { 273 glog.Infof("Retry validation after %v seconds.", sleepDuration/time.Second) 274 time.Sleep(sleepDuration) 275 } else { 276 clusterReady = true 277 break 278 } 279 } 280 if !clusterReady { 281 return "", fmt.Errorf("failed to setup cluster - %q", err) 282 } 283 } 284 glog.V(1).Infof("Successfully setup new kubernetes cluster version %s", version) 285 286 return kubeBaseDir, nil 287 } 288 289 func newKubeFramework(version string) (kubeFramework, error) { 290 var err error 291 kubeBaseDir := "" 292 if version != "" { 293 if len(strings.Split(version, ".")) != 3 { 294 return nil, fmt.Errorf("invalid kubernetes version specified - %q", version) 295 } 296 kubeBaseDir, err = downloadAndSetupCluster(version) 297 if err != nil { 298 return nil, err 299 } 300 } 301 302 // Setup kube client 303 masterIP, kubeClient, err := getKubeClient() 304 if err != nil { 305 return nil, err 306 } 307 return &realKubeFramework{ 308 kubeClient: kubeClient, 309 baseDir: kubeBaseDir, 310 version: version, 311 masterIP: masterIP, 312 }, nil 313 } 314 315 func (self *realKubeFramework) Client() *kclient.Client { 316 return self.kubeClient 317 } 318 319 func (self *realKubeFramework) loadObject(filePath string) (runtime.Object, error) { 320 data, err := ioutil.ReadFile(filePath) 321 if err != nil { 322 return nil, fmt.Errorf("failed to read object: %v", err) 323 } 324 return v1.Codec.Decode(data) 325 } 326 327 func (self *realKubeFramework) ParseRC(filePath string) (*api.ReplicationController, error) { 328 obj, err := self.loadObject(filePath) 329 if err != nil { 330 return nil, err 331 } 332 333 rc, ok := obj.(*api.ReplicationController) 334 if !ok { 335 return nil, fmt.Errorf("Failed to cast replicationController: %v", obj) 336 } 337 return rc, nil 338 } 339 340 func (self *realKubeFramework) ParseService(filePath string) (*api.Service, error) { 341 obj, err := self.loadObject(filePath) 342 if err != nil { 343 return nil, err 344 } 345 service, ok := obj.(*api.Service) 346 if !ok { 347 return nil, fmt.Errorf("Failed to cast service: %v", obj) 348 } 349 return service, nil 350 } 351 352 func (self *realKubeFramework) CreateService(ns string, service *api.Service) (*api.Service, error) { 353 service.Namespace = ns 354 newSvc, err := self.kubeClient.Services(ns).Create(service) 355 return newSvc, err 356 } 357 358 func (self *realKubeFramework) DeleteService(ns string, service *api.Service) error { 359 if _, err := self.kubeClient.Services(ns).Get(service.Name); err != nil { 360 glog.V(2).Infof("cannot find service %q. Skipping deletion.", service.Name) 361 return nil 362 } 363 364 return self.kubeClient.Services(ns).Delete(service.Name) 365 } 366 367 func (self *realKubeFramework) CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) { 368 rc.Namespace = ns 369 return self.kubeClient.ReplicationControllers(ns).Create(rc) 370 } 371 372 func (self *realKubeFramework) DeleteRC(ns string, inputRc *api.ReplicationController) error { 373 var list []*api.ReplicationController 374 labelValue := "heapster" 375 labelKeys := []string{"k8s-app", "name"} 376 for _, k := range labelKeys { 377 if val, e := inputRc.Labels[k]; e { 378 labelValue = val 379 } 380 } 381 for _, labelKey := range labelKeys { 382 selector := labels.Set(map[string]string{ 383 labelKey: labelValue, 384 }).AsSelector() 385 rcList, err := self.kubeClient.ReplicationControllers(ns).List(selector) 386 if err != nil { 387 return fmt.Errorf("cannot list RCs by label %s=%s: %v", labelKey, labelValue, err) 388 } 389 for i := range rcList.Items { 390 list = append(list, &rcList.Items[i]) 391 } 392 } 393 if len(list) < 1 { 394 glog.V(2).Infof("Found no RCs identified by '%s'. Skipping deletion.", labelValue) 395 return nil 396 } 397 398 for _, rc := range list { 399 rc.Spec.Replicas = 0 400 if _, err := self.kubeClient.ReplicationControllers(ns).Update(rc); err != nil { 401 return fmt.Errorf("unable to modify replica count for rc %v: %v", inputRc.Name, err) 402 } 403 404 if err := self.kubeClient.ReplicationControllers(ns).Delete(rc.Name); err != nil { 405 return fmt.Errorf("unable to delete rc %v: %v", inputRc.Name, err) 406 } 407 } 408 return nil 409 } 410 411 func (self *realKubeFramework) DestroyCluster() { 412 destroyCluster(self.baseDir) 413 } 414 415 func (self *realKubeFramework) GetProxyUrlForService(service *api.Service) string { 416 return fmt.Sprintf("%s/api/v1/proxy/namespaces/default/services/%s/", self.masterIP, service.Name) 417 } 418 419 func (self *realKubeFramework) GetNodes() ([]string, error) { 420 var nodes []string 421 nodeList, err := self.kubeClient.Nodes().List(labels.Everything(), fields.Everything()) 422 if err != nil { 423 return nodes, err 424 } 425 426 for _, node := range nodeList.Items { 427 nodes = append(nodes, node.Name) 428 } 429 return nodes, nil 430 } 431 432 func (self *realKubeFramework) GetRunningPods() ([]*api.Pod, error) { 433 podList, err := self.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything()) 434 if err != nil { 435 return nil, err 436 } 437 pods := []*api.Pod{} 438 for _, pod := range podList.Items { 439 if pod.Status.Phase == api.PodRunning && !strings.Contains(pod.Spec.NodeName, "kubernetes-master") { 440 pods = append(pods, &pod) 441 } 442 } 443 return pods, nil 444 } 445 446 func (self *realKubeFramework) GetRunningPodNames() ([]string, error) { 447 var pods []string 448 podList, err := self.GetRunningPods() 449 if err != nil { 450 return pods, err 451 } 452 for _, pod := range podList { 453 pods = append(pods, string(pod.Name)) 454 } 455 return pods, nil 456 } 457 458 func (rkf *realKubeFramework) WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error { 459 podsInterface := rkf.Client().Pods(ns) 460 for i := 0; i < int(timeout/time.Second); i++ { 461 selector := labels.Set(podLabels).AsSelector() 462 podList, err := podsInterface.List(selector, fields.Everything()) 463 if err != nil { 464 glog.V(1).Info(err) 465 return err 466 } 467 if len(podList.Items) > 0 { 468 podSpec := podList.Items[0] 469 if podSpec.Status.Phase == api.PodRunning { 470 return nil 471 } 472 } 473 time.Sleep(time.Second) 474 } 475 return fmt.Errorf("pod not in running state after %d", timeout/time.Second) 476 } 477 478 func (rkf *realKubeFramework) WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error { 479 for i := 0; i < int(timeout/time.Second); i++ { 480 e, err := rkf.Client().Endpoints(svc.Namespace).Get(svc.Name) 481 if err != nil { 482 return err 483 } 484 if len(e.Subsets) > 0 { 485 return nil 486 } 487 time.Sleep(time.Second) 488 } 489 490 return fmt.Errorf("Service %q not active after %d seconds - no endpoints found", svc.Name, timeout/time.Second) 491 492 }