github.com/timstclair/heapster@v0.20.0-alpha1/integration/framework.go (about) 1 // Copyright 2014 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "flag" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "strings" 25 "time" 26 27 "github.com/golang/glog" 28 "k8s.io/kubernetes/pkg/api" 29 "k8s.io/kubernetes/pkg/api/v1" 30 kclient "k8s.io/kubernetes/pkg/client/unversioned" 31 kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" 32 kclientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" 33 "k8s.io/kubernetes/pkg/fields" 34 "k8s.io/kubernetes/pkg/labels" 35 "k8s.io/kubernetes/pkg/runtime" 36 ) 37 38 type kubeFramework interface { 39 // Kube client 40 Client() *kclient.Client 41 42 // Parses and Returns a replication Controller object contained in 'filePath' 43 ParseRC(filePath string) (*api.ReplicationController, error) 44 45 // Parses and Returns a service object contained in 'filePath' 46 ParseService(filePath string) (*api.Service, error) 47 48 // Creates a kube service. 49 CreateService(ns string, service *api.Service) (*api.Service, error) 50 51 // Creates a namespace. 52 CreateNs(ns *api.Namespace) (*api.Namespace, error) 53 54 // Creates a kube replication controller. 55 CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) 56 57 // Deletes a namespace 58 DeleteNs(ns string) error 59 60 // Destroy cluster 61 DestroyCluster() 62 63 // Returns a url that provides access to a kubernetes service via the proxy on the apiserver. 64 // This url requires master auth. 65 GetProxyUrlForService(service *api.Service) string 66 67 // Returns the node hostnames. 68 GetNodes() ([]string, error) 69 70 // Returns pod names in the cluster. 71 // TODO: Remove, or mix with namespace 72 GetRunningPodNames() ([]string, error) 73 74 // Returns pods in the cluster. 75 GetRunningPods() ([]api.Pod, error) 76 77 WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error 78 WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error 79 } 80 81 type realKubeFramework struct { 82 // Kube client. 83 kubeClient *kclient.Client 84 85 // The version of the kube cluster 86 version string 87 88 // Master IP for this framework 89 masterIP string 90 91 // The base directory of current kubernetes release. 92 baseDir string 93 } 94 95 const imageUrlTemplate = "https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz" 96 97 var ( 98 kubeConfig = flag.String("kube_config", os.Getenv("HOME")+"/.kube/config", "Path to cluster info file.") 99 workDir = flag.String("work_dir", "/tmp/heapster_test", "Filesystem path where test files will be stored. Files will persist across runs to speed up tests.") 100 ) 101 102 func exists(path string) bool { 103 if _, err := os.Stat(path); err != nil { 104 return false 105 } 106 return true 107 } 108 109 const pathToGCEConfig = "cluster/gce/config-default.sh" 110 111 func disableClusterMonitoring(kubeBaseDir string) error { 112 kubeConfigFilePath := filepath.Join(kubeBaseDir, pathToGCEConfig) 113 input, err := ioutil.ReadFile(kubeConfigFilePath) 114 if err != nil { 115 return err 116 } 117 118 lines := strings.Split(string(input), "\n") 119 120 for i, line := range lines { 121 if strings.Contains(line, "ENABLE_CLUSTER_MONITORING") { 122 lines[i] = "ENABLE_CLUSTER_MONITORING=false" 123 } else if strings.Contains(line, "NUM_MINIONS=") { 124 lines[i] = "NUM_MINIONS=2" 125 } 126 } 127 output := strings.Join(lines, "\n") 128 return ioutil.WriteFile(kubeConfigFilePath, []byte(output), 0644) 129 } 130 131 func runKubeClusterCommand(kubeBaseDir, command string) ([]byte, error) { 132 cmd := exec.Command(filepath.Join(kubeBaseDir, "cluster", command)) 133 glog.V(2).Infof("about to run %v", cmd) 134 return cmd.CombinedOutput() 135 } 136 137 func setupNewCluster(kubeBaseDir string) error { 138 cmd := "kube-up.sh" 139 destroyCluster(kubeBaseDir) 140 out, err := runKubeClusterCommand(kubeBaseDir, cmd) 141 if err != nil { 142 glog.Errorf("failed to bring up cluster - %q\n%s", err, out) 143 return fmt.Errorf("failed to bring up cluster - %q", err) 144 } 145 glog.V(2).Info(string(out)) 146 glog.V(2).Infof("Giving the cluster 30 sec to stabilize") 147 time.Sleep(30 * time.Second) 148 return nil 149 } 150 151 func destroyCluster(kubeBaseDir string) error { 152 if kubeBaseDir == "" { 153 glog.Infof("Skipping cluster tear down since kubernetes repo base path is not set.") 154 return nil 155 } 156 glog.V(1).Info("Bringing down any existing kube cluster") 157 out, err := runKubeClusterCommand(kubeBaseDir, "kube-down.sh") 158 if err != nil { 159 glog.Errorf("failed to tear down cluster - %q\n%s", err, out) 160 return fmt.Errorf("failed to tear down kube cluster - %q", err) 161 } 162 163 return nil 164 } 165 166 func downloadRelease(workDir, version string) error { 167 // Temporary download path. 168 downloadPath := filepath.Join(workDir, "kube") 169 // Format url. 170 downloadUrl := fmt.Sprintf(imageUrlTemplate, version) 171 glog.V(1).Infof("About to download kube release using url: %q", downloadUrl) 172 173 // Download kube code and store it in a temp dir. 174 if err := exec.Command("wget", downloadUrl, "-O", downloadPath).Run(); err != nil { 175 return fmt.Errorf("failed to wget kubernetes release @ %q - %v", downloadUrl, err) 176 } 177 178 // Un-tar kube release. 179 if err := exec.Command("tar", "-xf", downloadPath, "-C", workDir).Run(); err != nil { 180 return fmt.Errorf("failed to un-tar kubernetes release at %q - %v", downloadPath, err) 181 } 182 return nil 183 } 184 185 func getKubeClient() (string, *kclient.Client, error) { 186 c, err := kclientcmd.LoadFromFile(*kubeConfig) 187 if err != nil { 188 return "", nil, fmt.Errorf("error loading kubeConfig: %v", err.Error()) 189 } 190 if c.CurrentContext == "" || len(c.Clusters) == 0 { 191 return "", nil, fmt.Errorf("invalid kubeConfig: %+v", *c) 192 } 193 config, err := kclientcmd.NewDefaultClientConfig( 194 *c, 195 &kclientcmd.ConfigOverrides{ 196 ClusterInfo: kclientcmdapi.Cluster{ 197 APIVersion: "v1", 198 }, 199 }).ClientConfig() 200 if err != nil { 201 return "", nil, fmt.Errorf("error parsing kubeConfig: %v", err.Error()) 202 } 203 kubeClient, err := kclient.New(config) 204 if err != nil { 205 return "", nil, fmt.Errorf("error creating client - %q", err) 206 } 207 208 return c.Clusters[c.CurrentContext].Server, kubeClient, nil 209 } 210 211 func validateCluster(baseDir string) bool { 212 glog.V(1).Info("validating existing cluster") 213 out, err := runKubeClusterCommand(baseDir, "validate-cluster.sh") 214 if err != nil { 215 glog.V(1).Infof("cluster validation failed - %q\n %s", err, out) 216 return false 217 } 218 return true 219 } 220 221 func requireNewCluster(baseDir, version string) bool { 222 // Setup kube client 223 _, kubeClient, err := getKubeClient() 224 if err != nil { 225 glog.V(1).Infof("kube client creation failed - %q", err) 226 return true 227 } 228 glog.V(1).Infof("checking if existing cluster can be used") 229 versionInfo, err := kubeClient.ServerVersion() 230 if err != nil { 231 glog.V(1).Infof("failed to get kube version info - %q", err) 232 return true 233 } 234 return !strings.Contains(versionInfo.GitVersion, version) 235 } 236 237 func downloadAndSetupCluster(version string) (baseDir string, err error) { 238 // Create a temp dir to store the kube release files. 239 tempDir := filepath.Join(*workDir, version) 240 if !exists(tempDir) { 241 if err := os.MkdirAll(tempDir, 0700); err != nil { 242 return "", fmt.Errorf("failed to create a temp dir at %s - %q", tempDir, err) 243 } 244 glog.V(1).Infof("Successfully setup work dir at %s", tempDir) 245 } 246 247 kubeBaseDir := filepath.Join(tempDir, "kubernetes") 248 249 if !exists(kubeBaseDir) { 250 if err := downloadRelease(tempDir, version); err != nil { 251 return "", err 252 } 253 glog.V(1).Infof("Successfully downloaded kubernetes release at %s", tempDir) 254 } 255 256 // Disable monitoring 257 if err := disableClusterMonitoring(kubeBaseDir); err != nil { 258 return "", fmt.Errorf("failed to disable cluster monitoring in kube cluster config - %q", err) 259 } 260 glog.V(1).Info("Disabled cluster monitoring") 261 if !requireNewCluster(kubeBaseDir, version) { 262 glog.V(1).Infof("skipping cluster setup since a cluster with required version already exists") 263 return kubeBaseDir, nil 264 } 265 266 // Setup kube cluster 267 glog.V(1).Infof("Setting up new kubernetes cluster version: %s", version) 268 if err := setupNewCluster(kubeBaseDir); err != nil { 269 // Cluster setup failed for some reason. 270 // Attempting to validate the cluster to see if it failed in the validate phase. 271 sleepDuration := 10 * time.Second 272 clusterReady := false 273 for i := 0; i < int(time.Minute/sleepDuration); i++ { 274 if !validateCluster(kubeBaseDir) { 275 glog.Infof("Retry validation after %v seconds.", sleepDuration/time.Second) 276 time.Sleep(sleepDuration) 277 } else { 278 clusterReady = true 279 break 280 } 281 } 282 if !clusterReady { 283 return "", fmt.Errorf("failed to setup cluster - %q", err) 284 } 285 } 286 glog.V(1).Infof("Successfully setup new kubernetes cluster version %s", version) 287 288 return kubeBaseDir, nil 289 } 290 291 func newKubeFramework(version string) (kubeFramework, error) { 292 var err error 293 kubeBaseDir := "" 294 if version != "" { 295 if len(strings.Split(version, ".")) != 3 { 296 return nil, fmt.Errorf("invalid kubernetes version specified - %q", version) 297 } 298 kubeBaseDir, err = downloadAndSetupCluster(version) 299 if err != nil { 300 return nil, err 301 } 302 } 303 304 // Setup kube client 305 masterIP, kubeClient, err := getKubeClient() 306 if err != nil { 307 return nil, err 308 } 309 return &realKubeFramework{ 310 kubeClient: kubeClient, 311 baseDir: kubeBaseDir, 312 version: version, 313 masterIP: masterIP, 314 }, nil 315 } 316 317 func (self *realKubeFramework) Client() *kclient.Client { 318 return self.kubeClient 319 } 320 321 func (self *realKubeFramework) loadObject(filePath string) (runtime.Object, error) { 322 data, err := ioutil.ReadFile(filePath) 323 if err != nil { 324 return nil, fmt.Errorf("failed to read object: %v", err) 325 } 326 return runtime.YAMLDecoder(v1.Codec).Decode(data) 327 } 328 329 func (self *realKubeFramework) ParseRC(filePath string) (*api.ReplicationController, error) { 330 obj, err := self.loadObject(filePath) 331 if err != nil { 332 return nil, err 333 } 334 335 rc, ok := obj.(*api.ReplicationController) 336 if !ok { 337 return nil, fmt.Errorf("Failed to cast replicationController: %v", obj) 338 } 339 return rc, nil 340 } 341 342 func (self *realKubeFramework) ParseService(filePath string) (*api.Service, error) { 343 obj, err := self.loadObject(filePath) 344 if err != nil { 345 return nil, err 346 } 347 service, ok := obj.(*api.Service) 348 if !ok { 349 return nil, fmt.Errorf("Failed to cast service: %v", obj) 350 } 351 return service, nil 352 } 353 354 func (self *realKubeFramework) CreateService(ns string, service *api.Service) (*api.Service, error) { 355 service.Namespace = ns 356 newSvc, err := self.kubeClient.Services(ns).Create(service) 357 return newSvc, err 358 } 359 360 func (self *realKubeFramework) DeleteNs(ns string) error { 361 362 _, err := self.kubeClient.Namespaces().Get(ns) 363 if err != nil { 364 glog.V(0).Infof("Cannot get namespace %q. Skipping deletion: %s", ns, err) 365 return nil 366 } 367 glog.V(0).Infof("Deleting namespace %s", ns) 368 self.kubeClient.Namespaces().Delete(ns) 369 370 for i := 0; i < 5; i++ { 371 glog.V(0).Infof("Checking for namespace %s", ns) 372 _, err := self.kubeClient.Namespaces().Get(ns) 373 if err != nil { 374 glog.V(0).Infof("%s doesn't exist", ns) 375 return nil 376 } 377 time.Sleep(10 * time.Second) 378 } 379 return fmt.Errorf("Namespace %s still exists", ns) 380 } 381 382 func (self *realKubeFramework) CreateNs(ns *api.Namespace) (*api.Namespace, error) { 383 return self.kubeClient.Namespaces().Create(ns) 384 } 385 386 func (self *realKubeFramework) CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) { 387 rc.Namespace = ns 388 return self.kubeClient.ReplicationControllers(ns).Create(rc) 389 } 390 391 func (self *realKubeFramework) DestroyCluster() { 392 destroyCluster(self.baseDir) 393 } 394 395 func (self *realKubeFramework) GetProxyUrlForService(service *api.Service) string { 396 return fmt.Sprintf("%s/api/v1/proxy/namespaces/default/services/%s/", self.masterIP, service.Name) 397 } 398 399 func (self *realKubeFramework) GetNodes() ([]string, error) { 400 var nodes []string 401 nodeList, err := self.kubeClient.Nodes().List(labels.Everything(), fields.Everything()) 402 if err != nil { 403 return nodes, err 404 } 405 406 for _, node := range nodeList.Items { 407 nodes = append(nodes, node.Name) 408 } 409 return nodes, nil 410 } 411 412 func (self *realKubeFramework) GetRunningPods() ([]api.Pod, error) { 413 glog.V(0).Infof("Getting running pods") 414 podList, err := self.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything()) 415 if err != nil { 416 return nil, err 417 } 418 pods := []api.Pod{} 419 for _, pod := range podList.Items { 420 if pod.Status.Phase == api.PodRunning && !strings.Contains(pod.Spec.NodeName, "kubernetes-master") { 421 pods = append(pods, pod) 422 } 423 } 424 return pods, nil 425 } 426 427 func (self *realKubeFramework) GetRunningPodNames() ([]string, error) { 428 var pods []string 429 podList, err := self.GetRunningPods() 430 if err != nil { 431 return pods, err 432 } 433 for _, pod := range podList { 434 pods = append(pods, string(pod.Name)) 435 } 436 return pods, nil 437 } 438 439 func (rkf *realKubeFramework) WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error { 440 glog.V(2).Infof("Waiting for pod %v in %s...", podLabels, ns) 441 podsInterface := rkf.Client().Pods(ns) 442 for i := 0; i < int(timeout/time.Second); i++ { 443 selector := labels.Set(podLabels).AsSelector() 444 podList, err := podsInterface.List(selector, fields.Everything()) 445 if err != nil { 446 glog.V(1).Info(err) 447 return err 448 } 449 if len(podList.Items) > 0 { 450 podSpec := podList.Items[0] 451 if podSpec.Status.Phase == api.PodRunning { 452 return nil 453 } 454 } 455 time.Sleep(time.Second) 456 } 457 return fmt.Errorf("pod not in running state after %d", timeout/time.Second) 458 } 459 460 func (rkf *realKubeFramework) WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error { 461 glog.V(2).Infof("Waiting for endpoints in service %s/%s", svc.Namespace, svc.Name) 462 for i := 0; i < int(timeout/time.Second); i++ { 463 e, err := rkf.Client().Endpoints(svc.Namespace).Get(svc.Name) 464 if err != nil { 465 return err 466 } 467 if len(e.Subsets) > 0 { 468 return nil 469 } 470 time.Sleep(time.Second) 471 } 472 473 return fmt.Errorf("Service %q not active after %d seconds - no endpoints found", svc.Name, timeout/time.Second) 474 475 }