github.com/aclisp/heapster@v0.19.2-0.20160613100040-51756f899a96/integration/framework.go (about) 1 // Copyright 2014 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "flag" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "strings" 25 "time" 26 27 "github.com/golang/glog" 28 "k8s.io/kubernetes/pkg/api" 29 kclient "k8s.io/kubernetes/pkg/client/unversioned" 30 kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" 31 kclientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" 32 "k8s.io/kubernetes/pkg/fields" 33 "k8s.io/kubernetes/pkg/labels" 34 "k8s.io/kubernetes/pkg/runtime" 35 ) 36 37 type kubeFramework interface { 38 // Kube client 39 Client() *kclient.Client 40 41 // Parses and Returns a replication Controller object contained in 'filePath' 42 ParseRC(filePath string) (*api.ReplicationController, error) 43 44 // Parses and Returns a service object contained in 'filePath' 45 ParseService(filePath string) (*api.Service, error) 46 47 // Creates a kube service. 48 CreateService(ns string, service *api.Service) (*api.Service, error) 49 50 // Creates a namespace. 51 CreateNs(ns *api.Namespace) (*api.Namespace, error) 52 53 // Creates a kube replication controller. 54 CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) 55 56 // Deletes a namespace 57 DeleteNs(ns string) error 58 59 // Destroy cluster 60 DestroyCluster() 61 62 // Returns a url that provides access to a kubernetes service via the proxy on the apiserver. 63 // This url requires master auth. 64 GetProxyUrlForService(service *api.Service) string 65 66 // Returns the node hostnames. 67 GetNodes() ([]string, error) 68 69 // Returns pod names in the cluster. 70 // TODO: Remove, or mix with namespace 71 GetRunningPodNames() ([]string, error) 72 73 // Returns pods in the cluster. 74 GetRunningPods() ([]api.Pod, error) 75 76 WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error 77 WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error 78 } 79 80 type realKubeFramework struct { 81 // Kube client. 82 kubeClient *kclient.Client 83 84 // The version of the kube cluster 85 version string 86 87 // Master IP for this framework 88 masterIP string 89 90 // The base directory of current kubernetes release. 91 baseDir string 92 } 93 94 const imageUrlTemplate = "https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz" 95 96 var ( 97 kubeConfig = flag.String("kube_config", os.Getenv("HOME")+"/.kube/config", "Path to cluster info file.") 98 workDir = flag.String("work_dir", "/tmp/heapster_test", "Filesystem path where test files will be stored. Files will persist across runs to speed up tests.") 99 ) 100 101 func exists(path string) bool { 102 if _, err := os.Stat(path); err != nil { 103 glog.V(2).Infof("%q does not exist", path) 104 return false 105 } 106 return true 107 } 108 109 const pathToGCEConfig = "cluster/gce/config-default.sh" 110 111 func disableClusterMonitoring(kubeBaseDir string) error { 112 kubeConfigFilePath := filepath.Join(kubeBaseDir, pathToGCEConfig) 113 input, err := ioutil.ReadFile(kubeConfigFilePath) 114 if err != nil { 115 return err 116 } 117 118 lines := strings.Split(string(input), "\n") 119 120 for i, line := range lines { 121 if strings.Contains(line, "ENABLE_CLUSTER_MONITORING") { 122 lines[i] = "ENABLE_CLUSTER_MONITORING=false" 123 } else if strings.Contains(line, "NUM_MINIONS=") { 124 lines[i] = "NUM_MINIONS=2" 125 } 126 } 127 output := strings.Join(lines, "\n") 128 return ioutil.WriteFile(kubeConfigFilePath, []byte(output), 0644) 129 } 130 131 func runKubeClusterCommand(kubeBaseDir, command string) ([]byte, error) { 132 cmd := exec.Command(filepath.Join(kubeBaseDir, "cluster", command)) 133 glog.V(2).Infof("about to run %v", cmd) 134 return cmd.CombinedOutput() 135 } 136 137 func setupNewCluster(kubeBaseDir string) error { 138 cmd := "kube-up.sh" 139 destroyCluster(kubeBaseDir) 140 out, err := runKubeClusterCommand(kubeBaseDir, cmd) 141 if err != nil { 142 glog.Errorf("failed to bring up cluster - %q\n%s", err, out) 143 return fmt.Errorf("failed to bring up cluster - %q", err) 144 } 145 glog.V(2).Info(string(out)) 146 glog.V(2).Infof("Giving the cluster 30 sec to stabilize") 147 time.Sleep(30 * time.Second) 148 return nil 149 } 150 151 func destroyCluster(kubeBaseDir string) error { 152 if kubeBaseDir == "" { 153 glog.Infof("Skipping cluster tear down since kubernetes repo base path is not set.") 154 return nil 155 } 156 glog.V(1).Info("Bringing down any existing kube cluster") 157 out, err := runKubeClusterCommand(kubeBaseDir, "kube-down.sh") 158 if err != nil { 159 glog.Errorf("failed to tear down cluster - %q\n%s", err, out) 160 return fmt.Errorf("failed to tear down kube cluster - %q", err) 161 } 162 163 return nil 164 } 165 166 func downloadRelease(workDir, version string) error { 167 // Temporary download path. 168 downloadPath := filepath.Join(workDir, "kube") 169 // Format url. 170 downloadUrl := fmt.Sprintf(imageUrlTemplate, version) 171 glog.V(1).Infof("About to download kube release using url: %q", downloadUrl) 172 173 // Download kube code and store it in a temp dir. 174 if err := exec.Command("wget", downloadUrl, "-O", downloadPath).Run(); err != nil { 175 return fmt.Errorf("failed to wget kubernetes release @ %q - %v", downloadUrl, err) 176 } 177 178 // Un-tar kube release. 179 if err := exec.Command("tar", "-xf", downloadPath, "-C", workDir).Run(); err != nil { 180 return fmt.Errorf("failed to un-tar kubernetes release at %q - %v", downloadPath, err) 181 } 182 return nil 183 } 184 185 func getKubeClient() (string, *kclient.Client, error) { 186 c, err := kclientcmd.LoadFromFile(*kubeConfig) 187 if err != nil { 188 return "", nil, fmt.Errorf("error loading kubeConfig: %v", err.Error()) 189 } 190 if c.CurrentContext == "" || len(c.Clusters) == 0 { 191 return "", nil, fmt.Errorf("invalid kubeConfig: %+v", *c) 192 } 193 config, err := kclientcmd.NewDefaultClientConfig( 194 *c, 195 &kclientcmd.ConfigOverrides{ 196 ClusterInfo: kclientcmdapi.Cluster{ 197 APIVersion: "v1", 198 }, 199 }).ClientConfig() 200 if err != nil { 201 return "", nil, fmt.Errorf("error parsing kubeConfig: %v", err.Error()) 202 } 203 kubeClient, err := kclient.New(config) 204 if err != nil { 205 return "", nil, fmt.Errorf("error creating client - %q", err) 206 } 207 208 return c.Clusters[c.CurrentContext].Server, kubeClient, nil 209 } 210 211 func validateCluster(baseDir string) bool { 212 glog.V(1).Info("validating existing cluster") 213 out, err := runKubeClusterCommand(baseDir, "validate-cluster.sh") 214 if err != nil { 215 glog.V(1).Infof("cluster validation failed - %q\n %s", err, out) 216 return false 217 } 218 return true 219 } 220 221 func requireNewCluster(baseDir, version string) bool { 222 // Setup kube client 223 _, kubeClient, err := getKubeClient() 224 if err != nil { 225 glog.V(1).Infof("kube client creation failed - %q", err) 226 return true 227 } 228 glog.V(1).Infof("checking if existing cluster can be used") 229 versionInfo, err := kubeClient.ServerVersion() 230 if err != nil { 231 glog.V(1).Infof("failed to get kube version info - %q", err) 232 return true 233 } 234 return !strings.Contains(versionInfo.GitVersion, version) 235 } 236 237 func requireDownload(baseDir string) bool { 238 // Check that cluster scripts are present. 239 return !exists(filepath.Join(baseDir, "cluster", "kube-up.sh")) || 240 !exists(filepath.Join(baseDir, "cluster", "kube-down.sh")) || 241 !exists(filepath.Join(baseDir, "cluster", "validate-cluster.sh")) 242 } 243 244 func downloadAndSetupCluster(version string) (baseDir string, err error) { 245 // Create a temp dir to store the kube release files. 246 tempDir := filepath.Join(*workDir, version) 247 if !exists(tempDir) { 248 if err := os.MkdirAll(tempDir, 0700); err != nil { 249 return "", fmt.Errorf("failed to create a temp dir at %s - %q", tempDir, err) 250 } 251 glog.V(1).Infof("Successfully setup work dir at %s", tempDir) 252 } 253 254 kubeBaseDir := filepath.Join(tempDir, "kubernetes") 255 256 if requireDownload(kubeBaseDir) { 257 if exists(kubeBaseDir) { 258 os.RemoveAll(kubeBaseDir) 259 } 260 if err := downloadRelease(tempDir, version); err != nil { 261 return "", err 262 } 263 glog.V(1).Infof("Successfully downloaded kubernetes release at %s", tempDir) 264 } 265 266 // Disable monitoring 267 if err := disableClusterMonitoring(kubeBaseDir); err != nil { 268 return "", fmt.Errorf("failed to disable cluster monitoring in kube cluster config - %q", err) 269 } 270 glog.V(1).Info("Disabled cluster monitoring") 271 if !requireNewCluster(kubeBaseDir, version) { 272 glog.V(1).Infof("skipping cluster setup since a cluster with required version already exists") 273 return kubeBaseDir, nil 274 } 275 276 // Setup kube cluster 277 glog.V(1).Infof("Setting up new kubernetes cluster version: %s", version) 278 if err := setupNewCluster(kubeBaseDir); err != nil { 279 // Cluster setup failed for some reason. 280 // Attempting to validate the cluster to see if it failed in the validate phase. 281 sleepDuration := 10 * time.Second 282 clusterReady := false 283 for i := 0; i < int(time.Minute/sleepDuration); i++ { 284 if !validateCluster(kubeBaseDir) { 285 glog.Infof("Retry validation after %v seconds.", sleepDuration/time.Second) 286 time.Sleep(sleepDuration) 287 } else { 288 clusterReady = true 289 break 290 } 291 } 292 if !clusterReady { 293 return "", fmt.Errorf("failed to setup cluster - %q", err) 294 } 295 } 296 glog.V(1).Infof("Successfully setup new kubernetes cluster version %s", version) 297 298 return kubeBaseDir, nil 299 } 300 301 func newKubeFramework(version string) (kubeFramework, error) { 302 var err error 303 kubeBaseDir := "" 304 if version != "" { 305 if len(strings.Split(version, ".")) != 3 { 306 return nil, fmt.Errorf("invalid kubernetes version specified - %q", version) 307 } 308 kubeBaseDir, err = downloadAndSetupCluster(version) 309 if err != nil { 310 return nil, err 311 } 312 } 313 314 // Setup kube client 315 masterIP, kubeClient, err := getKubeClient() 316 if err != nil { 317 return nil, err 318 } 319 return &realKubeFramework{ 320 kubeClient: kubeClient, 321 baseDir: kubeBaseDir, 322 version: version, 323 masterIP: masterIP, 324 }, nil 325 } 326 327 func (self *realKubeFramework) Client() *kclient.Client { 328 return self.kubeClient 329 } 330 331 func (self *realKubeFramework) loadObject(filePath string) (runtime.Object, error) { 332 data, err := ioutil.ReadFile(filePath) 333 if err != nil { 334 return nil, fmt.Errorf("failed to read object: %v", err) 335 } 336 obj, _, err := api.Codecs.UniversalDecoder().Decode(data, nil, nil) 337 return obj, err 338 } 339 340 func (self *realKubeFramework) ParseRC(filePath string) (*api.ReplicationController, error) { 341 obj, err := self.loadObject(filePath) 342 if err != nil { 343 return nil, err 344 } 345 346 rc, ok := obj.(*api.ReplicationController) 347 if !ok { 348 return nil, fmt.Errorf("Failed to cast replicationController: %v", obj) 349 } 350 return rc, nil 351 } 352 353 func (self *realKubeFramework) ParseService(filePath string) (*api.Service, error) { 354 obj, err := self.loadObject(filePath) 355 if err != nil { 356 return nil, err 357 } 358 service, ok := obj.(*api.Service) 359 if !ok { 360 return nil, fmt.Errorf("Failed to cast service: %v", obj) 361 } 362 return service, nil 363 } 364 365 func (self *realKubeFramework) CreateService(ns string, service *api.Service) (*api.Service, error) { 366 service.Namespace = ns 367 newSvc, err := self.kubeClient.Services(ns).Create(service) 368 return newSvc, err 369 } 370 371 func (self *realKubeFramework) DeleteNs(ns string) error { 372 373 _, err := self.kubeClient.Namespaces().Get(ns) 374 if err != nil { 375 glog.V(0).Infof("Cannot get namespace %q. Skipping deletion: %s", ns, err) 376 return nil 377 } 378 glog.V(0).Infof("Deleting namespace %s", ns) 379 self.kubeClient.Namespaces().Delete(ns) 380 381 for i := 0; i < 5; i++ { 382 glog.V(0).Infof("Checking for namespace %s", ns) 383 _, err := self.kubeClient.Namespaces().Get(ns) 384 if err != nil { 385 glog.V(0).Infof("%s doesn't exist", ns) 386 return nil 387 } 388 time.Sleep(10 * time.Second) 389 } 390 return fmt.Errorf("Namespace %s still exists", ns) 391 } 392 393 func (self *realKubeFramework) CreateNs(ns *api.Namespace) (*api.Namespace, error) { 394 return self.kubeClient.Namespaces().Create(ns) 395 } 396 397 func (self *realKubeFramework) CreateRC(ns string, rc *api.ReplicationController) (*api.ReplicationController, error) { 398 rc.Namespace = ns 399 return self.kubeClient.ReplicationControllers(ns).Create(rc) 400 } 401 402 func (self *realKubeFramework) DestroyCluster() { 403 destroyCluster(self.baseDir) 404 } 405 406 func (self *realKubeFramework) GetProxyUrlForService(service *api.Service) string { 407 return fmt.Sprintf("%s/api/v1/proxy/namespaces/default/services/%s/", self.masterIP, service.Name) 408 } 409 410 func (self *realKubeFramework) GetNodes() ([]string, error) { 411 var nodes []string 412 nodeList, err := self.kubeClient.Nodes().List(api.ListOptions{ 413 LabelSelector: labels.Everything(), 414 FieldSelector: fields.Everything(), 415 }) 416 if err != nil { 417 return nodes, err 418 } 419 420 for _, node := range nodeList.Items { 421 nodes = append(nodes, node.Name) 422 } 423 return nodes, nil 424 } 425 426 func (self *realKubeFramework) GetRunningPods() ([]api.Pod, error) { 427 glog.V(0).Infof("Getting running pods") 428 podList, err := self.kubeClient.Pods(api.NamespaceAll).List(api.ListOptions{ 429 LabelSelector: labels.Everything(), 430 FieldSelector: fields.Everything(), 431 }) 432 if err != nil { 433 return nil, err 434 } 435 pods := []api.Pod{} 436 for _, pod := range podList.Items { 437 if pod.Status.Phase == api.PodRunning && !strings.Contains(pod.Spec.NodeName, "kubernetes-master") { 438 pods = append(pods, pod) 439 } 440 } 441 return pods, nil 442 } 443 444 func (self *realKubeFramework) GetRunningPodNames() ([]string, error) { 445 var pods []string 446 podList, err := self.GetRunningPods() 447 if err != nil { 448 return pods, err 449 } 450 for _, pod := range podList { 451 pods = append(pods, string(pod.Name)) 452 } 453 return pods, nil 454 } 455 456 func (rkf *realKubeFramework) WaitUntilPodRunning(ns string, podLabels map[string]string, timeout time.Duration) error { 457 glog.V(2).Infof("Waiting for pod %v in %s...", podLabels, ns) 458 podsInterface := rkf.Client().Pods(ns) 459 for i := 0; i < int(timeout/time.Second); i++ { 460 podList, err := podsInterface.List(api.ListOptions{ 461 LabelSelector: labels.Set(podLabels).AsSelector(), 462 FieldSelector: fields.Everything(), 463 }) 464 if err != nil { 465 glog.V(1).Info(err) 466 return err 467 } 468 if len(podList.Items) > 0 { 469 podSpec := podList.Items[0] 470 if podSpec.Status.Phase == api.PodRunning { 471 return nil 472 } 473 } 474 time.Sleep(time.Second) 475 } 476 return fmt.Errorf("pod not in running state after %d", timeout/time.Second) 477 } 478 479 func (rkf *realKubeFramework) WaitUntilServiceActive(svc *api.Service, timeout time.Duration) error { 480 glog.V(2).Infof("Waiting for endpoints in service %s/%s", svc.Namespace, svc.Name) 481 for i := 0; i < int(timeout/time.Second); i++ { 482 e, err := rkf.Client().Endpoints(svc.Namespace).Get(svc.Name) 483 if err != nil { 484 return err 485 } 486 if len(e.Subsets) > 0 { 487 return nil 488 } 489 time.Sleep(time.Second) 490 } 491 492 return fmt.Errorf("Service %q not active after %d seconds - no endpoints found", svc.Name, timeout/time.Second) 493 494 }