github.com/we87/heapster@v0.18.2/integration/heapster_api_test.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "encoding/json" 19 "flag" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "reflect" 24 "strings" 25 "testing" 26 "time" 27 28 "github.com/golang/glog" 29 "github.com/stretchr/testify/require" 30 api_v1 "k8s.io/heapster/api/v1/types" 31 "k8s.io/heapster/model" 32 sink_api "k8s.io/heapster/sinks/api" 33 "k8s.io/heapster/sinks/cache" 34 kube_api "k8s.io/kubernetes/pkg/api" 35 apiErrors "k8s.io/kubernetes/pkg/api/errors" 36 ) 37 38 const ( 39 kTestZone = "us-central1-b" 40 targetTags = "kubernetes-minion" 41 heapsterBuildDir = "../deploy/docker" 42 ) 43 44 var ( 45 kubeVersions = flag.String("kube_versions", "", "Comma separated list of kube versions to test against. By default will run the test against an existing cluster") 46 heapsterControllerFile = flag.String("heapster_controller", "../deploy/kube-config/standalone/heapster-controller.json", "Path to heapster replication controller file.") 47 heapsterServiceFile = flag.String("heapster_service", "../deploy/kube-config/standalone/heapster-service.json", "Path to heapster service file.") 48 heapsterImage = flag.String("heapster_image", "heapster:e2e_test", "heapster docker image that needs to be tested.") 49 avoidBuild = flag.Bool("nobuild", false, "When true, a new heapster docker image will not be created and pushed to test cluster nodes.") 50 namespace = flag.String("namespace", "default", "namespace to be used for testing") 51 maxRetries = flag.Int("retries", 100, "Number of attempts before failing this test.") 52 runForever = flag.Bool("run_forever", false, "If true, the tests are run in a loop forever.") 53 ) 54 55 func deleteAll(fm kubeFramework, ns string, service *kube_api.Service, rc *kube_api.ReplicationController) error { 56 glog.V(2).Infof("Deleting rc %s/%s...", ns, rc.Name) 57 retries := 5 58 for { 59 if err := fm.DeleteRC(ns, rc); err != nil { 60 glog.V(2).Infof("Failed to delete rc: %v", err) 61 if retries <= 0 { 62 return err 63 } 64 } else { 65 break 66 } 67 retries-- 68 } 69 glog.V(2).Infof("Deleted rc %s/%s.", ns, rc.Name) 70 71 glog.V(2).Infof("Deleting service %s/%s.", ns, service.Name) 72 retries = 5 73 for { 74 if err := fm.DeleteService(ns, service); err != nil { 75 glog.V(2).Infof("Failed to delete service: %v", err) 76 if retries <= 0 { 77 return err 78 } 79 } else { 80 break 81 } 82 retries-- 83 } 84 glog.V(2).Infof("Deleted service %s/%s.", ns, service.Name) 85 return nil 86 } 87 88 func createAll(fm kubeFramework, ns string, service **kube_api.Service, rc **kube_api.ReplicationController) error { 89 glog.V(2).Infof("Creating rc %s/%s...", ns, (*rc).Name) 90 if newRc, err := fm.CreateRC(ns, *rc); err != nil { 91 glog.V(2).Infof("Failed to create rc: %v", err) 92 return err 93 } else { 94 *rc = newRc 95 } 96 glog.V(2).Infof("Created rc %s/%s.", ns, (*rc).Name) 97 98 glog.V(2).Infof("Creating service %s/%s...", ns, (*service).Name) 99 if newSvc, err := fm.CreateService(ns, *service); err != nil { 100 glog.V(2).Infof("Failed to create service: %v", err) 101 return err 102 } else { 103 *service = newSvc 104 } 105 glog.V(2).Infof("Created servuce %s/%s.", ns, (*service).Name) 106 107 return nil 108 } 109 110 func removeHeapsterImage(fm kubeFramework) error { 111 glog.V(2).Infof("Removing heapster image.") 112 if err := removeDockerImage(*heapsterImage); err != nil { 113 glog.Errorf("Failed to remove Heapster image: %v", err) 114 } else { 115 glog.V(2).Infof("Heapster image removed.") 116 } 117 if nodes, err := fm.GetNodes(); err == nil { 118 for _, node := range nodes { 119 host := strings.Split(node, ".")[0] 120 cleanupRemoteHost(host, kTestZone) 121 } 122 } else { 123 glog.Errorf("failed to cleanup nodes - %v", err) 124 } 125 return nil 126 } 127 128 func buildAndPushHeapsterImage(hostnames []string) error { 129 glog.V(2).Info("Building and pushing Heapster image...") 130 curwd, err := os.Getwd() 131 if err != nil { 132 return err 133 } 134 if err := os.Chdir(heapsterBuildDir); err != nil { 135 return err 136 } 137 if err := buildDockerImage(*heapsterImage); err != nil { 138 return err 139 } 140 for _, host := range hostnames { 141 if err := copyDockerImage(*heapsterImage, host, kTestZone); err != nil { 142 return err 143 } 144 } 145 glog.V(2).Info("Heapster image pushed.") 146 return os.Chdir(curwd) 147 } 148 149 func getHeapsterRcAndSvc(fm kubeFramework) (*kube_api.Service, *kube_api.ReplicationController, error) { 150 // Add test docker image 151 rc, err := fm.ParseRC(*heapsterControllerFile) 152 if err != nil { 153 return nil, nil, fmt.Errorf("failed to parse heapster controller - %v", err) 154 } 155 rc.Spec.Template.Spec.Containers[0].Image = *heapsterImage 156 rc.Spec.Template.Spec.Containers[0].ImagePullPolicy = kube_api.PullNever 157 // increase logging level 158 rc.Spec.Template.Spec.Containers[0].Env = append(rc.Spec.Template.Spec.Containers[0].Env, kube_api.EnvVar{Name: "FLAGS", Value: "--vmodule=*=3"}) 159 svc, err := fm.ParseService(*heapsterServiceFile) 160 if err != nil { 161 return nil, nil, fmt.Errorf("failed to parse heapster service - %v", err) 162 } 163 164 return svc, rc, nil 165 } 166 167 func buildAndPushDockerImages(fm kubeFramework) error { 168 if *avoidBuild { 169 return nil 170 } 171 nodes, err := fm.GetNodes() 172 if err != nil { 173 return err 174 } 175 hostnames := []string{} 176 for _, node := range nodes { 177 hostnames = append(hostnames, strings.Split(node, ".")[0]) 178 } 179 180 return buildAndPushHeapsterImage(hostnames) 181 } 182 183 const ( 184 metricsEndpoint = "/api/v1/metric-export" 185 metricsSchemaEndpoint = "/api/v1/metric-export-schema" 186 sinksEndpoint = "/api/v1/sinks" 187 ) 188 189 func getTimeseries(fm kubeFramework, svc *kube_api.Service) ([]*api_v1.Timeseries, error) { 190 body, err := fm.Client().Get(). 191 Namespace(svc.Namespace). 192 Prefix("proxy"). 193 Resource("services"). 194 Name(svc.Name). 195 Suffix(metricsEndpoint). 196 Do().Raw() 197 if err != nil { 198 return nil, err 199 } 200 var timeseries []*api_v1.Timeseries 201 if err := json.Unmarshal(body, ×eries); err != nil { 202 glog.V(2).Infof("response body: %v", string(body)) 203 return nil, err 204 } 205 return timeseries, nil 206 } 207 208 func getSchema(fm kubeFramework, svc *kube_api.Service) (*api_v1.TimeseriesSchema, error) { 209 body, err := fm.Client().Get(). 210 Namespace(svc.Namespace). 211 Prefix("proxy"). 212 Resource("services"). 213 Name(svc.Name). 214 Suffix(metricsSchemaEndpoint). 215 Do().Raw() 216 if err != nil { 217 return nil, err 218 } 219 var timeseriesSchema api_v1.TimeseriesSchema 220 if err := json.Unmarshal(body, ×eriesSchema); err != nil { 221 glog.V(2).Infof("response body: %v", string(body)) 222 return nil, err 223 } 224 return ×eriesSchema, nil 225 } 226 227 var expectedSystemContainers = map[string]struct{}{ 228 "machine": {}, 229 "kubelet": {}, 230 "kube-proxy": {}, 231 "system": {}, 232 "docker-daemon": {}, 233 } 234 235 func runHeapsterMetricsTest(fm kubeFramework, svc *kube_api.Service) error { 236 expectedPods, err := fm.GetRunningPodNames() 237 if err != nil { 238 return err 239 } 240 expectedNodes, err := fm.GetNodes() 241 if err != nil { 242 return err 243 } 244 245 timeseries, err := getTimeseries(fm, svc) 246 if err != nil { 247 return err 248 } 249 if len(timeseries) == 0 { 250 return fmt.Errorf("expected non zero timeseries") 251 } 252 schema, err := getSchema(fm, svc) 253 if err != nil { 254 return err 255 } 256 // Build a map of metric names to metric descriptors. 257 mdMap := map[string]*api_v1.MetricDescriptor{} 258 for idx := range schema.Metrics { 259 mdMap[schema.Metrics[idx].Name] = &schema.Metrics[idx] 260 } 261 actualPods := map[string]bool{} 262 actualNodes := map[string]bool{} 263 actualSystemContainers := map[string]map[string]struct{}{} 264 for _, ts := range timeseries { 265 // Verify the relevant labels are present. 266 // All common labels must be present. 267 for _, label := range sink_api.CommonLabels() { 268 _, exists := ts.Labels[label.Key] 269 if !exists { 270 return fmt.Errorf("timeseries: %v does not contain common label: %v", ts, label) 271 } 272 } 273 podName, podMetric := ts.Labels[sink_api.LabelPodName.Key] 274 if podMetric { 275 for _, label := range sink_api.PodLabels() { 276 _, exists := ts.Labels[label.Key] 277 if !exists { 278 return fmt.Errorf("timeseries: %v does not contain pod label: %v", ts, label) 279 } 280 } 281 } 282 if podMetric { 283 actualPods[podName] = true 284 } else { 285 if cName, ok := ts.Labels[sink_api.LabelContainerName.Key]; ok { 286 hostname, ok := ts.Labels[sink_api.LabelHostname.Key] 287 if !ok { 288 return fmt.Errorf("hostname label missing on container %+v", ts) 289 } 290 291 if cName == cache.NodeContainerName { 292 actualNodes[hostname] = true 293 } 294 if _, exists := expectedSystemContainers[cName]; exists { 295 if actualSystemContainers[cName] == nil { 296 actualSystemContainers[cName] = map[string]struct{}{} 297 } 298 actualSystemContainers[cName][hostname] = struct{}{} 299 } 300 } else { 301 return fmt.Errorf("container_name label missing on timeseries - %v", ts) 302 } 303 } 304 305 for metricName, points := range ts.Metrics { 306 md, exists := mdMap[metricName] 307 if !exists { 308 return fmt.Errorf("unexpected metric %q", metricName) 309 } 310 for _, point := range points { 311 for _, label := range md.Labels { 312 _, exists := point.Labels[label.Key] 313 if !exists { 314 return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label) 315 } 316 } 317 } 318 319 } 320 } 321 // Validate that system containers are running on all the nodes. 322 // This test could fail if one of the containers was down while the metrics sample was collected. 323 for cName, hosts := range actualSystemContainers { 324 for _, host := range expectedNodes { 325 if _, ok := hosts[host]; !ok { 326 return fmt.Errorf("System container %q not found on host: %q - %v", cName, host, actualSystemContainers) 327 } 328 } 329 } 330 331 if err := expectedItemsExist(expectedPods, actualPods); err != nil { 332 return fmt.Errorf("expected pods don't exist %v.\nExpected: %v\nActual:%v", err, expectedPods, actualPods) 333 } 334 if err := expectedItemsExist(expectedNodes, actualNodes); err != nil { 335 return fmt.Errorf("expected nodes don't exist %v.\nExpected: %v\nActual:%v", err, expectedNodes, actualNodes) 336 } 337 338 return nil 339 } 340 341 func expectedItemsExist(expectedItems []string, actualItems map[string]bool) error { 342 for _, item := range expectedItems { 343 if _, found := actualItems[item]; !found { 344 return fmt.Errorf("missing %s", item) 345 } 346 } 347 return nil 348 } 349 350 func getSinks(fm kubeFramework, svc *kube_api.Service) ([]string, error) { 351 body, err := fm.Client().Get(). 352 Namespace(svc.Namespace). 353 Prefix("proxy"). 354 Resource("services"). 355 Name(svc.Name). 356 Suffix(sinksEndpoint). 357 Do().Raw() 358 if err != nil { 359 return nil, err 360 } 361 var sinks []string 362 if err := json.Unmarshal(body, &sinks); err != nil { 363 glog.V(2).Infof("response body: %v", string(body)) 364 return nil, err 365 } 366 return sinks, nil 367 } 368 369 func setSinks(fm kubeFramework, svc *kube_api.Service, sinks []string) error { 370 data, err := json.Marshal(sinks) 371 if err != nil { 372 return err 373 } 374 return fm.Client().Post(). 375 Namespace(svc.Namespace). 376 Prefix("proxy"). 377 Resource("services"). 378 Name(svc.Name). 379 Suffix(sinksEndpoint). 380 SetHeader("Content-Type", "application/json"). 381 Body(data). 382 Do().Error() 383 } 384 385 func getErrorCauses(err error) string { 386 serr, ok := err.(*apiErrors.StatusError) 387 if !ok { 388 return "" 389 } 390 var causes []string 391 for _, c := range serr.ErrStatus.Details.Causes { 392 causes = append(causes, c.Message) 393 } 394 return strings.Join(causes, ", ") 395 } 396 397 func runSinksTest(fm kubeFramework, svc *kube_api.Service) error { 398 for _, newSinks := range [...][]string{ 399 {}, 400 { 401 "gcm", 402 }, 403 {}, 404 } { 405 if err := setSinks(fm, svc, newSinks); err != nil { 406 glog.Errorf("Could not set sinks. Causes: %s", getErrorCauses(err)) 407 return err 408 } 409 sinks, err := getSinks(fm, svc) 410 if err != nil { 411 return err 412 } 413 if !reflect.DeepEqual(sinks, newSinks) { 414 return fmt.Errorf("expected %v sinks, found %v", newSinks, sinks) 415 } 416 } 417 return nil 418 } 419 420 func getDataFromProxy(fm kubeFramework, svc *kube_api.Service, url string) ([]byte, error) { 421 glog.V(2).Infof("Querying heapster: %s", url) 422 return fm.Client().Get(). 423 Namespace(svc.Namespace). 424 Prefix("proxy"). 425 Resource("services"). 426 Name(svc.Name). 427 Suffix(url). 428 Do().Raw() 429 } 430 431 func getMetricResultList(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.MetricResultList, error) { 432 body, err := getDataFromProxy(fm, svc, url) 433 if err != nil { 434 return nil, err 435 } 436 var data api_v1.MetricResultList 437 if err := json.Unmarshal(body, &data); err != nil { 438 glog.V(2).Infof("response body: %v", string(body)) 439 return nil, err 440 } 441 if err := checkMetricResultListSanity(&data); err != nil { 442 return nil, err 443 } 444 return &data, nil 445 } 446 447 func getMetricResult(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.MetricResult, error) { 448 body, err := getDataFromProxy(fm, svc, url) 449 if err != nil { 450 return nil, err 451 } 452 var data api_v1.MetricResult 453 if err := json.Unmarshal(body, &data); err != nil { 454 glog.V(2).Infof("response body: %v", string(body)) 455 return nil, err 456 } 457 if err := checkMetricResultSanity(&data); err != nil { 458 return nil, err 459 } 460 return &data, nil 461 } 462 463 func getStringResult(fm kubeFramework, svc *kube_api.Service, url string) ([]string, error) { 464 body, err := getDataFromProxy(fm, svc, url) 465 if err != nil { 466 return nil, err 467 } 468 var data []string 469 if err := json.Unmarshal(body, &data); err != nil { 470 glog.V(2).Infof("response body: %v", string(body)) 471 return nil, err 472 } 473 if len(data) == 0 { 474 return nil, fmt.Errorf("empty string array") 475 } 476 return data, nil 477 } 478 479 func getStatsResponse(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.StatsResponse, error) { 480 body, err := getDataFromProxy(fm, svc, url) 481 if err != nil { 482 return nil, err 483 } 484 var data api_v1.StatsResponse 485 if err := json.Unmarshal(body, &data); err != nil { 486 glog.V(2).Infof("response body: %v", string(body)) 487 return nil, err 488 } 489 if len(data.Stats) == 0 { 490 return nil, fmt.Errorf("empty stats") 491 } 492 return &data, nil 493 } 494 495 func getEntityListEntry(fm kubeFramework, svc *kube_api.Service, url string) ([]model.EntityListEntry, error) { 496 body, err := getDataFromProxy(fm, svc, url) 497 if err != nil { 498 return nil, err 499 } 500 var data []model.EntityListEntry 501 if err := json.Unmarshal(body, &data); err != nil { 502 glog.V(2).Infof("response body: %v", string(body)) 503 return nil, err 504 } 505 if len(data) == 0 { 506 return nil, fmt.Errorf("empty data") 507 } 508 return data, nil 509 } 510 511 func checkMetricResultSanity(metrics *api_v1.MetricResult) error { 512 if len(metrics.Metrics) == 0 { 513 return fmt.Errorf("empty metrics") 514 } 515 if time.Now().Sub(metrics.LatestTimestamp).Seconds() > 120 { 516 return fmt.Errorf("corrupted last timestamp") 517 } 518 if time.Now().Sub(metrics.Metrics[0].Timestamp).Seconds() > 120 { 519 return fmt.Errorf("corrupted timestamp") 520 } 521 if metrics.Metrics[0].Value > 10000 { 522 return fmt.Errorf("value too big") 523 } 524 return nil 525 } 526 527 func checkMetricResultListSanity(metrics *api_v1.MetricResultList) error { 528 if len(metrics.Items) == 0 { 529 return fmt.Errorf("empty metrics") 530 } 531 for _, item := range metrics.Items { 532 err := checkMetricResultSanity(&item) 533 if err != nil { 534 return err 535 } 536 } 537 return nil 538 } 539 540 func runModelTest(fm kubeFramework, svc *kube_api.Service) error { 541 podList, err := fm.GetRunningPods() 542 if err != nil { 543 return err 544 } 545 if len(podList) == 0 { 546 return fmt.Errorf("empty pod list") 547 } 548 nodeList, err := fm.GetNodes() 549 if err != nil { 550 return err 551 } 552 if len(nodeList) == 0 { 553 return fmt.Errorf(("empty node list")) 554 } 555 556 metricUrlsToCheck := []string{} 557 batchMetricsUrlsToCheck := []string{} 558 stringUrlsToCheck := []string{} 559 entityListEntryUrlsToCheck := []string{} 560 statsUrlsToCheck := []string{} 561 562 metricUrlsToCheck = append(metricUrlsToCheck, 563 fmt.Sprintf("/api/v1/model/metrics/%s", "cpu-usage"), 564 ) 565 566 entityListEntryUrlsToCheck = append(entityListEntryUrlsToCheck, 567 "/api/v1/model/nodes", 568 "/api/v1/model/namespaces", 569 ) 570 571 stringUrlsToCheck = append(stringUrlsToCheck, 572 "/api/v1/model/", 573 "/api/v1/model/metrics", 574 ) 575 576 statsUrlsToCheck = append(statsUrlsToCheck, 577 "/api/v1/model/stats/", 578 ) 579 580 for _, node := range nodeList { 581 metricUrlsToCheck = append(metricUrlsToCheck, 582 fmt.Sprintf("/api/v1/model/nodes/%s/metrics/%s", node, "cpu-usage"), 583 ) 584 585 stringUrlsToCheck = append(stringUrlsToCheck, 586 fmt.Sprintf("/api/v1/model/nodes/%s", node), 587 fmt.Sprintf("/api/v1/model/nodes/%s/metrics", node), 588 ) 589 590 statsUrlsToCheck = append(statsUrlsToCheck, 591 fmt.Sprintf("/api/v1/model/nodes/%s/stats", node), 592 ) 593 594 entityListEntryUrlsToCheck = append(entityListEntryUrlsToCheck, 595 fmt.Sprintf("/api/v1/model/nodes/%s/pods", node), 596 ) 597 } 598 599 for _, pod := range podList { 600 containerName := pod.Spec.Containers[0].Name 601 602 metricUrlsToCheck = append(metricUrlsToCheck, 603 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics/%s", pod.Namespace, pod.Name, "cpu-usage"), 604 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics/%s", pod.Namespace, pod.Name, containerName, "cpu-usage"), 605 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics/%s", pod.Namespace, "cpu-usage"), 606 ) 607 608 batchMetricsUrlsToCheck = append(batchMetricsUrlsToCheck, 609 fmt.Sprintf("/api/v1/model/namespaces/%s/pod-list/%s,%s/metrics/%s", pod.Namespace, pod.Name, pod.Name, "cpu-usage")) 610 611 stringUrlsToCheck = append(stringUrlsToCheck, 612 fmt.Sprintf("/api/v1/model/namespaces/%s", pod.Namespace), 613 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics", pod.Namespace), 614 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s", pod.Namespace, pod.Name), 615 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics", pod.Namespace, pod.Name), 616 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s", pod.Namespace, pod.Name, containerName), 617 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics", pod.Namespace, pod.Name, containerName), 618 ) 619 620 entityListEntryUrlsToCheck = append(entityListEntryUrlsToCheck, 621 fmt.Sprintf("/api/v1/model/namespaces/%s/pods", pod.Namespace), 622 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers", pod.Namespace, pod.Name), 623 ) 624 625 statsUrlsToCheck = append(statsUrlsToCheck, 626 fmt.Sprintf("/api/v1/model/namespaces/%s/stats", pod.Namespace), 627 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/stats", pod.Namespace, pod.Name), 628 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/stats", pod.Namespace, pod.Name, containerName), 629 ) 630 } 631 632 for _, url := range metricUrlsToCheck { 633 _, err := getMetricResult(fm, svc, url) 634 if err != nil { 635 return fmt.Errorf("error while querying %s: %v", url, err) 636 } 637 } 638 639 for _, url := range batchMetricsUrlsToCheck { 640 _, err := getMetricResultList(fm, svc, url) 641 if err != nil { 642 return fmt.Errorf("error while querying %s: %v", url, err) 643 } 644 } 645 646 for _, url := range stringUrlsToCheck { 647 _, err := getStringResult(fm, svc, url) 648 if err != nil { 649 return fmt.Errorf("error while querying %s: %v", url, err) 650 } 651 } 652 653 for _, url := range statsUrlsToCheck { 654 _, err := getStatsResponse(fm, svc, url) 655 if err != nil { 656 return fmt.Errorf("error while querying %s: %v", url, err) 657 } 658 } 659 660 for _, url := range entityListEntryUrlsToCheck { 661 _, err := getEntityListEntry(fm, svc, url) 662 if err != nil { 663 return fmt.Errorf("error while querying %s: %v", url, err) 664 } 665 } 666 667 return nil 668 } 669 670 func apiTest(kubeVersion string) error { 671 fm, err := newKubeFramework(kubeVersion) 672 if err != nil { 673 return err 674 } 675 if err := buildAndPushDockerImages(fm); err != nil { 676 return err 677 } 678 // Create heapster pod and service. 679 svc, rc, err := getHeapsterRcAndSvc(fm) 680 if err != nil { 681 return err 682 } 683 ns := *namespace 684 if err := deleteAll(fm, ns, svc, rc); err != nil { 685 return err 686 } 687 if err := createAll(fm, ns, &svc, &rc); err != nil { 688 return err 689 } 690 if err := fm.WaitUntilPodRunning(ns, rc.Spec.Template.Labels, time.Minute); err != nil { 691 return err 692 } 693 if err := fm.WaitUntilServiceActive(svc, time.Minute); err != nil { 694 return err 695 } 696 testFuncs := []func() error{ 697 func() error { 698 glog.V(2).Infof("Heapster metrics test...") 699 err := runHeapsterMetricsTest(fm, svc) 700 if err == nil { 701 glog.V(2).Infof("Heapster metrics test: OK") 702 } else { 703 glog.V(2).Infof("Heapster metrics test error: %v", err) 704 } 705 return err 706 }, 707 func() error { 708 glog.V(2).Infof("Sinks test...") 709 err := runSinksTest(fm, svc) 710 if err == nil { 711 glog.V(2).Infof("Sinks test: OK") 712 } else { 713 glog.V(2).Infof("Sinks test error: %v", err) 714 } 715 return err 716 }, 717 func() error { 718 glog.V(2).Infof("Model test") 719 err := runModelTest(fm, svc) 720 if err == nil { 721 glog.V(2).Infof("Model test: OK") 722 } else { 723 glog.V(2).Infof("Model test error: %v", err) 724 } 725 return err 726 }, 727 } 728 attempts := *maxRetries 729 glog.Infof("Starting tests") 730 for { 731 var err error 732 for _, testFunc := range testFuncs { 733 if err = testFunc(); err != nil { 734 break 735 } 736 } 737 if *runForever { 738 continue 739 } 740 if err == nil { 741 glog.V(2).Infof("All tests passed.") 742 break 743 } 744 if attempts == 0 { 745 glog.V(2).Info("Too many attempts.") 746 return err 747 } 748 glog.V(2).Infof("Some tests failed. Retrying.") 749 attempts-- 750 time.Sleep(time.Second * 10) 751 } 752 deleteAll(fm, ns, svc, rc) 753 removeHeapsterImage(fm) 754 return nil 755 } 756 757 func runApiTest() error { 758 tempDir, err := ioutil.TempDir("", "deploy") 759 if err != nil { 760 return nil 761 } 762 defer os.RemoveAll(tempDir) 763 if *kubeVersions == "" { 764 return apiTest("") 765 } 766 kubeVersionsList := strings.Split(*kubeVersions, ",") 767 for _, kubeVersion := range kubeVersionsList { 768 if err := apiTest(kubeVersion); err != nil { 769 return err 770 } 771 } 772 return nil 773 } 774 775 func TestHeapster(t *testing.T) { 776 if testing.Short() { 777 t.Skip("skipping heapster kubernetes integration test.") 778 } 779 require.NoError(t, runApiTest()) 780 }