github.com/timstclair/heapster@v0.20.0-alpha1/integration/heapster_api_test.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "encoding/json" 19 "flag" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "strings" 24 "testing" 25 "time" 26 27 "github.com/golang/glog" 28 "github.com/stretchr/testify/require" 29 api_v1 "k8s.io/heapster/metrics/api/v1/types" 30 "k8s.io/heapster/metrics/core" 31 kube_api "k8s.io/kubernetes/pkg/api" 32 apiErrors "k8s.io/kubernetes/pkg/api/errors" 33 kube_api_unv "k8s.io/kubernetes/pkg/api/unversioned" 34 ) 35 36 const ( 37 targetTags = "kubernetes-minion" 38 heapsterBuildDir = "../deploy/docker" 39 ) 40 41 var ( 42 testZone = flag.String("test_zone", "us-central1-b", "GCE zone where the test will be executed") 43 kubeVersions = flag.String("kube_versions", "", "Comma separated list of kube versions to test against. By default will run the test against an existing cluster") 44 heapsterControllerFile = flag.String("heapster_controller", "../deploy/kube-config/standalone-test/heapster-controller.yaml", "Path to heapster replication controller file.") 45 heapsterServiceFile = flag.String("heapster_service", "../deploy/kube-config/standalone-test/heapster-service.yaml", "Path to heapster service file.") 46 heapsterImage = flag.String("heapster_image", "heapster:e2e_test", "heapster docker image that needs to be tested.") 47 avoidBuild = flag.Bool("nobuild", false, "When true, a new heapster docker image will not be created and pushed to test cluster nodes.") 48 namespace = flag.String("namespace", "heapster-e2e-tests", "namespace to be used for testing, it will be deleted at the beginning of the test if exists") 49 maxRetries = flag.Int("retries", 20, "Number of attempts before failing this test.") 50 runForever = flag.Bool("run_forever", false, "If true, the tests are run in a loop forever.") 51 ) 52 53 func deleteAll(fm kubeFramework, ns string, service *kube_api.Service, rc *kube_api.ReplicationController) error { 54 glog.V(2).Infof("Deleting ns %s...", ns) 55 err := fm.DeleteNs(ns) 56 if err != nil { 57 glog.V(2).Infof("Failed to delete %s", ns) 58 return err 59 } 60 glog.V(2).Infof("Deleted ns %s.", ns) 61 return nil 62 } 63 64 func createAll(fm kubeFramework, ns string, service **kube_api.Service, rc **kube_api.ReplicationController) error { 65 glog.V(2).Infof("Creating ns %s...", ns) 66 namespace := kube_api.Namespace{ 67 TypeMeta: kube_api_unv.TypeMeta{ 68 Kind: "Namespace", 69 APIVersion: "v1", 70 }, 71 ObjectMeta: kube_api.ObjectMeta{ 72 Name: ns, 73 }, 74 } 75 if _, err := fm.CreateNs(&namespace); err != nil { 76 glog.V(2).Infof("Failed to create ns: %v", err) 77 return err 78 } 79 80 glog.V(2).Infof("Created ns %s.", ns) 81 82 glog.V(2).Infof("Creating rc %s/%s...", ns, (*rc).Name) 83 if newRc, err := fm.CreateRC(ns, *rc); err != nil { 84 glog.V(2).Infof("Failed to create rc: %v", err) 85 return err 86 } else { 87 *rc = newRc 88 } 89 glog.V(2).Infof("Created rc %s/%s.", ns, (*rc).Name) 90 91 glog.V(2).Infof("Creating service %s/%s...", ns, (*service).Name) 92 if newSvc, err := fm.CreateService(ns, *service); err != nil { 93 glog.V(2).Infof("Failed to create service: %v", err) 94 return err 95 } else { 96 *service = newSvc 97 } 98 glog.V(2).Infof("Created service %s/%s.", ns, (*service).Name) 99 100 return nil 101 } 102 103 func removeHeapsterImage(fm kubeFramework, zone string) error { 104 glog.V(2).Infof("Removing heapster image.") 105 if err := removeDockerImage(*heapsterImage); err != nil { 106 glog.Errorf("Failed to remove Heapster image: %v", err) 107 } else { 108 glog.V(2).Infof("Heapster image removed.") 109 } 110 if nodes, err := fm.GetNodes(); err == nil { 111 for _, node := range nodes { 112 host := strings.Split(node, ".")[0] 113 cleanupRemoteHost(host, zone) 114 } 115 } else { 116 glog.Errorf("failed to cleanup nodes - %v", err) 117 } 118 return nil 119 } 120 121 func buildAndPushHeapsterImage(hostnames []string, zone string) error { 122 glog.V(2).Info("Building and pushing Heapster image...") 123 curwd, err := os.Getwd() 124 if err != nil { 125 return err 126 } 127 if err := os.Chdir(heapsterBuildDir); err != nil { 128 return err 129 } 130 if err := buildDockerImage(*heapsterImage); err != nil { 131 return err 132 } 133 for _, host := range hostnames { 134 if err := copyDockerImage(*heapsterImage, host, zone); err != nil { 135 return err 136 } 137 } 138 glog.V(2).Info("Heapster image pushed.") 139 return os.Chdir(curwd) 140 } 141 142 func getHeapsterRcAndSvc(fm kubeFramework) (*kube_api.Service, *kube_api.ReplicationController, error) { 143 // Add test docker image 144 rc, err := fm.ParseRC(*heapsterControllerFile) 145 if err != nil { 146 return nil, nil, fmt.Errorf("failed to parse heapster controller - %v", err) 147 } 148 for i := range rc.Spec.Template.Spec.Containers { 149 rc.Spec.Template.Spec.Containers[i].Image = *heapsterImage 150 rc.Spec.Template.Spec.Containers[i].ImagePullPolicy = kube_api.PullNever 151 // increase logging level 152 rc.Spec.Template.Spec.Containers[i].Env = append(rc.Spec.Template.Spec.Containers[0].Env, kube_api.EnvVar{Name: "FLAGS", Value: "--vmodule=*=3"}) 153 } 154 155 svc, err := fm.ParseService(*heapsterServiceFile) 156 if err != nil { 157 return nil, nil, fmt.Errorf("failed to parse heapster service - %v", err) 158 } 159 160 return svc, rc, nil 161 } 162 163 func buildAndPushDockerImages(fm kubeFramework, zone string) error { 164 if *avoidBuild { 165 return nil 166 } 167 nodes, err := fm.GetNodes() 168 if err != nil { 169 return err 170 } 171 hostnames := []string{} 172 for _, node := range nodes { 173 hostnames = append(hostnames, strings.Split(node, ".")[0]) 174 } 175 176 return buildAndPushHeapsterImage(hostnames, zone) 177 } 178 179 const ( 180 metricsEndpoint = "/api/v1/metric-export" 181 metricsSchemaEndpoint = "/api/v1/metric-export-schema" 182 sinksEndpoint = "/api/v1/sinks" 183 ) 184 185 func getTimeseries(fm kubeFramework, svc *kube_api.Service) ([]*api_v1.Timeseries, error) { 186 body, err := fm.Client().Get(). 187 Namespace(svc.Namespace). 188 Prefix("proxy"). 189 Resource("services"). 190 Name(svc.Name). 191 Suffix(metricsEndpoint). 192 Do().Raw() 193 if err != nil { 194 return nil, err 195 } 196 var timeseries []*api_v1.Timeseries 197 if err := json.Unmarshal(body, ×eries); err != nil { 198 glog.V(2).Infof("Timeseries error: %v %v", err, string(body)) 199 return nil, err 200 } 201 return timeseries, nil 202 } 203 204 func getSchema(fm kubeFramework, svc *kube_api.Service) (*api_v1.TimeseriesSchema, error) { 205 body, err := fm.Client().Get(). 206 Namespace(svc.Namespace). 207 Prefix("proxy"). 208 Resource("services"). 209 Name(svc.Name). 210 Suffix(metricsSchemaEndpoint). 211 Do().Raw() 212 if err != nil { 213 return nil, err 214 } 215 var timeseriesSchema api_v1.TimeseriesSchema 216 if err := json.Unmarshal(body, ×eriesSchema); err != nil { 217 glog.V(2).Infof("Metrics schema error: %v %v", err, string(body)) 218 return nil, err 219 } 220 return ×eriesSchema, nil 221 } 222 223 var expectedSystemContainers = map[string]struct{}{ 224 "machine": {}, 225 "kubelet": {}, 226 "kube-proxy": {}, 227 "system": {}, 228 "docker-daemon": {}, 229 } 230 231 func isContainerBaseImageExpected(ts *api_v1.Timeseries) bool { 232 _, exists := expectedSystemContainers[ts.Labels[core.LabelContainerName.Key]] 233 return !exists 234 } 235 236 func runHeapsterMetricsTest(fm kubeFramework, svc *kube_api.Service) error { 237 expectedPods, err := fm.GetRunningPodNames() 238 if err != nil { 239 return err 240 } 241 glog.V(0).Infof("Expected pods: %v", expectedPods) 242 243 expectedNodes, err := fm.GetNodes() 244 if err != nil { 245 return err 246 } 247 glog.V(0).Infof("Expected nodes: %v", expectedNodes) 248 249 timeseries, err := getTimeseries(fm, svc) 250 if err != nil { 251 return err 252 } 253 if len(timeseries) == 0 { 254 return fmt.Errorf("expected non zero timeseries") 255 } 256 schema, err := getSchema(fm, svc) 257 if err != nil { 258 return err 259 } 260 // Build a map of metric names to metric descriptors. 261 mdMap := map[string]*api_v1.MetricDescriptor{} 262 for idx := range schema.Metrics { 263 mdMap[schema.Metrics[idx].Name] = &schema.Metrics[idx] 264 } 265 actualPods := map[string]bool{} 266 actualNodes := map[string]bool{} 267 actualSystemContainers := map[string]map[string]struct{}{} 268 for _, ts := range timeseries { 269 // Verify the relevant labels are present. 270 // All common labels must be present. 271 podName, podMetric := ts.Labels[core.LabelPodName.Key] 272 273 for _, label := range core.CommonLabels() { 274 _, exists := ts.Labels[label.Key] 275 if !exists { 276 return fmt.Errorf("timeseries: %v does not contain common label: %v", ts, label) 277 } 278 } 279 if podMetric { 280 for _, label := range core.PodLabels() { 281 _, exists := ts.Labels[label.Key] 282 if !exists { 283 return fmt.Errorf("timeseries: %v does not contain pod label: %v", ts, label) 284 } 285 } 286 } 287 288 if podMetric { 289 actualPods[podName] = true 290 } else { 291 if cName, ok := ts.Labels[core.LabelContainerName.Key]; ok { 292 hostname, ok := ts.Labels[core.LabelHostname.Key] 293 if !ok { 294 return fmt.Errorf("hostname label missing on container %+v", ts) 295 } 296 297 if cName == "machine" { 298 actualNodes[hostname] = true 299 } else { 300 for _, label := range core.ContainerLabels() { 301 if label == core.LabelContainerBaseImage && !isContainerBaseImageExpected(ts) { 302 continue 303 } 304 _, exists := ts.Labels[label.Key] 305 if !exists { 306 return fmt.Errorf("timeseries: %v does not contain container label: %v", ts, label) 307 } 308 } 309 } 310 311 if _, exists := expectedSystemContainers[cName]; exists { 312 if actualSystemContainers[cName] == nil { 313 actualSystemContainers[cName] = map[string]struct{}{} 314 } 315 actualSystemContainers[cName][hostname] = struct{}{} 316 } 317 } else { 318 return fmt.Errorf("container_name label missing on timeseries - %v", ts) 319 } 320 } 321 322 for metricName, points := range ts.Metrics { 323 md, exists := mdMap[metricName] 324 if !exists { 325 return fmt.Errorf("unexpected metric %q", metricName) 326 } 327 328 for _, point := range points { 329 for _, label := range md.Labels { 330 _, exists := point.Labels[label.Key] 331 if !exists { 332 return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label) 333 } 334 } 335 } 336 337 } 338 } 339 // Validate that system containers are running on all the nodes. 340 // This test could fail if one of the containers was down while the metrics sample was collected. 341 for cName, hosts := range actualSystemContainers { 342 for _, host := range expectedNodes { 343 if _, ok := hosts[host]; !ok { 344 return fmt.Errorf("System container %q not found on host: %q - %v", cName, host, actualSystemContainers) 345 } 346 } 347 } 348 349 if err := expectedItemsExist(expectedPods, actualPods); err != nil { 350 return fmt.Errorf("expected pods don't exist %v.\nExpected: %v\nActual:%v", err, expectedPods, actualPods) 351 } 352 if err := expectedItemsExist(expectedNodes, actualNodes); err != nil { 353 return fmt.Errorf("expected nodes don't exist %v.\nExpected: %v\nActual:%v", err, expectedNodes, actualNodes) 354 } 355 356 return nil 357 } 358 359 func expectedItemsExist(expectedItems []string, actualItems map[string]bool) error { 360 for _, item := range expectedItems { 361 if _, found := actualItems[item]; !found { 362 return fmt.Errorf("missing %s", item) 363 } 364 } 365 return nil 366 } 367 368 func getErrorCauses(err error) string { 369 serr, ok := err.(*apiErrors.StatusError) 370 if !ok { 371 return "" 372 } 373 var causes []string 374 for _, c := range serr.ErrStatus.Details.Causes { 375 causes = append(causes, c.Message) 376 } 377 return strings.Join(causes, ", ") 378 } 379 380 func getDataFromProxy(fm kubeFramework, svc *kube_api.Service, url string) ([]byte, error) { 381 glog.V(2).Infof("Querying heapster: %s", url) 382 return fm.Client().Get(). 383 Namespace(svc.Namespace). 384 Prefix("proxy"). 385 Resource("services"). 386 Name(svc.Name). 387 Suffix(url). 388 Do().Raw() 389 } 390 391 func getMetricResultList(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.MetricResultList, error) { 392 body, err := getDataFromProxy(fm, svc, url) 393 if err != nil { 394 return nil, err 395 } 396 var data api_v1.MetricResultList 397 if err := json.Unmarshal(body, &data); err != nil { 398 glog.V(2).Infof("response body: %v", string(body)) 399 return nil, err 400 } 401 if err := checkMetricResultListSanity(&data); err != nil { 402 return nil, err 403 } 404 return &data, nil 405 } 406 407 func getMetricResult(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.MetricResult, error) { 408 body, err := getDataFromProxy(fm, svc, url) 409 if err != nil { 410 return nil, err 411 } 412 var data api_v1.MetricResult 413 if err := json.Unmarshal(body, &data); err != nil { 414 glog.V(2).Infof("response body: %v", string(body)) 415 return nil, err 416 } 417 if err := checkMetricResultSanity(&data); err != nil { 418 return nil, err 419 } 420 return &data, nil 421 } 422 423 func getStringResult(fm kubeFramework, svc *kube_api.Service, url string) ([]string, error) { 424 body, err := getDataFromProxy(fm, svc, url) 425 if err != nil { 426 return nil, err 427 } 428 var data []string 429 if err := json.Unmarshal(body, &data); err != nil { 430 glog.V(2).Infof("response body: %v", string(body)) 431 return nil, err 432 } 433 if len(data) == 0 { 434 return nil, fmt.Errorf("empty string array") 435 } 436 return data, nil 437 } 438 439 func checkMetricResultSanity(metrics *api_v1.MetricResult) error { 440 bytes, err := json.Marshal(*metrics) 441 if err != nil { 442 return err 443 } 444 stringVersion := string(bytes) 445 446 if len(metrics.Metrics) == 0 { 447 return fmt.Errorf("empty metrics: %s", stringVersion) 448 } 449 // There should be recent metrics in the response. 450 if time.Now().Sub(metrics.LatestTimestamp).Seconds() > 120 { 451 return fmt.Errorf("corrupted last timestamp: %s", stringVersion) 452 } 453 // Metrics don't have to be sorted, so the oldest one can be first. 454 if time.Now().Sub(metrics.Metrics[0].Timestamp).Hours() > 1 { 455 return fmt.Errorf("corrupted timestamp: %s", stringVersion) 456 } 457 if metrics.Metrics[0].Value > 10000 { 458 return fmt.Errorf("value too big: %s", stringVersion) 459 } 460 return nil 461 } 462 463 func checkMetricResultListSanity(metrics *api_v1.MetricResultList) error { 464 if len(metrics.Items) == 0 { 465 return fmt.Errorf("empty metrics") 466 } 467 for _, item := range metrics.Items { 468 err := checkMetricResultSanity(&item) 469 if err != nil { 470 return err 471 } 472 } 473 return nil 474 } 475 476 func runModelTest(fm kubeFramework, svc *kube_api.Service) error { 477 podList, err := fm.GetRunningPods() 478 if err != nil { 479 return err 480 } 481 if len(podList) == 0 { 482 return fmt.Errorf("empty pod list") 483 } 484 nodeList, err := fm.GetNodes() 485 if err != nil { 486 return err 487 } 488 if len(nodeList) == 0 { 489 return fmt.Errorf(("empty node list")) 490 } 491 podNamesList := make([]string, 0, len(podList)) 492 for _, pod := range podList { 493 podNamesList = append(podNamesList, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) 494 } 495 496 glog.V(0).Infof("Expected nodes:\n%s", strings.Join(nodeList, "\n")) 497 glog.V(0).Infof("Expected pods:\n%s", strings.Join(podNamesList, "\n")) 498 allkeys, err := getStringResult(fm, svc, "/api/v1/model/debug/allkeys") 499 if err != nil { 500 return fmt.Errorf("Failed to get debug information about keys: %v", err) 501 } 502 glog.V(0).Infof("Available Heapster metric sets:\n%s", strings.Join(allkeys, "\n")) 503 504 metricUrlsToCheck := []string{} 505 batchMetricsUrlsToCheck := []string{} 506 stringUrlsToCheck := []string{} 507 508 /* TODO: enable once cluster aggregator is added. 509 metricUrlsToCheck = append(metricUrlsToCheck, 510 fmt.Sprintf("/api/v1/model/metrics/%s", "cpu-usage"), 511 ) 512 */ 513 514 /* TODO: add once Cluster metrics aggregator is added. 515 "/api/v1/model/metrics", 516 "/api/v1/model/" 517 */ 518 stringUrlsToCheck = append(stringUrlsToCheck) 519 520 for _, node := range nodeList { 521 metricUrlsToCheck = append(metricUrlsToCheck, 522 fmt.Sprintf("/api/v1/model/nodes/%s/metrics/%s", node, "cpu-usage"), 523 ) 524 525 stringUrlsToCheck = append(stringUrlsToCheck, 526 fmt.Sprintf("/api/v1/model/nodes/%s/metrics", node), 527 ) 528 } 529 530 for _, pod := range podList { 531 containerName := pod.Spec.Containers[0].Name 532 533 metricUrlsToCheck = append(metricUrlsToCheck, 534 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics/%s", pod.Namespace, pod.Name, "cpu-usage"), 535 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics/%s", pod.Namespace, pod.Name, containerName, "cpu-usage"), 536 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics/%s", pod.Namespace, "cpu-usage"), 537 ) 538 539 batchMetricsUrlsToCheck = append(batchMetricsUrlsToCheck, 540 fmt.Sprintf("/api/v1/model/namespaces/%s/pod-list/%s,%s/metrics/%s", pod.Namespace, pod.Name, pod.Name, "cpu-usage")) 541 542 stringUrlsToCheck = append(stringUrlsToCheck, 543 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics", pod.Namespace), 544 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics", pod.Namespace, pod.Name), 545 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics", pod.Namespace, pod.Name, containerName), 546 ) 547 } 548 549 for _, url := range metricUrlsToCheck { 550 _, err := getMetricResult(fm, svc, url) 551 if err != nil { 552 return fmt.Errorf("error while querying %s: %v", url, err) 553 } 554 } 555 556 for _, url := range batchMetricsUrlsToCheck { 557 _, err := getMetricResultList(fm, svc, url) 558 if err != nil { 559 return fmt.Errorf("error while querying %s: %v", url, err) 560 } 561 } 562 563 for _, url := range stringUrlsToCheck { 564 _, err := getStringResult(fm, svc, url) 565 if err != nil { 566 return fmt.Errorf("error while querying %s: %v", url, err) 567 } 568 } 569 return nil 570 } 571 572 func apiTest(kubeVersion string, zone string) error { 573 fm, err := newKubeFramework(kubeVersion) 574 if err != nil { 575 return err 576 } 577 if err := buildAndPushDockerImages(fm, zone); err != nil { 578 return err 579 } 580 // Create heapster pod and service. 581 svc, rc, err := getHeapsterRcAndSvc(fm) 582 if err != nil { 583 return err 584 } 585 ns := *namespace 586 if err := deleteAll(fm, ns, svc, rc); err != nil { 587 return err 588 } 589 if err := createAll(fm, ns, &svc, &rc); err != nil { 590 return err 591 } 592 if err := fm.WaitUntilPodRunning(ns, rc.Spec.Template.Labels, time.Minute); err != nil { 593 return err 594 } 595 if err := fm.WaitUntilServiceActive(svc, time.Minute); err != nil { 596 return err 597 } 598 testFuncs := []func() error{ 599 func() error { 600 glog.V(2).Infof("Heapster metrics test...") 601 err := runHeapsterMetricsTest(fm, svc) 602 if err == nil { 603 glog.V(2).Infof("Heapster metrics test: OK") 604 } else { 605 glog.V(2).Infof("Heapster metrics test error: %v", err) 606 } 607 return err 608 }, 609 /* 610 TODO(mwielgus): Enable once dynamic sink setting is enabled. 611 func() error { 612 glog.V(2).Infof("Sinks test...") 613 err := runSinksTest(fm, svc) 614 if err == nil { 615 glog.V(2).Infof("Sinks test: OK") 616 } else { 617 glog.V(2).Infof("Sinks test error: %v", err) 618 } 619 return err 620 }, */ 621 func() error { 622 glog.V(2).Infof("Model test") 623 err := runModelTest(fm, svc) 624 if err == nil { 625 glog.V(2).Infof("Model test: OK") 626 } else { 627 glog.V(2).Infof("Model test error: %v", err) 628 } 629 return err 630 }, 631 } 632 attempts := *maxRetries 633 glog.Infof("Starting tests") 634 for { 635 var err error 636 for _, testFunc := range testFuncs { 637 if err = testFunc(); err != nil { 638 break 639 } 640 } 641 if *runForever { 642 continue 643 } 644 if err == nil { 645 glog.V(2).Infof("All tests passed.") 646 break 647 } 648 if attempts == 0 { 649 glog.V(2).Info("Too many attempts.") 650 return err 651 } 652 glog.V(2).Infof("Some tests failed. Retrying.") 653 attempts-- 654 time.Sleep(time.Second * 10) 655 } 656 deleteAll(fm, ns, svc, rc) 657 removeHeapsterImage(fm, zone) 658 return nil 659 } 660 661 func runApiTest() error { 662 tempDir, err := ioutil.TempDir("", "deploy") 663 if err != nil { 664 return nil 665 } 666 defer os.RemoveAll(tempDir) 667 if *kubeVersions == "" { 668 return apiTest("", *testZone) 669 } 670 kubeVersionsList := strings.Split(*kubeVersions, ",") 671 for _, kubeVersion := range kubeVersionsList { 672 if err := apiTest(kubeVersion, *testZone); err != nil { 673 return err 674 } 675 } 676 return nil 677 } 678 679 func TestHeapster(t *testing.T) { 680 if testing.Short() { 681 t.Skip("skipping heapster kubernetes integration test.") 682 } 683 require.NoError(t, runApiTest()) 684 }