github.com/qsunny/k8s@v0.0.0-20220101153623-e6dca256d5bf/heapster-1.3.0/integration/heapster_api_test.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "encoding/json" 19 "flag" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "strings" 24 "testing" 25 "time" 26 27 "github.com/golang/glog" 28 "github.com/stretchr/testify/require" 29 apiErrors "k8s.io/apimachinery/pkg/api/errors" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/labels" 32 "k8s.io/apimachinery/pkg/util/sets" 33 kube_v1 "k8s.io/client-go/pkg/api/v1" 34 api_v1 "k8s.io/heapster/metrics/api/v1/types" 35 metrics_api "k8s.io/heapster/metrics/apis/metrics/v1alpha1" 36 "k8s.io/heapster/metrics/core" 37 ) 38 39 const ( 40 targetTags = "kubernetes-minion" 41 heapsterBuildDir = "../" 42 ) 43 44 var ( 45 testZone = flag.String("test_zone", "us-central1-b", "GCE zone where the test will be executed") 46 kubeVersions = flag.String("kube_versions", "", "Comma separated list of kube versions to test against. By default will run the test against an existing cluster") 47 heapsterControllerFile = flag.String("heapster_controller", "../deploy/kube-config/standalone-test/heapster-controller.yaml", "Path to heapster replication controller file.") 48 heapsterServiceFile = flag.String("heapster_service", "../deploy/kube-config/standalone-test/heapster-service.yaml", "Path to heapster service file.") 49 heapsterImage = flag.String("heapster_image", "heapster:e2e_test", "heapster docker image that needs to be tested.") 50 avoidBuild = flag.Bool("nobuild", false, "When true, a new heapster docker image will not be created and pushed to test cluster nodes.") 51 namespace = flag.String("namespace", "heapster-e2e-tests", "namespace to be used for testing, it will be deleted at the beginning of the test if exists") 52 maxRetries = flag.Int("retries", 20, "Number of attempts before failing this test.") 53 runForever = flag.Bool("run_forever", false, "If true, the tests are run in a loop forever.") 54 testUser = flag.String("test_user", "", "GCE user when copy file to host") 55 ) 56 57 func deleteAll(fm kubeFramework, ns string, service *kube_v1.Service, rc *kube_v1.ReplicationController) error { 58 glog.V(2).Infof("Deleting ns %s...", ns) 59 err := fm.DeleteNs(ns) 60 if err != nil { 61 glog.V(2).Infof("Failed to delete %s", ns) 62 return err 63 } 64 glog.V(2).Infof("Deleted ns %s.", ns) 65 return nil 66 } 67 68 func createAll(fm kubeFramework, ns string, service **kube_v1.Service, rc **kube_v1.ReplicationController) error { 69 glog.V(2).Infof("Creating ns %s...", ns) 70 namespace := kube_v1.Namespace{ 71 TypeMeta: metav1.TypeMeta{ 72 Kind: "Namespace", 73 APIVersion: "v1", 74 }, 75 ObjectMeta: metav1.ObjectMeta{ 76 Name: ns, 77 }, 78 } 79 if _, err := fm.CreateNs(&namespace); err != nil { 80 glog.V(2).Infof("Failed to create ns: %v", err) 81 return err 82 } 83 84 glog.V(2).Infof("Created ns %s.", ns) 85 86 glog.V(2).Infof("Creating rc %s/%s...", ns, (*rc).Name) 87 if newRc, err := fm.CreateRC(ns, *rc); err != nil { 88 glog.V(2).Infof("Failed to create rc: %v", err) 89 return err 90 } else { 91 *rc = newRc 92 } 93 glog.V(2).Infof("Created rc %s/%s.", ns, (*rc).Name) 94 95 glog.V(2).Infof("Creating service %s/%s...", ns, (*service).Name) 96 if newSvc, err := fm.CreateService(ns, *service); err != nil { 97 glog.V(2).Infof("Failed to create service: %v", err) 98 return err 99 } else { 100 *service = newSvc 101 } 102 glog.V(2).Infof("Created service %s/%s.", ns, (*service).Name) 103 104 return nil 105 } 106 107 func removeHeapsterImage(fm kubeFramework, zone string) error { 108 glog.V(2).Infof("Removing heapster image.") 109 if err := removeDockerImage(*heapsterImage); err != nil { 110 glog.Errorf("Failed to remove Heapster image: %v", err) 111 } else { 112 glog.V(2).Infof("Heapster image removed.") 113 } 114 if nodes, err := fm.GetNodeNames(); err == nil { 115 for _, node := range nodes { 116 host := strings.Split(node, ".")[0] 117 cleanupRemoteHost(host, zone) 118 } 119 } else { 120 glog.Errorf("failed to cleanup nodes - %v", err) 121 } 122 return nil 123 } 124 125 func buildAndPushHeapsterImage(hostnames []string, zone string) error { 126 glog.V(2).Info("Building and pushing Heapster image...") 127 curwd, err := os.Getwd() 128 if err != nil { 129 return err 130 } 131 if err := os.Chdir(heapsterBuildDir); err != nil { 132 return err 133 } 134 if err := buildDockerImage(*heapsterImage); err != nil { 135 return err 136 } 137 for _, host := range hostnames { 138 if err := copyDockerImage(*heapsterImage, host, zone); err != nil { 139 return err 140 } 141 } 142 glog.V(2).Info("Heapster image pushed.") 143 return os.Chdir(curwd) 144 } 145 146 func getHeapsterRcAndSvc(fm kubeFramework) (*kube_v1.Service, *kube_v1.ReplicationController, error) { 147 // Add test docker image 148 rc, err := fm.ParseRC(*heapsterControllerFile) 149 if err != nil { 150 return nil, nil, fmt.Errorf("failed to parse heapster controller - %v", err) 151 } 152 for i := range rc.Spec.Template.Spec.Containers { 153 rc.Spec.Template.Spec.Containers[i].Image = *heapsterImage 154 rc.Spec.Template.Spec.Containers[i].ImagePullPolicy = kube_v1.PullNever 155 // increase logging level 156 rc.Spec.Template.Spec.Containers[i].Env = append(rc.Spec.Template.Spec.Containers[0].Env, kube_v1.EnvVar{Name: "FLAGS", Value: "--vmodule=*=3"}) 157 } 158 159 svc, err := fm.ParseService(*heapsterServiceFile) 160 if err != nil { 161 return nil, nil, fmt.Errorf("failed to parse heapster service - %v", err) 162 } 163 164 return svc, rc, nil 165 } 166 167 func buildAndPushDockerImages(fm kubeFramework, zone string) error { 168 if *avoidBuild { 169 return nil 170 } 171 nodes, err := fm.GetNodeNames() 172 if err != nil { 173 return err 174 } 175 hostnames := []string{} 176 for _, node := range nodes { 177 nodeHost := strings.Split(node, ".")[0] 178 if *testUser != "" { 179 nodeHost = fmt.Sprintf("%s@%s", *testUser, nodeHost) 180 } 181 hostnames = append(hostnames, nodeHost) 182 } 183 184 return buildAndPushHeapsterImage(hostnames, zone) 185 } 186 187 const ( 188 metricsEndpoint = "/api/v1/metric-export" 189 metricsSchemaEndpoint = "/api/v1/metric-export-schema" 190 ) 191 192 func getTimeseries(fm kubeFramework, svc *kube_v1.Service) ([]*api_v1.Timeseries, error) { 193 body, err := fm.Client().Core().RESTClient().Get(). 194 Namespace(svc.Namespace). 195 Prefix("proxy"). 196 Resource("services"). 197 Name(svc.Name). 198 Suffix(metricsEndpoint). 199 Do().Raw() 200 if err != nil { 201 return nil, err 202 } 203 var timeseries []*api_v1.Timeseries 204 if err := json.Unmarshal(body, ×eries); err != nil { 205 glog.V(2).Infof("Timeseries error: %v %v", err, string(body)) 206 return nil, err 207 } 208 return timeseries, nil 209 } 210 211 func getSchema(fm kubeFramework, svc *kube_v1.Service) (*api_v1.TimeseriesSchema, error) { 212 body, err := fm.Client().Core().RESTClient().Get(). 213 Namespace(svc.Namespace). 214 Prefix("proxy"). 215 Resource("services"). 216 Name(svc.Name). 217 Suffix(metricsSchemaEndpoint). 218 Do().Raw() 219 if err != nil { 220 return nil, err 221 } 222 var timeseriesSchema api_v1.TimeseriesSchema 223 if err := json.Unmarshal(body, ×eriesSchema); err != nil { 224 glog.V(2).Infof("Metrics schema error: %v %v", err, string(body)) 225 return nil, err 226 } 227 return ×eriesSchema, nil 228 } 229 230 var expectedSystemContainers = map[string]struct{}{ 231 "machine": {}, 232 "kubelet": {}, 233 "kube-proxy": {}, 234 // TODO(piosz): Uncomment once https://github.com/kubernetes/kubernetes/issues/37453 is fixed 235 // "system": {}, 236 "docker-daemon": {}, 237 } 238 239 func isContainerBaseImageExpected(ts *api_v1.Timeseries) bool { 240 cName := ts.Labels[core.LabelContainerName.Key] 241 // TODO(piosz): remove this if once https://github.com/kubernetes/kubernetes/issues/37453 is fixed 242 if cName == "system" { 243 return false 244 } 245 _, exists := expectedSystemContainers[cName] 246 return !exists 247 } 248 249 func runMetricExportTest(fm kubeFramework, svc *kube_v1.Service) error { 250 podList, err := fm.GetPodsRunningOnNodes() 251 if err != nil { 252 return err 253 } 254 expectedPods := make([]string, 0, len(podList)) 255 for _, pod := range podList { 256 expectedPods = append(expectedPods, pod.Name) 257 } 258 glog.V(0).Infof("Expected pods: %v", expectedPods) 259 260 expectedNodes, err := fm.GetNodeNames() 261 if err != nil { 262 return err 263 } 264 glog.V(0).Infof("Expected nodes: %v", expectedNodes) 265 266 timeseries, err := getTimeseries(fm, svc) 267 if err != nil { 268 return err 269 } 270 if len(timeseries) == 0 { 271 return fmt.Errorf("expected non zero timeseries") 272 } 273 schema, err := getSchema(fm, svc) 274 if err != nil { 275 return err 276 } 277 // Build a map of metric names to metric descriptors. 278 mdMap := map[string]*api_v1.MetricDescriptor{} 279 for idx := range schema.Metrics { 280 mdMap[schema.Metrics[idx].Name] = &schema.Metrics[idx] 281 } 282 actualPods := map[string]bool{} 283 actualNodes := map[string]bool{} 284 actualSystemContainers := map[string]map[string]struct{}{} 285 for _, ts := range timeseries { 286 // Verify the relevant labels are present. 287 // All common labels must be present. 288 podName, podMetric := ts.Labels[core.LabelPodName.Key] 289 290 for _, label := range core.CommonLabels() { 291 _, exists := ts.Labels[label.Key] 292 if !exists { 293 return fmt.Errorf("timeseries: %v does not contain common label: %v", ts, label) 294 } 295 } 296 if podMetric { 297 for _, label := range core.PodLabels() { 298 _, exists := ts.Labels[label.Key] 299 if !exists { 300 return fmt.Errorf("timeseries: %v does not contain pod label: %v", ts, label) 301 } 302 } 303 } 304 305 if podMetric { 306 actualPods[podName] = true 307 // Extra explicit check that the expecte metrics are there: 308 requiredLabels := []string{ 309 core.LabelPodNamespaceUID.Key, 310 core.LabelPodId.Key, 311 core.LabelHostID.Key, 312 // container name is checked later 313 } 314 for _, label := range requiredLabels { 315 _, exists := ts.Labels[label] 316 if !exists { 317 return fmt.Errorf("timeseries: %v does not contain required label: %v", ts, label) 318 } 319 } 320 321 } else { 322 if cName, ok := ts.Labels[core.LabelContainerName.Key]; ok { 323 hostname, ok := ts.Labels[core.LabelHostname.Key] 324 if !ok { 325 return fmt.Errorf("hostname label missing on container %+v", ts) 326 } 327 328 if cName == "machine" { 329 actualNodes[hostname] = true 330 } else { 331 for _, label := range core.ContainerLabels() { 332 if label == core.LabelContainerBaseImage && !isContainerBaseImageExpected(ts) { 333 continue 334 } 335 _, exists := ts.Labels[label.Key] 336 if !exists { 337 return fmt.Errorf("timeseries: %v does not contain container label: %v", ts, label) 338 } 339 } 340 } 341 342 if _, exists := expectedSystemContainers[cName]; exists { 343 if actualSystemContainers[cName] == nil { 344 actualSystemContainers[cName] = map[string]struct{}{} 345 } 346 actualSystemContainers[cName][hostname] = struct{}{} 347 } 348 } else { 349 return fmt.Errorf("container_name label missing on timeseries - %v", ts) 350 } 351 } 352 353 // Explicitly check for resource id 354 explicitRequirement := map[string][]string{ 355 core.MetricFilesystemUsage.MetricDescriptor.Name: {core.LabelResourceID.Key}, 356 core.MetricFilesystemLimit.MetricDescriptor.Name: {core.LabelResourceID.Key}, 357 core.MetricFilesystemAvailable.Name: {core.LabelResourceID.Key}, 358 core.MetricFilesystemInodes.Name: {core.LabelResourceID.Key}, 359 core.MetricFilesystemInodesFree.Name: {core.LabelResourceID.Key}} 360 361 for metricName, points := range ts.Metrics { 362 md, exists := mdMap[metricName] 363 if !exists { 364 return fmt.Errorf("unexpected metric %q", metricName) 365 } 366 367 for _, point := range points { 368 for _, label := range md.Labels { 369 _, exists := point.Labels[label.Key] 370 if !exists { 371 return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label) 372 } 373 } 374 } 375 376 required := explicitRequirement[metricName] 377 for _, label := range required { 378 for _, point := range points { 379 _, exists := point.Labels[label] 380 if !exists { 381 return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label) 382 } 383 } 384 } 385 } 386 } 387 // Validate that system containers are running on all the nodes. 388 // This test could fail if one of the containers was down while the metrics sample was collected. 389 for cName, hosts := range actualSystemContainers { 390 for _, host := range expectedNodes { 391 if _, ok := hosts[host]; !ok { 392 return fmt.Errorf("System container %q not found on host: %q - %v", cName, host, actualSystemContainers) 393 } 394 } 395 } 396 397 if err := expectedItemsExist(expectedPods, actualPods); err != nil { 398 return fmt.Errorf("expected pods don't exist %v.\nExpected: %v\nActual:%v", err, expectedPods, actualPods) 399 } 400 if err := expectedItemsExist(expectedNodes, actualNodes); err != nil { 401 return fmt.Errorf("expected nodes don't exist %v.\nExpected: %v\nActual:%v", err, expectedNodes, actualNodes) 402 } 403 404 return nil 405 } 406 407 func expectedItemsExist(expectedItems []string, actualItems map[string]bool) error { 408 for _, item := range expectedItems { 409 if _, found := actualItems[item]; !found { 410 return fmt.Errorf("missing %s", item) 411 } 412 } 413 return nil 414 } 415 416 func getErrorCauses(err error) string { 417 serr, ok := err.(*apiErrors.StatusError) 418 if !ok { 419 return "" 420 } 421 var causes []string 422 for _, c := range serr.ErrStatus.Details.Causes { 423 causes = append(causes, c.Message) 424 } 425 return strings.Join(causes, ", ") 426 } 427 428 var labelSelectorEverything = labels.Everything() 429 430 func getDataFromProxy(fm kubeFramework, svc *kube_v1.Service, url string) ([]byte, error) { 431 glog.V(2).Infof("Querying heapster: %s", url) 432 return fm.Client().Core().RESTClient().Get(). 433 Namespace(svc.Namespace). 434 Prefix("proxy"). 435 Resource("services"). 436 Name(svc.Name). 437 Suffix(url). 438 Do().Raw() 439 } 440 441 func getDataFromProxyWithSelector(fm kubeFramework, svc *kube_v1.Service, url string, labelSelector *labels.Selector) ([]byte, error) { 442 glog.V(2).Infof("Querying heapster: %s", url) 443 return fm.Client().Core().RESTClient().Get(). 444 Namespace(svc.Namespace). 445 Prefix("proxy"). 446 Resource("services"). 447 Name(svc.Name). 448 Suffix(url).LabelsSelectorParam(*labelSelector). 449 Do().Raw() 450 } 451 452 func getMetricResultList(fm kubeFramework, svc *kube_v1.Service, url string) (*api_v1.MetricResultList, error) { 453 body, err := getDataFromProxy(fm, svc, url) 454 if err != nil { 455 return nil, err 456 } 457 var data api_v1.MetricResultList 458 if err := json.Unmarshal(body, &data); err != nil { 459 glog.V(2).Infof("response body: %v", string(body)) 460 return nil, err 461 } 462 if err := checkMetricResultListSanity(&data); err != nil { 463 return nil, err 464 } 465 return &data, nil 466 } 467 468 func getMetricResult(fm kubeFramework, svc *kube_v1.Service, url string) (*api_v1.MetricResult, error) { 469 body, err := getDataFromProxy(fm, svc, url) 470 if err != nil { 471 return nil, err 472 } 473 var data api_v1.MetricResult 474 if err := json.Unmarshal(body, &data); err != nil { 475 glog.V(2).Infof("response body: %v", string(body)) 476 return nil, err 477 } 478 if err := checkMetricResultSanity(&data); err != nil { 479 return nil, err 480 } 481 return &data, nil 482 } 483 484 func getStringResult(fm kubeFramework, svc *kube_v1.Service, url string) ([]string, error) { 485 body, err := getDataFromProxy(fm, svc, url) 486 if err != nil { 487 return nil, err 488 } 489 var data []string 490 if err := json.Unmarshal(body, &data); err != nil { 491 glog.V(2).Infof("response body: %v", string(body)) 492 return nil, err 493 } 494 if len(data) == 0 { 495 return nil, fmt.Errorf("empty string array") 496 } 497 return data, nil 498 } 499 500 func checkMetricResultSanity(metrics *api_v1.MetricResult) error { 501 bytes, err := json.Marshal(*metrics) 502 if err != nil { 503 return err 504 } 505 stringVersion := string(bytes) 506 507 if len(metrics.Metrics) == 0 { 508 return fmt.Errorf("empty metrics: %s", stringVersion) 509 } 510 // There should be recent metrics in the response. 511 if time.Now().Sub(metrics.LatestTimestamp).Seconds() > 120 { 512 return fmt.Errorf("corrupted last timestamp: %s", stringVersion) 513 } 514 // Metrics don't have to be sorted, so the oldest one can be first. 515 if time.Now().Sub(metrics.Metrics[0].Timestamp).Hours() > 1 { 516 return fmt.Errorf("corrupted timestamp: %s", stringVersion) 517 } 518 if metrics.Metrics[0].Value > 10000 { 519 return fmt.Errorf("value too big: %s", stringVersion) 520 } 521 return nil 522 } 523 524 func checkMetricResultListSanity(metrics *api_v1.MetricResultList) error { 525 if len(metrics.Items) == 0 { 526 return fmt.Errorf("empty metrics") 527 } 528 for _, item := range metrics.Items { 529 err := checkMetricResultSanity(&item) 530 if err != nil { 531 return err 532 } 533 } 534 return nil 535 } 536 537 func runModelTest(fm kubeFramework, svc *kube_v1.Service) error { 538 podList, err := fm.GetPodsRunningOnNodes() 539 if err != nil { 540 return err 541 } 542 if len(podList) == 0 { 543 return fmt.Errorf("empty pod list") 544 } 545 nodeList, err := fm.GetNodeNames() 546 if err != nil { 547 return err 548 } 549 if len(nodeList) == 0 { 550 return fmt.Errorf("empty node list") 551 } 552 podNamesList := make([]string, 0, len(podList)) 553 for _, pod := range podList { 554 podNamesList = append(podNamesList, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) 555 } 556 557 glog.V(0).Infof("Expected pods: %v", podNamesList) 558 glog.V(0).Infof("Expected nodes: %v", nodeList) 559 allkeys, err := getStringResult(fm, svc, "/api/v1/model/debug/allkeys") 560 if err != nil { 561 return fmt.Errorf("Failed to get debug information about keys: %v", err) 562 } 563 glog.V(0).Infof("Available Heapster metric sets: %v", allkeys) 564 565 metricUrlsToCheck := []string{} 566 batchMetricsUrlsToCheck := []string{} 567 stringUrlsToCheck := []string{} 568 569 /* TODO: enable once cluster aggregator is added. 570 metricUrlsToCheck = append(metricUrlsToCheck, 571 fmt.Sprintf("/api/v1/model/metrics/%s", "cpu-usage"), 572 ) 573 */ 574 575 /* TODO: add once Cluster metrics aggregator is added. 576 "/api/v1/model/metrics", 577 "/api/v1/model/" 578 */ 579 stringUrlsToCheck = append(stringUrlsToCheck) 580 581 for _, node := range nodeList { 582 metricUrlsToCheck = append(metricUrlsToCheck, 583 fmt.Sprintf("/api/v1/model/nodes/%s/metrics/%s", node, "cpu/usage_rate"), 584 fmt.Sprintf("/api/v1/model/nodes/%s/metrics/%s", node, "cpu-usage"), 585 ) 586 587 stringUrlsToCheck = append(stringUrlsToCheck, 588 fmt.Sprintf("/api/v1/model/nodes/%s/metrics", node), 589 ) 590 } 591 592 for _, pod := range podList { 593 containerName := pod.Spec.Containers[0].Name 594 595 metricUrlsToCheck = append(metricUrlsToCheck, 596 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics/%s", pod.Namespace, pod.Name, "cpu/usage_rate"), 597 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics/%s", pod.Namespace, pod.Name, "cpu-usage"), 598 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics/%s", pod.Namespace, pod.Name, containerName, "cpu/usage_rate"), 599 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics/%s", pod.Namespace, pod.Name, containerName, "cpu-usage"), 600 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics/%s", pod.Namespace, "cpu/usage_rate"), 601 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics/%s", pod.Namespace, "cpu-usage"), 602 ) 603 604 batchMetricsUrlsToCheck = append(batchMetricsUrlsToCheck, 605 fmt.Sprintf("/api/v1/model/namespaces/%s/pod-list/%s,%s/metrics/%s", pod.Namespace, pod.Name, pod.Name, "cpu/usage_rate"), 606 fmt.Sprintf("/api/v1/model/namespaces/%s/pod-list/%s,%s/metrics/%s", pod.Namespace, pod.Name, pod.Name, "cpu-usage"), 607 ) 608 609 stringUrlsToCheck = append(stringUrlsToCheck, 610 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics", pod.Namespace), 611 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics", pod.Namespace, pod.Name), 612 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics", pod.Namespace, pod.Name, containerName), 613 ) 614 } 615 616 for _, url := range metricUrlsToCheck { 617 _, err := getMetricResult(fm, svc, url) 618 if err != nil { 619 return fmt.Errorf("error while querying %s: %v", url, err) 620 } 621 } 622 623 for _, url := range batchMetricsUrlsToCheck { 624 _, err := getMetricResultList(fm, svc, url) 625 if err != nil { 626 return fmt.Errorf("error while querying %s: %v", url, err) 627 } 628 } 629 630 for _, url := range stringUrlsToCheck { 631 _, err := getStringResult(fm, svc, url) 632 if err != nil { 633 return fmt.Errorf("error while querying %s: %v", url, err) 634 } 635 } 636 return nil 637 } 638 639 const ( 640 apiPrefix = "apis" 641 metricsApiGroupName = "metrics" 642 metricsApiVersion = "v1alpha1" 643 ) 644 645 var baseMetricsUrl = fmt.Sprintf("%s/%s/%s", apiPrefix, metricsApiGroupName, metricsApiVersion) 646 647 func checkUsage(res kube_v1.ResourceList) error { 648 if _, found := res[kube_v1.ResourceCPU]; !found { 649 return fmt.Errorf("Cpu not found") 650 } 651 if _, found := res[kube_v1.ResourceMemory]; !found { 652 return fmt.Errorf("Memory not found") 653 } 654 return nil 655 } 656 657 func getPodMetrics(fm kubeFramework, svc *kube_v1.Service, pod kube_v1.Pod) (*metrics_api.PodMetrics, error) { 658 url := fmt.Sprintf("%s/namespaces/%s/pods/%s", baseMetricsUrl, pod.Namespace, pod.Name) 659 body, err := getDataFromProxy(fm, svc, url) 660 if err != nil { 661 return nil, err 662 } 663 var data metrics_api.PodMetrics 664 if err := json.Unmarshal(body, &data); err != nil { 665 glog.V(2).Infof("response body: %v", string(body)) 666 return nil, err 667 } 668 return &data, nil 669 } 670 671 func getAllPodsInNamespaceMetrics(fm kubeFramework, svc *kube_v1.Service, namespace string) (metrics_api.PodMetricsList, error) { 672 url := fmt.Sprintf("%s/namespaces/%s/pods/", baseMetricsUrl, namespace) 673 return getPodMetricsList(fm, svc, url, &labelSelectorEverything) 674 } 675 676 func getAllPodsMetrics(fm kubeFramework, svc *kube_v1.Service) (metrics_api.PodMetricsList, error) { 677 url := fmt.Sprintf("%s/pods/", baseMetricsUrl) 678 selector := labels.Everything() 679 return getPodMetricsList(fm, svc, url, &selector) 680 } 681 682 func getLabelSelectedPodMetrics(fm kubeFramework, svc *kube_v1.Service, namespace string, labelSelector *labels.Selector) (metrics_api.PodMetricsList, error) { 683 url := fmt.Sprintf("%s/namespaces/%s/pods/", baseMetricsUrl, namespace) 684 return getPodMetricsList(fm, svc, url, labelSelector) 685 } 686 687 func getPodMetricsList(fm kubeFramework, svc *kube_v1.Service, url string, labelSelector *labels.Selector) (metrics_api.PodMetricsList, error) { 688 body, err := getDataFromProxyWithSelector(fm, svc, url, labelSelector) 689 if err != nil { 690 return metrics_api.PodMetricsList{}, err 691 } 692 var data metrics_api.PodMetricsList 693 if err := json.Unmarshal(body, &data); err != nil { 694 glog.V(2).Infof("response body: %v", string(body)) 695 return metrics_api.PodMetricsList{}, err 696 } 697 return data, nil 698 } 699 700 func checkSinglePodMetrics(metrics *metrics_api.PodMetrics, pod *kube_v1.Pod) error { 701 if metrics.Name != pod.Name { 702 return fmt.Errorf("Wrong pod name: expected %v, got %v", pod.Name, metrics.Name) 703 } 704 if metrics.Namespace != pod.Namespace { 705 return fmt.Errorf("Wrong pod namespace: expected %v, got %v", pod.Namespace, metrics.Namespace) 706 } 707 if len(pod.Spec.Containers) != len(metrics.Containers) { 708 return fmt.Errorf("Wrong number of containers in returned metrics: expected %v, got %v", len(pod.Spec.Containers), len(metrics.Containers)) 709 } 710 for _, c := range metrics.Containers { 711 if err := checkUsage(c.Usage); err != nil { 712 return err 713 } 714 } 715 return nil 716 } 717 718 func getSingleNodeMetrics(fm kubeFramework, svc *kube_v1.Service, node string) (*metrics_api.NodeMetrics, error) { 719 url := fmt.Sprintf("%s/nodes/%s", baseMetricsUrl, node) 720 body, err := getDataFromProxy(fm, svc, url) 721 if err != nil { 722 return nil, err 723 } 724 var data metrics_api.NodeMetrics 725 if err := json.Unmarshal(body, &data); err != nil { 726 glog.V(2).Infof("response body: %v", string(body)) 727 return nil, err 728 } 729 return &data, nil 730 } 731 732 func getNodeMetricsList(fm kubeFramework, svc *kube_v1.Service, url string, labelSelector *labels.Selector) (metrics_api.NodeMetricsList, error) { 733 body, err := getDataFromProxyWithSelector(fm, svc, url, labelSelector) 734 if err != nil { 735 return metrics_api.NodeMetricsList{}, err 736 } 737 var data metrics_api.NodeMetricsList 738 if err := json.Unmarshal(body, &data); err != nil { 739 glog.V(2).Infof("response body: %v", string(body)) 740 return metrics_api.NodeMetricsList{}, err 741 } 742 return data, nil 743 } 744 745 func getLabelSelectedNodeMetrics(fm kubeFramework, svc *kube_v1.Service, labelSelector *labels.Selector) (metrics_api.NodeMetricsList, error) { 746 url := fmt.Sprintf("%s/nodes", baseMetricsUrl) 747 return getNodeMetricsList(fm, svc, url, labelSelector) 748 } 749 750 func getAllNodeMetrics(fm kubeFramework, svc *kube_v1.Service) (metrics_api.NodeMetricsList, error) { 751 url := fmt.Sprintf("%s/nodes", baseMetricsUrl) 752 selector := labels.Everything() 753 return getNodeMetricsList(fm, svc, url, &selector) 754 } 755 756 func runSingleNodeMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 757 nodeList, err := fm.GetNodeNames() 758 if err != nil { 759 return err 760 } 761 if len(nodeList) == 0 { 762 return fmt.Errorf("empty node list") 763 } 764 765 for _, node := range nodeList { 766 metrics, err := getSingleNodeMetrics(fm, svc, node) 767 if err != nil { 768 return err 769 } 770 if metrics.Name != node { 771 return fmt.Errorf("Wrong node name: expected %v, got %v", node, metrics.Name) 772 } 773 if err := checkUsage(metrics.Usage); err != nil { 774 return err 775 } 776 } 777 return nil 778 } 779 780 func runLabelSelectorNodeMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 781 nodeList, err := fm.GetNodes() 782 if err != nil { 783 return err 784 } 785 if len(nodeList.Items) == 0 { 786 return fmt.Errorf("empty node list") 787 } 788 labelMap := make(map[string]map[string]kube_v1.Node) 789 for _, n := range nodeList.Items { 790 for label, value := range n.Labels { 791 selector := label + "=" + value 792 if _, found := labelMap[selector]; !found { 793 labelMap[selector] = make(map[string]kube_v1.Node) 794 } 795 labelMap[selector][n.Name] = n 796 } 797 } 798 799 for selector, nodesWithLabel := range labelMap { 800 sel, err := labels.Parse(selector) 801 if err != nil { 802 return err 803 } 804 metrics, err := getLabelSelectedNodeMetrics(fm, svc, &sel) 805 if err != nil { 806 return err 807 } 808 if len(metrics.Items) != len(nodesWithLabel) { 809 return fmt.Errorf("Wrong number of label selected node metrics: expected %v, got %v", len(nodesWithLabel), len(metrics.Items)) 810 } 811 for _, nodeMetric := range metrics.Items { 812 node := nodesWithLabel[nodeMetric.Name] 813 if nodeMetric.Name != node.Name { 814 return fmt.Errorf("Wrong node name: expected %v, got %v", node.Name, nodeMetric.Name) 815 } 816 if err := checkUsage(nodeMetric.Usage); err != nil { 817 return err 818 } 819 } 820 } 821 return nil 822 } 823 824 func runAllNodesMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 825 nodeList, err := fm.GetNodeNames() 826 if err != nil { 827 return err 828 } 829 if len(nodeList) == 0 { 830 return fmt.Errorf("empty node list") 831 } 832 833 nodeNames := sets.NewString(nodeList...) 834 metrics, err := getAllNodeMetrics(fm, svc) 835 if err != nil { 836 return err 837 } 838 839 if len(metrics.Items) != len(nodeList) { 840 return fmt.Errorf("Wrong number of all node metrics: expected %v, got %v", len(nodeList), len(metrics.Items)) 841 } 842 for _, nodeMetrics := range metrics.Items { 843 if !nodeNames.Has(nodeMetrics.Name) { 844 return fmt.Errorf("Unexpected node name: %v, expected one of: %v", nodeMetrics.Name, nodeList) 845 } 846 if err := checkUsage(nodeMetrics.Usage); err != nil { 847 return err 848 } 849 } 850 return nil 851 } 852 853 func runSinglePodMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 854 podList, err := fm.GetAllRunningPods() 855 if err != nil { 856 return err 857 } 858 if len(podList) == 0 { 859 return fmt.Errorf("empty pod list") 860 } 861 for _, pod := range podList { 862 metrics, err := getPodMetrics(fm, svc, pod) 863 if err != nil { 864 return err 865 } 866 err = checkSinglePodMetrics(metrics, &pod) 867 if err != nil { 868 return err 869 } 870 } 871 return nil 872 } 873 874 func runAllPodsInNamespaceMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 875 podList, err := fm.GetAllRunningPods() 876 if err != nil { 877 return err 878 } 879 if len(podList) == 0 { 880 return fmt.Errorf("empty pod list") 881 } 882 nsToPods := make(map[string]map[string]kube_v1.Pod) 883 for _, pod := range podList { 884 if _, found := nsToPods[pod.Namespace]; !found { 885 nsToPods[pod.Namespace] = make(map[string]kube_v1.Pod) 886 } 887 nsToPods[pod.Namespace][pod.Name] = pod 888 } 889 890 for ns, podMap := range nsToPods { 891 metrics, err := getAllPodsInNamespaceMetrics(fm, svc, ns) 892 if err != nil { 893 return err 894 } 895 896 if len(metrics.Items) != len(nsToPods[ns]) { 897 return fmt.Errorf("Wrong number of metrics of all pods in a namespace: expected %v, got %v", len(nsToPods[ns]), len(metrics.Items)) 898 } 899 for _, podMetric := range metrics.Items { 900 pod := podMap[podMetric.Name] 901 err := checkSinglePodMetrics(&podMetric, &pod) 902 if err != nil { 903 return err 904 } 905 } 906 } 907 return nil 908 } 909 910 func runAllPodsMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 911 podList, err := fm.GetAllRunningPods() 912 if err != nil { 913 return err 914 } 915 if len(podList) == 0 { 916 return fmt.Errorf("empty pod list") 917 } 918 pods := make(map[string]kube_v1.Pod) 919 for _, p := range podList { 920 pods[p.Namespace+"/"+p.Name] = p 921 } 922 923 metrics, err := getAllPodsMetrics(fm, svc) 924 if err != nil { 925 return err 926 } 927 928 if len(metrics.Items) != len(podList) { 929 return fmt.Errorf("Wrong number of all pod metrics: expected %v, got %v", len(podList), len(metrics.Items)) 930 } 931 for _, podMetric := range metrics.Items { 932 pod := pods[podMetric.Namespace+"/"+podMetric.Name] 933 err := checkSinglePodMetrics(&podMetric, &pod) 934 if err != nil { 935 return err 936 } 937 } 938 return nil 939 } 940 941 func runLabelSelectorPodMetricsApiTest(fm kubeFramework, svc *kube_v1.Service) error { 942 podList, err := fm.GetAllRunningPods() 943 if err != nil { 944 return err 945 } 946 if len(podList) == 0 { 947 return fmt.Errorf("empty pod list") 948 } 949 nsToPods := make(map[string][]kube_v1.Pod) 950 for _, pod := range podList { 951 nsToPods[pod.Namespace] = append(nsToPods[pod.Namespace], pod) 952 } 953 954 for ns, podsInNamespace := range nsToPods { 955 labelMap := make(map[string]map[string]kube_v1.Pod) 956 for _, p := range podsInNamespace { 957 for label, name := range p.Labels { 958 selector := label + "=" + name 959 if _, found := labelMap[selector]; !found { 960 labelMap[selector] = make(map[string]kube_v1.Pod) 961 } 962 labelMap[selector][p.Name] = p 963 } 964 } 965 for selector, podsWithLabel := range labelMap { 966 sel, err := labels.Parse(selector) 967 if err != nil { 968 return err 969 } 970 metrics, err := getLabelSelectedPodMetrics(fm, svc, ns, &sel) 971 if err != nil { 972 return err 973 } 974 if len(metrics.Items) != len(podsWithLabel) { 975 return fmt.Errorf("Wrong number of label selected pod metrics: expected %v, got %v", len(podsWithLabel), len(metrics.Items)) 976 } 977 for _, podMetric := range metrics.Items { 978 pod := podsWithLabel[podMetric.Name] 979 err := checkSinglePodMetrics(&podMetric, &pod) 980 if err != nil { 981 return err 982 } 983 } 984 } 985 } 986 return nil 987 } 988 989 func apiTest(kubeVersion string, zone string) error { 990 fm, err := newKubeFramework(kubeVersion) 991 if err != nil { 992 return err 993 } 994 if err := buildAndPushDockerImages(fm, zone); err != nil { 995 return err 996 } 997 // Create heapster pod and service. 998 svc, rc, err := getHeapsterRcAndSvc(fm) 999 if err != nil { 1000 return err 1001 } 1002 ns := *namespace 1003 if err := deleteAll(fm, ns, svc, rc); err != nil { 1004 return err 1005 } 1006 if err := createAll(fm, ns, &svc, &rc); err != nil { 1007 return err 1008 } 1009 if err := fm.WaitUntilPodRunning(ns, rc.Spec.Template.Labels, time.Minute); err != nil { 1010 return err 1011 } 1012 if err := fm.WaitUntilServiceActive(svc, time.Minute); err != nil { 1013 return err 1014 } 1015 testFuncs := []func() error{ 1016 func() error { 1017 glog.V(2).Infof("Heapster metric export test...") 1018 err := runMetricExportTest(fm, svc) 1019 if err == nil { 1020 glog.V(2).Infof("Heapster metric export test: OK") 1021 } else { 1022 glog.V(2).Infof("Heapster metric export test: error: %v", err) 1023 } 1024 return err 1025 }, 1026 func() error { 1027 glog.V(2).Infof("Model test") 1028 err := runModelTest(fm, svc) 1029 if err == nil { 1030 glog.V(2).Infof("Model test: OK") 1031 } else { 1032 glog.V(2).Infof("Model test: error: %v", err) 1033 } 1034 return err 1035 }, 1036 func() error { 1037 glog.V(2).Infof("Metrics API test - single pod") 1038 err := runSinglePodMetricsApiTest(fm, svc) 1039 if err == nil { 1040 glog.V(2).Infof("Metrics API test - single pod: OK") 1041 } else { 1042 glog.V(2).Infof("Metrics API test - single pod: error: %v", err) 1043 } 1044 return err 1045 }, 1046 func() error { 1047 glog.V(2).Infof("Metrics API test - All pods in a namespace") 1048 err := runAllPodsInNamespaceMetricsApiTest(fm, svc) 1049 if err == nil { 1050 glog.V(2).Infof("Metrics API test - All pods in a namespace: OK") 1051 } else { 1052 glog.V(2).Infof("Metrics API test - All pods in a namespace: error: %v", err) 1053 } 1054 return err 1055 }, 1056 func() error { 1057 glog.V(2).Infof("Metrics API test - all pods") 1058 err := runAllPodsMetricsApiTest(fm, svc) 1059 if err == nil { 1060 glog.V(2).Infof("Metrics API test - all pods: OK") 1061 } else { 1062 glog.V(2).Infof("Metrics API test - all pods: error: %v", err) 1063 } 1064 return err 1065 }, 1066 func() error { 1067 glog.V(2).Infof("Metrics API test - label selector for pods") 1068 err := runLabelSelectorPodMetricsApiTest(fm, svc) 1069 if err == nil { 1070 glog.V(2).Infof("Metrics API test - label selector for pods: OK") 1071 } else { 1072 glog.V(2).Infof("Metrics API test - label selector for pods: error: %v", err) 1073 } 1074 return err 1075 }, 1076 func() error { 1077 glog.V(2).Infof("Metrics API test - single node") 1078 err := runSingleNodeMetricsApiTest(fm, svc) 1079 if err == nil { 1080 glog.V(2).Infof("Metrics API test - single node: OK") 1081 } else { 1082 glog.V(2).Infof("Metrics API test - single node: error: %v", err) 1083 } 1084 return err 1085 }, 1086 func() error { 1087 glog.V(2).Infof("Metrics API test - label selector for nodes") 1088 err := runLabelSelectorNodeMetricsApiTest(fm, svc) 1089 if err == nil { 1090 glog.V(2).Infof("Metrics API test - label selector for nodes: OK") 1091 } else { 1092 glog.V(2).Infof("Metrics API test - label selector for nodes: error: %v", err) 1093 } 1094 return err 1095 }, 1096 func() error { 1097 glog.V(2).Infof("Metrics API test - all nodes") 1098 err := runAllNodesMetricsApiTest(fm, svc) 1099 if err == nil { 1100 glog.V(2).Infof("Metrics API test - all nodes: OK") 1101 } else { 1102 glog.V(2).Infof("Metrics API test - all nodes: error: %v", err) 1103 } 1104 return err 1105 }, 1106 } 1107 attempts := *maxRetries 1108 glog.Infof("Starting tests") 1109 for { 1110 var err error 1111 for _, testFunc := range testFuncs { 1112 if err = testFunc(); err != nil { 1113 break 1114 } 1115 } 1116 if *runForever { 1117 continue 1118 } 1119 if err == nil { 1120 glog.V(2).Infof("All tests passed.") 1121 break 1122 } 1123 if attempts == 0 { 1124 glog.V(2).Info("Too many attempts.") 1125 return err 1126 } 1127 glog.V(2).Infof("Some tests failed. Retrying.") 1128 attempts-- 1129 time.Sleep(time.Second * 10) 1130 } 1131 deleteAll(fm, ns, svc, rc) 1132 removeHeapsterImage(fm, zone) 1133 return nil 1134 } 1135 1136 func runApiTest() error { 1137 tempDir, err := ioutil.TempDir("", "deploy") 1138 if err != nil { 1139 return nil 1140 } 1141 defer os.RemoveAll(tempDir) 1142 if *kubeVersions == "" { 1143 return apiTest("", *testZone) 1144 } 1145 kubeVersionsList := strings.Split(*kubeVersions, ",") 1146 for _, kubeVersion := range kubeVersionsList { 1147 if err := apiTest(kubeVersion, *testZone); err != nil { 1148 return err 1149 } 1150 } 1151 return nil 1152 } 1153 1154 func TestHeapster(t *testing.T) { 1155 if testing.Short() { 1156 t.Skip("skipping heapster kubernetes integration test.") 1157 } 1158 require.NoError(t, runApiTest()) 1159 }