github.com/intelsdi-x/heapster@v1.2.1-0.20221107230227-7f05c7c5dbad/integration/heapster_api_test.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "encoding/json" 19 "flag" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "strings" 24 "testing" 25 "time" 26 27 "github.com/golang/glog" 28 "github.com/stretchr/testify/require" 29 api_v1 "k8s.io/heapster/metrics/api/v1/types" 30 metrics_api "k8s.io/heapster/metrics/apis/metrics/v1alpha1" 31 "k8s.io/heapster/metrics/core" 32 kube_api "k8s.io/kubernetes/pkg/api" 33 apiErrors "k8s.io/kubernetes/pkg/api/errors" 34 kube_api_unv "k8s.io/kubernetes/pkg/api/unversioned" 35 kube_v1 "k8s.io/kubernetes/pkg/api/v1" 36 "k8s.io/kubernetes/pkg/labels" 37 "k8s.io/kubernetes/pkg/util/sets" 38 ) 39 40 const ( 41 targetTags = "kubernetes-minion" 42 heapsterBuildDir = "../deploy/docker" 43 ) 44 45 var ( 46 testZone = flag.String("test_zone", "us-central1-b", "GCE zone where the test will be executed") 47 kubeVersions = flag.String("kube_versions", "", "Comma separated list of kube versions to test against. By default will run the test against an existing cluster") 48 heapsterControllerFile = flag.String("heapster_controller", "../deploy/kube-config/standalone-test/heapster-controller.yaml", "Path to heapster replication controller file.") 49 heapsterServiceFile = flag.String("heapster_service", "../deploy/kube-config/standalone-test/heapster-service.yaml", "Path to heapster service file.") 50 heapsterImage = flag.String("heapster_image", "heapster:e2e_test", "heapster docker image that needs to be tested.") 51 avoidBuild = flag.Bool("nobuild", false, "When true, a new heapster docker image will not be created and pushed to test cluster nodes.") 52 namespace = flag.String("namespace", "heapster-e2e-tests", "namespace to be used for testing, it will be deleted at the beginning of the test if exists") 53 maxRetries = flag.Int("retries", 20, "Number of attempts before failing this test.") 54 runForever = flag.Bool("run_forever", false, "If true, the tests are run in a loop forever.") 55 ) 56 57 func deleteAll(fm kubeFramework, ns string, service *kube_api.Service, rc *kube_api.ReplicationController) error { 58 glog.V(2).Infof("Deleting ns %s...", ns) 59 err := fm.DeleteNs(ns) 60 if err != nil { 61 glog.V(2).Infof("Failed to delete %s", ns) 62 return err 63 } 64 glog.V(2).Infof("Deleted ns %s.", ns) 65 return nil 66 } 67 68 func createAll(fm kubeFramework, ns string, service **kube_api.Service, rc **kube_api.ReplicationController) error { 69 glog.V(2).Infof("Creating ns %s...", ns) 70 namespace := kube_api.Namespace{ 71 TypeMeta: kube_api_unv.TypeMeta{ 72 Kind: "Namespace", 73 APIVersion: "v1", 74 }, 75 ObjectMeta: kube_api.ObjectMeta{ 76 Name: ns, 77 }, 78 } 79 if _, err := fm.CreateNs(&namespace); err != nil { 80 glog.V(2).Infof("Failed to create ns: %v", err) 81 return err 82 } 83 84 glog.V(2).Infof("Created ns %s.", ns) 85 86 glog.V(2).Infof("Creating rc %s/%s...", ns, (*rc).Name) 87 if newRc, err := fm.CreateRC(ns, *rc); err != nil { 88 glog.V(2).Infof("Failed to create rc: %v", err) 89 return err 90 } else { 91 *rc = newRc 92 } 93 glog.V(2).Infof("Created rc %s/%s.", ns, (*rc).Name) 94 95 glog.V(2).Infof("Creating service %s/%s...", ns, (*service).Name) 96 if newSvc, err := fm.CreateService(ns, *service); err != nil { 97 glog.V(2).Infof("Failed to create service: %v", err) 98 return err 99 } else { 100 *service = newSvc 101 } 102 glog.V(2).Infof("Created service %s/%s.", ns, (*service).Name) 103 104 return nil 105 } 106 107 func removeHeapsterImage(fm kubeFramework, zone string) error { 108 glog.V(2).Infof("Removing heapster image.") 109 if err := removeDockerImage(*heapsterImage); err != nil { 110 glog.Errorf("Failed to remove Heapster image: %v", err) 111 } else { 112 glog.V(2).Infof("Heapster image removed.") 113 } 114 if nodes, err := fm.GetNodeNames(); err == nil { 115 for _, node := range nodes { 116 host := strings.Split(node, ".")[0] 117 cleanupRemoteHost(host, zone) 118 } 119 } else { 120 glog.Errorf("failed to cleanup nodes - %v", err) 121 } 122 return nil 123 } 124 125 func buildAndPushHeapsterImage(hostnames []string, zone string) error { 126 glog.V(2).Info("Building and pushing Heapster image...") 127 curwd, err := os.Getwd() 128 if err != nil { 129 return err 130 } 131 if err := os.Chdir(heapsterBuildDir); err != nil { 132 return err 133 } 134 if err := buildDockerImage(*heapsterImage); err != nil { 135 return err 136 } 137 for _, host := range hostnames { 138 if err := copyDockerImage(*heapsterImage, host, zone); err != nil { 139 return err 140 } 141 } 142 glog.V(2).Info("Heapster image pushed.") 143 return os.Chdir(curwd) 144 } 145 146 func getHeapsterRcAndSvc(fm kubeFramework) (*kube_api.Service, *kube_api.ReplicationController, error) { 147 // Add test docker image 148 rc, err := fm.ParseRC(*heapsterControllerFile) 149 if err != nil { 150 return nil, nil, fmt.Errorf("failed to parse heapster controller - %v", err) 151 } 152 for i := range rc.Spec.Template.Spec.Containers { 153 rc.Spec.Template.Spec.Containers[i].Image = *heapsterImage 154 rc.Spec.Template.Spec.Containers[i].ImagePullPolicy = kube_api.PullNever 155 // increase logging level 156 rc.Spec.Template.Spec.Containers[i].Env = append(rc.Spec.Template.Spec.Containers[0].Env, kube_api.EnvVar{Name: "FLAGS", Value: "--vmodule=*=3"}) 157 } 158 159 svc, err := fm.ParseService(*heapsterServiceFile) 160 if err != nil { 161 return nil, nil, fmt.Errorf("failed to parse heapster service - %v", err) 162 } 163 164 return svc, rc, nil 165 } 166 167 func buildAndPushDockerImages(fm kubeFramework, zone string) error { 168 if *avoidBuild { 169 return nil 170 } 171 nodes, err := fm.GetNodeNames() 172 if err != nil { 173 return err 174 } 175 hostnames := []string{} 176 for _, node := range nodes { 177 hostnames = append(hostnames, strings.Split(node, ".")[0]) 178 } 179 180 return buildAndPushHeapsterImage(hostnames, zone) 181 } 182 183 const ( 184 metricsEndpoint = "/api/v1/metric-export" 185 metricsSchemaEndpoint = "/api/v1/metric-export-schema" 186 ) 187 188 func getTimeseries(fm kubeFramework, svc *kube_api.Service) ([]*api_v1.Timeseries, error) { 189 body, err := fm.Client().Get(). 190 Namespace(svc.Namespace). 191 Prefix("proxy"). 192 Resource("services"). 193 Name(svc.Name). 194 Suffix(metricsEndpoint). 195 Do().Raw() 196 if err != nil { 197 return nil, err 198 } 199 var timeseries []*api_v1.Timeseries 200 if err := json.Unmarshal(body, ×eries); err != nil { 201 glog.V(2).Infof("Timeseries error: %v %v", err, string(body)) 202 return nil, err 203 } 204 return timeseries, nil 205 } 206 207 func getSchema(fm kubeFramework, svc *kube_api.Service) (*api_v1.TimeseriesSchema, error) { 208 body, err := fm.Client().Get(). 209 Namespace(svc.Namespace). 210 Prefix("proxy"). 211 Resource("services"). 212 Name(svc.Name). 213 Suffix(metricsSchemaEndpoint). 214 Do().Raw() 215 if err != nil { 216 return nil, err 217 } 218 var timeseriesSchema api_v1.TimeseriesSchema 219 if err := json.Unmarshal(body, ×eriesSchema); err != nil { 220 glog.V(2).Infof("Metrics schema error: %v %v", err, string(body)) 221 return nil, err 222 } 223 return ×eriesSchema, nil 224 } 225 226 var expectedSystemContainers = map[string]struct{}{ 227 "machine": {}, 228 "kubelet": {}, 229 "kube-proxy": {}, 230 // TODO(piosz): Uncomment once https://github.com/kubernetes/kubernetes/issues/37453 is fixed 231 // "system": {}, 232 "docker-daemon": {}, 233 } 234 235 func isContainerBaseImageExpected(ts *api_v1.Timeseries) bool { 236 cName := ts.Labels[core.LabelContainerName.Key] 237 // TODO(piosz): remove this if once https://github.com/kubernetes/kubernetes/issues/37453 is fixed 238 if cName == "system" { 239 return false 240 } 241 _, exists := expectedSystemContainers[cName] 242 return !exists 243 } 244 245 func runMetricExportTest(fm kubeFramework, svc *kube_api.Service) error { 246 podList, err := fm.GetPodsRunningOnNodes() 247 if err != nil { 248 return err 249 } 250 expectedPods := make([]string, 0, len(podList)) 251 for _, pod := range podList { 252 expectedPods = append(expectedPods, pod.Name) 253 } 254 glog.V(0).Infof("Expected pods: %v", expectedPods) 255 256 expectedNodes, err := fm.GetNodeNames() 257 if err != nil { 258 return err 259 } 260 glog.V(0).Infof("Expected nodes: %v", expectedNodes) 261 262 timeseries, err := getTimeseries(fm, svc) 263 if err != nil { 264 return err 265 } 266 if len(timeseries) == 0 { 267 return fmt.Errorf("expected non zero timeseries") 268 } 269 schema, err := getSchema(fm, svc) 270 if err != nil { 271 return err 272 } 273 // Build a map of metric names to metric descriptors. 274 mdMap := map[string]*api_v1.MetricDescriptor{} 275 for idx := range schema.Metrics { 276 mdMap[schema.Metrics[idx].Name] = &schema.Metrics[idx] 277 } 278 actualPods := map[string]bool{} 279 actualNodes := map[string]bool{} 280 actualSystemContainers := map[string]map[string]struct{}{} 281 for _, ts := range timeseries { 282 // Verify the relevant labels are present. 283 // All common labels must be present. 284 podName, podMetric := ts.Labels[core.LabelPodName.Key] 285 286 for _, label := range core.CommonLabels() { 287 _, exists := ts.Labels[label.Key] 288 if !exists { 289 return fmt.Errorf("timeseries: %v does not contain common label: %v", ts, label) 290 } 291 } 292 if podMetric { 293 for _, label := range core.PodLabels() { 294 _, exists := ts.Labels[label.Key] 295 if !exists { 296 return fmt.Errorf("timeseries: %v does not contain pod label: %v", ts, label) 297 } 298 } 299 } 300 301 if podMetric { 302 actualPods[podName] = true 303 // Extra explicit check that the expecte metrics are there: 304 requiredLabels := []string{ 305 core.LabelPodNamespaceUID.Key, 306 core.LabelPodId.Key, 307 core.LabelHostID.Key, 308 // container name is checked later 309 } 310 for _, label := range requiredLabels { 311 _, exists := ts.Labels[label] 312 if !exists { 313 return fmt.Errorf("timeseries: %v does not contain required label: %v", ts, label) 314 } 315 } 316 317 } else { 318 if cName, ok := ts.Labels[core.LabelContainerName.Key]; ok { 319 hostname, ok := ts.Labels[core.LabelHostname.Key] 320 if !ok { 321 return fmt.Errorf("hostname label missing on container %+v", ts) 322 } 323 324 if cName == "machine" { 325 actualNodes[hostname] = true 326 } else { 327 for _, label := range core.ContainerLabels() { 328 if label == core.LabelContainerBaseImage && !isContainerBaseImageExpected(ts) { 329 continue 330 } 331 _, exists := ts.Labels[label.Key] 332 if !exists { 333 return fmt.Errorf("timeseries: %v does not contain container label: %v", ts, label) 334 } 335 } 336 } 337 338 if _, exists := expectedSystemContainers[cName]; exists { 339 if actualSystemContainers[cName] == nil { 340 actualSystemContainers[cName] = map[string]struct{}{} 341 } 342 actualSystemContainers[cName][hostname] = struct{}{} 343 } 344 } else { 345 return fmt.Errorf("container_name label missing on timeseries - %v", ts) 346 } 347 } 348 349 // Explicitly check for resource id 350 explicitRequirement := map[string][]string{ 351 core.MetricFilesystemUsage.MetricDescriptor.Name: {core.LabelResourceID.Key}, 352 core.MetricFilesystemLimit.MetricDescriptor.Name: {core.LabelResourceID.Key}, 353 core.MetricFilesystemAvailable.Name: {core.LabelResourceID.Key}} 354 355 for metricName, points := range ts.Metrics { 356 md, exists := mdMap[metricName] 357 if !exists { 358 return fmt.Errorf("unexpected metric %q", metricName) 359 } 360 361 for _, point := range points { 362 for _, label := range md.Labels { 363 _, exists := point.Labels[label.Key] 364 if !exists { 365 return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label) 366 } 367 } 368 } 369 370 required := explicitRequirement[metricName] 371 for _, label := range required { 372 for _, point := range points { 373 _, exists := point.Labels[label] 374 if !exists { 375 return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label) 376 } 377 } 378 } 379 } 380 } 381 // Validate that system containers are running on all the nodes. 382 // This test could fail if one of the containers was down while the metrics sample was collected. 383 for cName, hosts := range actualSystemContainers { 384 for _, host := range expectedNodes { 385 if _, ok := hosts[host]; !ok { 386 return fmt.Errorf("System container %q not found on host: %q - %v", cName, host, actualSystemContainers) 387 } 388 } 389 } 390 391 if err := expectedItemsExist(expectedPods, actualPods); err != nil { 392 return fmt.Errorf("expected pods don't exist %v.\nExpected: %v\nActual:%v", err, expectedPods, actualPods) 393 } 394 if err := expectedItemsExist(expectedNodes, actualNodes); err != nil { 395 return fmt.Errorf("expected nodes don't exist %v.\nExpected: %v\nActual:%v", err, expectedNodes, actualNodes) 396 } 397 398 return nil 399 } 400 401 func expectedItemsExist(expectedItems []string, actualItems map[string]bool) error { 402 for _, item := range expectedItems { 403 if _, found := actualItems[item]; !found { 404 return fmt.Errorf("missing %s", item) 405 } 406 } 407 return nil 408 } 409 410 func getErrorCauses(err error) string { 411 serr, ok := err.(*apiErrors.StatusError) 412 if !ok { 413 return "" 414 } 415 var causes []string 416 for _, c := range serr.ErrStatus.Details.Causes { 417 causes = append(causes, c.Message) 418 } 419 return strings.Join(causes, ", ") 420 } 421 422 var labelSelectorEverything = labels.Everything() 423 424 func getDataFromProxy(fm kubeFramework, svc *kube_api.Service, url string) ([]byte, error) { 425 glog.V(2).Infof("Querying heapster: %s", url) 426 return fm.Client().Get(). 427 Namespace(svc.Namespace). 428 Prefix("proxy"). 429 Resource("services"). 430 Name(svc.Name). 431 Suffix(url). 432 Do().Raw() 433 } 434 435 func getDataFromProxyWithSelector(fm kubeFramework, svc *kube_api.Service, url string, labelSelector *labels.Selector) ([]byte, error) { 436 glog.V(2).Infof("Querying heapster: %s", url) 437 return fm.Client().Get(). 438 Namespace(svc.Namespace). 439 Prefix("proxy"). 440 Resource("services"). 441 Name(svc.Name). 442 Suffix(url).LabelsSelectorParam(*labelSelector). 443 Do().Raw() 444 } 445 446 func getMetricResultList(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.MetricResultList, error) { 447 body, err := getDataFromProxy(fm, svc, url) 448 if err != nil { 449 return nil, err 450 } 451 var data api_v1.MetricResultList 452 if err := json.Unmarshal(body, &data); err != nil { 453 glog.V(2).Infof("response body: %v", string(body)) 454 return nil, err 455 } 456 if err := checkMetricResultListSanity(&data); err != nil { 457 return nil, err 458 } 459 return &data, nil 460 } 461 462 func getMetricResult(fm kubeFramework, svc *kube_api.Service, url string) (*api_v1.MetricResult, error) { 463 body, err := getDataFromProxy(fm, svc, url) 464 if err != nil { 465 return nil, err 466 } 467 var data api_v1.MetricResult 468 if err := json.Unmarshal(body, &data); err != nil { 469 glog.V(2).Infof("response body: %v", string(body)) 470 return nil, err 471 } 472 if err := checkMetricResultSanity(&data); err != nil { 473 return nil, err 474 } 475 return &data, nil 476 } 477 478 func getStringResult(fm kubeFramework, svc *kube_api.Service, url string) ([]string, error) { 479 body, err := getDataFromProxy(fm, svc, url) 480 if err != nil { 481 return nil, err 482 } 483 var data []string 484 if err := json.Unmarshal(body, &data); err != nil { 485 glog.V(2).Infof("response body: %v", string(body)) 486 return nil, err 487 } 488 if len(data) == 0 { 489 return nil, fmt.Errorf("empty string array") 490 } 491 return data, nil 492 } 493 494 func checkMetricResultSanity(metrics *api_v1.MetricResult) error { 495 bytes, err := json.Marshal(*metrics) 496 if err != nil { 497 return err 498 } 499 stringVersion := string(bytes) 500 501 if len(metrics.Metrics) == 0 { 502 return fmt.Errorf("empty metrics: %s", stringVersion) 503 } 504 // There should be recent metrics in the response. 505 if time.Now().Sub(metrics.LatestTimestamp).Seconds() > 120 { 506 return fmt.Errorf("corrupted last timestamp: %s", stringVersion) 507 } 508 // Metrics don't have to be sorted, so the oldest one can be first. 509 if time.Now().Sub(metrics.Metrics[0].Timestamp).Hours() > 1 { 510 return fmt.Errorf("corrupted timestamp: %s", stringVersion) 511 } 512 if metrics.Metrics[0].Value > 10000 { 513 return fmt.Errorf("value too big: %s", stringVersion) 514 } 515 return nil 516 } 517 518 func checkMetricResultListSanity(metrics *api_v1.MetricResultList) error { 519 if len(metrics.Items) == 0 { 520 return fmt.Errorf("empty metrics") 521 } 522 for _, item := range metrics.Items { 523 err := checkMetricResultSanity(&item) 524 if err != nil { 525 return err 526 } 527 } 528 return nil 529 } 530 531 func runModelTest(fm kubeFramework, svc *kube_api.Service) error { 532 podList, err := fm.GetPodsRunningOnNodes() 533 if err != nil { 534 return err 535 } 536 if len(podList) == 0 { 537 return fmt.Errorf("empty pod list") 538 } 539 nodeList, err := fm.GetNodeNames() 540 if err != nil { 541 return err 542 } 543 if len(nodeList) == 0 { 544 return fmt.Errorf("empty node list") 545 } 546 podNamesList := make([]string, 0, len(podList)) 547 for _, pod := range podList { 548 podNamesList = append(podNamesList, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) 549 } 550 551 glog.V(0).Infof("Expected pods: %v", podNamesList) 552 glog.V(0).Infof("Expected nodes: %v", nodeList) 553 allkeys, err := getStringResult(fm, svc, "/api/v1/model/debug/allkeys") 554 if err != nil { 555 return fmt.Errorf("Failed to get debug information about keys: %v", err) 556 } 557 glog.V(0).Infof("Available Heapster metric sets: %v", allkeys) 558 559 metricUrlsToCheck := []string{} 560 batchMetricsUrlsToCheck := []string{} 561 stringUrlsToCheck := []string{} 562 563 /* TODO: enable once cluster aggregator is added. 564 metricUrlsToCheck = append(metricUrlsToCheck, 565 fmt.Sprintf("/api/v1/model/metrics/%s", "cpu-usage"), 566 ) 567 */ 568 569 /* TODO: add once Cluster metrics aggregator is added. 570 "/api/v1/model/metrics", 571 "/api/v1/model/" 572 */ 573 stringUrlsToCheck = append(stringUrlsToCheck) 574 575 for _, node := range nodeList { 576 metricUrlsToCheck = append(metricUrlsToCheck, 577 fmt.Sprintf("/api/v1/model/nodes/%s/metrics/%s", node, "cpu/usage_rate"), 578 fmt.Sprintf("/api/v1/model/nodes/%s/metrics/%s", node, "cpu-usage"), 579 ) 580 581 stringUrlsToCheck = append(stringUrlsToCheck, 582 fmt.Sprintf("/api/v1/model/nodes/%s/metrics", node), 583 ) 584 } 585 586 for _, pod := range podList { 587 containerName := pod.Spec.Containers[0].Name 588 589 metricUrlsToCheck = append(metricUrlsToCheck, 590 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics/%s", pod.Namespace, pod.Name, "cpu/usage_rate"), 591 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics/%s", pod.Namespace, pod.Name, "cpu-usage"), 592 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics/%s", pod.Namespace, pod.Name, containerName, "cpu/usage_rate"), 593 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics/%s", pod.Namespace, pod.Name, containerName, "cpu-usage"), 594 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics/%s", pod.Namespace, "cpu/usage_rate"), 595 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics/%s", pod.Namespace, "cpu-usage"), 596 ) 597 598 batchMetricsUrlsToCheck = append(batchMetricsUrlsToCheck, 599 fmt.Sprintf("/api/v1/model/namespaces/%s/pod-list/%s,%s/metrics/%s", pod.Namespace, pod.Name, pod.Name, "cpu/usage_rate"), 600 fmt.Sprintf("/api/v1/model/namespaces/%s/pod-list/%s,%s/metrics/%s", pod.Namespace, pod.Name, pod.Name, "cpu-usage"), 601 ) 602 603 stringUrlsToCheck = append(stringUrlsToCheck, 604 fmt.Sprintf("/api/v1/model/namespaces/%s/metrics", pod.Namespace), 605 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/metrics", pod.Namespace, pod.Name), 606 fmt.Sprintf("/api/v1/model/namespaces/%s/pods/%s/containers/%s/metrics", pod.Namespace, pod.Name, containerName), 607 ) 608 } 609 610 for _, url := range metricUrlsToCheck { 611 _, err := getMetricResult(fm, svc, url) 612 if err != nil { 613 return fmt.Errorf("error while querying %s: %v", url, err) 614 } 615 } 616 617 for _, url := range batchMetricsUrlsToCheck { 618 _, err := getMetricResultList(fm, svc, url) 619 if err != nil { 620 return fmt.Errorf("error while querying %s: %v", url, err) 621 } 622 } 623 624 for _, url := range stringUrlsToCheck { 625 _, err := getStringResult(fm, svc, url) 626 if err != nil { 627 return fmt.Errorf("error while querying %s: %v", url, err) 628 } 629 } 630 return nil 631 } 632 633 const ( 634 apiPrefix = "apis" 635 metricsApiGroupName = "metrics" 636 metricsApiVersion = "v1alpha1" 637 ) 638 639 var baseMetricsUrl = fmt.Sprintf("%s/%s/%s", apiPrefix, metricsApiGroupName, metricsApiVersion) 640 641 func checkUsage(res kube_v1.ResourceList) error { 642 if _, found := res[kube_v1.ResourceCPU]; !found { 643 return fmt.Errorf("Cpu not found") 644 } 645 if _, found := res[kube_v1.ResourceMemory]; !found { 646 return fmt.Errorf("Memory not found") 647 } 648 return nil 649 } 650 651 func getPodMetrics(fm kubeFramework, svc *kube_api.Service, pod kube_api.Pod) (*metrics_api.PodMetrics, error) { 652 url := fmt.Sprintf("%s/namespaces/%s/pods/%s", baseMetricsUrl, pod.Namespace, pod.Name) 653 body, err := getDataFromProxy(fm, svc, url) 654 if err != nil { 655 return nil, err 656 } 657 var data metrics_api.PodMetrics 658 if err := json.Unmarshal(body, &data); err != nil { 659 glog.V(2).Infof("response body: %v", string(body)) 660 return nil, err 661 } 662 return &data, nil 663 } 664 665 func getAllPodsInNamespaceMetrics(fm kubeFramework, svc *kube_api.Service, namespace string) (metrics_api.PodMetricsList, error) { 666 url := fmt.Sprintf("%s/namespaces/%s/pods/", baseMetricsUrl, namespace) 667 return getPodMetricsList(fm, svc, url, &labelSelectorEverything) 668 } 669 670 func getAllPodsMetrics(fm kubeFramework, svc *kube_api.Service) (metrics_api.PodMetricsList, error) { 671 url := fmt.Sprintf("%s/pods/", baseMetricsUrl) 672 selector := labels.Everything() 673 return getPodMetricsList(fm, svc, url, &selector) 674 } 675 676 func getLabelSelectedPodMetrics(fm kubeFramework, svc *kube_api.Service, namespace string, labelSelector *labels.Selector) (metrics_api.PodMetricsList, error) { 677 url := fmt.Sprintf("%s/namespaces/%s/pods/", baseMetricsUrl, namespace) 678 return getPodMetricsList(fm, svc, url, labelSelector) 679 } 680 681 func getPodMetricsList(fm kubeFramework, svc *kube_api.Service, url string, labelSelector *labels.Selector) (metrics_api.PodMetricsList, error) { 682 body, err := getDataFromProxyWithSelector(fm, svc, url, labelSelector) 683 if err != nil { 684 return metrics_api.PodMetricsList{}, err 685 } 686 var data metrics_api.PodMetricsList 687 if err := json.Unmarshal(body, &data); err != nil { 688 glog.V(2).Infof("response body: %v", string(body)) 689 return metrics_api.PodMetricsList{}, err 690 } 691 return data, nil 692 } 693 694 func checkSinglePodMetrics(metrics *metrics_api.PodMetrics, pod *kube_api.Pod) error { 695 if metrics.Name != pod.Name { 696 return fmt.Errorf("Wrong pod name: expected %v, got %v", pod.Name, metrics.Name) 697 } 698 if metrics.Namespace != pod.Namespace { 699 return fmt.Errorf("Wrong pod namespace: expected %v, got %v", pod.Namespace, metrics.Namespace) 700 } 701 if len(pod.Spec.Containers) != len(metrics.Containers) { 702 return fmt.Errorf("Wrong number of containers in returned metrics: expected %v, got %v", len(pod.Spec.Containers), len(metrics.Containers)) 703 } 704 for _, c := range metrics.Containers { 705 if err := checkUsage(c.Usage); err != nil { 706 return err 707 } 708 } 709 return nil 710 } 711 712 func getSingleNodeMetrics(fm kubeFramework, svc *kube_api.Service, node string) (*metrics_api.NodeMetrics, error) { 713 url := fmt.Sprintf("%s/nodes/%s", baseMetricsUrl, node) 714 body, err := getDataFromProxy(fm, svc, url) 715 if err != nil { 716 return nil, err 717 } 718 var data metrics_api.NodeMetrics 719 if err := json.Unmarshal(body, &data); err != nil { 720 glog.V(2).Infof("response body: %v", string(body)) 721 return nil, err 722 } 723 return &data, nil 724 } 725 726 func getNodeMetricsList(fm kubeFramework, svc *kube_api.Service, url string, labelSelector *labels.Selector) (metrics_api.NodeMetricsList, error) { 727 body, err := getDataFromProxyWithSelector(fm, svc, url, labelSelector) 728 if err != nil { 729 return metrics_api.NodeMetricsList{}, err 730 } 731 var data metrics_api.NodeMetricsList 732 if err := json.Unmarshal(body, &data); err != nil { 733 glog.V(2).Infof("response body: %v", string(body)) 734 return metrics_api.NodeMetricsList{}, err 735 } 736 return data, nil 737 } 738 739 func getLabelSelectedNodeMetrics(fm kubeFramework, svc *kube_api.Service, labelSelector *labels.Selector) (metrics_api.NodeMetricsList, error) { 740 url := fmt.Sprintf("%s/nodes", baseMetricsUrl) 741 return getNodeMetricsList(fm, svc, url, labelSelector) 742 } 743 744 func getAllNodeMetrics(fm kubeFramework, svc *kube_api.Service) (metrics_api.NodeMetricsList, error) { 745 url := fmt.Sprintf("%s/nodes", baseMetricsUrl) 746 selector := labels.Everything() 747 return getNodeMetricsList(fm, svc, url, &selector) 748 } 749 750 func runSingleNodeMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 751 nodeList, err := fm.GetNodeNames() 752 if err != nil { 753 return err 754 } 755 if len(nodeList) == 0 { 756 return fmt.Errorf("empty node list") 757 } 758 759 for _, node := range nodeList { 760 metrics, err := getSingleNodeMetrics(fm, svc, node) 761 if err != nil { 762 return err 763 } 764 if metrics.Name != node { 765 return fmt.Errorf("Wrong node name: expected %v, got %v", node, metrics.Name) 766 } 767 if err := checkUsage(metrics.Usage); err != nil { 768 return err 769 } 770 } 771 return nil 772 } 773 774 func runLabelSelectorNodeMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 775 nodeList, err := fm.GetNodes() 776 if err != nil { 777 return err 778 } 779 if len(nodeList.Items) == 0 { 780 return fmt.Errorf("empty node list") 781 } 782 labelMap := make(map[string]map[string]kube_api.Node) 783 for _, n := range nodeList.Items { 784 for label, value := range n.Labels { 785 selector := label + "=" + value 786 if _, found := labelMap[selector]; !found { 787 labelMap[selector] = make(map[string]kube_api.Node) 788 } 789 labelMap[selector][n.Name] = n 790 } 791 } 792 793 for selector, nodesWithLabel := range labelMap { 794 sel, err := labels.Parse(selector) 795 if err != nil { 796 return err 797 } 798 metrics, err := getLabelSelectedNodeMetrics(fm, svc, &sel) 799 if err != nil { 800 return err 801 } 802 if len(metrics.Items) != len(nodesWithLabel) { 803 return fmt.Errorf("Wrong number of label selected node metrics: expected %v, got %v", len(nodesWithLabel), len(metrics.Items)) 804 } 805 for _, nodeMetric := range metrics.Items { 806 node := nodesWithLabel[nodeMetric.Name] 807 if nodeMetric.Name != node.Name { 808 return fmt.Errorf("Wrong node name: expected %v, got %v", node.Name, nodeMetric.Name) 809 } 810 if err := checkUsage(nodeMetric.Usage); err != nil { 811 return err 812 } 813 } 814 } 815 return nil 816 } 817 818 func runAllNodesMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 819 nodeList, err := fm.GetNodeNames() 820 if err != nil { 821 return err 822 } 823 if len(nodeList) == 0 { 824 return fmt.Errorf("empty node list") 825 } 826 827 nodeNames := sets.NewString(nodeList...) 828 metrics, err := getAllNodeMetrics(fm, svc) 829 if err != nil { 830 return err 831 } 832 833 if len(metrics.Items) != len(nodeList) { 834 return fmt.Errorf("Wrong number of all node metrics: expected %v, got %v", len(nodeList), len(metrics.Items)) 835 } 836 for _, nodeMetrics := range metrics.Items { 837 if !nodeNames.Has(nodeMetrics.Name) { 838 return fmt.Errorf("Unexpected node name: %v, expected one of: %v", nodeMetrics.Name, nodeList) 839 } 840 if err := checkUsage(nodeMetrics.Usage); err != nil { 841 return err 842 } 843 } 844 return nil 845 } 846 847 func runSinglePodMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 848 podList, err := fm.GetAllRunningPods() 849 if err != nil { 850 return err 851 } 852 if len(podList) == 0 { 853 return fmt.Errorf("empty pod list") 854 } 855 for _, pod := range podList { 856 metrics, err := getPodMetrics(fm, svc, pod) 857 if err != nil { 858 return err 859 } 860 err = checkSinglePodMetrics(metrics, &pod) 861 if err != nil { 862 return err 863 } 864 } 865 return nil 866 } 867 868 func runAllPodsInNamespaceMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 869 podList, err := fm.GetAllRunningPods() 870 if err != nil { 871 return err 872 } 873 if len(podList) == 0 { 874 return fmt.Errorf("empty pod list") 875 } 876 nsToPods := make(map[string]map[string]kube_api.Pod) 877 for _, pod := range podList { 878 if _, found := nsToPods[pod.Namespace]; !found { 879 nsToPods[pod.Namespace] = make(map[string]kube_api.Pod) 880 } 881 nsToPods[pod.Namespace][pod.Name] = pod 882 } 883 884 for ns, podMap := range nsToPods { 885 metrics, err := getAllPodsInNamespaceMetrics(fm, svc, ns) 886 if err != nil { 887 return err 888 } 889 890 if len(metrics.Items) != len(nsToPods[ns]) { 891 return fmt.Errorf("Wrong number of metrics of all pods in a namespace: expected %v, got %v", len(nsToPods[ns]), len(metrics.Items)) 892 } 893 for _, podMetric := range metrics.Items { 894 pod := podMap[podMetric.Name] 895 err := checkSinglePodMetrics(&podMetric, &pod) 896 if err != nil { 897 return err 898 } 899 } 900 } 901 return nil 902 } 903 904 func runAllPodsMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 905 podList, err := fm.GetAllRunningPods() 906 if err != nil { 907 return err 908 } 909 if len(podList) == 0 { 910 return fmt.Errorf("empty pod list") 911 } 912 pods := make(map[string]kube_api.Pod) 913 for _, p := range podList { 914 pods[p.Namespace+"/"+p.Name] = p 915 } 916 917 metrics, err := getAllPodsMetrics(fm, svc) 918 if err != nil { 919 return err 920 } 921 922 if len(metrics.Items) != len(podList) { 923 return fmt.Errorf("Wrong number of all pod metrics: expected %v, got %v", len(podList), len(metrics.Items)) 924 } 925 for _, podMetric := range metrics.Items { 926 pod := pods[podMetric.Namespace+"/"+podMetric.Name] 927 err := checkSinglePodMetrics(&podMetric, &pod) 928 if err != nil { 929 return err 930 } 931 } 932 return nil 933 } 934 935 func runLabelSelectorPodMetricsApiTest(fm kubeFramework, svc *kube_api.Service) error { 936 podList, err := fm.GetAllRunningPods() 937 if err != nil { 938 return err 939 } 940 if len(podList) == 0 { 941 return fmt.Errorf("empty pod list") 942 } 943 nsToPods := make(map[string][]kube_api.Pod) 944 for _, pod := range podList { 945 nsToPods[pod.Namespace] = append(nsToPods[pod.Namespace], pod) 946 } 947 948 for ns, podsInNamespace := range nsToPods { 949 labelMap := make(map[string]map[string]kube_api.Pod) 950 for _, p := range podsInNamespace { 951 for label, name := range p.Labels { 952 selector := label + "=" + name 953 if _, found := labelMap[selector]; !found { 954 labelMap[selector] = make(map[string]kube_api.Pod) 955 } 956 labelMap[selector][p.Name] = p 957 } 958 } 959 for selector, podsWithLabel := range labelMap { 960 sel, err := labels.Parse(selector) 961 if err != nil { 962 return err 963 } 964 metrics, err := getLabelSelectedPodMetrics(fm, svc, ns, &sel) 965 if err != nil { 966 return err 967 } 968 if len(metrics.Items) != len(podsWithLabel) { 969 return fmt.Errorf("Wrong number of label selected pod metrics: expected %v, got %v", len(podsWithLabel), len(metrics.Items)) 970 } 971 for _, podMetric := range metrics.Items { 972 pod := podsWithLabel[podMetric.Name] 973 err := checkSinglePodMetrics(&podMetric, &pod) 974 if err != nil { 975 return err 976 } 977 } 978 } 979 } 980 return nil 981 } 982 983 func apiTest(kubeVersion string, zone string) error { 984 fm, err := newKubeFramework(kubeVersion) 985 if err != nil { 986 return err 987 } 988 if err := buildAndPushDockerImages(fm, zone); err != nil { 989 return err 990 } 991 // Create heapster pod and service. 992 svc, rc, err := getHeapsterRcAndSvc(fm) 993 if err != nil { 994 return err 995 } 996 ns := *namespace 997 if err := deleteAll(fm, ns, svc, rc); err != nil { 998 return err 999 } 1000 if err := createAll(fm, ns, &svc, &rc); err != nil { 1001 return err 1002 } 1003 if err := fm.WaitUntilPodRunning(ns, rc.Spec.Template.Labels, time.Minute); err != nil { 1004 return err 1005 } 1006 if err := fm.WaitUntilServiceActive(svc, time.Minute); err != nil { 1007 return err 1008 } 1009 testFuncs := []func() error{ 1010 func() error { 1011 glog.V(2).Infof("Heapster metric export test...") 1012 err := runMetricExportTest(fm, svc) 1013 if err == nil { 1014 glog.V(2).Infof("Heapster metric export test: OK") 1015 } else { 1016 glog.V(2).Infof("Heapster metric export test: error: %v", err) 1017 } 1018 return err 1019 }, 1020 func() error { 1021 glog.V(2).Infof("Model test") 1022 err := runModelTest(fm, svc) 1023 if err == nil { 1024 glog.V(2).Infof("Model test: OK") 1025 } else { 1026 glog.V(2).Infof("Model test: error: %v", err) 1027 } 1028 return err 1029 }, 1030 func() error { 1031 glog.V(2).Infof("Metrics API test - single pod") 1032 err := runSinglePodMetricsApiTest(fm, svc) 1033 if err == nil { 1034 glog.V(2).Infof("Metrics API test - single pod: OK") 1035 } else { 1036 glog.V(2).Infof("Metrics API test - single pod: error: %v", err) 1037 } 1038 return err 1039 }, 1040 func() error { 1041 glog.V(2).Infof("Metrics API test - All pods in a namespace") 1042 err := runAllPodsInNamespaceMetricsApiTest(fm, svc) 1043 if err == nil { 1044 glog.V(2).Infof("Metrics API test - All pods in a namespace: OK") 1045 } else { 1046 glog.V(2).Infof("Metrics API test - All pods in a namespace: error: %v", err) 1047 } 1048 return err 1049 }, 1050 func() error { 1051 glog.V(2).Infof("Metrics API test - all pods") 1052 err := runAllPodsMetricsApiTest(fm, svc) 1053 if err == nil { 1054 glog.V(2).Infof("Metrics API test - all pods: OK") 1055 } else { 1056 glog.V(2).Infof("Metrics API test - all pods: error: %v", err) 1057 } 1058 return err 1059 }, 1060 func() error { 1061 glog.V(2).Infof("Metrics API test - label selector for pods") 1062 err := runLabelSelectorPodMetricsApiTest(fm, svc) 1063 if err == nil { 1064 glog.V(2).Infof("Metrics API test - label selector for pods: OK") 1065 } else { 1066 glog.V(2).Infof("Metrics API test - label selector for pods: error: %v", err) 1067 } 1068 return err 1069 }, 1070 func() error { 1071 glog.V(2).Infof("Metrics API test - single node") 1072 err := runSingleNodeMetricsApiTest(fm, svc) 1073 if err == nil { 1074 glog.V(2).Infof("Metrics API test - single node: OK") 1075 } else { 1076 glog.V(2).Infof("Metrics API test - single node: error: %v", err) 1077 } 1078 return err 1079 }, 1080 func() error { 1081 glog.V(2).Infof("Metrics API test - label selector for nodes") 1082 err := runLabelSelectorNodeMetricsApiTest(fm, svc) 1083 if err == nil { 1084 glog.V(2).Infof("Metrics API test - label selector for nodes: OK") 1085 } else { 1086 glog.V(2).Infof("Metrics API test - label selector for nodes: error: %v", err) 1087 } 1088 return err 1089 }, 1090 func() error { 1091 glog.V(2).Infof("Metrics API test - all nodes") 1092 err := runAllNodesMetricsApiTest(fm, svc) 1093 if err == nil { 1094 glog.V(2).Infof("Metrics API test - all nodes: OK") 1095 } else { 1096 glog.V(2).Infof("Metrics API test - all nodes: error: %v", err) 1097 } 1098 return err 1099 }, 1100 } 1101 attempts := *maxRetries 1102 glog.Infof("Starting tests") 1103 for { 1104 var err error 1105 for _, testFunc := range testFuncs { 1106 if err = testFunc(); err != nil { 1107 break 1108 } 1109 } 1110 if *runForever { 1111 continue 1112 } 1113 if err == nil { 1114 glog.V(2).Infof("All tests passed.") 1115 break 1116 } 1117 if attempts == 0 { 1118 glog.V(2).Info("Too many attempts.") 1119 return err 1120 } 1121 glog.V(2).Infof("Some tests failed. Retrying.") 1122 attempts-- 1123 time.Sleep(time.Second * 10) 1124 } 1125 deleteAll(fm, ns, svc, rc) 1126 removeHeapsterImage(fm, zone) 1127 return nil 1128 } 1129 1130 func runApiTest() error { 1131 tempDir, err := ioutil.TempDir("", "deploy") 1132 if err != nil { 1133 return nil 1134 } 1135 defer os.RemoveAll(tempDir) 1136 if *kubeVersions == "" { 1137 return apiTest("", *testZone) 1138 } 1139 kubeVersionsList := strings.Split(*kubeVersions, ",") 1140 for _, kubeVersion := range kubeVersionsList { 1141 if err := apiTest(kubeVersion, *testZone); err != nil { 1142 return err 1143 } 1144 } 1145 return nil 1146 } 1147 1148 func TestHeapster(t *testing.T) { 1149 if testing.Short() { 1150 t.Skip("skipping heapster kubernetes integration test.") 1151 } 1152 require.NoError(t, runApiTest()) 1153 }