github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/kubetest/e2e.go (about) 1 /* 2 Copyright 2014 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "fmt" 21 "io/ioutil" 22 "log" 23 "os" 24 "os/exec" 25 "path/filepath" 26 "regexp" 27 "strings" 28 "time" 29 ) 30 31 // Add more default --test_args as we migrate them 32 func argFields(args, dump, ipRange string) []string { 33 f := strings.Fields(args) 34 if dump != "" { 35 f = setFieldDefault(f, "--report-dir", dump) 36 // Disable logdump within ginkgo as it'll be done in kubetest anyway now. 37 f = setFieldDefault(f, "--disable-log-dump", "true") 38 } 39 if ipRange != "" { 40 f = setFieldDefault(f, "--cluster-ip-range", ipRange) 41 } 42 return f 43 } 44 45 func run(deploy deployer, o options) error { 46 if o.checkSkew { 47 os.Setenv("KUBECTL", "./cluster/kubectl.sh --match-server-version") 48 } else { 49 os.Setenv("KUBECTL", "./cluster/kubectl.sh") 50 } 51 os.Setenv("KUBE_CONFIG_FILE", "config-test.sh") 52 os.Setenv("KUBE_RUNTIME_CONFIG", o.runtimeConfig) 53 54 dump := o.dump 55 if dump != "" { 56 if !filepath.IsAbs(dump) { // Directory may change 57 wd, err := os.Getwd() 58 if err != nil { 59 return fmt.Errorf("failed to os.Getwd(): %v", err) 60 } 61 dump = filepath.Join(wd, dump) 62 } 63 } 64 65 if o.up { 66 if o.federation { 67 if err := xmlWrap("Federation TearDown Previous", fedDown); err != nil { 68 return fmt.Errorf("error tearing down previous federation control plane: %v", err) 69 } 70 } 71 if err := xmlWrap("TearDown Previous", deploy.Down); err != nil { 72 return fmt.Errorf("error tearing down previous cluster: %s", err) 73 } 74 } 75 76 var err error 77 var errs []error 78 79 // Ensures that the cleanup/down action is performed exactly once. 80 var ( 81 downDone = false 82 federationDownDone = false 83 ) 84 85 var ( 86 beforeResources []byte 87 upResources []byte 88 downResources []byte 89 afterResources []byte 90 ) 91 92 if o.checkLeaks { 93 errs = appendError(errs, xmlWrap("listResources Before", func() error { 94 beforeResources, err = listResources() 95 return err 96 })) 97 } 98 99 if o.up { 100 // If we tried to bring the cluster up, make a courtesy 101 // attempt to bring it down so we're not leaving resources around. 102 if o.down { 103 defer xmlWrap("Deferred TearDown", func() error { 104 if !downDone { 105 return deploy.Down() 106 } 107 return nil 108 }) 109 // Deferred statements are executed in last-in-first-out order, so 110 // federation down defer must appear after the cluster teardown in 111 // order to execute that before cluster teardown. 112 if o.federation { 113 defer xmlWrap("Deferred Federation TearDown", func() error { 114 if !federationDownDone { 115 return fedDown() 116 } 117 return nil 118 }) 119 } 120 } 121 // Start the cluster using this version. 122 if err := xmlWrap("Up", deploy.Up); err != nil { 123 if dump != "" { 124 xmlWrap("DumpClusterLogs (--up failed)", func() error { 125 // This frequently means the cluster does not exist. 126 // Thus DumpClusterLogs() typically fails. 127 // Therefore always return null for this scenarios. 128 // TODO(fejta): report a green E in testgrid if it errors. 129 deploy.DumpClusterLogs(dump, o.logexporterGCSPath) 130 return nil 131 }) 132 } 133 return fmt.Errorf("starting e2e cluster: %s", err) 134 } 135 if o.federation { 136 if err := xmlWrap("Federation Up", fedUp); err != nil { 137 xmlWrap("dumpFederationLogs", func() error { 138 return dumpFederationLogs(dump) 139 }) 140 return fmt.Errorf("error starting federation: %s", err) 141 } 142 } 143 144 if !o.nodeTests { 145 // Check that the api is reachable before proceeding with further steps. 146 errs = appendError(errs, xmlWrap("Check APIReachability", getKubectlVersion)) 147 if dump != "" { 148 errs = appendError(errs, xmlWrap("list nodes", func() error { 149 return listNodes(dump) 150 })) 151 } 152 } 153 } 154 155 if o.checkLeaks { 156 errs = appendError(errs, xmlWrap("listResources Up", func() error { 157 upResources, err = listResources() 158 return err 159 })) 160 } 161 162 if o.upgradeArgs != "" { 163 if err := xmlWrap("test setup", deploy.TestSetup); err != nil { 164 errs = appendError(errs, err) 165 } else { 166 errs = appendError(errs, xmlWrap("UpgradeTest", func() error { 167 return skewTest(argFields(o.upgradeArgs, dump, o.clusterIPRange), "upgrade", o.checkSkew) 168 })) 169 } 170 } 171 172 testArgs := argFields(o.testArgs, dump, o.clusterIPRange) 173 if o.test { 174 if err := xmlWrap("test setup", deploy.TestSetup); err != nil { 175 errs = appendError(errs, err) 176 } else if o.nodeTests { 177 nodeArgs := strings.Fields(o.nodeArgs) 178 errs = appendError(errs, xmlWrap("Node Tests", func() error { 179 return nodeTest(nodeArgs, o.testArgs, o.nodeTestArgs, o.gcpProject, o.gcpZone) 180 })) 181 } else { 182 errs = appendError(errs, xmlWrap("kubectl version", getKubectlVersion)) 183 if o.skew { 184 errs = appendError(errs, xmlWrap("SkewTest", func() error { 185 return skewTest(testArgs, "skew", o.checkSkew) 186 })) 187 } else { 188 if err := xmlWrap("IsUp", deploy.IsUp); err != nil { 189 errs = appendError(errs, err) 190 } else { 191 if o.federation { 192 errs = appendError(errs, xmlWrap("FederationTest", func() error { 193 return federationTest(testArgs) 194 })) 195 } else { 196 errs = appendError(errs, xmlWrap("Test", func() error { 197 return test(testArgs) 198 })) 199 } 200 } 201 } 202 } 203 } 204 205 if o.kubemark { 206 errs = appendError(errs, xmlWrap("Kubemark Overall", func() error { 207 return kubemarkTest(testArgs, dump, o.kubemarkNodes) 208 })) 209 } 210 211 if o.charts { 212 errs = appendError(errs, xmlWrap("Helm Charts", chartsTest)) 213 } 214 215 if o.perfTests { 216 errs = appendError(errs, xmlWrap("Perf Tests", perfTest)) 217 } 218 219 if dump != "" { 220 errs = appendError(errs, xmlWrap("DumpClusterLogs", func() error { 221 return deploy.DumpClusterLogs(dump, o.logexporterGCSPath) 222 })) 223 if o.federation { 224 errs = appendError(errs, xmlWrap("dumpFederationLogs", func() error { 225 return dumpFederationLogs(dump) 226 })) 227 } 228 } 229 230 if o.checkLeaks { 231 errs = appendError(errs, xmlWrap("listResources Down", func() error { 232 downResources, err = listResources() 233 return err 234 })) 235 } 236 237 if o.down { 238 if o.federation { 239 errs = appendError(errs, xmlWrap("Federation TearDown", func() error { 240 if !federationDownDone { 241 err := fedDown() 242 if err != nil { 243 return err 244 } 245 federationDownDone = true 246 } 247 return nil 248 })) 249 } 250 errs = appendError(errs, xmlWrap("TearDown", func() error { 251 if !downDone { 252 err := deploy.Down() 253 if err != nil { 254 return err 255 } 256 downDone = true 257 } 258 return nil 259 })) 260 } 261 262 if o.checkLeaks { 263 log.Print("Sleeping for 30 seconds...") // Wait for eventually consistent listing 264 time.Sleep(30 * time.Second) 265 if err := xmlWrap("listResources After", func() error { 266 afterResources, err = listResources() 267 return err 268 }); err != nil { 269 errs = append(errs, err) 270 } else { 271 errs = appendError(errs, xmlWrap("diffResources", func() error { 272 return diffResources(beforeResources, upResources, downResources, afterResources, dump) 273 })) 274 } 275 } 276 if len(errs) == 0 && o.publish != "" { 277 errs = appendError(errs, xmlWrap("Publish version", func() error { 278 // Use plaintext version file packaged with kubernetes.tar.gz 279 v, err := ioutil.ReadFile("version") 280 if err != nil { 281 return err 282 } 283 log.Printf("Set %s version to %s", o.publish, string(v)) 284 return finishRunning(exec.Command("gsutil", "cp", "version", o.publish)) 285 })) 286 } 287 288 if len(errs) != 0 { 289 return fmt.Errorf("encountered %d errors: %v", len(errs), errs) 290 } 291 return nil 292 } 293 294 func getKubectlVersion() error { 295 retries := 5 296 for { 297 _, err := output(exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "version")) 298 if err == nil { 299 return nil 300 } 301 retries-- 302 if retries == 0 { 303 return err 304 } 305 log.Print("Failed to reach api. Sleeping for 10 seconds before retrying...") 306 time.Sleep(10 * time.Second) 307 } 308 } 309 310 func listNodes(dump string) error { 311 b, err := output(exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml")) 312 if err != nil { 313 return err 314 } 315 return ioutil.WriteFile(filepath.Join(dump, "nodes.yaml"), b, 0644) 316 } 317 318 func diffResources(before, clusterUp, clusterDown, after []byte, location string) error { 319 if location == "" { 320 var err error 321 location, err = ioutil.TempDir("", "e2e-check-resources") 322 if err != nil { 323 return fmt.Errorf("Could not create e2e-check-resources temp dir: %s", err) 324 } 325 } 326 327 var mode os.FileMode = 0664 328 bp := filepath.Join(location, "gcp-resources-before.txt") 329 up := filepath.Join(location, "gcp-resources-cluster-up.txt") 330 cdp := filepath.Join(location, "gcp-resources-cluster-down.txt") 331 ap := filepath.Join(location, "gcp-resources-after.txt") 332 dp := filepath.Join(location, "gcp-resources-diff.txt") 333 334 if err := ioutil.WriteFile(bp, before, mode); err != nil { 335 return err 336 } 337 if err := ioutil.WriteFile(up, clusterUp, mode); err != nil { 338 return err 339 } 340 if err := ioutil.WriteFile(cdp, clusterDown, mode); err != nil { 341 return err 342 } 343 if err := ioutil.WriteFile(ap, after, mode); err != nil { 344 return err 345 } 346 347 stdout, cerr := output(exec.Command("diff", "-sw", "-U0", "-F^\\[.*\\]$", bp, ap)) 348 if err := ioutil.WriteFile(dp, stdout, mode); err != nil { 349 return err 350 } 351 if cerr == nil { // No diffs 352 return nil 353 } 354 lines := strings.Split(string(stdout), "\n") 355 if len(lines) < 3 { // Ignore the +++ and --- header lines 356 return nil 357 } 358 lines = lines[2:] 359 360 var added, report []string 361 resourceTypeRE := regexp.MustCompile(`^@@.+\s(\[\s\S+\s\])$`) 362 for _, l := range lines { 363 if matches := resourceTypeRE.FindStringSubmatch(l); matches != nil { 364 report = append(report, matches[1]) 365 } 366 if strings.HasPrefix(l, "+") && len(strings.TrimPrefix(l, "+")) > 0 { 367 added = append(added, l) 368 report = append(report, l) 369 } 370 } 371 if len(added) > 0 { 372 return fmt.Errorf("Error: %d leaked resources\n%v", len(added), strings.Join(report, "\n")) 373 } 374 return nil 375 } 376 377 func listResources() ([]byte, error) { 378 log.Printf("Listing resources...") 379 stdout, err := output(exec.Command("./cluster/gce/list-resources.sh")) 380 if err != nil { 381 return stdout, fmt.Errorf("Failed to list resources (%s):\n%s", err, string(stdout)) 382 } 383 return stdout, err 384 } 385 386 func clusterSize(deploy deployer) (int, error) { 387 if err := deploy.TestSetup(); err != nil { 388 return -1, err 389 } 390 o, err := output(exec.Command("kubectl", "get", "nodes", "--no-headers")) 391 if err != nil { 392 log.Printf("kubectl get nodes failed: %s\n%s", wrapError(err).Error(), string(o)) 393 return -1, err 394 } 395 stdout := strings.TrimSpace(string(o)) 396 log.Printf("Cluster nodes:\n%s", stdout) 397 return len(strings.Split(stdout, "\n")), nil 398 } 399 400 // commandError will provide stderr output (if available) from structured 401 // exit errors 402 type commandError struct { 403 err error 404 } 405 406 func wrapError(err error) *commandError { 407 if err == nil { 408 return nil 409 } 410 return &commandError{err: err} 411 } 412 413 func (e *commandError) Error() string { 414 if e == nil { 415 return "" 416 } 417 exitErr, ok := e.err.(*exec.ExitError) 418 if !ok { 419 return e.err.Error() 420 } 421 422 stderr := "" 423 if exitErr.Stderr != nil { 424 stderr = string(stderr) 425 } 426 return fmt.Sprintf("%q: %q", exitErr.Error(), stderr) 427 } 428 429 func isUp(d deployer) error { 430 n, err := clusterSize(d) 431 if err != nil { 432 return err 433 } 434 if n <= 0 { 435 return fmt.Errorf("cluster found, but %d nodes reported", n) 436 } 437 return nil 438 } 439 440 func waitForNodes(d deployer, nodes int, timeout time.Duration) error { 441 for stop := time.Now().Add(timeout); time.Now().Before(stop); time.Sleep(30 * time.Second) { 442 n, err := clusterSize(d) 443 if err != nil { 444 log.Printf("Can't get cluster size, sleeping: %v", err) 445 continue 446 } 447 if n < nodes { 448 log.Printf("%d (current nodes) < %d (requested instances), sleeping", n, nodes) 449 continue 450 } 451 return nil 452 } 453 return fmt.Errorf("waiting for nodes timed out") 454 } 455 456 func defaultDumpClusterLogs(localArtifactsDir, logexporterGCSPath string) error { 457 logDumpPath := "./cluster/log-dump/log-dump.sh" 458 // cluster/log-dump/log-dump.sh only exists in the Kubernetes tree 459 // post-1.3. If it doesn't exist, print a debug log but do not report an error. 460 if _, err := os.Stat(logDumpPath); err != nil { 461 log.Printf("Could not find %s. This is expected if running tests against a Kubernetes 1.3 or older tree.", logDumpPath) 462 if cwd, err := os.Getwd(); err == nil { 463 log.Printf("CWD: %v", cwd) 464 } 465 return nil 466 } 467 var cmd *exec.Cmd 468 if logexporterGCSPath != "" { 469 log.Printf("Dumping logs from nodes to GCS directly at path: %v", logexporterGCSPath) 470 cmd = exec.Command(logDumpPath, localArtifactsDir, logexporterGCSPath) 471 } else { 472 log.Printf("Dumping logs locally to: %v", localArtifactsDir) 473 cmd = exec.Command(logDumpPath, localArtifactsDir) 474 } 475 return finishRunning(cmd) 476 } 477 478 func dumpFederationLogs(location string) error { 479 logDumpPath := "./federation/cluster/log-dump.sh" 480 // federation/cluster/log-dump.sh only exists in the Kubernetes tree 481 // post-1.6. If it doesn't exist, do nothing and do not report an error. 482 if _, err := os.Stat(logDumpPath); err == nil { 483 log.Printf("Dumping Federation logs to: %v", location) 484 return finishRunning(exec.Command(logDumpPath, location)) 485 } 486 log.Printf("Could not find %s. This is expected if running tests against a Kubernetes 1.6 or older tree.", logDumpPath) 487 return nil 488 } 489 490 func perfTest() error { 491 // Run perf tests 492 // TODO(fejta): GOPATH may be split by : 493 cmdline := fmt.Sprintf("%s/src/k8s.io/perf-tests/clusterloader/run-e2e.sh", os.Getenv("GOPATH")) 494 if err := finishRunning(exec.Command(cmdline)); err != nil { 495 return err 496 } 497 return nil 498 } 499 500 func chartsTest() error { 501 // Run helm tests. 502 cmdline := fmt.Sprintf("%s/src/k8s.io/charts/test/helm-test-e2e.sh", os.Getenv("GOPATH")) 503 if err := finishRunning(exec.Command(cmdline)); err != nil { 504 return err 505 } 506 return nil 507 } 508 509 func nodeTest(nodeArgs []string, testArgs, nodeTestArgs, project, zone string) error { 510 // Run node e2e tests. 511 // TODO(krzyzacy): remove once nodeTest is stable 512 if wd, err := os.Getwd(); err == nil { 513 log.Printf("cwd : %s", wd) 514 } 515 516 sshKeyPath := os.Getenv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE") 517 if _, err := os.Stat(sshKeyPath); err != nil { 518 return fmt.Errorf("Cannot find ssh key from: %v, err : %v", sshKeyPath, err) 519 } 520 521 // prep node args 522 runner := []string{ 523 "run", 524 fmt.Sprintf("%s/src/k8s.io/kubernetes/test/e2e_node/runner/remote/run_remote.go", os.Getenv("GOPATH")), 525 "--cleanup", 526 "--logtostderr", 527 "--vmodule=*=4", 528 "--ssh-env=gce", 529 fmt.Sprintf("--results-dir=%s/_artifacts", os.Getenv("WORKSPACE")), 530 fmt.Sprintf("--project=%s", project), 531 fmt.Sprintf("--zone=%s", zone), 532 fmt.Sprintf("--ssh-user=%s", os.Getenv("USER")), 533 fmt.Sprintf("--ssh-key=%s", sshKeyPath), 534 fmt.Sprintf("--ginkgo-flags=%s", testArgs), 535 fmt.Sprintf("--test_args=%s", nodeTestArgs), 536 fmt.Sprintf("--test-timeout=%s", timeout.String()), 537 } 538 539 runner = append(runner, nodeArgs...) 540 541 if err := finishRunning(exec.Command("go", runner...)); err != nil { 542 return err 543 } 544 return nil 545 } 546 547 func kubemarkTest(testArgs []string, dump, numNodes string) error { 548 // Stop previously running kubemark cluster (if any). 549 if err := xmlWrap("Kubemark TearDown Previous", func() error { 550 return finishRunning(exec.Command("./test/kubemark/stop-kubemark.sh")) 551 }); err != nil { 552 return err 553 } 554 // If we tried to bring the Kubemark cluster up, make a courtesy 555 // attempt to bring it down so we're not leaving resources around. 556 // 557 // TODO: We should try calling stop-kubemark exactly once. Though to 558 // stop the leaking resources for now, we want to be on the safe side 559 // and call it explicitly in defer if the other one is not called. 560 defer xmlWrap("Kubemark TearDown (Deferred)", func() error { 561 return finishRunning(exec.Command("./test/kubemark/stop-kubemark.sh")) 562 }) 563 564 // Start kubemark cluster. 565 if err := xmlWrap("Kubemark Up", func() error { 566 return finishRunning(exec.Command("./test/kubemark/start-kubemark.sh")) 567 }); err != nil { 568 if dump != "" { 569 xmlWrap("Kubemark MasterLogDump (--up failed)", func() error { 570 return finishRunning(exec.Command("./test/kubemark/master-log-dump.sh", dump)) 571 }) 572 } 573 return err 574 } 575 576 // Run tests on the kubemark cluster. 577 if err := xmlWrap("Kubemark Test", func() error { 578 testArgs = setFieldDefault(testArgs, "--ginkgo.focus", "starting\\s30\\pods") 579 return finishRunning(exec.Command("./test/kubemark/run-e2e-tests.sh", testArgs...)) 580 }); err != nil { 581 if dump != "" { 582 xmlWrap("Kubemark MasterLogDump (--test failed)", func() error { 583 return finishRunning(exec.Command("./test/kubemark/master-log-dump.sh", dump)) 584 }) 585 } 586 return err 587 } 588 589 // Dump logs from kubemark master. 590 xmlWrap("Kubemark MasterLogDump", func() error { 591 return finishRunning(exec.Command("./test/kubemark/master-log-dump.sh", dump)) 592 }) 593 594 // Stop the kubemark cluster. 595 if err := xmlWrap("Kubemark TearDown", func() error { 596 return finishRunning(exec.Command("./test/kubemark/stop-kubemark.sh")) 597 }); err != nil { 598 return err 599 } 600 601 return nil 602 } 603 604 // Runs tests in the kubernetes_skew directory, appending --repor-prefix flag to the run 605 func skewTest(args []string, prefix string, checkSkew bool) error { 606 // TODO(fejta): run this inside this kubetest process, do not spawn a new one. 607 popS, err := pushd("../kubernetes_skew") 608 if err != nil { 609 return err 610 } 611 defer popS() 612 args = appendField(args, "--report-prefix", prefix) 613 return finishRunning(exec.Command( 614 "kubetest", 615 "--test", 616 "--test_args="+strings.Join(args, " "), 617 fmt.Sprintf("--v=%t", verbose), 618 fmt.Sprintf("--check-version-skew=%t", checkSkew), 619 )) 620 } 621 622 func test(testArgs []string) error { 623 return finishRunning(exec.Command("./hack/ginkgo-e2e.sh", testArgs...)) 624 }