github.com/jenkins-x/test-infra@v0.0.7/kubetest/e2e.go (about) 1 /* 2 Copyright 2014 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "fmt" 21 "io/ioutil" 22 "log" 23 "os" 24 "os/exec" 25 "path/filepath" 26 "regexp" 27 "strings" 28 "sync" 29 "time" 30 31 "k8s.io/test-infra/kubetest/e2e" 32 "k8s.io/test-infra/kubetest/process" 33 "k8s.io/test-infra/kubetest/util" 34 ) 35 36 // Add more default --test_args as we migrate them 37 func argFields(args, dump, ipRange string) []string { 38 f := strings.Fields(args) 39 if dump != "" { 40 f = util.SetFieldDefault(f, "--report-dir", dump) 41 // Disable logdump within ginkgo as it'll be done in kubetest anyway now. 42 f = util.SetFieldDefault(f, "--disable-log-dump", "true") 43 } 44 if ipRange != "" { 45 f = util.SetFieldDefault(f, "--cluster-ip-range", ipRange) 46 } 47 return f 48 } 49 50 func run(deploy deployer, o options) error { 51 cmd, err := deploy.KubectlCommand() 52 if err != nil { 53 return err 54 } 55 if cmd == nil { 56 cmd = exec.Command("./cluster/kubectl.sh") 57 } 58 if o.checkSkew { 59 cmd.Args = append(cmd.Args, "--match-server-version") 60 } 61 os.Setenv("KUBECTL", strings.Join(cmd.Args, " ")) 62 63 os.Setenv("KUBE_CONFIG_FILE", "config-test.sh") 64 os.Setenv("KUBE_RUNTIME_CONFIG", o.runtimeConfig) 65 66 var errs []error 67 68 dump, err := util.OptionalAbsPath(o.dump) 69 if err != nil { 70 return fmt.Errorf("failed handling --dump path: %v", err) 71 } 72 73 dumpPreTestLogs, err := util.OptionalAbsPath(o.dumpPreTestLogs) 74 if err != nil { 75 return fmt.Errorf("failed handling --dump-pre-test-logs path: %v", err) 76 } 77 78 if o.up { 79 if o.federation { 80 if err := control.XMLWrap(&suite, "Federation TearDown Previous", fedDown); err != nil { 81 return fmt.Errorf("error tearing down previous federation control plane: %v", err) 82 } 83 } 84 if err := control.XMLWrap(&suite, "TearDown Previous", deploy.Down); err != nil { 85 return fmt.Errorf("error tearing down previous cluster: %s", err) 86 } 87 } 88 89 // Ensures that the cleanup/down action is performed exactly once. 90 var ( 91 downDone = false 92 federationDownDone = false 93 ) 94 95 var ( 96 beforeResources []byte 97 upResources []byte 98 downResources []byte 99 afterResources []byte 100 ) 101 102 if o.checkLeaks { 103 errs = util.AppendError(errs, control.XMLWrap(&suite, "listResources Before", func() error { 104 beforeResources, err = listResources() 105 return err 106 })) 107 } 108 109 if o.up { 110 // If we tried to bring the cluster up, make a courtesy 111 // attempt to bring it down so we're not leaving resources around. 112 if o.down { 113 defer control.XMLWrap(&suite, "Deferred TearDown", func() error { 114 if !downDone { 115 return deploy.Down() 116 } 117 return nil 118 }) 119 // Deferred statements are executed in last-in-first-out order, so 120 // federation down defer must appear after the cluster teardown in 121 // order to execute that before cluster teardown. 122 if o.federation { 123 defer control.XMLWrap(&suite, "Deferred Federation TearDown", func() error { 124 if !federationDownDone { 125 return fedDown() 126 } 127 return nil 128 }) 129 } 130 } 131 // Start the cluster using this version. 132 if err := control.XMLWrap(&suite, "Up", deploy.Up); err != nil { 133 if dump != "" { 134 control.XMLWrap(&suite, "DumpClusterLogs (--up failed)", func() error { 135 // This frequently means the cluster does not exist. 136 // Thus DumpClusterLogs() typically fails. 137 // Therefore always return null for this scenarios. 138 // TODO(fejta): report a green E in testgrid if it errors. 139 deploy.DumpClusterLogs(dump, o.logexporterGCSPath) 140 return nil 141 }) 142 } 143 return fmt.Errorf("starting e2e cluster: %s", err) 144 } 145 if o.federation { 146 if err := control.XMLWrap(&suite, "Federation Up", fedUp); err != nil { 147 control.XMLWrap(&suite, "dumpFederationLogs", func() error { 148 return dumpFederationLogs(dump) 149 }) 150 return fmt.Errorf("error starting federation: %s", err) 151 } 152 } 153 // If node testing is enabled, check that the api is reachable before 154 // proceeding with further steps. This is accomplished by listing the nodes. 155 if !o.nodeTests { 156 errs = util.AppendError(errs, control.XMLWrap(&suite, "Check APIReachability", func() error { return getKubectlVersion(deploy) })) 157 if dump != "" { 158 errs = util.AppendError(errs, control.XMLWrap(&suite, "list nodes", func() error { 159 return listNodes(deploy, dump) 160 })) 161 } 162 } 163 } 164 165 if o.checkLeaks { 166 errs = util.AppendError(errs, control.XMLWrap(&suite, "listResources Up", func() error { 167 upResources, err = listResources() 168 return err 169 })) 170 } 171 172 if o.upgradeArgs != "" { 173 if err := control.XMLWrap(&suite, "test setup", deploy.TestSetup); err != nil { 174 errs = util.AppendError(errs, err) 175 } else { 176 errs = util.AppendError(errs, control.XMLWrap(&suite, "UpgradeTest", func() error { 177 // upgrade tests really only run one spec 178 var env []string 179 for _, v := range os.Environ() { 180 if !strings.HasPrefix(v, "GINKGO_PARALLEL") { 181 env = append(env, v) 182 } 183 } 184 return skewTestEnv(env, argFields(o.upgradeArgs, dump, o.clusterIPRange), "upgrade", o.checkSkew) 185 })) 186 } 187 } 188 189 if dumpPreTestLogs != "" { 190 errs = append(errs, dumpRemoteLogs(deploy, o, dumpPreTestLogs, "pre-test")...) 191 } 192 193 testArgs := argFields(o.testArgs, dump, o.clusterIPRange) 194 if o.test { 195 if err := control.XMLWrap(&suite, "test setup", deploy.TestSetup); err != nil { 196 errs = util.AppendError(errs, err) 197 } else if o.nodeTests { 198 nodeArgs := strings.Fields(o.nodeArgs) 199 errs = util.AppendError(errs, control.XMLWrap(&suite, "Node Tests", func() error { 200 return nodeTest(nodeArgs, o.testArgs, o.nodeTestArgs, o.gcpProject, o.gcpZone) 201 })) 202 } else if err := control.XMLWrap(&suite, "IsUp", deploy.IsUp); err != nil { 203 errs = util.AppendError(errs, err) 204 } else if o.federation { 205 errs = util.AppendError(errs, control.XMLWrap(&suite, "FederationTest", func() error { 206 return federationTest(testArgs) 207 })) 208 } else { 209 if o.deployment != "conformance" { 210 errs = util.AppendError(errs, control.XMLWrap(&suite, "kubectl version", func() error { return getKubectlVersion(deploy) })) 211 } 212 213 if o.skew { 214 errs = util.AppendError(errs, control.XMLWrap(&suite, "SkewTest", func() error { 215 return skewTest(testArgs, "skew", o.checkSkew) 216 })) 217 } else { 218 var tester e2e.Tester 219 tester = &GinkgoScriptTester{} 220 if testBuilder, ok := deploy.(e2e.TestBuilder); ok { 221 tester, err = testBuilder.BuildTester(toBuildTesterOptions(&o)) 222 errs = util.AppendError(errs, err) 223 } 224 if tester != nil { 225 errs = util.AppendError(errs, control.XMLWrap(&suite, "Test", func() error { 226 return tester.Run(control, testArgs) 227 })) 228 } 229 } 230 } 231 } 232 233 if o.testCmd != "" { 234 if err := control.XMLWrap(&suite, "test setup", deploy.TestSetup); err != nil { 235 errs = util.AppendError(errs, err) 236 } else { 237 errs = util.AppendError(errs, control.XMLWrap(&suite, o.testCmdName, func() error { 238 cmdLine := os.ExpandEnv(o.testCmd) 239 return control.FinishRunning(exec.Command(cmdLine, o.testCmdArgs...)) 240 })) 241 } 242 } 243 244 // TODO(bentheelder): consider remapping charts, etc to testCmd 245 246 var kubemarkWg sync.WaitGroup 247 var kubemarkDownErr error 248 if o.kubemark { 249 errs = util.AppendError(errs, control.XMLWrap(&suite, "Kubemark Overall", func() error { 250 return kubemarkTest(testArgs, dump, o, deploy) 251 })) 252 kubemarkWg.Add(1) 253 go kubemarkDown(&kubemarkDownErr, &kubemarkWg) 254 } 255 256 if o.charts { 257 errs = util.AppendError(errs, control.XMLWrap(&suite, "Helm Charts", chartsTest)) 258 } 259 260 if dump != "" { 261 errs = append(errs, dumpRemoteLogs(deploy, o, dump, "")...) 262 } 263 264 if o.checkLeaks { 265 errs = util.AppendError(errs, control.XMLWrap(&suite, "listResources Down", func() error { 266 downResources, err = listResources() 267 return err 268 })) 269 } 270 271 if o.down { 272 if o.federation { 273 errs = util.AppendError(errs, control.XMLWrap(&suite, "Federation TearDown", func() error { 274 if !federationDownDone { 275 err := fedDown() 276 if err != nil { 277 return err 278 } 279 federationDownDone = true 280 } 281 return nil 282 })) 283 } 284 errs = util.AppendError(errs, control.XMLWrap(&suite, "TearDown", func() error { 285 if !downDone { 286 err := deploy.Down() 287 if err != nil { 288 return err 289 } 290 downDone = true 291 } 292 return nil 293 })) 294 } 295 296 // Wait for kubemarkDown step to finish before going further. 297 kubemarkWg.Wait() 298 errs = util.AppendError(errs, kubemarkDownErr) 299 300 // Save the state if we upped a new cluster without downing it 301 // or we are turning up federated clusters without turning up 302 // the federation control plane. 303 if o.save != "" && ((!o.down && o.up) || (!o.federation && o.up && o.deployment != "none")) { 304 errs = util.AppendError(errs, control.XMLWrap(&suite, "Save Cluster State", func() error { 305 return saveState(o.save) 306 })) 307 } 308 309 if o.checkLeaks { 310 log.Print("Sleeping for 30 seconds...") // Wait for eventually consistent listing 311 time.Sleep(30 * time.Second) 312 if err := control.XMLWrap(&suite, "listResources After", func() error { 313 afterResources, err = listResources() 314 return err 315 }); err != nil { 316 errs = append(errs, err) 317 } else { 318 errs = util.AppendError(errs, control.XMLWrap(&suite, "diffResources", func() error { 319 return diffResources(beforeResources, upResources, downResources, afterResources, dump) 320 })) 321 } 322 } 323 if len(errs) == 0 { 324 if pub, ok := deploy.(publisher); ok { 325 errs = util.AppendError(errs, pub.Publish()) 326 } 327 } 328 if len(errs) == 0 && o.publish != "" { 329 errs = util.AppendError(errs, control.XMLWrap(&suite, "Publish version", func() error { 330 // Use plaintext version file packaged with kubernetes.tar.gz 331 v, err := ioutil.ReadFile("version") 332 if err != nil { 333 return err 334 } 335 log.Printf("Set %s version to %s", o.publish, string(v)) 336 return gcsWrite(o.publish, v) 337 })) 338 } 339 340 if len(errs) != 0 { 341 return fmt.Errorf("encountered %d errors: %v", len(errs), errs) 342 } 343 return nil 344 } 345 346 func getKubectlVersion(dp deployer) error { 347 cmd, err := dp.KubectlCommand() 348 if err != nil { 349 return err 350 } 351 if cmd == nil { 352 cmd = exec.Command("./cluster/kubectl.sh") 353 } 354 cmd.Args = append(cmd.Args, "--match-server-version=false", "version") 355 copied := *cmd 356 retries := 5 357 for { 358 _, err := control.Output(&copied) 359 if err == nil { 360 return nil 361 } 362 retries-- 363 if retries == 0 { 364 return err 365 } 366 log.Printf("Failed to reach api. Sleeping for 10 seconds before retrying... (%v)", copied.Args) 367 time.Sleep(10 * time.Second) 368 } 369 } 370 371 func dumpRemoteLogs(deploy deployer, o options, path, reason string) []error { 372 if reason != "" { 373 reason += " " 374 } 375 376 var errs []error 377 378 errs = util.AppendError(errs, control.XMLWrap(&suite, reason+"DumpClusterLogs", func() error { 379 return deploy.DumpClusterLogs(path, o.logexporterGCSPath) 380 })) 381 if o.federation { 382 errs = util.AppendError(errs, control.XMLWrap(&suite, reason+"dumpFederationLogs", func() error { 383 return dumpFederationLogs(path) 384 })) 385 } 386 387 return errs 388 } 389 390 func listNodes(dp deployer, dump string) error { 391 cmd, err := dp.KubectlCommand() 392 if err != nil { 393 return err 394 } 395 if cmd == nil { 396 cmd = exec.Command("./cluster/kubectl.sh") 397 } 398 cmd.Args = append(cmd.Args, "--match-server-version=false", "get", "nodes", "-oyaml") 399 b, err := control.Output(cmd) 400 if err != nil { 401 return err 402 } 403 return ioutil.WriteFile(filepath.Join(dump, "nodes.yaml"), b, 0644) 404 } 405 406 func listKubemarkNodes(dp deployer, dump string) error { 407 cmd, err := dp.KubectlCommand() 408 if err != nil { 409 return err 410 } 411 if cmd == nil { 412 cmd = exec.Command("./cluster/kubectl.sh") 413 } 414 cmd.Args = append(cmd.Args, "--match-server-version=false", "--kubeconfig=./test/kubemark/resources/kubeconfig.kubemark", "get", "nodes", "-oyaml") 415 b, err := control.Output(cmd) 416 if err != nil { 417 return err 418 } 419 return ioutil.WriteFile(filepath.Join(dump, "kubemark_nodes.yaml"), b, 0644) 420 } 421 422 func diffResources(before, clusterUp, clusterDown, after []byte, location string) error { 423 if location == "" { 424 var err error 425 location, err = ioutil.TempDir("", "e2e-check-resources") 426 if err != nil { 427 return fmt.Errorf("Could not create e2e-check-resources temp dir: %s", err) 428 } 429 } 430 431 var mode os.FileMode = 0664 432 bp := filepath.Join(location, "gcp-resources-before.txt") 433 up := filepath.Join(location, "gcp-resources-cluster-up.txt") 434 cdp := filepath.Join(location, "gcp-resources-cluster-down.txt") 435 ap := filepath.Join(location, "gcp-resources-after.txt") 436 dp := filepath.Join(location, "gcp-resources-diff.txt") 437 438 if err := ioutil.WriteFile(bp, before, mode); err != nil { 439 return err 440 } 441 if err := ioutil.WriteFile(up, clusterUp, mode); err != nil { 442 return err 443 } 444 if err := ioutil.WriteFile(cdp, clusterDown, mode); err != nil { 445 return err 446 } 447 if err := ioutil.WriteFile(ap, after, mode); err != nil { 448 return err 449 } 450 451 stdout, cerr := control.Output(exec.Command("diff", "-sw", "-U0", "-F^\\[.*\\]$", bp, ap)) 452 if err := ioutil.WriteFile(dp, stdout, mode); err != nil { 453 return err 454 } 455 if cerr == nil { // No diffs 456 return nil 457 } 458 lines := strings.Split(string(stdout), "\n") 459 if len(lines) < 3 { // Ignore the +++ and --- header lines 460 return nil 461 } 462 lines = lines[2:] 463 464 var added, report []string 465 resourceTypeRE := regexp.MustCompile(`^@@.+\s(\[\s\S+\s\])$`) 466 for _, l := range lines { 467 if matches := resourceTypeRE.FindStringSubmatch(l); matches != nil { 468 report = append(report, matches[1]) 469 } 470 if strings.HasPrefix(l, "+") && len(strings.TrimPrefix(l, "+")) > 0 { 471 added = append(added, l) 472 report = append(report, l) 473 } 474 } 475 if len(added) > 0 { 476 return fmt.Errorf("Error: %d leaked resources\n%v", len(added), strings.Join(report, "\n")) 477 } 478 return nil 479 } 480 481 func listResources() ([]byte, error) { 482 log.Printf("Listing resources...") 483 stdout, err := control.Output(exec.Command("./cluster/gce/list-resources.sh")) 484 if err != nil { 485 return stdout, fmt.Errorf("Failed to list resources (%s):\n%s", err, string(stdout)) 486 } 487 return stdout, err 488 } 489 490 func clusterSize(deploy deployer) (int, error) { 491 if err := deploy.TestSetup(); err != nil { 492 return -1, err 493 } 494 o, err := control.Output(exec.Command("kubectl", "get", "nodes", "--no-headers")) 495 if err != nil { 496 log.Printf("kubectl get nodes failed: %s\n%s", wrapError(err).Error(), string(o)) 497 return -1, err 498 } 499 stdout := strings.TrimSpace(string(o)) 500 log.Printf("Cluster nodes:\n%s", stdout) 501 return len(strings.Split(stdout, "\n")), nil 502 } 503 504 // commandError will provide stderr output (if available) from structured 505 // exit errors 506 type commandError struct { 507 err error 508 } 509 510 func wrapError(err error) *commandError { 511 if err == nil { 512 return nil 513 } 514 return &commandError{err: err} 515 } 516 517 func (e *commandError) Error() string { 518 if e == nil { 519 return "" 520 } 521 exitErr, ok := e.err.(*exec.ExitError) 522 if !ok { 523 return e.err.Error() 524 } 525 526 stderr := "" 527 if exitErr.Stderr != nil { 528 stderr = string(stderr) 529 } 530 return fmt.Sprintf("%q: %q", exitErr.Error(), stderr) 531 } 532 533 func isUp(d deployer) error { 534 n, err := clusterSize(d) 535 if err != nil { 536 return err 537 } 538 if n <= 0 { 539 return fmt.Errorf("cluster found, but %d nodes reported", n) 540 } 541 return nil 542 } 543 544 func defaultDumpClusterLogs(localArtifactsDir, logexporterGCSPath string) error { 545 logDumpPath := "./cluster/log-dump/log-dump.sh" 546 // cluster/log-dump/log-dump.sh only exists in the Kubernetes tree 547 // post-1.3. If it doesn't exist, print a debug log but do not report an error. 548 if _, err := os.Stat(logDumpPath); err != nil { 549 log.Printf("Could not find %s. This is expected if running tests against a Kubernetes 1.3 or older tree.", logDumpPath) 550 if cwd, err := os.Getwd(); err == nil { 551 log.Printf("CWD: %v", cwd) 552 } 553 return nil 554 } 555 var cmd *exec.Cmd 556 if logexporterGCSPath != "" { 557 log.Printf("Dumping logs from nodes to GCS directly at path: %v", logexporterGCSPath) 558 cmd = exec.Command(logDumpPath, localArtifactsDir, logexporterGCSPath) 559 } else { 560 log.Printf("Dumping logs locally to: %v", localArtifactsDir) 561 cmd = exec.Command(logDumpPath, localArtifactsDir) 562 } 563 return control.FinishRunning(cmd) 564 } 565 566 func dumpFederationLogs(location string) error { 567 // TODO(shashidharatd): Remove below logic of choosing the scripts to run from federation 568 // repo once the k8s deployment in federation jobs moves to kubernetes-anywhere 569 var logDumpPath string 570 if useFederationRepo() { 571 logDumpPath = "../federation/deploy/cluster/log-dump.sh" 572 } else { 573 logDumpPath = "./federation/cluster/log-dump.sh" 574 } 575 // federation/cluster/log-dump.sh only exists in the Kubernetes tree 576 // post-1.6. If it doesn't exist, do nothing and do not report an error. 577 if _, err := os.Stat(logDumpPath); err == nil { 578 log.Printf("Dumping Federation logs to: %v", location) 579 return control.FinishRunning(exec.Command(logDumpPath, location)) 580 } 581 log.Printf("Could not find %s. This is expected if running tests against a Kubernetes 1.6 or older tree.", logDumpPath) 582 return nil 583 } 584 585 func chartsTest() error { 586 // Run helm tests. 587 cmdline := util.K8s("charts", "test", "helm-test-e2e.sh") 588 return control.FinishRunning(exec.Command(cmdline)) 589 } 590 591 func nodeTest(nodeArgs []string, testArgs, nodeTestArgs, project, zone string) error { 592 // Run node e2e tests. 593 // TODO(krzyzacy): remove once nodeTest is stable 594 if wd, err := os.Getwd(); err == nil { 595 log.Printf("cwd : %s", wd) 596 } 597 598 sshKeyPath := os.Getenv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE") 599 if _, err := os.Stat(sshKeyPath); err != nil { 600 return fmt.Errorf("Cannot find ssh key from: %v, err : %v", sshKeyPath, err) 601 } 602 603 // prep node args 604 runner := []string{ 605 "run", 606 util.K8s("kubernetes", "test", "e2e_node", "runner", "remote", "run_remote.go"), 607 "--cleanup", 608 "--logtostderr", 609 "--vmodule=*=4", 610 "--ssh-env=gce", 611 fmt.Sprintf("--results-dir=%s/_artifacts", os.Getenv("WORKSPACE")), 612 fmt.Sprintf("--project=%s", project), 613 fmt.Sprintf("--zone=%s", zone), 614 fmt.Sprintf("--ssh-user=%s", os.Getenv("USER")), 615 fmt.Sprintf("--ssh-key=%s", sshKeyPath), 616 fmt.Sprintf("--ginkgo-flags=%s", testArgs), 617 fmt.Sprintf("--test_args=%s", nodeTestArgs), 618 fmt.Sprintf("--test-timeout=%s", timeout.String()), 619 } 620 621 runner = append(runner, nodeArgs...) 622 623 return control.FinishRunning(exec.Command("go", runner...)) 624 } 625 626 func kubemarkTest(testArgs []string, dump string, o options, deploy deployer) error { 627 // Stop previously running kubemark cluster (if any). 628 if err := control.XMLWrap(&suite, "Kubemark TearDown Previous", func() error { 629 return control.FinishRunning(exec.Command("./test/kubemark/stop-kubemark.sh")) 630 }); err != nil { 631 return err 632 } 633 634 if err := control.XMLWrap(&suite, "IsUp", deploy.IsUp); err != nil { 635 return err 636 } 637 638 // Start kubemark cluster. 639 if err := control.XMLWrap(&suite, "Kubemark Up", func() error { 640 return control.FinishRunning(exec.Command("./test/kubemark/start-kubemark.sh")) 641 }); err != nil { 642 if dump != "" { 643 control.XMLWrap(&suite, "Kubemark MasterLogDump (--up failed)", func() error { 644 return control.FinishRunning(exec.Command("./test/kubemark/master-log-dump.sh", dump)) 645 }) 646 } 647 return err 648 } 649 650 // Check kubemark apiserver reachability by listing all nodes. 651 if dump != "" { 652 control.XMLWrap(&suite, "list kubemark nodes", func() error { 653 return listKubemarkNodes(deploy, dump) 654 }) 655 } 656 657 // Run tests on the kubemark cluster. 658 if err := control.XMLWrap(&suite, "Kubemark Test", func() error { 659 testArgs = util.SetFieldDefault(testArgs, "--ginkgo.focus", "starting\\s30\\pods") 660 661 // detect master IP 662 if err := os.Setenv("MASTER_NAME", os.Getenv("INSTANCE_PREFIX")+"-kubemark-master"); err != nil { 663 return err 664 } 665 666 masterIP, err := control.Output(exec.Command( 667 "gcloud", "compute", "addresses", "describe", 668 os.Getenv("MASTER_NAME")+"-ip", 669 "--project="+o.gcpProject, 670 "--region="+o.gcpZone[:len(o.gcpZone)-2], 671 "--format=value(address)")) 672 if err != nil { 673 return fmt.Errorf("failed to get masterIP: %v", err) 674 } 675 if err := os.Setenv("KUBE_MASTER_IP", strings.TrimSpace(string(masterIP))); err != nil { 676 return err 677 } 678 679 if os.Getenv("ENABLE_KUBEMARK_CLUSTER_AUTOSCALER") == "true" { 680 testArgs = append(testArgs, "--kubemark-external-kubeconfig="+os.Getenv("DEFAULT_KUBECONFIG")) 681 } 682 683 cwd, err := os.Getwd() 684 if err != nil { 685 return err 686 } 687 688 // TODO(krzyzacy): unsure if the envs in kubemark/util.sh makes a difference to e2e tests 689 // will verify and remove (or uncomment) next 690 //util := os.Getenv("WORKSPACE") + "/kubernetes/cluster/kubemark/util.sh" 691 //testArgs = append([]string{"-c", "source", util, " ; ./hack/ginkgo-e2e.sh"}, testArgs...) 692 cmd := exec.Command("./hack/ginkgo-e2e.sh", testArgs...) 693 cmd.Env = append( 694 os.Environ(), 695 "KUBERNETES_PROVIDER=kubemark", 696 "KUBE_CONFIG_FILE=config-default.sh", 697 fmt.Sprintf("KUBECONFIG=%s/test/kubemark/resources/kubeconfig.kubemark", cwd), 698 "KUBE_MASTER_URL=https://"+os.Getenv("KUBE_MASTER_IP"), 699 ) 700 701 return control.FinishRunning(cmd) 702 }); err != nil { 703 if dump != "" { 704 control.XMLWrap(&suite, "Kubemark MasterLogDump (--test failed)", func() error { 705 return control.FinishRunning(exec.Command("./test/kubemark/master-log-dump.sh", dump)) 706 }) 707 } 708 return err 709 } 710 711 // Dump logs from kubemark master. 712 control.XMLWrap(&suite, "Kubemark MasterLogDump", func() error { 713 return control.FinishRunning(exec.Command("./test/kubemark/master-log-dump.sh", dump)) 714 }) 715 716 // 'Stop kubemark cluster' step has now been moved outside this function 717 // to make it asynchronous with other steps (to speed test execution). 718 return nil 719 } 720 721 // Brings down the kubemark cluster. 722 func kubemarkDown(err *error, wg *sync.WaitGroup) { 723 defer wg.Done() 724 *err = control.XMLWrap(&suite, "Kubemark TearDown", func() error { 725 return control.FinishRunning(exec.Command("./test/kubemark/stop-kubemark.sh")) 726 }) 727 } 728 729 // Runs tests in the kubernetes_skew directory, appending --report-prefix flag to the run 730 func skewTest(args []string, prefix string, checkSkew bool) error { 731 return skewTestEnv(nil, args, prefix, checkSkew) 732 } 733 734 // Runs tests in the kubernetes_skew directory, appending --report-prefix flag to the run 735 func skewTestEnv(env, args []string, prefix string, checkSkew bool) error { 736 // TODO(fejta): run this inside this kubetest process, do not spawn a new one. 737 popS, err := util.Pushd("../kubernetes_skew") 738 if err != nil { 739 return err 740 } 741 defer popS() 742 args = util.AppendField(args, "--report-prefix", prefix) 743 cmd := exec.Command( 744 "kubetest", 745 "--test", 746 "--test_args="+strings.Join(args, " "), 747 fmt.Sprintf("--check-version-skew=%t", checkSkew), 748 ) 749 cmd.Env = env 750 return control.FinishRunning(cmd) 751 } 752 753 // GinkgoScriptTester implements Tester by calling the hack/ginkgo-e2e.sh script 754 type GinkgoScriptTester struct { 755 } 756 757 // Run executes ./hack/ginkgo-e2e.sh 758 func (t *GinkgoScriptTester) Run(control *process.Control, testArgs []string) error { 759 return control.FinishRunning(exec.Command("./hack/ginkgo-e2e.sh", testArgs...)) 760 } 761 762 // toBuildTesterOptions builds the BuildTesterOptions data structure for passing to BuildTester 763 func toBuildTesterOptions(o *options) *e2e.BuildTesterOptions { 764 return &e2e.BuildTesterOptions{ 765 FocusRegex: o.focusRegex, 766 SkipRegex: o.skipRegex, 767 Parallelism: o.ginkgoParallel.Get(), 768 } 769 }