github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/kubetest/main.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "encoding/json" 21 "encoding/xml" 22 "errors" 23 "flag" 24 "fmt" 25 "io/ioutil" 26 "log" 27 "os" 28 "os/exec" 29 "os/signal" 30 "path/filepath" 31 "regexp" 32 "strconv" 33 "strings" 34 "time" 35 36 "k8s.io/test-infra/boskos/client" 37 ) 38 39 // Hardcoded in ginkgo-e2e.sh 40 const defaultGinkgoParallel = 25 41 42 var ( 43 artifacts = filepath.Join(os.Getenv("WORKSPACE"), "_artifacts") 44 interrupt = time.NewTimer(time.Duration(0)) // interrupt testing at this time. 45 terminate = time.NewTimer(time.Duration(0)) // terminate testing at this time. 46 verbose = false 47 timeout = time.Duration(0) 48 boskos = client.NewClient(os.Getenv("JOB_NAME"), "http://boskos") 49 ) 50 51 type options struct { 52 build buildStrategy 53 charts bool 54 checkLeaks bool 55 checkSkew bool 56 cluster string 57 clusterIPRange string 58 deployment string 59 down bool 60 dump string 61 extract extractStrategies 62 federation bool 63 gcpCloudSdk string 64 gcpMasterImage string 65 gcpNetwork string 66 gcpNodeImage string 67 gcpNodes string 68 gcpProject string 69 gcpProjectType string 70 gcpServiceAccount string 71 gcpRegion string 72 gcpZone string 73 ginkgoParallel ginkgoParallelValue 74 kubemark bool 75 kubemarkMasterSize string 76 kubemarkNodes string // TODO(fejta): switch to int after migration 77 logexporterGCSPath string 78 metadataSources string 79 multiClusters multiClusterDeployment 80 multipleFederations bool 81 nodeArgs string 82 nodeTestArgs string 83 nodeTests bool 84 perfTests bool 85 provider string 86 publish string 87 runtimeConfig string 88 save string 89 skew bool 90 stage stageStrategy 91 test bool 92 testArgs string 93 up bool 94 upgradeArgs string 95 } 96 97 func defineFlags() *options { 98 o := options{} 99 flag.Var(&o.build, "build", "Rebuild k8s binaries, optionally forcing (release|quick|bazel) stategy") 100 flag.BoolVar(&o.charts, "charts", false, "If true, run charts tests") 101 flag.BoolVar(&o.checkSkew, "check-version-skew", true, "Verify client and server versions match") 102 flag.BoolVar(&o.checkLeaks, "check-leaked-resources", false, "Ensure project ends with the same resources") 103 flag.StringVar(&o.cluster, "cluster", "", "Cluster name. Must be set for --deployment=gke (TODO: other deployments).") 104 flag.StringVar(&o.clusterIPRange, "cluster-ip-range", "", "Specifies CLUSTER_IP_RANGE value during --up and --test (only relevant for --deployment=bash). Auto-calculated if empty.") 105 flag.StringVar(&o.deployment, "deployment", "bash", "Choices: none/bash/gke/kops/kubernetes-anywhere/node") 106 flag.BoolVar(&o.down, "down", false, "If true, tear down the cluster before exiting.") 107 flag.StringVar(&o.dump, "dump", "", "If set, dump cluster logs to this location on test or cluster-up failure") 108 flag.Var(&o.extract, "extract", "Extract k8s binaries from the specified release location") 109 flag.BoolVar(&o.federation, "federation", false, "If true, start/tear down the federation control plane along with the clusters. To only start/tear down the federation control plane, specify --deployment=none") 110 flag.Var(&o.ginkgoParallel, "ginkgo-parallel", fmt.Sprintf("Run Ginkgo tests in parallel, default %d runners. Use --ginkgo-parallel=N to specify an exact count.", defaultGinkgoParallel)) 111 flag.StringVar(&o.gcpCloudSdk, "gcp-cloud-sdk", "", "Install/upgrade google-cloud-sdk to the gs:// path if set") 112 flag.StringVar(&o.gcpProject, "gcp-project", "", "For use with gcloud commands") 113 flag.StringVar(&o.gcpProjectType, "gcp-project-type", "", "Explicitly indicate which project type to select from boskos") 114 flag.StringVar(&o.gcpServiceAccount, "gcp-service-account", "", "Service account to activate before using gcloud") 115 flag.StringVar(&o.gcpZone, "gcp-zone", "", "For use with gcloud commands") 116 flag.StringVar(&o.gcpRegion, "gcp-region", "", "For use with gcloud commands") 117 flag.StringVar(&o.gcpNetwork, "gcp-network", "", "Cluster network. Must be set for --deployment=gke (TODO: other deployments).") 118 flag.StringVar(&o.gcpMasterImage, "gcp-master-image", "", "Master image type (cos|debian on GCE, n/a on GKE)") 119 flag.StringVar(&o.gcpNodeImage, "gcp-node-image", "", "Node image type (cos|container_vm on GKE, cos|debian on GCE)") 120 flag.StringVar(&o.gcpNodes, "gcp-nodes", "", "(--provider=gce only) Number of nodes to create.") 121 flag.BoolVar(&o.kubemark, "kubemark", false, "If true, run kubemark tests.") 122 flag.StringVar(&o.kubemarkMasterSize, "kubemark-master-size", "", "Kubemark master size (only relevant if --kubemark=true). Auto-calculated based on '--kubemark-nodes' if left empty.") 123 flag.StringVar(&o.kubemarkNodes, "kubemark-nodes", "5", "Number of kubemark nodes to start (only relevant if --kubemark=true).") 124 flag.StringVar(&o.logexporterGCSPath, "logexporter-gcs-path", "", "Path to the GCS artifacts directory to dump logs from nodes. Logexporter gets enabled if this is non-empty") 125 flag.StringVar(&o.metadataSources, "metadata-sources", "images.json", "Comma-separated list of files inside ./artifacts to merge into metadata.json") 126 flag.Var(&o.multiClusters, "multi-clusters", "If set, bring up/down multiple clusters specified. Format is [Zone1:]Cluster1[,[ZoneN:]ClusterN]]*. Zone is optional and default zone is used if zone is not specified") 127 flag.BoolVar(&o.multipleFederations, "multiple-federations", false, "If true, enable running multiple federation control planes in parallel") 128 flag.StringVar(&o.nodeArgs, "node-args", "", "Args for node e2e tests.") 129 flag.StringVar(&o.nodeTestArgs, "node-test-args", "", "Test args specifically for node e2e tests.") 130 flag.BoolVar(&o.nodeTests, "node-tests", false, "If true, run node-e2e tests.") 131 flag.BoolVar(&o.perfTests, "perf-tests", false, "If true, run tests from perf-tests repo.") 132 flag.StringVar(&o.provider, "provider", "", "Kubernetes provider such as gce, gke, aws, etc") 133 flag.StringVar(&o.publish, "publish", "", "Publish version to the specified gs:// path on success") 134 flag.StringVar(&o.runtimeConfig, "runtime-config", "batch/v2alpha1=true", "If set, API versions can be turned on or off while bringing up the API server.") 135 flag.StringVar(&o.stage.dockerRegistry, "registry", "", "Push images to the specified docker registry (e.g. gcr.io/a-test-project)") 136 flag.StringVar(&o.save, "save", "", "Save credentials to gs:// path on --up if set (or load from there if not --up)") 137 flag.BoolVar(&o.skew, "skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go") 138 flag.Var(&o.stage, "stage", "Upload binaries to gs://bucket/devel/job-suffix if set") 139 flag.StringVar(&o.stage.versionSuffix, "stage-suffix", "", "Append suffix to staged version when set") 140 flag.BoolVar(&o.test, "test", false, "Run Ginkgo tests.") 141 flag.StringVar(&o.testArgs, "test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") 142 flag.DurationVar(&timeout, "timeout", time.Duration(0), "Terminate testing after the timeout duration (s/m/h)") 143 flag.BoolVar(&o.up, "up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.") 144 flag.StringVar(&o.upgradeArgs, "upgrade_args", "", "If set, run upgrade tests before other tests") 145 146 flag.BoolVar(&verbose, "v", false, "If true, print all command output.") 147 return &o 148 } 149 150 type testCase struct { 151 XMLName xml.Name `xml:"testcase"` 152 ClassName string `xml:"classname,attr"` 153 Name string `xml:"name,attr"` 154 Time float64 `xml:"time,attr"` 155 Failure string `xml:"failure,omitempty"` 156 Skipped string `xml:"skipped,omitempty"` 157 } 158 159 type testSuite struct { 160 XMLName xml.Name `xml:"testsuite"` 161 Failures int `xml:"failures,attr"` 162 Tests int `xml:"tests,attr"` 163 Time float64 `xml:"time,attr"` 164 Cases []testCase 165 } 166 167 var suite testSuite 168 169 func validWorkingDirectory() error { 170 cwd, err := os.Getwd() 171 if err != nil { 172 return fmt.Errorf("could not get pwd: %v", err) 173 } 174 acwd, err := filepath.Abs(cwd) 175 if err != nil { 176 return fmt.Errorf("failed to convert %s to an absolute path: %v", cwd, err) 177 } 178 // This also matches "kubernetes_skew" for upgrades. 179 if !strings.Contains(filepath.Base(acwd), "kubernetes") { 180 return fmt.Errorf("must run from kubernetes directory root: %v", acwd) 181 } 182 return nil 183 } 184 185 func writeXML(dump string, start time.Time) { 186 suite.Time = time.Since(start).Seconds() 187 out, err := xml.MarshalIndent(&suite, "", " ") 188 if err != nil { 189 log.Fatalf("Could not marshal XML: %s", err) 190 } 191 path := filepath.Join(dump, "junit_runner.xml") 192 f, err := os.Create(path) 193 if err != nil { 194 log.Fatalf("Could not create file: %s", err) 195 } 196 defer f.Close() 197 if _, err := f.WriteString(xml.Header); err != nil { 198 log.Fatalf("Error writing XML header: %s", err) 199 } 200 if _, err := f.Write(out); err != nil { 201 log.Fatalf("Error writing XML data: %s", err) 202 } 203 log.Printf("Saved XML output to %s.", path) 204 } 205 206 type deployer interface { 207 Up() error 208 IsUp() error 209 DumpClusterLogs(localPath, gcsPath string) error 210 TestSetup() error 211 Down() error 212 } 213 214 func getDeployer(o *options) (deployer, error) { 215 switch o.deployment { 216 case "bash": 217 return newBash(&o.clusterIPRange), nil 218 case "gke": 219 return newGKE(o.provider, o.gcpProject, o.gcpZone, o.gcpRegion, o.gcpNetwork, o.gcpNodeImage, o.cluster, &o.testArgs, &o.upgradeArgs) 220 case "kops": 221 return newKops() 222 case "kubernetes-anywhere": 223 if o.multiClusters.Enabled() { 224 return newKubernetesAnywhereMultiCluster(o.gcpProject, o.gcpZone, o.multiClusters) 225 } 226 return newKubernetesAnywhere(o.gcpProject, o.gcpZone) 227 case "node": 228 return nodeDeploy{}, nil 229 case "none": 230 return noneDeploy{}, nil 231 default: 232 return nil, fmt.Errorf("unknown deployment strategy %q", o.deployment) 233 } 234 } 235 236 func validateFlags(o *options) error { 237 if o.multiClusters.Enabled() && o.deployment != "kubernetes-anywhere" { 238 return errors.New("--multi-clusters flag cannot be passed with deployments other than 'kubernetes-anywhere'") 239 } 240 return nil 241 } 242 243 func main() { 244 log.SetFlags(log.LstdFlags | log.Lshortfile) 245 o := defineFlags() 246 flag.Parse() 247 err := complete(o) 248 249 if err := validateFlags(o); err != nil { 250 log.Fatalf("Flags validation failed. err: %v", err) 251 } 252 253 if boskos.HasResource() { 254 if berr := boskos.ReleaseAll("dirty"); berr != nil { 255 log.Fatalf("[Boskos] Fail To Release: %v, kubetest err: %v", berr, err) 256 } 257 } 258 259 if err != nil { 260 log.Fatalf("Something went wrong: %v", err) 261 } 262 } 263 264 func complete(o *options) error { 265 if !terminate.Stop() { 266 <-terminate.C // Drain the value if necessary. 267 } 268 if !interrupt.Stop() { 269 <-interrupt.C // Drain value 270 } 271 272 if timeout > 0 { 273 log.Printf("Limiting testing to %s", timeout) 274 interrupt.Reset(timeout) 275 } 276 277 if o.dump != "" { 278 defer writeMetadata(o.dump, o.metadataSources) 279 defer writeXML(o.dump, time.Now()) 280 } 281 if o.logexporterGCSPath != "" { 282 o.testArgs += fmt.Sprintf(" --logexporter-gcs-path=%s", o.logexporterGCSPath) 283 } 284 if err := prepare(o); err != nil { 285 return fmt.Errorf("failed to prepare test environment: %v", err) 286 } 287 if err := prepareFederation(o); err != nil { 288 return fmt.Errorf("failed to prepare federation test environment: %v", err) 289 } 290 // Get the deployer before we acquire k8s so any additional flag 291 // verifications happen early. 292 deploy, err := getDeployer(o) 293 if err != nil { 294 return fmt.Errorf("error creating deployer: %v", err) 295 } 296 if err := acquireKubernetes(o); err != nil { 297 return fmt.Errorf("failed to acquire k8s binaries: %v", err) 298 } 299 if err := validWorkingDirectory(); err != nil { 300 return fmt.Errorf("called from invalid working directory: %v", err) 301 } 302 303 if o.down { 304 // listen for signals such as ^C and gracefully attempt to clean up 305 c := make(chan os.Signal, 1) 306 signal.Notify(c, os.Interrupt) 307 go func() { 308 for range c { 309 log.Print("Captured ^C, gracefully attempting to cleanup resources..") 310 var fedErr, err error 311 if o.federation { 312 if fedErr = fedDown(); fedErr != nil { 313 log.Printf("Tearing down federation failed: %v", fedErr) 314 } 315 } 316 if err = deploy.Down(); err != nil { 317 log.Printf("Tearing down deployment failed: %v", err) 318 } 319 if fedErr != nil || err != nil { 320 os.Exit(1) 321 } 322 } 323 }() 324 } 325 326 if err := run(deploy, *o); err != nil { 327 return err 328 } 329 330 // Save the state if we upped a new cluster without downing it 331 // or we are turning up federated clusters without turning up 332 // the federation control plane. 333 if o.save != "" && ((!o.down && o.up) || (!o.federation && o.up && o.deployment != "none")) { 334 if err := saveState(o.save); err != nil { 335 return err 336 } 337 } 338 339 // Publish the successfully tested version when requested 340 if o.publish != "" { 341 if err := publish(o.publish); err != nil { 342 return err 343 } 344 } 345 return nil 346 } 347 348 func acquireKubernetes(o *options) error { 349 // Potentially build kubernetes 350 if o.build.Enabled() { 351 if err := xmlWrap("Build", o.build.Build); err != nil { 352 return err 353 } 354 } 355 356 // Potentially stage build binaries somewhere on GCS 357 if o.stage.Enabled() { 358 if err := xmlWrap("Stage", func() error { 359 return o.stage.Stage(o.federation) 360 }); err != nil { 361 return err 362 } 363 } 364 365 // Potentially download existing binaries and extract them. 366 if o.extract.Enabled() { 367 err := xmlWrap("Extract", func() error { 368 // Should we restore a previous state? 369 // Restore if we are not upping the cluster or we are bringing up 370 // a federation control plane without the federated clusters. 371 if o.save != "" { 372 if !o.up { 373 // Restore version and .kube/config from --up 374 log.Printf("Overwriting extract strategy to load kubeconfig and version from %s", o.save) 375 o.extract = extractStrategies{extractStrategy{mode: load, option: o.save}} 376 } else if o.federation && o.up && o.deployment == "none" { 377 // Only restore .kube/config from previous --up, use the regular 378 // extraction strategy to restore version. 379 log.Printf("Load kubeconfig from %s", o.save) 380 loadKubeconfig(o.save) 381 } 382 } 383 // New deployment, extract new version 384 return o.extract.Extract(o.gcpProject, o.gcpZone) 385 }) 386 if err != nil { 387 return err 388 } 389 } 390 return nil 391 } 392 393 // Returns the k8s version name 394 func findVersion() string { 395 // The version may be in a version file 396 if _, err := os.Stat("version"); err == nil { 397 b, err := ioutil.ReadFile("version") 398 if err == nil { 399 return strings.TrimSpace(string(b)) 400 } 401 log.Printf("Failed to read version: %v", err) 402 } 403 404 // We can also get it from the git repo. 405 if _, err := os.Stat("hack/lib/version.sh"); err == nil { 406 // TODO(fejta): do this in go. At least we removed the upload-to-gcs.sh dep. 407 gross := `. hack/lib/version.sh && KUBE_ROOT=. kube::version::get_version_vars && echo "${KUBE_GIT_VERSION-}"` 408 b, err := output(exec.Command("bash", "-c", gross)) 409 if err == nil { 410 return strings.TrimSpace(string(b)) 411 } 412 log.Printf("Failed to get_version_vars: %v", err) 413 } 414 415 return "unknown" // Sad trombone 416 } 417 418 // maybeMergeMetadata will add new keyvals into the map; quietly eats errors. 419 func maybeMergeJSON(meta map[string]string, path string) { 420 if data, err := ioutil.ReadFile(path); err == nil { 421 json.Unmarshal(data, &meta) 422 } 423 } 424 425 // Write metadata.json, including version and env arg data. 426 func writeMetadata(path, metadataSources string) error { 427 m := make(map[string]string) 428 429 // Look for any sources of metadata and load 'em 430 for _, f := range strings.Split(metadataSources, ",") { 431 maybeMergeJSON(m, filepath.Join(path, f)) 432 } 433 434 ver := findVersion() 435 m["version"] = ver // TODO(fejta): retire 436 m["job-version"] = ver 437 re := regexp.MustCompile(`^BUILD_METADATA_(.+)$`) 438 for _, e := range os.Environ() { 439 p := strings.SplitN(e, "=", 2) 440 r := re.FindStringSubmatch(p[0]) 441 if r == nil { 442 continue 443 } 444 k, v := strings.ToLower(r[1]), p[1] 445 m[k] = v 446 } 447 f, err := os.Create(filepath.Join(path, "metadata.json")) 448 if err != nil { 449 return err 450 } 451 defer f.Close() 452 e := json.NewEncoder(f) 453 return e.Encode(m) 454 } 455 456 // Install cloudsdk tarball to location, updating PATH 457 func installGcloud(tarball string, location string) error { 458 459 if err := os.MkdirAll(location, 0775); err != nil { 460 return err 461 } 462 463 if err := finishRunning(exec.Command("tar", "xzf", tarball, "-C", location)); err != nil { 464 return err 465 } 466 467 if err := finishRunning(exec.Command(filepath.Join(location, "google-cloud-sdk", "install.sh"), "--disable-installation-options", "--bash-completion=false", "--path-update=false", "--usage-reporting=false")); err != nil { 468 return err 469 } 470 471 if err := insertPath(filepath.Join(location, "google-cloud-sdk", "bin")); err != nil { 472 return err 473 } 474 475 if err := finishRunning(exec.Command("gcloud", "components", "install", "alpha")); err != nil { 476 return err 477 } 478 479 if err := finishRunning(exec.Command("gcloud", "components", "install", "beta")); err != nil { 480 return err 481 } 482 483 if err := finishRunning(exec.Command("gcloud", "info")); err != nil { 484 return err 485 } 486 return nil 487 } 488 489 func migrateGcpEnvAndOptions(o *options) error { 490 var network string 491 var zone string 492 switch o.provider { 493 case "gke": 494 network = "KUBE_GKE_NETWORK" 495 zone = "ZONE" 496 default: 497 network = "KUBE_GCE_NETWORK" 498 zone = "KUBE_GCE_ZONE" 499 } 500 return migrateOptions([]migratedOption{ 501 { 502 env: "PROJECT", 503 option: &o.gcpProject, 504 name: "--gcp-project", 505 }, 506 { 507 env: zone, 508 option: &o.gcpZone, 509 name: "--gcp-zone", 510 }, 511 { 512 env: "REGION", 513 option: &o.gcpRegion, 514 name: "--gcp-region", 515 }, 516 { 517 env: "GOOGLE_APPLICATION_CREDENTIALS", 518 option: &o.gcpServiceAccount, 519 name: "--gcp-service-account", 520 }, 521 { 522 env: network, 523 option: &o.gcpNetwork, 524 name: "--gcp-network", 525 }, 526 { 527 env: "KUBE_NODE_OS_DISTRIBUTION", 528 option: &o.gcpNodeImage, 529 name: "--gcp-node-image", 530 }, 531 { 532 env: "KUBE_MASTER_OS_DISTRIBUTION", 533 option: &o.gcpMasterImage, 534 name: "--gcp-master-image", 535 }, 536 { 537 env: "NUM_NODES", 538 option: &o.gcpNodes, 539 name: "--gcp-nodes", 540 }, 541 { 542 env: "CLOUDSDK_BUCKET", 543 option: &o.gcpCloudSdk, 544 name: "--gcp-cloud-sdk", 545 skipPush: true, 546 }, 547 }) 548 } 549 550 func prepareGcp(o *options) error { 551 if err := migrateGcpEnvAndOptions(o); err != nil { 552 return err 553 } 554 if o.provider == "gce" { 555 if distro := os.Getenv("KUBE_OS_DISTRIBUTION"); distro != "" { 556 log.Printf("Please use --gcp-master-image=%s --gcp-node-image=%s (instead of deprecated KUBE_OS_DISTRIBUTION)", 557 distro, distro) 558 // Note: KUBE_OS_DISTRIBUTION takes precedence over 559 // KUBE_{MASTER,NODE}_OS_DISTRIBUTION, so override here 560 // after the migration above. 561 o.gcpNodeImage = distro 562 o.gcpMasterImage = distro 563 if err := os.Setenv("KUBE_NODE_OS_DISTRIBUTION", distro); err != nil { 564 return fmt.Errorf("could not set KUBE_NODE_OS_DISTRIBUTION=%s: %v", distro, err) 565 } 566 if err := os.Setenv("KUBE_MASTER_OS_DISTRIBUTION", distro); err != nil { 567 return fmt.Errorf("could not set KUBE_MASTER_OS_DISTRIBUTION=%s: %v", distro, err) 568 } 569 } 570 } else if o.provider == "gke" { 571 if o.deployment == "" { 572 o.deployment = "gke" 573 } 574 if o.deployment != "gke" { 575 return fmt.Errorf("--provider=gke implies --deployment=gke") 576 } 577 if o.gcpNodeImage == "" { 578 return fmt.Errorf("--gcp-node-image must be set for GKE") 579 } 580 if o.gcpMasterImage != "" { 581 return fmt.Errorf("--gcp-master-image cannot be set on GKE") 582 } 583 if o.gcpNodes != "" { 584 return fmt.Errorf("--gcp-nodes cannot be set on GKE, use --gke-shape instead") 585 } 586 587 // TODO(kubernetes/test-infra#3536): This is used by the 588 // ginkgo-e2e.sh wrapper. 589 nod := o.gcpNodeImage 590 if nod == "container_vm" { 591 // gcloud container clusters create understands 592 // "container_vm", e2es understand "debian". 593 nod = "debian" 594 } 595 os.Setenv("NODE_OS_DISTRIBUTION", nod) 596 } 597 if o.gcpProject == "" { 598 var resType string 599 if o.gcpProjectType != "" { 600 resType = o.gcpProjectType 601 } else if o.provider == "gke" { 602 resType = "gke-project" 603 } else { 604 resType = "gce-project" 605 } 606 607 log.Printf("provider %v, will acquire resource %v from boskos", o.provider, resType) 608 609 p, err := boskos.Acquire(resType, "free", "busy") 610 if err != nil { 611 return fmt.Errorf("--provider=%s boskos failed to acquire project: %v", o.provider, err) 612 } 613 614 if p == "" { 615 return fmt.Errorf("boskos does not have a free %s at the moment", resType) 616 } 617 618 go func(c *client.Client, proj string) { 619 for range time.Tick(time.Minute * 5) { 620 if err := c.UpdateOne(p, "busy"); err != nil { 621 log.Printf("[Boskos] Update %s failed with %v", p, err) 622 } 623 } 624 }(boskos, p) 625 o.gcpProject = p 626 } 627 628 if err := os.Setenv("CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS", "1"); err != nil { 629 return fmt.Errorf("could not set CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS=1: %v", err) 630 } 631 632 if err := finishRunning(exec.Command("gcloud", "config", "set", "project", o.gcpProject)); err != nil { 633 return fmt.Errorf("fail to set project %s : err %v", o.gcpProject, err) 634 } 635 636 // TODO(krzyzacy):Remove this when we retire migrateGcpEnvAndOptions 637 // Note that a lot of scripts are still depend on this env in k/k repo. 638 if err := os.Setenv("PROJECT", o.gcpProject); err != nil { 639 return fmt.Errorf("fail to set env var PROJECT %s : err %v", o.gcpProject, err) 640 } 641 642 // gcloud creds may have changed 643 if err := activateServiceAccount(o.gcpServiceAccount); err != nil { 644 return err 645 } 646 647 // Ensure ssh keys exist 648 log.Print("Checking existing of GCP ssh keys...") 649 k := filepath.Join(home(".ssh"), "google_compute_engine") 650 if _, err := os.Stat(k); err != nil { 651 return err 652 } 653 pk := k + ".pub" 654 if _, err := os.Stat(pk); err != nil { 655 return err 656 } 657 658 log.Printf("Checking presence of public key in %s", o.gcpProject) 659 if out, err := output(exec.Command("gcloud", "compute", "--project="+o.gcpProject, "project-info", "describe")); err != nil { 660 return err 661 } else if b, err := ioutil.ReadFile(pk); err != nil { 662 return err 663 } else if !strings.Contains(string(b), string(out)) { 664 log.Print("Uploading public ssh key to project metadata...") 665 if err = finishRunning(exec.Command("gcloud", "compute", "--project="+o.gcpProject, "config-ssh")); err != nil { 666 return err 667 } 668 } 669 670 // Install custom gcloud verion if necessary 671 if o.gcpCloudSdk != "" { 672 for i := 0; i < 3; i++ { 673 if err := finishRunning(exec.Command("gsutil", "-mq", "cp", "-r", o.gcpCloudSdk, home())); err == nil { 674 break // Success! 675 } 676 time.Sleep(1 << uint(i) * time.Second) 677 } 678 for _, f := range []string{home(".gsutil"), home("repo"), home("cloudsdk")} { 679 if _, err := os.Stat(f); err == nil || !os.IsNotExist(err) { 680 if err = os.RemoveAll(f); err != nil { 681 return err 682 } 683 } 684 } 685 686 install := home("repo", "google-cloud-sdk.tar.gz") 687 if strings.HasSuffix(o.gcpCloudSdk, ".tar.gz") { 688 install = home(filepath.Base(o.gcpCloudSdk)) 689 } else { 690 if err := os.Rename(home(filepath.Base(o.gcpCloudSdk)), home("repo")); err != nil { 691 return err 692 } 693 694 // Controls which gcloud components to install. 695 pop, err := pushEnv("CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL", "file://"+home("repo", "components-2.json")) 696 if err != nil { 697 return err 698 } 699 defer pop() 700 } 701 702 if err := installGcloud(install, home("cloudsdk")); err != nil { 703 return err 704 } 705 // gcloud creds may have changed 706 if err := activateServiceAccount(o.gcpServiceAccount); err != nil { 707 return err 708 } 709 } 710 return nil 711 } 712 713 func prepareAws(o *options) error { 714 // gcloud creds may have changed 715 if err := activateServiceAccount(o.gcpServiceAccount); err != nil { 716 return err 717 } 718 return finishRunning(exec.Command("pip", "install", "awscli")) 719 } 720 721 // Activate GOOGLE_APPLICATION_CREDENTIALS if set or do nothing. 722 func activateServiceAccount(path string) error { 723 if path == "" { 724 return nil 725 } 726 return finishRunning(exec.Command("gcloud", "auth", "activate-service-account", "--key-file="+path)) 727 } 728 729 // Make all artifacts world readable. 730 // The root user winds up owning the files when the container exists. 731 // Ensure that other users can read these files at that time. 732 func chmodArtifacts() error { 733 return finishRunning(exec.Command("chmod", "-R", "o+r", artifacts)) 734 } 735 736 func prepare(o *options) error { 737 if err := migrateOptions([]migratedOption{ 738 { 739 env: "KUBERNETES_PROVIDER", 740 option: &o.provider, 741 name: "--provider", 742 }, 743 { 744 env: "CLUSTER_NAME", 745 option: &o.cluster, 746 name: "--cluster", 747 }, 748 }); err != nil { 749 return err 750 } 751 if err := prepareGinkgoParallel(&o.ginkgoParallel); err != nil { 752 return err 753 } 754 755 switch o.provider { 756 case "gce", "gke", "kubernetes-anywhere", "node": 757 if err := prepareGcp(o); err != nil { 758 return err 759 } 760 case "aws": 761 if err := prepareAws(o); err != nil { 762 return err 763 } 764 } 765 766 if o.kubemark { 767 if err := migrateOptions([]migratedOption{ 768 { 769 env: "KUBEMARK_NUM_NODES", 770 option: &o.kubemarkNodes, 771 name: "--kubemark-nodes", 772 }, 773 { 774 env: "KUBEMARK_MASTER_SIZE", 775 option: &o.kubemarkMasterSize, 776 name: "--kubemark-master-size", 777 }, 778 }); err != nil { 779 return err 780 } 781 } 782 783 if err := os.MkdirAll(artifacts, 0777); err != nil { // Create artifacts 784 return err 785 } 786 787 return nil 788 } 789 790 func prepareFederation(o *options) error { 791 if o.multipleFederations { 792 // TODO(fejta): use boskos to grab a federation cluster 793 // Note: EXECUTOR_NUMBER and NODE_NAME are Jenkins 794 // specific environment variables. So this doesn't work 795 // when we move away from Jenkins. 796 execNum := os.Getenv("EXECUTOR_NUMBER") 797 if execNum == "" { 798 execNum = "0" 799 } 800 suffix := fmt.Sprintf("%s-%s", os.Getenv("NODE_NAME"), execNum) 801 federationName := fmt.Sprintf("e2e-f8n-%s", suffix) 802 federationSystemNamespace := fmt.Sprintf("f8n-system-%s", suffix) 803 err := os.Setenv("FEDERATION_NAME", federationName) 804 if err != nil { 805 return err 806 } 807 return os.Setenv("FEDERATION_NAMESPACE", federationSystemNamespace) 808 } 809 return nil 810 } 811 812 type ginkgoParallelValue struct { 813 v int // 0 == not set (defaults to 1) 814 } 815 816 func (v *ginkgoParallelValue) IsBoolFlag() bool { 817 return true 818 } 819 820 func (v *ginkgoParallelValue) String() string { 821 if v.v == 0 { 822 return "1" 823 } 824 return strconv.Itoa(v.v) 825 } 826 827 func (v *ginkgoParallelValue) Set(s string) error { 828 if s == "" { 829 v.v = 0 830 return nil 831 } 832 if s == "true" { 833 v.v = defaultGinkgoParallel 834 return nil 835 } 836 p, err := strconv.Atoi(s) 837 if err != nil { 838 return fmt.Errorf("--ginkgo-parallel must be an integer, found %q", s) 839 } 840 if p < 1 { 841 return fmt.Errorf("--ginkgo-parallel must be >= 1, found %d", p) 842 } 843 v.v = p 844 return nil 845 } 846 847 func (v *ginkgoParallelValue) Get() int { 848 if v.v == 0 { 849 return 1 850 } 851 return v.v 852 } 853 854 var _ flag.Value = &ginkgoParallelValue{} 855 856 // Hand migrate this option. GINKGO_PARALLEL => GINKGO_PARALLEL_NODES=25 857 func prepareGinkgoParallel(v *ginkgoParallelValue) error { 858 if p := os.Getenv("GINKGO_PARALLEL"); strings.ToLower(p) == "y" { 859 log.Printf("Please use kubetest --ginkgo-parallel (instead of deprecated GINKGO_PARALLEL=y)") 860 if err := v.Set("true"); err != nil { 861 return err 862 } 863 os.Unsetenv("GINKGO_PARALLEL") 864 } 865 if p := os.Getenv("GINKGO_PARALLEL_NODES"); p != "" { 866 log.Printf("Please use kubetest --ginkgo-parallel=%s (instead of deprecated GINKGO_PARALLEL_NODES=%s)", p, p) 867 if err := v.Set(p); err != nil { 868 return err 869 } 870 } 871 os.Setenv("GINKGO_PARALLEL_NODES", v.String()) 872 return nil 873 } 874 875 func publish(pub string) error { 876 v, err := ioutil.ReadFile("version") 877 if err != nil { 878 return err 879 } 880 log.Printf("Set %s version to %s", pub, string(v)) 881 return finishRunning(exec.Command("gsutil", "cp", "version", pub)) 882 }