github.com/abayer/test-infra@v0.0.5/kubetest/kops.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "errors" 24 "flag" 25 "fmt" 26 "io/ioutil" 27 "log" 28 "math/rand" 29 "os" 30 "os/exec" 31 "os/user" 32 "path/filepath" 33 "strconv" 34 "strings" 35 "time" 36 37 "github.com/aws/aws-sdk-go/aws" 38 "github.com/aws/aws-sdk-go/aws/session" 39 "github.com/aws/aws-sdk-go/service/ec2" 40 "golang.org/x/crypto/ssh" 41 "k8s.io/test-infra/kubetest/e2e" 42 "k8s.io/test-infra/kubetest/util" 43 ) 44 45 // kopsAWSMasterSize is the default ec2 instance type for kops on aws 46 const kopsAWSMasterSize = "c4.large" 47 48 var ( 49 50 // kops specific flags. 51 kopsPath = flag.String("kops", "", "(kops only) Path to the kops binary. kops will be downloaded from kops-base-url if not set.") 52 kopsCluster = flag.String("kops-cluster", "", "(kops only) Deprecated. Cluster name for kops; if not set defaults to --cluster.") 53 kopsState = flag.String("kops-state", "", "(kops only) s3:// path to kops state store. Must be set.") 54 kopsSSHUser = flag.String("kops-ssh-user", os.Getenv("USER"), "(kops only) Username for SSH connections to nodes.") 55 kopsSSHKey = flag.String("kops-ssh-key", "", "(kops only) Path to ssh key-pair for each node (defaults '~/.ssh/kube_aws_rsa' if unset.)") 56 kopsKubeVersion = flag.String("kops-kubernetes-version", "", "(kops only) If set, the version of Kubernetes to deploy (can be a URL to a GCS path where the release is stored) (Defaults to kops default, latest stable release.).") 57 kopsZones = flag.String("kops-zones", "", "(kops only) zones for kops deployment, comma delimited.") 58 kopsNodes = flag.Int("kops-nodes", 2, "(kops only) Number of nodes to create.") 59 kopsUpTimeout = flag.Duration("kops-up-timeout", 20*time.Minute, "(kops only) Time limit between 'kops config / kops update' and a response from the Kubernetes API.") 60 kopsAdminAccess = flag.String("kops-admin-access", "", "(kops only) If set, restrict apiserver access to this CIDR range.") 61 kopsImage = flag.String("kops-image", "", "(kops only) Image (AMI) for nodes to use. (Defaults to kops default, a Debian image with a custom kubernetes kernel.)") 62 kopsArgs = flag.String("kops-args", "", "(kops only) Additional space-separated args to pass unvalidated to 'kops create cluster', e.g. '--kops-args=\"--dns private --node-size t2.micro\"'") 63 kopsPriorityPath = flag.String("kops-priority-path", "", "(kops only) Insert into PATH if set") 64 kopsBaseURL = flag.String("kops-base-url", "", "(kops only) Base URL for a prebuilt version of kops") 65 kopsVersion = flag.String("kops-version", "", "(kops only) URL to a file containing a valid kops-base-url") 66 kopsDiskSize = flag.Int("kops-disk-size", 48, "(kops only) Disk size to use for nodes and masters") 67 kopsPublish = flag.String("kops-publish", "", "(kops only) Publish kops version to the specified gs:// path on success") 68 kopsMasterSize = flag.String("kops-master-size", kopsAWSMasterSize, "(kops only) master instance type") 69 kopsMasterCount = flag.Int("kops-master-count", 1, "(kops only) Number of masters to run") 70 kopsEtcdVersion = flag.String("kops-etcd-version", "", "(kops only) Etcd Version") 71 72 kopsMultipleZones = flag.Bool("kops-multiple-zones", false, "(kops only) run tests in multiple zones") 73 74 awsRegions = []string{ 75 "ap-south-1", 76 "eu-west-2", 77 "eu-west-1", 78 "ap-northeast-2", 79 "ap-northeast-1", 80 "sa-east-1", 81 "ca-central-1", 82 // not supporting Singapore since they do not seem to have capacity for c4.large 83 //"ap-southeast-1", 84 "ap-southeast-2", 85 "eu-central-1", 86 "us-east-1", 87 "us-east-2", 88 "us-west-1", 89 "us-west-2", 90 // not supporting Paris yet as AWS does not have all instance types available 91 //"eu-west-3", 92 } 93 ) 94 95 type kops struct { 96 path string 97 kubeVersion string 98 zones []string 99 nodes int 100 adminAccess string 101 cluster string 102 image string 103 args string 104 kubecfg string 105 diskSize int 106 107 // sshUser is the username to use when SSHing to nodes (for example for log capture) 108 sshUser string 109 // sshPublicKey is the path to the SSH public key matching sshPrivateKey 110 sshPublicKey string 111 // sshPrivateKey is the path to the SSH private key matching sshPublicKey 112 sshPrivateKey string 113 114 // GCP project we should use 115 gcpProject string 116 117 // Cloud provider in use (gce, aws) 118 provider string 119 120 // kopsVersion is the version of kops we are running (used for publishing) 121 kopsVersion string 122 123 // kopsPublish is the path where we will publish kopsVersion, after a successful test 124 kopsPublish string 125 126 // masterCount denotes how many masters to start 127 masterCount int 128 129 // etcdVersion is the etcd version to run 130 etcdVersion string 131 132 // masterSize is the EC2 instance type for the master 133 masterSize string 134 135 // multipleZones denotes using more than one zone 136 multipleZones bool 137 } 138 139 var _ deployer = kops{} 140 141 func migrateKopsEnv() error { 142 return util.MigrateOptions([]util.MigratedOption{ 143 { 144 Env: "KOPS_STATE_STORE", 145 Option: kopsState, 146 Name: "--kops-state", 147 SkipPush: true, 148 }, 149 { 150 Env: "AWS_SSH_KEY", 151 Option: kopsSSHKey, 152 Name: "--kops-ssh-key", 153 SkipPush: true, 154 }, 155 { 156 Env: "PRIORITY_PATH", 157 Option: kopsPriorityPath, 158 Name: "--kops-priority-path", 159 SkipPush: true, 160 }, 161 }) 162 } 163 164 func newKops(provider, gcpProject, cluster string) (*kops, error) { 165 tmpdir, err := ioutil.TempDir("", "kops") 166 if err != nil { 167 return nil, err 168 } 169 170 if err := migrateKopsEnv(); err != nil { 171 return nil, err 172 } 173 174 if *kopsCluster != "" { 175 cluster = *kopsCluster 176 } 177 if cluster == "" { 178 return nil, fmt.Errorf("--cluster or --kops-cluster must be set to a valid cluster name for kops deployment") 179 } 180 if *kopsState == "" { 181 return nil, fmt.Errorf("--kops-state must be set to a valid S3 path for kops deployment") 182 } 183 if *kopsPriorityPath != "" { 184 if err := util.InsertPath(*kopsPriorityPath); err != nil { 185 return nil, err 186 } 187 } 188 189 // TODO(fejta): consider explicitly passing these env items where needed. 190 sshKey := *kopsSSHKey 191 if sshKey == "" { 192 usr, err := user.Current() 193 if err != nil { 194 return nil, err 195 } 196 sshKey = filepath.Join(usr.HomeDir, ".ssh/kube_aws_rsa") 197 } 198 if err := os.Setenv("KOPS_STATE_STORE", *kopsState); err != nil { 199 return nil, err 200 } 201 202 sshUser := *kopsSSHUser 203 if sshUser != "" { 204 if err := os.Setenv("KUBE_SSH_USER", sshUser); err != nil { 205 return nil, err 206 } 207 } 208 209 // Repoint KUBECONFIG to an isolated kubeconfig in our temp directory 210 kubecfg := filepath.Join(tmpdir, "kubeconfig") 211 f, err := os.Create(kubecfg) 212 if err != nil { 213 return nil, err 214 } 215 defer f.Close() 216 if err := f.Chmod(0600); err != nil { 217 return nil, err 218 } 219 if err := os.Setenv("KUBECONFIG", kubecfg); err != nil { 220 return nil, err 221 } 222 223 // Set KUBERNETES_CONFORMANCE_TEST so the auth info is picked up 224 // from kubectl instead of bash inference. 225 if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil { 226 return nil, err 227 } 228 // Set KUBERNETES_CONFORMANCE_PROVIDER to override the 229 // cloudprovider for KUBERNETES_CONFORMANCE_TEST. 230 // This value is set by the provider flag that is passed into kubetest. 231 // HACK: until we merge #7408, there's a bug in the ginkgo-e2e.sh script we have to work around 232 // TODO(justinsb): remove this hack once #7408 merges 233 // if err := os.Setenv("KUBERNETES_CONFORMANCE_PROVIDER", provider); err != nil { 234 if err := os.Setenv("KUBERNETES_CONFORMANCE_PROVIDER", "aws"); err != nil { 235 return nil, err 236 } 237 // AWS_SSH_KEY is required by the AWS e2e tests. 238 if err := os.Setenv("AWS_SSH_KEY", sshKey); err != nil { 239 return nil, err 240 } 241 242 // zones are required by the kops e2e tests. 243 var zones []string 244 245 // if zones is set to zero and gcp project is not set then pick random aws zone 246 if *kopsZones == "" && provider == "aws" { 247 zones, err = getRandomAWSZones(*kopsMasterCount, *kopsMultipleZones) 248 if err != nil { 249 return nil, err 250 } 251 } else { 252 zones = strings.Split(*kopsZones, ",") 253 } 254 255 // set ZONES for e2e.go 256 if err := os.Setenv("ZONE", zones[0]); err != nil { 257 return nil, err 258 } 259 260 if len(zones) == 0 { 261 return nil, errors.New("no zones found") 262 } else if zones[0] == "" { 263 return nil, errors.New("zone cannot be a empty string") 264 } 265 266 log.Printf("executing kops with zones: %q", zones) 267 268 // Set kops-base-url from kops-version 269 if *kopsVersion != "" { 270 if *kopsBaseURL != "" { 271 return nil, fmt.Errorf("cannot set --kops-version and --kops-base-url") 272 } 273 274 var b bytes.Buffer 275 if err := httpRead(*kopsVersion, &b); err != nil { 276 return nil, err 277 } 278 latest := strings.TrimSpace(b.String()) 279 280 log.Printf("Got latest kops version from %v: %v", *kopsVersion, latest) 281 if latest == "" { 282 return nil, fmt.Errorf("version URL %v was empty", *kopsVersion) 283 } 284 *kopsBaseURL = latest 285 } 286 287 // kops looks at KOPS_BASE_URL env var, so export it here 288 if *kopsBaseURL != "" { 289 if err := os.Setenv("KOPS_BASE_URL", *kopsBaseURL); err != nil { 290 return nil, err 291 } 292 } 293 294 // Download kops from kopsBaseURL if kopsPath is not set 295 if *kopsPath == "" { 296 if *kopsBaseURL == "" { 297 return nil, errors.New("--kops or --kops-base-url must be set") 298 } 299 300 kopsBinURL := *kopsBaseURL + "/linux/amd64/kops" 301 log.Printf("Download kops binary from %s", kopsBinURL) 302 kopsBin := filepath.Join(tmpdir, "kops") 303 f, err := os.Create(kopsBin) 304 if err != nil { 305 return nil, fmt.Errorf("error creating file %q: %v", kopsBin, err) 306 } 307 defer f.Close() 308 if err := httpRead(kopsBinURL, f); err != nil { 309 return nil, err 310 } 311 if err := util.EnsureExecutable(kopsBin); err != nil { 312 return nil, err 313 } 314 *kopsPath = kopsBin 315 } 316 317 return &kops{ 318 path: *kopsPath, 319 kubeVersion: *kopsKubeVersion, 320 sshPrivateKey: sshKey, 321 sshPublicKey: sshKey + ".pub", 322 sshUser: sshUser, 323 zones: zones, 324 nodes: *kopsNodes, 325 adminAccess: *kopsAdminAccess, 326 cluster: cluster, 327 image: *kopsImage, 328 args: *kopsArgs, 329 kubecfg: kubecfg, 330 provider: provider, 331 gcpProject: gcpProject, 332 diskSize: *kopsDiskSize, 333 kopsVersion: *kopsBaseURL, 334 kopsPublish: *kopsPublish, 335 masterCount: *kopsMasterCount, 336 etcdVersion: *kopsEtcdVersion, 337 masterSize: *kopsMasterSize, 338 }, nil 339 } 340 341 func (k kops) isGoogleCloud() bool { 342 return k.provider == "gce" 343 } 344 345 func (k kops) Up() error { 346 // If we downloaded kubernetes, pass that version to kops 347 if k.kubeVersion == "" { 348 // TODO(justinsb): figure out a refactor that allows us to get this from acquireKubernetes cleanly 349 kubeReleaseURL := os.Getenv("KUBERNETES_RELEASE_URL") 350 kubeRelease := os.Getenv("KUBERNETES_RELEASE") 351 if kubeReleaseURL != "" && kubeRelease != "" { 352 if !strings.HasSuffix(kubeReleaseURL, "/") { 353 kubeReleaseURL += "/" 354 } 355 k.kubeVersion = kubeReleaseURL + kubeRelease 356 } 357 } 358 359 var featureFlags []string 360 var overrides []string 361 362 createArgs := []string{ 363 "create", "cluster", 364 "--name", k.cluster, 365 "--ssh-public-key", k.sshPublicKey, 366 "--node-count", strconv.Itoa(k.nodes), 367 "--node-volume-size", strconv.Itoa(k.diskSize), 368 "--master-volume-size", strconv.Itoa(k.diskSize), 369 "--master-count", strconv.Itoa(k.masterCount), 370 "--zones", strings.Join(k.zones, ","), 371 } 372 373 // We are defaulting the master size to c4.large on AWS because m3.larges are getting less previlent. 374 // When we are using GCE, then we need to handle the flag differently. 375 // If we are not using gce then add the masters size flag, or if we are using gce, and the 376 // master size is not set to the aws default, then add the master size flag. 377 if !k.isGoogleCloud() || (k.isGoogleCloud() && k.masterSize != kopsAWSMasterSize) { 378 createArgs = append(createArgs, "--master-size", k.masterSize) 379 } 380 381 if k.kubeVersion != "" { 382 createArgs = append(createArgs, "--kubernetes-version", k.kubeVersion) 383 } 384 if k.adminAccess != "" { 385 createArgs = append(createArgs, "--admin-access", k.adminAccess) 386 // Enable nodeport access from the same IP (we expect it to be the test IPs) 387 overrides = append(overrides, "cluster.spec.nodePortAccess="+k.adminAccess) 388 } 389 if k.image != "" { 390 createArgs = append(createArgs, "--image", k.image) 391 } 392 if k.gcpProject != "" { 393 createArgs = append(createArgs, "--project", k.gcpProject) 394 } 395 if k.isGoogleCloud() { 396 featureFlags = append(featureFlags, "AlphaAllowGCE") 397 createArgs = append(createArgs, "--cloud", "gce") 398 } else { 399 // append cloud type to allow for use of new regions without updates 400 createArgs = append(createArgs, "--cloud", "aws") 401 } 402 if k.args != "" { 403 createArgs = append(createArgs, strings.Split(k.args, " ")...) 404 } 405 if k.etcdVersion != "" { 406 overrides = append(overrides, "cluster.spec.etcdClusters[*].version="+k.etcdVersion) 407 } 408 if len(overrides) != 0 { 409 featureFlags = append(featureFlags, "SpecOverrideFlag") 410 createArgs = append(createArgs, "--override", strings.Join(overrides, ",")) 411 } 412 if len(featureFlags) != 0 { 413 os.Setenv("KOPS_FEATURE_FLAGS", strings.Join(featureFlags, ",")) 414 } 415 if err := control.FinishRunning(exec.Command(k.path, createArgs...)); err != nil { 416 return fmt.Errorf("kops configuration failed: %v", err) 417 } 418 if err := control.FinishRunning(exec.Command(k.path, "update", "cluster", k.cluster, "--yes")); err != nil { 419 return fmt.Errorf("kops bringup failed: %v", err) 420 } 421 422 // We require repeated successes, so we know that the cluster is stable 423 // (e.g. in HA scenarios, or where we're using multiple DNS servers) 424 requiredConsecutiveSuccesses := 4 425 // TODO(zmerlynn): More cluster validation. This should perhaps be 426 // added to kops and not here, but this is a fine place to loop 427 // for now. 428 return waitForReadyNodes(k.nodes+1, *kopsUpTimeout, requiredConsecutiveSuccesses) 429 } 430 431 func (k kops) IsUp() error { 432 return isUp(k) 433 } 434 435 func (k kops) DumpClusterLogs(localPath, gcsPath string) error { 436 privateKeyPath := k.sshPrivateKey 437 if strings.HasPrefix(privateKeyPath, "~/") { 438 privateKeyPath = filepath.Join(os.Getenv("HOME"), privateKeyPath[2:]) 439 } 440 key, err := ioutil.ReadFile(privateKeyPath) 441 if err != nil { 442 return fmt.Errorf("error reading private key %q: %v", k.sshPrivateKey, err) 443 } 444 445 signer, err := ssh.ParsePrivateKey(key) 446 if err != nil { 447 return fmt.Errorf("error parsing private key %q: %v", k.sshPrivateKey, err) 448 } 449 450 sshConfig := &ssh.ClientConfig{ 451 User: k.sshUser, 452 Auth: []ssh.AuthMethod{ 453 ssh.PublicKeys(signer), 454 }, 455 HostKeyCallback: ssh.InsecureIgnoreHostKey(), 456 } 457 458 sshClientFactory := &sshClientFactoryImplementation{ 459 sshConfig: sshConfig, 460 } 461 logDumper, err := newLogDumper(sshClientFactory, localPath) 462 if err != nil { 463 return err 464 } 465 466 ctx, cancel := context.WithCancel(context.TODO()) 467 defer cancel() 468 469 finished := make(chan error) 470 go func() { 471 finished <- k.dumpAllNodes(ctx, logDumper) 472 }() 473 474 for { 475 select { 476 case <-interrupt.C: 477 cancel() 478 case err := <-finished: 479 return err 480 } 481 } 482 } 483 484 // dumpAllNodes connects to every node and dumps the logs 485 func (k *kops) dumpAllNodes(ctx context.Context, d *logDumper) error { 486 // Make sure kubeconfig is set, in particular before calling DumpAllNodes, which calls kubectlGetNodes 487 if err := k.TestSetup(); err != nil { 488 return fmt.Errorf("error setting up kubeconfig: %v", err) 489 } 490 491 var additionalIPs []string 492 dump, err := k.runKopsDump() 493 if err != nil { 494 log.Printf("unable to get cluster status from kops: %v", err) 495 } else { 496 for _, instance := range dump.Instances { 497 name := instance.Name 498 499 if len(instance.PublicAddresses) == 0 { 500 log.Printf("ignoring instance in kops status with no public address: %v", name) 501 continue 502 } 503 504 additionalIPs = append(additionalIPs, instance.PublicAddresses[0]) 505 } 506 } 507 508 if err := d.DumpAllNodes(ctx, additionalIPs); err != nil { 509 return err 510 } 511 512 return nil 513 } 514 515 func (k kops) TestSetup() error { 516 info, err := os.Stat(k.kubecfg) 517 if err != nil { 518 if os.IsNotExist(err) { 519 log.Printf("kubeconfig file %s not found", k.kubecfg) 520 } else { 521 return err 522 } 523 } else if info.Size() > 0 { 524 // Assume that if we already have it, it's good. 525 return nil 526 } 527 528 if err := control.FinishRunning(exec.Command(k.path, "export", "kubecfg", k.cluster)); err != nil { 529 return fmt.Errorf("failure from 'kops export kubecfg %s': %v", k.cluster, err) 530 } 531 532 // Double-check that the file was exported 533 info, err = os.Stat(k.kubecfg) 534 if err != nil { 535 return fmt.Errorf("kubeconfig file %s was not exported", k.kubecfg) 536 } 537 if info.Size() == 0 { 538 return fmt.Errorf("exported kubeconfig file %s was empty", k.kubecfg) 539 } 540 541 return nil 542 } 543 544 // BuildTester returns a standard ginkgo-script tester, except for GCE where we build an e2e.Tester 545 func (k kops) BuildTester(o *e2e.BuildTesterOptions) (e2e.Tester, error) { 546 // Start by only enabling this on GCE 547 if !k.isGoogleCloud() { 548 return &GinkgoScriptTester{}, nil 549 } 550 551 log.Printf("running ginkgo tests directly") 552 553 t := e2e.NewGinkgoTester(o) 554 t.KubeRoot = "." 555 556 t.Kubeconfig = k.kubecfg 557 t.Provider = k.provider 558 559 if k.provider == "gce" { 560 t.GCEProject = k.gcpProject 561 if len(k.zones) > 0 { 562 zone := k.zones[0] 563 t.GCEZone = zone 564 565 // us-central1-a => us-central1 566 lastDash := strings.LastIndex(zone, "-") 567 if lastDash == -1 { 568 return nil, fmt.Errorf("unexpected format for GCE zone: %q", zone) 569 } 570 t.GCERegion = zone[0:lastDash] 571 } 572 } 573 574 return t, nil 575 } 576 577 func (k kops) Down() error { 578 // We do a "kops get" first so the exit status of "kops delete" is 579 // more sensical in the case of a non-existent cluster. ("kops 580 // delete" will exit with status 1 on a non-existent cluster) 581 err := control.FinishRunning(exec.Command(k.path, "get", "clusters", k.cluster)) 582 if err != nil { 583 // This is expected if the cluster doesn't exist. 584 return nil 585 } 586 return control.FinishRunning(exec.Command(k.path, "delete", "cluster", k.cluster, "--yes")) 587 } 588 589 func (k kops) GetClusterCreated(gcpProject string) (time.Time, error) { 590 return time.Time{}, errors.New("not implemented") 591 } 592 593 // kopsDump is the format of data as dumped by `kops toolbox dump -ojson` 594 type kopsDump struct { 595 Instances []*kopsDumpInstance `json:"instances"` 596 } 597 598 // String implements fmt.Stringer 599 func (o *kopsDump) String() string { 600 return util.JSONForDebug(o) 601 } 602 603 // kopsDumpInstance is the format of an instance (machine) in a kops dump 604 type kopsDumpInstance struct { 605 Name string `json:"name"` 606 PublicAddresses []string `json:"publicAddresses"` 607 } 608 609 // String implements fmt.Stringer 610 func (o *kopsDumpInstance) String() string { 611 return util.JSONForDebug(o) 612 } 613 614 // runKopsDump runs a kops toolbox dump to dump the status of the cluster 615 func (k *kops) runKopsDump() (*kopsDump, error) { 616 o, err := control.Output(exec.Command(k.path, "toolbox", "dump", "--name", k.cluster, "-ojson")) 617 if err != nil { 618 log.Printf("error running kops toolbox dump: %s\n%s", wrapError(err).Error(), string(o)) 619 return nil, err 620 } 621 622 dump := &kopsDump{} 623 if err := json.Unmarshal(o, dump); err != nil { 624 return nil, fmt.Errorf("error parsing kops toolbox dump output: %v", err) 625 } 626 627 return dump, nil 628 } 629 630 // kops deployer implements publisher 631 var _ publisher = &kops{} 632 633 // kops deployer implements e2e.TestBuilder 634 var _ e2e.TestBuilder = &kops{} 635 636 // Publish will publish a success file, it is called if the tests were successful 637 func (k kops) Publish() error { 638 if k.kopsPublish == "" { 639 // No publish destination set 640 return nil 641 } 642 643 if k.kopsVersion == "" { 644 return errors.New("kops-version not set; cannot publish") 645 } 646 647 return control.XMLWrap(&suite, "Publish kops version", func() error { 648 log.Printf("Set %s version to %s", k.kopsPublish, k.kopsVersion) 649 return gcsWrite(k.kopsPublish, []byte(k.kopsVersion)) 650 }) 651 } 652 653 // getRandomAWSZones looks up all regions, and the availability zones for those regions. A random 654 // region is then chosen and the AZ's for that region is returned. At least masterCount zones will be 655 // returned, all in the same region. 656 func getRandomAWSZones(masterCount int, multipleZones bool) ([]string, error) { 657 658 // TODO(chrislovecnm): get the number of ec2 instances in the region and ensure that there are not too many running 659 for _, i := range rand.Perm(len(awsRegions)) { 660 ec2Session, err := getAWSEC2Session(awsRegions[i]) 661 if err != nil { 662 return nil, err 663 } 664 665 // az for a region. AWS Go API does not allow us to make a single call 666 zoneResults, err := ec2Session.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}) 667 if err != nil { 668 return nil, fmt.Errorf("unable to call aws api DescribeAvailabilityZones for %q: %v", awsRegions[i], err) 669 } 670 671 var selectedZones []string 672 if len(zoneResults.AvailabilityZones) >= masterCount && multipleZones { 673 for _, z := range zoneResults.AvailabilityZones { 674 selectedZones = append(selectedZones, *z.ZoneName) 675 } 676 677 log.Printf("Launching cluster in region: %q", awsRegions[i]) 678 return selectedZones, nil 679 } else if !multipleZones { 680 z := zoneResults.AvailabilityZones[rand.Intn(len(zoneResults.AvailabilityZones))] 681 selectedZones = append(selectedZones, *z.ZoneName) 682 log.Printf("Launching cluster in region: %q", awsRegions[i]) 683 return selectedZones, nil 684 } 685 } 686 687 return nil, fmt.Errorf("unable to find region with %d zones", masterCount) 688 } 689 690 // getAWSEC2Session creates an returns a EC2 API session. 691 func getAWSEC2Session(region string) (*ec2.EC2, error) { 692 config := aws.NewConfig().WithRegion(region) 693 694 // This avoids a confusing error message when we fail to get credentials 695 config = config.WithCredentialsChainVerboseErrors(true) 696 697 s, err := session.NewSession(config) 698 if err != nil { 699 return nil, fmt.Errorf("unable to build aws API session with region: %q: %v", region, err) 700 } 701 702 return ec2.New(s, config), nil 703 704 }