github.com/jenkins-x/test-infra@v0.0.7/kubetest/kops.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "errors" 24 "flag" 25 "fmt" 26 "io/ioutil" 27 "log" 28 "math/rand" 29 "os" 30 "os/exec" 31 "os/user" 32 "path/filepath" 33 "strconv" 34 "strings" 35 "time" 36 37 "github.com/aws/aws-sdk-go/aws" 38 "github.com/aws/aws-sdk-go/aws/session" 39 "github.com/aws/aws-sdk-go/service/ec2" 40 "golang.org/x/crypto/ssh" 41 "k8s.io/test-infra/kubetest/e2e" 42 "k8s.io/test-infra/kubetest/util" 43 ) 44 45 // kopsAWSMasterSize is the default ec2 instance type for kops on aws 46 const kopsAWSMasterSize = "c4.large" 47 48 var ( 49 50 // kops specific flags. 51 kopsPath = flag.String("kops", "", "(kops only) Path to the kops binary. kops will be downloaded from kops-base-url if not set.") 52 kopsCluster = flag.String("kops-cluster", "", "(kops only) Deprecated. Cluster name for kops; if not set defaults to --cluster.") 53 kopsState = flag.String("kops-state", "", "(kops only) s3:// path to kops state store. Must be set.") 54 kopsSSHUser = flag.String("kops-ssh-user", os.Getenv("USER"), "(kops only) Username for SSH connections to nodes.") 55 kopsSSHKey = flag.String("kops-ssh-key", "", "(kops only) Path to ssh key-pair for each node (defaults '~/.ssh/kube_aws_rsa' if unset.)") 56 kopsKubeVersion = flag.String("kops-kubernetes-version", "", "(kops only) If set, the version of Kubernetes to deploy (can be a URL to a GCS path where the release is stored) (Defaults to kops default, latest stable release.).") 57 kopsZones = flag.String("kops-zones", "", "(kops only) zones for kops deployment, comma delimited.") 58 kopsNodes = flag.Int("kops-nodes", 2, "(kops only) Number of nodes to create.") 59 kopsUpTimeout = flag.Duration("kops-up-timeout", 20*time.Minute, "(kops only) Time limit between 'kops config / kops update' and a response from the Kubernetes API.") 60 kopsAdminAccess = flag.String("kops-admin-access", "", "(kops only) If set, restrict apiserver access to this CIDR range.") 61 kopsImage = flag.String("kops-image", "", "(kops only) Image (AMI) for nodes to use. (Defaults to kops default, a Debian image with a custom kubernetes kernel.)") 62 kopsArgs = flag.String("kops-args", "", "(kops only) Additional space-separated args to pass unvalidated to 'kops create cluster', e.g. '--kops-args=\"--dns private --node-size t2.micro\"'") 63 kopsPriorityPath = flag.String("kops-priority-path", "", "(kops only) Insert into PATH if set") 64 kopsBaseURL = flag.String("kops-base-url", "", "(kops only) Base URL for a prebuilt version of kops") 65 kopsVersion = flag.String("kops-version", "", "(kops only) URL to a file containing a valid kops-base-url") 66 kopsDiskSize = flag.Int("kops-disk-size", 48, "(kops only) Disk size to use for nodes and masters") 67 kopsPublish = flag.String("kops-publish", "", "(kops only) Publish kops version to the specified gs:// path on success") 68 kopsMasterSize = flag.String("kops-master-size", kopsAWSMasterSize, "(kops only) master instance type") 69 kopsMasterCount = flag.Int("kops-master-count", 1, "(kops only) Number of masters to run") 70 kopsEtcdVersion = flag.String("kops-etcd-version", "", "(kops only) Etcd Version") 71 72 kopsMultipleZones = flag.Bool("kops-multiple-zones", false, "(kops only) run tests in multiple zones") 73 74 awsRegions = []string{ 75 "ap-south-1", 76 "eu-west-2", 77 "eu-west-1", 78 "ap-northeast-2", 79 "ap-northeast-1", 80 "sa-east-1", 81 "ca-central-1", 82 // not supporting Singapore since they do not seem to have capacity for c4.large 83 //"ap-southeast-1", 84 "ap-southeast-2", 85 "eu-central-1", 86 "us-east-1", 87 "us-east-2", 88 "us-west-1", 89 "us-west-2", 90 // not supporting Paris yet as AWS does not have all instance types available 91 //"eu-west-3", 92 } 93 ) 94 95 type kops struct { 96 path string 97 kubeVersion string 98 zones []string 99 nodes int 100 adminAccess string 101 cluster string 102 image string 103 args string 104 kubecfg string 105 diskSize int 106 107 // sshUser is the username to use when SSHing to nodes (for example for log capture) 108 sshUser string 109 // sshPublicKey is the path to the SSH public key matching sshPrivateKey 110 sshPublicKey string 111 // sshPrivateKey is the path to the SSH private key matching sshPublicKey 112 sshPrivateKey string 113 114 // GCP project we should use 115 gcpProject string 116 117 // Cloud provider in use (gce, aws) 118 provider string 119 120 // kopsVersion is the version of kops we are running (used for publishing) 121 kopsVersion string 122 123 // kopsPublish is the path where we will publish kopsVersion, after a successful test 124 kopsPublish string 125 126 // masterCount denotes how many masters to start 127 masterCount int 128 129 // etcdVersion is the etcd version to run 130 etcdVersion string 131 132 // masterSize is the EC2 instance type for the master 133 masterSize string 134 135 // multipleZones denotes using more than one zone 136 multipleZones bool 137 } 138 139 var _ deployer = kops{} 140 141 func migrateKopsEnv() error { 142 return util.MigrateOptions([]util.MigratedOption{ 143 { 144 Env: "KOPS_STATE_STORE", 145 Option: kopsState, 146 Name: "--kops-state", 147 SkipPush: true, 148 }, 149 { 150 Env: "AWS_SSH_KEY", 151 Option: kopsSSHKey, 152 Name: "--kops-ssh-key", 153 SkipPush: true, 154 }, 155 { 156 Env: "PRIORITY_PATH", 157 Option: kopsPriorityPath, 158 Name: "--kops-priority-path", 159 SkipPush: true, 160 }, 161 }) 162 } 163 164 func newKops(provider, gcpProject, cluster string) (*kops, error) { 165 tmpdir, err := ioutil.TempDir("", "kops") 166 if err != nil { 167 return nil, err 168 } 169 170 if err := migrateKopsEnv(); err != nil { 171 return nil, err 172 } 173 174 if *kopsCluster != "" { 175 cluster = *kopsCluster 176 } 177 if cluster == "" { 178 return nil, fmt.Errorf("--cluster or --kops-cluster must be set to a valid cluster name for kops deployment") 179 } 180 if *kopsState == "" { 181 return nil, fmt.Errorf("--kops-state must be set to a valid S3 path for kops deployment") 182 } 183 if *kopsPriorityPath != "" { 184 if err := util.InsertPath(*kopsPriorityPath); err != nil { 185 return nil, err 186 } 187 } 188 189 // TODO(fejta): consider explicitly passing these env items where needed. 190 sshKey := *kopsSSHKey 191 if sshKey == "" { 192 usr, err := user.Current() 193 if err != nil { 194 return nil, err 195 } 196 sshKey = filepath.Join(usr.HomeDir, ".ssh/kube_aws_rsa") 197 } 198 if err := os.Setenv("KOPS_STATE_STORE", *kopsState); err != nil { 199 return nil, err 200 } 201 202 sshUser := *kopsSSHUser 203 if sshUser != "" { 204 if err := os.Setenv("KUBE_SSH_USER", sshUser); err != nil { 205 return nil, err 206 } 207 } 208 209 // Repoint KUBECONFIG to an isolated kubeconfig in our temp directory 210 kubecfg := filepath.Join(tmpdir, "kubeconfig") 211 f, err := os.Create(kubecfg) 212 if err != nil { 213 return nil, err 214 } 215 defer f.Close() 216 if err := f.Chmod(0600); err != nil { 217 return nil, err 218 } 219 if err := os.Setenv("KUBECONFIG", kubecfg); err != nil { 220 return nil, err 221 } 222 223 // Set KUBERNETES_CONFORMANCE_TEST so the auth info is picked up 224 // from kubectl instead of bash inference. 225 if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil { 226 return nil, err 227 } 228 // Set KUBERNETES_CONFORMANCE_PROVIDER to override the 229 // cloudprovider for KUBERNETES_CONFORMANCE_TEST. 230 // This value is set by the provider flag that is passed into kubetest. 231 // HACK: until we merge #7408, there's a bug in the ginkgo-e2e.sh script we have to work around 232 // TODO(justinsb): remove this hack once #7408 merges 233 // if err := os.Setenv("KUBERNETES_CONFORMANCE_PROVIDER", provider); err != nil { 234 if err := os.Setenv("KUBERNETES_CONFORMANCE_PROVIDER", "aws"); err != nil { 235 return nil, err 236 } 237 // AWS_SSH_KEY is required by the AWS e2e tests. 238 if err := os.Setenv("AWS_SSH_KEY", sshKey); err != nil { 239 return nil, err 240 } 241 242 // zones are required by the kops e2e tests. 243 var zones []string 244 245 // if zones is set to zero and gcp project is not set then pick random aws zone 246 if *kopsZones == "" && provider == "aws" { 247 zones, err = getRandomAWSZones(*kopsMasterCount, *kopsMultipleZones) 248 if err != nil { 249 return nil, err 250 } 251 } else { 252 zones = strings.Split(*kopsZones, ",") 253 } 254 255 // set ZONES for e2e.go 256 if err := os.Setenv("ZONE", zones[0]); err != nil { 257 return nil, err 258 } 259 260 if len(zones) == 0 { 261 return nil, errors.New("no zones found") 262 } else if zones[0] == "" { 263 return nil, errors.New("zone cannot be a empty string") 264 } 265 266 log.Printf("executing kops with zones: %q", zones) 267 268 // Set kops-base-url from kops-version 269 if *kopsVersion != "" { 270 if *kopsBaseURL != "" { 271 return nil, fmt.Errorf("cannot set --kops-version and --kops-base-url") 272 } 273 274 var b bytes.Buffer 275 if err := httpRead(*kopsVersion, &b); err != nil { 276 return nil, err 277 } 278 latest := strings.TrimSpace(b.String()) 279 280 log.Printf("Got latest kops version from %v: %v", *kopsVersion, latest) 281 if latest == "" { 282 return nil, fmt.Errorf("version URL %v was empty", *kopsVersion) 283 } 284 *kopsBaseURL = latest 285 } 286 287 // kops looks at KOPS_BASE_URL env var, so export it here 288 if *kopsBaseURL != "" { 289 if err := os.Setenv("KOPS_BASE_URL", *kopsBaseURL); err != nil { 290 return nil, err 291 } 292 } 293 294 // Download kops from kopsBaseURL if kopsPath is not set 295 if *kopsPath == "" { 296 if *kopsBaseURL == "" { 297 return nil, errors.New("--kops or --kops-base-url must be set") 298 } 299 300 kopsBinURL := *kopsBaseURL + "/linux/amd64/kops" 301 log.Printf("Download kops binary from %s", kopsBinURL) 302 kopsBin := filepath.Join(tmpdir, "kops") 303 f, err := os.Create(kopsBin) 304 if err != nil { 305 return nil, fmt.Errorf("error creating file %q: %v", kopsBin, err) 306 } 307 defer f.Close() 308 if err := httpRead(kopsBinURL, f); err != nil { 309 return nil, err 310 } 311 if err := util.EnsureExecutable(kopsBin); err != nil { 312 return nil, err 313 } 314 *kopsPath = kopsBin 315 } 316 317 return &kops{ 318 path: *kopsPath, 319 kubeVersion: *kopsKubeVersion, 320 sshPrivateKey: sshKey, 321 sshPublicKey: sshKey + ".pub", 322 sshUser: sshUser, 323 zones: zones, 324 nodes: *kopsNodes, 325 adminAccess: *kopsAdminAccess, 326 cluster: cluster, 327 image: *kopsImage, 328 args: *kopsArgs, 329 kubecfg: kubecfg, 330 provider: provider, 331 gcpProject: gcpProject, 332 diskSize: *kopsDiskSize, 333 kopsVersion: *kopsBaseURL, 334 kopsPublish: *kopsPublish, 335 masterCount: *kopsMasterCount, 336 etcdVersion: *kopsEtcdVersion, 337 masterSize: *kopsMasterSize, 338 }, nil 339 } 340 341 func (k kops) isGoogleCloud() bool { 342 return k.provider == "gce" 343 } 344 345 func (k kops) Up() error { 346 // If we downloaded kubernetes, pass that version to kops 347 if k.kubeVersion == "" { 348 // TODO(justinsb): figure out a refactor that allows us to get this from acquireKubernetes cleanly 349 kubeReleaseURL := os.Getenv("KUBERNETES_RELEASE_URL") 350 kubeRelease := os.Getenv("KUBERNETES_RELEASE") 351 if kubeReleaseURL != "" && kubeRelease != "" { 352 if !strings.HasSuffix(kubeReleaseURL, "/") { 353 kubeReleaseURL += "/" 354 } 355 k.kubeVersion = kubeReleaseURL + kubeRelease 356 } 357 } 358 359 var featureFlags []string 360 var overrides []string 361 362 createArgs := []string{ 363 "create", "cluster", 364 "--name", k.cluster, 365 "--ssh-public-key", k.sshPublicKey, 366 "--node-count", strconv.Itoa(k.nodes), 367 "--node-volume-size", strconv.Itoa(k.diskSize), 368 "--master-volume-size", strconv.Itoa(k.diskSize), 369 "--master-count", strconv.Itoa(k.masterCount), 370 "--zones", strings.Join(k.zones, ","), 371 } 372 373 // We are defaulting the master size to c4.large on AWS because m3.larges are getting less previlent. 374 // When we are using GCE, then we need to handle the flag differently. 375 // If we are not using gce then add the masters size flag, or if we are using gce, and the 376 // master size is not set to the aws default, then add the master size flag. 377 if !k.isGoogleCloud() || (k.isGoogleCloud() && k.masterSize != kopsAWSMasterSize) { 378 createArgs = append(createArgs, "--master-size", k.masterSize) 379 } 380 381 if k.kubeVersion != "" { 382 createArgs = append(createArgs, "--kubernetes-version", k.kubeVersion) 383 } 384 if k.adminAccess != "" { 385 createArgs = append(createArgs, "--admin-access", k.adminAccess) 386 // Enable nodeport access from the same IP (we expect it to be the test IPs) 387 overrides = append(overrides, "cluster.spec.nodePortAccess="+k.adminAccess) 388 } 389 if k.image != "" { 390 createArgs = append(createArgs, "--image", k.image) 391 } 392 if k.gcpProject != "" { 393 createArgs = append(createArgs, "--project", k.gcpProject) 394 } 395 if k.isGoogleCloud() { 396 featureFlags = append(featureFlags, "AlphaAllowGCE") 397 createArgs = append(createArgs, "--cloud", "gce") 398 } else { 399 // append cloud type to allow for use of new regions without updates 400 createArgs = append(createArgs, "--cloud", "aws") 401 } 402 if k.args != "" { 403 createArgs = append(createArgs, strings.Split(k.args, " ")...) 404 } 405 if k.etcdVersion != "" { 406 overrides = append(overrides, "cluster.spec.etcdClusters[*].version="+k.etcdVersion) 407 } 408 if len(overrides) != 0 { 409 featureFlags = append(featureFlags, "SpecOverrideFlag") 410 createArgs = append(createArgs, "--override", strings.Join(overrides, ",")) 411 } 412 if len(featureFlags) != 0 { 413 os.Setenv("KOPS_FEATURE_FLAGS", strings.Join(featureFlags, ",")) 414 } 415 if err := control.FinishRunning(exec.Command(k.path, createArgs...)); err != nil { 416 return fmt.Errorf("kops configuration failed: %v", err) 417 } 418 if err := control.FinishRunning(exec.Command(k.path, "update", "cluster", k.cluster, "--yes")); err != nil { 419 return fmt.Errorf("kops bringup failed: %v", err) 420 } 421 422 // We require repeated successes, so we know that the cluster is stable 423 // (e.g. in HA scenarios, or where we're using multiple DNS servers) 424 // We use a relatively high number as DNS can take a while to 425 // propagate across multiple servers / caches 426 requiredConsecutiveSuccesses := 10 427 428 // TODO(zmerlynn): More cluster validation. This should perhaps be 429 // added to kops and not here, but this is a fine place to loop 430 // for now. 431 return waitForReadyNodes(k.nodes+1, *kopsUpTimeout, requiredConsecutiveSuccesses) 432 } 433 434 func (k kops) IsUp() error { 435 return isUp(k) 436 } 437 438 func (k kops) DumpClusterLogs(localPath, gcsPath string) error { 439 privateKeyPath := k.sshPrivateKey 440 if strings.HasPrefix(privateKeyPath, "~/") { 441 privateKeyPath = filepath.Join(os.Getenv("HOME"), privateKeyPath[2:]) 442 } 443 key, err := ioutil.ReadFile(privateKeyPath) 444 if err != nil { 445 return fmt.Errorf("error reading private key %q: %v", k.sshPrivateKey, err) 446 } 447 448 signer, err := ssh.ParsePrivateKey(key) 449 if err != nil { 450 return fmt.Errorf("error parsing private key %q: %v", k.sshPrivateKey, err) 451 } 452 453 sshConfig := &ssh.ClientConfig{ 454 User: k.sshUser, 455 Auth: []ssh.AuthMethod{ 456 ssh.PublicKeys(signer), 457 }, 458 HostKeyCallback: ssh.InsecureIgnoreHostKey(), 459 } 460 461 sshClientFactory := &sshClientFactoryImplementation{ 462 sshConfig: sshConfig, 463 } 464 logDumper, err := newLogDumper(sshClientFactory, localPath) 465 if err != nil { 466 return err 467 } 468 469 ctx, cancel := context.WithCancel(context.TODO()) 470 defer cancel() 471 472 finished := make(chan error) 473 go func() { 474 finished <- k.dumpAllNodes(ctx, logDumper) 475 }() 476 477 for { 478 select { 479 case <-interrupt.C: 480 cancel() 481 case err := <-finished: 482 return err 483 } 484 } 485 } 486 487 // dumpAllNodes connects to every node and dumps the logs 488 func (k *kops) dumpAllNodes(ctx context.Context, d *logDumper) error { 489 // Make sure kubeconfig is set, in particular before calling DumpAllNodes, which calls kubectlGetNodes 490 if err := k.TestSetup(); err != nil { 491 return fmt.Errorf("error setting up kubeconfig: %v", err) 492 } 493 494 var additionalIPs []string 495 dump, err := k.runKopsDump() 496 if err != nil { 497 log.Printf("unable to get cluster status from kops: %v", err) 498 } else { 499 for _, instance := range dump.Instances { 500 name := instance.Name 501 502 if len(instance.PublicAddresses) == 0 { 503 log.Printf("ignoring instance in kops status with no public address: %v", name) 504 continue 505 } 506 507 additionalIPs = append(additionalIPs, instance.PublicAddresses[0]) 508 } 509 } 510 511 if err := d.DumpAllNodes(ctx, additionalIPs); err != nil { 512 return err 513 } 514 515 return nil 516 } 517 518 func (k kops) TestSetup() error { 519 info, err := os.Stat(k.kubecfg) 520 if err != nil { 521 if os.IsNotExist(err) { 522 log.Printf("kubeconfig file %s not found", k.kubecfg) 523 } else { 524 return err 525 } 526 } else if info.Size() > 0 { 527 // Assume that if we already have it, it's good. 528 return nil 529 } 530 531 if err := control.FinishRunning(exec.Command(k.path, "export", "kubecfg", k.cluster)); err != nil { 532 return fmt.Errorf("failure from 'kops export kubecfg %s': %v", k.cluster, err) 533 } 534 535 // Double-check that the file was exported 536 info, err = os.Stat(k.kubecfg) 537 if err != nil { 538 return fmt.Errorf("kubeconfig file %s was not exported", k.kubecfg) 539 } 540 if info.Size() == 0 { 541 return fmt.Errorf("exported kubeconfig file %s was empty", k.kubecfg) 542 } 543 544 return nil 545 } 546 547 // BuildTester returns a standard ginkgo-script tester, except for GCE where we build an e2e.Tester 548 func (k kops) BuildTester(o *e2e.BuildTesterOptions) (e2e.Tester, error) { 549 // Start by only enabling this on GCE 550 if !k.isGoogleCloud() { 551 return &GinkgoScriptTester{}, nil 552 } 553 554 log.Printf("running ginkgo tests directly") 555 556 t := e2e.NewGinkgoTester(o) 557 t.KubeRoot = "." 558 559 t.Kubeconfig = k.kubecfg 560 t.Provider = k.provider 561 562 if k.provider == "gce" { 563 t.GCEProject = k.gcpProject 564 if len(k.zones) > 0 { 565 zone := k.zones[0] 566 t.GCEZone = zone 567 568 // us-central1-a => us-central1 569 lastDash := strings.LastIndex(zone, "-") 570 if lastDash == -1 { 571 return nil, fmt.Errorf("unexpected format for GCE zone: %q", zone) 572 } 573 t.GCERegion = zone[0:lastDash] 574 } 575 } 576 577 return t, nil 578 } 579 580 func (k kops) Down() error { 581 // We do a "kops get" first so the exit status of "kops delete" is 582 // more sensical in the case of a non-existent cluster. ("kops 583 // delete" will exit with status 1 on a non-existent cluster) 584 err := control.FinishRunning(exec.Command(k.path, "get", "clusters", k.cluster)) 585 if err != nil { 586 // This is expected if the cluster doesn't exist. 587 return nil 588 } 589 return control.FinishRunning(exec.Command(k.path, "delete", "cluster", k.cluster, "--yes")) 590 } 591 592 func (k kops) GetClusterCreated(gcpProject string) (time.Time, error) { 593 return time.Time{}, errors.New("not implemented") 594 } 595 596 // kopsDump is the format of data as dumped by `kops toolbox dump -ojson` 597 type kopsDump struct { 598 Instances []*kopsDumpInstance `json:"instances"` 599 } 600 601 // String implements fmt.Stringer 602 func (o *kopsDump) String() string { 603 return util.JSONForDebug(o) 604 } 605 606 // kopsDumpInstance is the format of an instance (machine) in a kops dump 607 type kopsDumpInstance struct { 608 Name string `json:"name"` 609 PublicAddresses []string `json:"publicAddresses"` 610 } 611 612 // String implements fmt.Stringer 613 func (o *kopsDumpInstance) String() string { 614 return util.JSONForDebug(o) 615 } 616 617 // runKopsDump runs a kops toolbox dump to dump the status of the cluster 618 func (k *kops) runKopsDump() (*kopsDump, error) { 619 o, err := control.Output(exec.Command(k.path, "toolbox", "dump", "--name", k.cluster, "-ojson")) 620 if err != nil { 621 log.Printf("error running kops toolbox dump: %s\n%s", wrapError(err).Error(), string(o)) 622 return nil, err 623 } 624 625 dump := &kopsDump{} 626 if err := json.Unmarshal(o, dump); err != nil { 627 return nil, fmt.Errorf("error parsing kops toolbox dump output: %v", err) 628 } 629 630 return dump, nil 631 } 632 633 // kops deployer implements publisher 634 var _ publisher = &kops{} 635 636 // kops deployer implements e2e.TestBuilder 637 var _ e2e.TestBuilder = &kops{} 638 639 // Publish will publish a success file, it is called if the tests were successful 640 func (k kops) Publish() error { 641 if k.kopsPublish == "" { 642 // No publish destination set 643 return nil 644 } 645 646 if k.kopsVersion == "" { 647 return errors.New("kops-version not set; cannot publish") 648 } 649 650 return control.XMLWrap(&suite, "Publish kops version", func() error { 651 log.Printf("Set %s version to %s", k.kopsPublish, k.kopsVersion) 652 return gcsWrite(k.kopsPublish, []byte(k.kopsVersion)) 653 }) 654 } 655 656 func (_ kops) KubectlCommand() (*exec.Cmd, error) { return nil, nil } 657 658 // getRandomAWSZones looks up all regions, and the availability zones for those regions. A random 659 // region is then chosen and the AZ's for that region is returned. At least masterCount zones will be 660 // returned, all in the same region. 661 func getRandomAWSZones(masterCount int, multipleZones bool) ([]string, error) { 662 663 // TODO(chrislovecnm): get the number of ec2 instances in the region and ensure that there are not too many running 664 for _, i := range rand.Perm(len(awsRegions)) { 665 ec2Session, err := getAWSEC2Session(awsRegions[i]) 666 if err != nil { 667 return nil, err 668 } 669 670 // az for a region. AWS Go API does not allow us to make a single call 671 zoneResults, err := ec2Session.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}) 672 if err != nil { 673 return nil, fmt.Errorf("unable to call aws api DescribeAvailabilityZones for %q: %v", awsRegions[i], err) 674 } 675 676 var selectedZones []string 677 if len(zoneResults.AvailabilityZones) >= masterCount && multipleZones { 678 for _, z := range zoneResults.AvailabilityZones { 679 selectedZones = append(selectedZones, *z.ZoneName) 680 } 681 682 log.Printf("Launching cluster in region: %q", awsRegions[i]) 683 return selectedZones, nil 684 } else if !multipleZones { 685 z := zoneResults.AvailabilityZones[rand.Intn(len(zoneResults.AvailabilityZones))] 686 selectedZones = append(selectedZones, *z.ZoneName) 687 log.Printf("Launching cluster in region: %q", awsRegions[i]) 688 return selectedZones, nil 689 } 690 } 691 692 return nil, fmt.Errorf("unable to find region with %d zones", masterCount) 693 } 694 695 // getAWSEC2Session creates an returns a EC2 API session. 696 func getAWSEC2Session(region string) (*ec2.EC2, error) { 697 config := aws.NewConfig().WithRegion(region) 698 699 // This avoids a confusing error message when we fail to get credentials 700 config = config.WithCredentialsChainVerboseErrors(true) 701 702 s, err := session.NewSession(config) 703 if err != nil { 704 return nil, fmt.Errorf("unable to build aws API session with region: %q: %v", region, err) 705 } 706 707 return ec2.New(s, config), nil 708 709 }