github.com/jenkins-x/test-infra@v0.0.7/kubetest/gke.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package main / gke.go provides the Google Container Engine (GKE) 18 // kubetest deployer via newGKE(). 19 // 20 // TODO(zmerlynn): Pull this out to a separate package? 21 package main 22 23 import ( 24 "encoding/json" 25 "flag" 26 "fmt" 27 "io/ioutil" 28 "log" 29 "os" 30 "os/exec" 31 "regexp" 32 "sort" 33 "strconv" 34 "strings" 35 "time" 36 37 "k8s.io/test-infra/kubetest/util" 38 ) 39 40 const ( 41 defaultPool = "default" 42 e2eAllow = "tcp:22,tcp:80,tcp:8080,tcp:30000-32767,udp:30000-32767" 43 defaultCreate = "container clusters create --quiet" 44 ) 45 46 var ( 47 gkeAdditionalZones = flag.String("gke-additional-zones", "", "(gke only) List of additional Google Compute Engine zones to use. Clusters are created symmetrically across zones by default, see --gke-shape for details.") 48 gkeNodeLocations = flag.String("gke-node-locations", "", "(gke only) List of Google Compute Engine zones to use.") 49 gkeEnvironment = flag.String("gke-environment", "", "(gke only) Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL") 50 gkeShape = flag.String("gke-shape", `{"default":{"Nodes":3,"MachineType":"n1-standard-2"}}`, `(gke only) A JSON description of node pools to create. The node pool 'default' is required and used for initial cluster creation. All node pools are symmetric across zones, so the cluster total node count is {total nodes in --gke-shape} * {1 + (length of --gke-additional-zones)}. Example: '{"default":{"Nodes":999,"MachineType:":"n1-standard-1"},"heapster":{"Nodes":1, "MachineType":"n1-standard-8", "ExtraArgs": []}}`) 51 gkeCreateArgs = flag.String("gke-create-args", "", "(gke only) (deprecated, use a modified --gke-create-command') Additional arguments passed directly to 'gcloud container clusters create'") 52 gkeCommandGroup = flag.String("gke-command-group", "", "(gke only) Use a different gcloud track (e.g. 'alpha') for all 'gcloud container' commands. Note: This is added to --gke-create-command on create. You should only use --gke-command-group if you need to change the gcloud track for *every* gcloud container command.") 53 gkeCreateCommand = flag.String("gke-create-command", defaultCreate, "(gke only) gcloud subcommand used to create a cluster. Modify if you need to pass arbitrary arguments to create.") 54 gkeCustomSubnet = flag.String("gke-custom-subnet", "", "(gke only) if specified, we create a custom subnet with the specified options and use it for the gke cluster. The format should be '<subnet-name> --region=<subnet-gcp-region> --range=<subnet-cidr> <any other optional params>'.") 55 gkeSingleZoneNodeInstanceGroup = flag.Bool("gke-single-zone-node-instance-group", true, "(gke only) Add instance groups from a single zone to the NODE_INSTANCE_GROUP env variable.") 56 57 // poolRe matches instance group URLs of the form `https://www.googleapis.com/compute/v1/projects/some-project/zones/a-zone/instanceGroupManagers/gke-some-cluster-some-pool-90fcb815-grp`. Match meaning: 58 // m[0]: path starting with zones/ 59 // m[1]: zone 60 // m[2]: pool name (passed to e2es) 61 // m[3]: unique hash (used as nonce for firewall rules) 62 poolRe = regexp.MustCompile(`zones/([^/]+)/instanceGroupManagers/(gke-.*-([0-9a-f]{8})-grp)$`) 63 64 urlRe = regexp.MustCompile(`https://.*/`) 65 ) 66 67 type gkeNodePool struct { 68 Nodes int 69 MachineType string 70 ExtraArgs []string 71 } 72 73 type gkeDeployer struct { 74 project string 75 zone string 76 region string 77 location string 78 additionalZones string 79 nodeLocations string 80 cluster string 81 shape map[string]gkeNodePool 82 network string 83 subnetwork string 84 subnetworkRegion string 85 image string 86 imageFamily string 87 imageProject string 88 commandGroup []string 89 createCommand []string 90 singleZoneNodeInstanceGroup bool 91 92 setup bool 93 kubecfg string 94 instanceGroups []*ig 95 } 96 97 type ig struct { 98 path string 99 zone string 100 name string 101 uniq string 102 } 103 104 var _ deployer = &gkeDeployer{} 105 106 func newGKE(provider, project, zone, region, network, image, imageFamily, imageProject, cluster string, testArgs *string, upgradeArgs *string) (*gkeDeployer, error) { 107 if provider != "gke" { 108 return nil, fmt.Errorf("--provider must be 'gke' for GKE deployment, found %q", provider) 109 } 110 g := &gkeDeployer{} 111 112 if cluster == "" { 113 return nil, fmt.Errorf("--cluster must be set for GKE deployment") 114 } 115 g.cluster = cluster 116 117 if project == "" { 118 return nil, fmt.Errorf("--gcp-project must be set for GKE deployment") 119 } 120 g.project = project 121 122 if zone == "" && region == "" { 123 return nil, fmt.Errorf("--gcp-zone or --gcp-region must be set for GKE deployment") 124 } else if zone != "" && region != "" { 125 return nil, fmt.Errorf("--gcp-zone and --gcp-region cannot both be set") 126 } 127 if zone != "" { 128 g.zone = zone 129 g.location = "--zone=" + zone 130 } else if region != "" { 131 g.region = region 132 g.location = "--region=" + region 133 } 134 135 if network == "" { 136 return nil, fmt.Errorf("--gcp-network must be set for GKE deployment") 137 } 138 g.network = network 139 140 if image == "" { 141 return nil, fmt.Errorf("--gcp-node-image must be set for GKE deployment") 142 } 143 if strings.ToUpper(image) == "CUSTOM" { 144 if imageFamily == "" || imageProject == "" { 145 return nil, fmt.Errorf("--image-family and --image-project must be set for GKE deployment if --gcp-node-image=CUSTOM") 146 } 147 } 148 g.imageFamily = imageFamily 149 g.imageProject = imageProject 150 g.image = image 151 152 g.additionalZones = *gkeAdditionalZones 153 g.nodeLocations = *gkeNodeLocations 154 155 err := json.Unmarshal([]byte(*gkeShape), &g.shape) 156 if err != nil { 157 return nil, fmt.Errorf("--gke-shape must be valid JSON, unmarshal error: %v, JSON: %q", err, *gkeShape) 158 } 159 if _, ok := g.shape[defaultPool]; !ok { 160 return nil, fmt.Errorf("--gke-shape must include a node pool named 'default', found %q", *gkeShape) 161 } 162 163 g.commandGroup = strings.Fields(*gkeCommandGroup) 164 165 g.createCommand = append([]string{}, g.commandGroup...) 166 g.createCommand = append(g.createCommand, strings.Fields(*gkeCreateCommand)...) 167 createArgs := strings.Fields(*gkeCreateArgs) 168 if len(createArgs) > 0 { 169 log.Printf("--gke-create-args is deprecated, please use '--gke-create-command=%s %s'", defaultCreate, *gkeCreateArgs) 170 } 171 g.createCommand = append(g.createCommand, createArgs...) 172 173 if err := util.MigrateOptions([]util.MigratedOption{{ 174 Env: "CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER", 175 Option: gkeEnvironment, 176 Name: "--gke-environment", 177 }}); err != nil { 178 return nil, err 179 } 180 181 var endpoint string 182 switch env := *gkeEnvironment; { 183 case env == "test": 184 endpoint = "https://test-container.sandbox.googleapis.com/" 185 case env == "staging": 186 endpoint = "https://staging-container.sandbox.googleapis.com/" 187 case env == "prod": 188 endpoint = "https://container.googleapis.com/" 189 case urlRe.MatchString(env): 190 endpoint = env 191 default: 192 return nil, fmt.Errorf("--gke-environment must be one of {test,staging,prod} or match %v, found %q", urlRe, env) 193 } 194 if err := os.Setenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER", endpoint); err != nil { 195 return nil, err 196 } 197 198 // Override kubecfg to a temporary file rather than trashing the user's. 199 f, err := ioutil.TempFile("", "gke-kubecfg") 200 if err != nil { 201 return nil, err 202 } 203 defer f.Close() 204 kubecfg := f.Name() 205 if err := f.Chmod(0600); err != nil { 206 return nil, err 207 } 208 g.kubecfg = kubecfg 209 210 // We want no KUBERNETES_PROVIDER, but to set 211 // KUBERNETES_CONFORMANCE_PROVIDER and 212 // KUBERNETES_CONFORMANCE_TEST. This prevents ginkgo-e2e.sh from 213 // using the cluster/gke functions. 214 // 215 // We do this in the deployer constructor so that 216 // cluster/gce/list-resources.sh outputs the same provider for the 217 // extent of the binary. (It seems like it belongs in TestSetup, 218 // but that way leads to madness.) 219 // 220 // TODO(zmerlynn): This is gross. 221 if err := os.Unsetenv("KUBERNETES_PROVIDER"); err != nil { 222 return nil, err 223 } 224 if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil { 225 return nil, err 226 } 227 if err := os.Setenv("KUBERNETES_CONFORMANCE_PROVIDER", "gke"); err != nil { 228 return nil, err 229 } 230 231 // TODO(zmerlynn): Another snafu of cluster/gke/list-resources.sh: 232 // Set KUBE_GCE_INSTANCE_PREFIX so that we don't accidentally pick 233 // up CLUSTER_NAME later. 234 if err := os.Setenv("KUBE_GCE_INSTANCE_PREFIX", "gke-"+g.cluster); err != nil { 235 return nil, err 236 } 237 238 // set --num-nodes flag for ginkgo, since NUM_NODES is not set for gke deployer. 239 numNodes := strconv.Itoa(g.shape[defaultPool].Nodes) 240 // testArgs can be empty, and we need to support this case 241 *testArgs = strings.Join(util.SetFieldDefault(strings.Fields(*testArgs), "--num-nodes", numNodes), " ") 242 243 if *upgradeArgs != "" { 244 // --upgrade-target will be passed to e2e upgrade framework to get a valid update version. 245 // See usage from https://github.com/kubernetes/kubernetes/blob/master/hack/get-build.sh for supported targets. 246 // Here we special case for gke-latest and will extract an actual valid gke version. 247 // - gke-latest will be resolved to the latest gke version, and 248 // - gke-latest-1.7 will be resolved to the latest 1.7 patch version supported on gke. 249 fields, val, exist := util.ExtractField(strings.Fields(*upgradeArgs), "--upgrade-target") 250 if exist { 251 if strings.HasPrefix(val, "gke-latest") { 252 releasePrefix := "" 253 if strings.HasPrefix(val, "gke-latest-") { 254 releasePrefix = strings.TrimPrefix(val, "gke-latest-") 255 } 256 if val, err = getLatestGKEVersion(project, zone, region, releasePrefix); err != nil { 257 return nil, fmt.Errorf("fail to get latest gke version : %v", err) 258 } 259 } 260 fields = util.SetFieldDefault(fields, "--upgrade-target", val) 261 } 262 *upgradeArgs = strings.Join(util.SetFieldDefault(fields, "--num-nodes", numNodes), " ") 263 } 264 265 g.singleZoneNodeInstanceGroup = *gkeSingleZoneNodeInstanceGroup 266 267 return g, nil 268 } 269 270 func (g *gkeDeployer) Up() error { 271 // Create network if it doesn't exist. 272 if control.NoOutput(exec.Command("gcloud", "compute", "networks", "describe", g.network, 273 "--project="+g.project, 274 "--format=value(name)")) != nil { 275 // Assume error implies non-existent. 276 log.Printf("Couldn't describe network '%s', assuming it doesn't exist and creating it", g.network) 277 if err := control.FinishRunning(exec.Command("gcloud", "compute", "networks", "create", g.network, 278 "--project="+g.project, 279 "--subnet-mode=auto")); err != nil { 280 return err 281 } 282 } 283 // Create a custom subnet in that network if it was asked for. 284 if *gkeCustomSubnet != "" { 285 customSubnetFields := strings.Fields(*gkeCustomSubnet) 286 createSubnetCommand := []string{"compute", "networks", "subnets", "create"} 287 createSubnetCommand = append(createSubnetCommand, "--project="+g.project, "--network="+g.network) 288 createSubnetCommand = append(createSubnetCommand, customSubnetFields...) 289 if err := control.FinishRunning(exec.Command("gcloud", createSubnetCommand...)); err != nil { 290 return err 291 } 292 g.subnetwork = customSubnetFields[0] 293 g.subnetworkRegion = customSubnetFields[1] 294 } 295 296 def := g.shape[defaultPool] 297 args := make([]string, len(g.createCommand)) 298 copy(args, g.createCommand) 299 args = append(args, 300 "--project="+g.project, 301 g.location, 302 "--machine-type="+def.MachineType, 303 "--image-type="+g.image, 304 "--num-nodes="+strconv.Itoa(def.Nodes), 305 "--network="+g.network, 306 ) 307 args = append(args, def.ExtraArgs...) 308 if strings.ToUpper(g.image) == "CUSTOM" { 309 args = append(args, "--image-family="+g.imageFamily) 310 args = append(args, "--image-project="+g.imageProject) 311 } 312 if g.subnetwork != "" { 313 args = append(args, "--subnetwork="+g.subnetwork) 314 } 315 if g.additionalZones != "" { 316 args = append(args, "--additional-zones="+g.additionalZones) 317 if err := os.Setenv("MULTIZONE", "true"); err != nil { 318 return fmt.Errorf("error setting MULTIZONE env variable: %v", err) 319 } 320 321 } 322 if g.nodeLocations != "" { 323 args = append(args, "--node-locations="+g.nodeLocations) 324 numNodeLocations := strings.Split(g.nodeLocations, ",") 325 if len(numNodeLocations) > 1 { 326 if err := os.Setenv("MULTIZONE", "true"); err != nil { 327 return fmt.Errorf("error setting MULTIZONE env variable: %v", err) 328 } 329 } 330 } 331 // TODO(zmerlynn): The version should be plumbed through Extract 332 // or a separate flag rather than magic env variables. 333 if v := os.Getenv("CLUSTER_API_VERSION"); v != "" { 334 args = append(args, "--cluster-version="+v) 335 } 336 args = append(args, g.cluster) 337 if err := control.FinishRunning(exec.Command("gcloud", args...)); err != nil { 338 return fmt.Errorf("error creating cluster: %v", err) 339 } 340 for poolName, pool := range g.shape { 341 if poolName == defaultPool { 342 continue 343 } 344 poolArgs := []string{"node-pools", "create", poolName, 345 "--cluster=" + g.cluster, 346 "--project=" + g.project, 347 g.location, 348 "--machine-type=" + pool.MachineType, 349 "--num-nodes=" + strconv.Itoa(pool.Nodes)} 350 poolArgs = append(poolArgs, pool.ExtraArgs...) 351 if err := control.FinishRunning(exec.Command("gcloud", g.containerArgs(poolArgs...)...)); err != nil { 352 return fmt.Errorf("error creating node pool %q: %v", poolName, err) 353 } 354 } 355 return nil 356 } 357 358 func (g *gkeDeployer) IsUp() error { 359 return isUp(g) 360 } 361 362 // DumpClusterLogs for GKE generates a small script that wraps 363 // log-dump.sh with the appropriate shell-fu to get the cluster 364 // dumped. 365 // 366 // TODO(zmerlynn): This whole path is really gross, but this seemed 367 // the least gross hack to get this done. 368 // 369 // TODO(shyamjvs): Make this work with multizonal and regional clusters. 370 func (g *gkeDeployer) DumpClusterLogs(localPath, gcsPath string) error { 371 // gkeLogDumpTemplate is a template of a shell script where 372 // - %[1]s is the project 373 // - %[2]s is the zone 374 // - %[3]s is a filter composed of the instance groups 375 // - %[4]s is the log-dump.sh command line 376 const gkeLogDumpTemplate = ` 377 function log_dump_custom_get_instances() { 378 if [[ $1 == "master" ]]; then 379 return 0 380 fi 381 382 gcloud compute instances list '--project=%[1]s' '--filter=%[4]s' '--format=get(name)' 383 } 384 export -f log_dump_custom_get_instances 385 # Set below vars that log-dump.sh expects in order to use scp with gcloud. 386 export PROJECT=%[1]s 387 export ZONE='%[2]s' 388 export KUBERNETES_PROVIDER=gke 389 export KUBE_NODE_OS_DISTRIBUTION='%[3]s' 390 %[5]s 391 ` 392 // Prevent an obvious injection. 393 if strings.Contains(localPath, "'") || strings.Contains(gcsPath, "'") { 394 return fmt.Errorf("%q or %q contain single quotes - nice try", localPath, gcsPath) 395 } 396 397 // Generate a slice of filters to be OR'd together below 398 if err := g.getInstanceGroups(); err != nil { 399 return err 400 } 401 var filters []string 402 for _, ig := range g.instanceGroups { 403 filters = append(filters, fmt.Sprintf("(metadata.created-by:*%s)", ig.path)) 404 } 405 406 // Generate the log-dump.sh command-line 407 var dumpCmd string 408 if gcsPath == "" { 409 dumpCmd = fmt.Sprintf("./cluster/log-dump/log-dump.sh '%s'", localPath) 410 } else { 411 dumpCmd = fmt.Sprintf("./cluster/log-dump/log-dump.sh '%s' '%s'", localPath, gcsPath) 412 } 413 return control.FinishRunning(exec.Command("bash", "-c", fmt.Sprintf(gkeLogDumpTemplate, 414 g.project, 415 g.zone, 416 os.Getenv("NODE_OS_DISTRIBUTION"), 417 strings.Join(filters, " OR "), 418 dumpCmd))) 419 } 420 421 func (g *gkeDeployer) TestSetup() error { 422 if g.setup { 423 // Ensure setup is a singleton. 424 return nil 425 } 426 if err := g.getKubeConfig(); err != nil { 427 return err 428 } 429 if err := g.getInstanceGroups(); err != nil { 430 return err 431 } 432 if err := g.ensureFirewall(); err != nil { 433 return err 434 } 435 if err := g.setupEnv(); err != nil { 436 return err 437 } 438 g.setup = true 439 return nil 440 } 441 442 func (g *gkeDeployer) getKubeConfig() error { 443 info, err := os.Stat(g.kubecfg) 444 if err != nil { 445 return err 446 } 447 if info.Size() > 0 { 448 // Assume that if we already have it, it's good. 449 return nil 450 } 451 if err := os.Setenv("KUBECONFIG", g.kubecfg); err != nil { 452 return err 453 } 454 if err := control.FinishRunning(exec.Command("gcloud", g.containerArgs("clusters", "get-credentials", g.cluster, 455 "--project="+g.project, 456 g.location)...)); err != nil { 457 return fmt.Errorf("error executing get-credentials: %v", err) 458 } 459 return nil 460 } 461 462 // setupEnv is to appease ginkgo-e2e.sh and other pieces of the e2e infrastructure. It 463 // would be nice to handle this elsewhere, and not with env 464 // variables. c.f. kubernetes/test-infra#3330. 465 func (g *gkeDeployer) setupEnv() error { 466 // If singleZoneNodeInstanceGroup is true, set NODE_INSTANCE_GROUP to the 467 // names of instance groups that are in the same zone as the lexically first 468 // instance group. Otherwise set NODE_INSTANCE_GROUP to the names of all 469 // instance groups. 470 var filt []string 471 zone := g.instanceGroups[0].zone 472 for _, ig := range g.instanceGroups { 473 if !g.singleZoneNodeInstanceGroup || ig.zone == zone { 474 filt = append(filt, ig.name) 475 } 476 } 477 if err := os.Setenv("NODE_INSTANCE_GROUP", strings.Join(filt, ",")); err != nil { 478 return fmt.Errorf("error setting NODE_INSTANCE_GROUP: %v", err) 479 } 480 return nil 481 } 482 483 func (g *gkeDeployer) ensureFirewall() error { 484 firewall, err := g.getClusterFirewall() 485 if err != nil { 486 return fmt.Errorf("error getting unique firewall: %v", err) 487 } 488 if control.NoOutput(exec.Command("gcloud", "compute", "firewall-rules", "describe", firewall, 489 "--project="+g.project, 490 "--format=value(name)")) == nil { 491 // Assume that if this unique firewall exists, it's good to go. 492 return nil 493 } 494 log.Printf("Couldn't describe firewall '%s', assuming it doesn't exist and creating it", firewall) 495 496 tagOut, err := exec.Command("gcloud", "compute", "instances", "list", 497 "--project="+g.project, 498 "--filter=metadata.created-by:*"+g.instanceGroups[0].path, 499 "--limit=1", 500 "--format=get(tags.items)").Output() 501 if err != nil { 502 return fmt.Errorf("instances list failed: %s", util.ExecError(err)) 503 } 504 tag := strings.TrimSpace(string(tagOut)) 505 if tag == "" { 506 return fmt.Errorf("instances list returned no instances (or instance has no tags)") 507 } 508 509 if err := control.FinishRunning(exec.Command("gcloud", "compute", "firewall-rules", "create", firewall, 510 "--project="+g.project, 511 "--network="+g.network, 512 "--allow="+e2eAllow, 513 "--target-tags="+tag)); err != nil { 514 return fmt.Errorf("error creating e2e firewall: %v", err) 515 } 516 return nil 517 } 518 519 func (g *gkeDeployer) getInstanceGroups() error { 520 if len(g.instanceGroups) > 0 { 521 return nil 522 } 523 igs, err := exec.Command("gcloud", g.containerArgs("clusters", "describe", g.cluster, 524 "--format=value(instanceGroupUrls)", 525 "--project="+g.project, 526 g.location)...).Output() 527 if err != nil { 528 return fmt.Errorf("instance group URL fetch failed: %s", util.ExecError(err)) 529 } 530 igURLs := strings.Split(strings.TrimSpace(string(igs)), ";") 531 if len(igURLs) == 0 { 532 return fmt.Errorf("no instance group URLs returned by gcloud, output %q", string(igs)) 533 } 534 sort.Strings(igURLs) 535 for _, igURL := range igURLs { 536 m := poolRe.FindStringSubmatch(igURL) 537 if len(m) == 0 { 538 return fmt.Errorf("instanceGroupUrl %q did not match regex %v", igURL, poolRe) 539 } 540 g.instanceGroups = append(g.instanceGroups, &ig{path: m[0], zone: m[1], name: m[2], uniq: m[3]}) 541 } 542 return nil 543 } 544 545 func (g *gkeDeployer) getClusterFirewall() (string, error) { 546 if err := g.getInstanceGroups(); err != nil { 547 return "", err 548 } 549 // We want to ensure that there's an e2e-ports-* firewall rule 550 // that maps to the cluster nodes, but the target tag for the 551 // nodes can be slow to get. Use the hash from the lexically first 552 // node pool instead. 553 return "e2e-ports-" + g.instanceGroups[0].uniq, nil 554 } 555 556 // This function ensures that all firewall-rules are deleted from specific network. 557 // We also want to keep in logs that there were some resources leaking. 558 func (g *gkeDeployer) cleanupNetworkFirewalls() (int, error) { 559 fws, err := exec.Command("gcloud", "compute", "firewall-rules", "list", 560 "--format=value(name)", 561 "--project="+g.project, 562 "--filter=network:"+g.network).Output() 563 if err != nil { 564 return 0, fmt.Errorf("firewall rules list failed: %s", util.ExecError(err)) 565 } 566 if len(fws) > 0 { 567 fwList := strings.Split(strings.TrimSpace(string(fws)), "\n") 568 log.Printf("Network %s has %v undeleted firewall rules %v", g.network, len(fwList), fwList) 569 commandArgs := []string{"compute", "firewall-rules", "delete", "-q"} 570 commandArgs = append(commandArgs, fwList...) 571 commandArgs = append(commandArgs, "--project="+g.project) 572 errFirewall := control.FinishRunning(exec.Command("gcloud", commandArgs...)) 573 if errFirewall != nil { 574 return 0, fmt.Errorf("error deleting firewall: %v", errFirewall) 575 } 576 return len(fwList), nil 577 } 578 return 0, nil 579 } 580 581 func (g *gkeDeployer) Down() error { 582 firewall, err := g.getClusterFirewall() 583 if err != nil { 584 // This is expected if the cluster doesn't exist. 585 return nil 586 } 587 g.instanceGroups = nil 588 589 // We best-effort try all of these and report errors as appropriate. 590 errCluster := control.FinishRunning(exec.Command( 591 "gcloud", g.containerArgs("clusters", "delete", "-q", g.cluster, 592 "--project="+g.project, 593 g.location)...)) 594 var errFirewall error 595 if control.NoOutput(exec.Command("gcloud", "compute", "firewall-rules", "describe", firewall, 596 "--project="+g.project, 597 "--format=value(name)")) == nil { 598 log.Printf("Found rules for firewall '%s', deleting them", firewall) 599 errFirewall = control.FinishRunning(exec.Command("gcloud", "compute", "firewall-rules", "delete", "-q", firewall, 600 "--project="+g.project)) 601 } else { 602 log.Printf("Found no rules for firewall '%s', assuming resources are clean", firewall) 603 } 604 numLeakedFWRules, errCleanFirewalls := g.cleanupNetworkFirewalls() 605 var errSubnet error 606 if g.subnetwork != "" { 607 errSubnet = control.FinishRunning(exec.Command("gcloud", "compute", "networks", "subnets", "delete", "-q", g.subnetwork, 608 g.subnetworkRegion, "--project="+g.project)) 609 } 610 errNetwork := control.FinishRunning(exec.Command("gcloud", "compute", "networks", "delete", "-q", g.network, 611 "--project="+g.project)) 612 if errCluster != nil { 613 return fmt.Errorf("error deleting cluster: %v", errCluster) 614 } 615 if errFirewall != nil { 616 return fmt.Errorf("error deleting firewall: %v", errFirewall) 617 } 618 if errCleanFirewalls != nil { 619 return fmt.Errorf("error cleaning-up firewalls: %v", errCleanFirewalls) 620 } 621 if errSubnet != nil { 622 return fmt.Errorf("error deleting subnetwork: %v", errSubnet) 623 } 624 if errNetwork != nil { 625 return fmt.Errorf("error deleting network: %v", errNetwork) 626 } 627 if numLeakedFWRules > 0 { 628 return fmt.Errorf("leaked firewall rules") 629 } 630 return nil 631 } 632 633 func (g *gkeDeployer) containerArgs(args ...string) []string { 634 return append(append(append([]string{}, g.commandGroup...), "container"), args...) 635 } 636 637 func (g *gkeDeployer) GetClusterCreated(gcpProject string) (time.Time, error) { 638 res, err := control.Output(exec.Command( 639 "gcloud", 640 "compute", 641 "instance-groups", 642 "list", 643 "--project="+gcpProject, 644 "--format=json(name,creationTimestamp)")) 645 if err != nil { 646 return time.Time{}, fmt.Errorf("list instance-group failed : %v", err) 647 } 648 649 created, err := getLatestClusterUpTime(string(res)) 650 if err != nil { 651 return time.Time{}, fmt.Errorf("parse time failed : got gcloud res %s, err %v", string(res), err) 652 } 653 return created, nil 654 } 655 656 func (_ *gkeDeployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil }