github.com/jenkins-x/jx/v2@v2.1.155/pkg/cmd/gc/gc_gke.go (about) 1 package gc 2 3 import ( 4 "strings" 5 6 "github.com/pkg/errors" 7 8 "github.com/jenkins-x/jx/v2/pkg/cmd/helper" 9 10 "github.com/spf13/cobra" 11 12 "encoding/json" 13 14 "fmt" 15 16 "io/ioutil" 17 18 "os" 19 20 "github.com/jenkins-x/jx-logging/pkg/log" 21 "github.com/jenkins-x/jx/v2/pkg/cmd/opts" 22 "github.com/jenkins-x/jx/v2/pkg/cmd/templates" 23 "github.com/jenkins-x/jx/v2/pkg/util" 24 ) 25 26 // GCGKEOptions is the start of the data required to perform the operation. As new fields are added, add them here instead of 27 // referencing the cmd.Flags() 28 type GCGKEOptions struct { 29 *opts.CommonOptions 30 Flags GCGKEFlags 31 RevisionHistoryLimit int 32 } 33 34 // GCGKEFlags contains the flags for the command 35 type GCGKEFlags struct { 36 ProjectID string 37 RunNow bool 38 } 39 40 var ( 41 GCGKELong = templates.LongDesc(` 42 Garbage collect Google Container Engine resources that are not deleted when a delete cluster is performed 43 44 This command will generate the gcloud command to run and delete external loadbalancers and persistent disks 45 that are no longer in use. 46 47 `) 48 49 GCGKEExample = templates.Examples(` 50 jx garbage collect gke 51 jx gc gke 52 `) 53 54 ServiceAccountSuffixes = []string{"-vt", "-ko", "-tf", "-dn", "-ex", "-jb", "-st", "-tk", "-vo", "-bc", "-tekton"} 55 ) 56 57 type Rules struct { 58 Rules []Rule 59 } 60 61 type Rule struct { 62 Name string `json:"name"` 63 TargetTags []string `json:"targetTags"` 64 } 65 66 type cluster struct { 67 Name string `json:"name"` 68 } 69 70 type address struct { 71 Name string `json:"name"` 72 Region string `json:"region"` 73 } 74 75 type zone struct { 76 Name string `json:"name"` 77 } 78 79 type disk struct { 80 Name string `json:"name"` 81 } 82 83 type serviceAccount struct { 84 DisplayName string `json:"displayName"` 85 Email string `json:"email"` 86 } 87 88 type iamPolicy struct { 89 Bindings []iamBinding `json:"bindings"` 90 } 91 92 type iamBinding struct { 93 Members []string `json:"members"` 94 Role string `json:"role"` 95 } 96 97 func (o *GCGKEOptions) addFlags(cmd *cobra.Command) { 98 cmd.Flags().StringVarP(&o.Flags.ProjectID, "project", "p", "", "The google project id to create the GC script for") 99 cmd.Flags().BoolVarP(&o.Flags.RunNow, "run-now", "", false, "Execute the script") 100 } 101 102 // NewCmdGCGKE is a command object for the "step" command 103 func NewCmdGCGKE(commonOpts *opts.CommonOptions) *cobra.Command { 104 options := &GCGKEOptions{ 105 CommonOptions: commonOpts, 106 } 107 108 cmd := &cobra.Command{ 109 Use: "gke", 110 Short: "garbage collection for gke", 111 Long: GCGKELong, 112 Example: GCGKEExample, 113 Run: func(cmd *cobra.Command, args []string) { 114 options.Cmd = cmd 115 options.Args = args 116 err := options.Run() 117 helper.CheckErr(err) 118 }, 119 } 120 121 options.CommonOptions.AddBaseFlags(cmd) 122 options.addFlags(cmd) 123 124 return cmd 125 } 126 127 // Run implements this command 128 func (o *GCGKEOptions) Run() error { 129 130 if o.Flags.ProjectID == "" { 131 if o.BatchMode { 132 projectID, err := o.getCurrentGoogleProjectId() 133 if err != nil { 134 return err 135 } 136 o.Flags.ProjectID = projectID 137 } else { 138 projectID, err := o.GetGoogleProjectID("") 139 if err != nil { 140 return err 141 } 142 o.Flags.ProjectID = projectID 143 } 144 } 145 146 dir, err := os.Getwd() 147 if err != nil { 148 return err 149 } 150 151 gkeSa := os.Getenv("GKE_SA_KEY_FILE") 152 if gkeSa != "" { 153 err = o.GCloud().Login(gkeSa, true) 154 if err != nil { 155 return err 156 } 157 } 158 159 path := util.UrlJoin(dir, "gc_gke.sh") 160 exists, err := util.FileExists(path) 161 if err != nil { 162 return errors.Wrapf(err, "checking if file %s exists", path) 163 } 164 if exists { 165 err = os.Remove(path) 166 if err != nil { 167 return err 168 } 169 } 170 171 message := `#!/bin/bash 172 173 set -euo pipefail 174 175 ################################################################################################### 176 # 177 # WARNING: this command is experimental and the generated script should be executed at the users own risk. We use this 178 # generated command on the Jenkins X project itself but it has not been tested on other clusters. 179 # 180 ################################################################################################### 181 182 # Project %s 183 184 %s 185 186 %s 187 188 %s 189 190 %s 191 192 ` 193 log.Logger().Warn("This command is experimental and the generated script should be executed at the users own risk\n") 194 log.Logger().Warn("We will generate a script for you to review and execute, this command will not delete any resources by itself\n") 195 log.Logger().Info("It may take a few minutes to create the script\n") 196 197 fw, err := o.cleanUpFirewalls() 198 if err != nil { 199 return err 200 } 201 202 disks, err := o.cleanUpPersistentDisks() 203 if err != nil { 204 return err 205 } 206 207 addr, err := o.cleanUpAddresses() 208 if err != nil { 209 return err 210 } 211 212 serviceAccounts, err := o.cleanUpServiceAccounts() 213 if err != nil { 214 return err 215 } 216 217 data := fmt.Sprintf(message, o.Flags.ProjectID, fw, strings.Join(disks, "\n"), strings.Join(addr, "\n"), strings.Join(serviceAccounts, "\n")) 218 data = strings.Replace(data, "[", "", -1) 219 data = strings.Replace(data, "]", "", -1) 220 221 err = ioutil.WriteFile("gc_gke.sh", []byte(data), util.DefaultWritePermissions) 222 if err != nil { 223 return err 224 } 225 log.Logger().Info("Script 'gc_gke.sh' created!") 226 if o.Flags.RunNow { 227 log.Logger().Info("Executing 'gc_gke.sh'") 228 err = o.RunCommand("gc_gke.sh") 229 log.Logger().Info("Done") 230 } 231 return err 232 } 233 234 func (o *GCGKEOptions) cleanUpFirewalls() (string, error) { 235 co := &opts.CommonOptions{} 236 data, err := co.GetCommandOutput("", "gcloud", "compute", "firewall-rules", "list", "--format", "json", "--project", o.Flags.ProjectID) 237 if err != nil { 238 return "", err 239 } 240 241 var rules []Rule 242 err = json.Unmarshal([]byte(data), &rules) 243 if err != nil { 244 return "", err 245 } 246 247 out, err := co.GetCommandOutput("", "gcloud", "container", "clusters", "list", "--project", o.Flags.ProjectID) 248 if err != nil { 249 return "", err 250 } 251 252 lines := strings.Split(out, "\n") 253 var existingClusters []string 254 for _, l := range lines { 255 if strings.Contains(l, "NAME") { 256 continue 257 } 258 if strings.TrimSpace(l) == "" { 259 break 260 } 261 fields := strings.Fields(l) 262 existingClusters = append(existingClusters, fields[0]) 263 } 264 265 var nameToDelete []string 266 for _, rule := range rules { 267 name := strings.TrimPrefix(rule.Name, "gke-") 268 269 if !contains(existingClusters, name) { 270 for _, tagFull := range rule.TargetTags { 271 tag := strings.TrimPrefix(tagFull, "gke-") 272 273 if !contains(existingClusters, tag) { 274 nameToDelete = append(nameToDelete, rule.Name) 275 } 276 } 277 } 278 } 279 280 if nameToDelete != nil { 281 args := "gcloud compute firewall-rules delete --quiet --project " + o.Flags.ProjectID 282 for _, name := range nameToDelete { 283 args = args + " " + name 284 } 285 args = args + " || true" 286 return args, nil 287 } 288 289 return "# No firewalls found for deletion", nil 290 } 291 292 func (o *GCGKEOptions) cleanUpPersistentDisks() ([]string, error) { 293 zones, err := o.getZones() 294 if err != nil { 295 return nil, err 296 } 297 var line []string 298 299 for _, z := range zones { 300 disks, err := o.getUnusedDisksForZone(z) 301 if err != nil { 302 return nil, err 303 } 304 305 for _, d := range disks { 306 if strings.HasPrefix(d.Name, "gke-") { 307 line = append(line, fmt.Sprintf("gcloud compute disks delete --zone=%s --quiet %s --project %s || true", z.Name, d.Name, o.Flags.ProjectID)) 308 } 309 } 310 } 311 312 if len(line) == 0 { 313 line = append(line, "# No disks found for deletion\n") 314 } 315 316 return line, nil 317 } 318 319 func (o *GCGKEOptions) cleanUpAddresses() ([]string, error) { 320 321 cmd := "gcloud compute addresses list --filter=\"status:RESERVED\" --format=json --project " + o.Flags.ProjectID 322 data, err := o.GetCommandOutput("", "bash", "-c", cmd) 323 if err != nil { 324 return nil, err 325 } 326 327 var addresses []address 328 err = json.Unmarshal([]byte(data), &addresses) 329 if err != nil { 330 return nil, err 331 } 332 333 var line []string 334 if len(addresses) > 0 { 335 for _, address := range addresses { 336 var scope string 337 if address.Region != "" { 338 region := getLastString(strings.Split(address.Region, "/")) 339 scope = fmt.Sprintf("--region %s", region) 340 } else { 341 scope = "--global" 342 } 343 line = append(line, fmt.Sprintf("gcloud compute addresses delete %s %s --project %s || true", address.Name, scope, o.Flags.ProjectID)) 344 } 345 return line, nil 346 } 347 348 if len(line) == 0 { 349 line = append(line, "# No addresses found for deletion\n") 350 } 351 352 return line, nil 353 } 354 355 func (o *GCGKEOptions) cleanUpServiceAccounts() ([]string, error) { 356 serviceAccounts, err := o.getServiceAccounts() 357 if err != nil { 358 return nil, err 359 } 360 361 clusters, err := o.getClusters() 362 if err != nil { 363 return nil, err 364 } 365 366 serviceAccounts, err = o.getFilteredServiceAccounts(serviceAccounts, clusters) 367 if err != nil { 368 return nil, err 369 } 370 var line []string 371 372 if len(serviceAccounts) > 0 { 373 for _, sa := range serviceAccounts { 374 log.Logger().Debugf("About to delete service account %s", sa) 375 line = append(line, fmt.Sprintf("gcloud iam service-accounts delete %s --quiet --project %s || true", sa.Email, o.Flags.ProjectID)) 376 } 377 } 378 379 if len(line) == 0 { 380 line = append(line, "# No service accounts found for deletion\n") 381 } 382 383 policy, err := o.getIamPolicy() 384 if err != nil { 385 return nil, err 386 } 387 388 iam, err := o.determineUnusedIamBindings(policy) 389 if err != nil { 390 return nil, err 391 } 392 393 if len(iam) == 0 { 394 line = append(line, "# No iam policy bindings found for deletion\n") 395 } 396 397 line = append(line, iam...) 398 399 if len(line) == 0 { 400 line = append(line, "# No service accounts found for deletion\n") 401 } 402 403 return line, nil 404 } 405 406 func contains(s []string, e string) bool { 407 for _, a := range s { 408 if strings.HasPrefix(e, a) { 409 return true 410 } 411 } 412 return false 413 } 414 415 func getLastString(s []string) string { 416 return s[len(s)-1] 417 } 418 419 func (o *GCGKEOptions) getZones() ([]zone, error) { 420 cmd := "gcloud compute zones list --format=json --project " + o.Flags.ProjectID 421 data, err := o.GetCommandOutput("", "bash", "-c", cmd) 422 if err != nil { 423 return nil, err 424 } 425 426 var zones []zone 427 err = json.Unmarshal([]byte(data), &zones) 428 if err != nil { 429 return nil, err 430 } 431 432 return zones, nil 433 } 434 435 func (o *GCGKEOptions) getClusters() ([]cluster, error) { 436 cmd := "gcloud container clusters list --format=json --project " + o.Flags.ProjectID 437 data, err := o.GetCommandOutput("", "bash", "-c", cmd) 438 if err != nil { 439 return nil, err 440 } 441 442 var clusters []cluster 443 err = json.Unmarshal([]byte(data), &clusters) 444 if err != nil { 445 return nil, err 446 } 447 448 return clusters, nil 449 } 450 451 func (o *GCGKEOptions) getUnusedDisksForZone(z zone) ([]disk, error) { 452 diskCmd := fmt.Sprintf("gcloud compute disks list --filter=\"NOT users:* AND zone:(%s)\" --format=json --project %s", z.Name, o.Flags.ProjectID) 453 data, err := o.GetCommandOutput("", "bash", "-c", diskCmd) 454 if err != nil { 455 return nil, err 456 } 457 458 var disks []disk 459 err = json.Unmarshal([]byte(data), &disks) 460 if err != nil { 461 return nil, err 462 } 463 464 return disks, nil 465 } 466 467 func (o *GCGKEOptions) getFilteredServiceAccounts(serviceAccounts []serviceAccount, clusters []cluster) ([]serviceAccount, error) { 468 469 filteredServiceAccounts := []serviceAccount{} 470 for _, sa := range serviceAccounts { 471 if isServiceAccount(sa.DisplayName) { 472 if o.shouldRemoveServiceAccount(sa.DisplayName, clusters) { 473 log.Logger().Debugf("Adding service account to filtered service account list %s", sa.DisplayName) 474 filteredServiceAccounts = append(filteredServiceAccounts, sa) 475 } 476 } 477 } 478 return filteredServiceAccounts, nil 479 } 480 481 func (o *GCGKEOptions) shouldRemoveServiceAccount(saDisplayName string, clusters []cluster) bool { 482 sz := len(saDisplayName) 483 clusterName := saDisplayName[:sz-3] 484 if strings.HasPrefix(clusterName, "pr") { 485 // clusters with '-' in names such as BDD test clusters 486 // e.g. pr-331-170-gitop-vt 487 clusterNameParts := strings.Split(clusterName, "-") 488 if len(clusterNameParts) > 2 { 489 clusterNamePrefix := clusterNameParts[0] + "-" + clusterNameParts[1] + "-" + clusterNameParts[2] 490 if !o.clusterExistsWithPrefix(clusters, clusterNamePrefix) { 491 log.Logger().Debugf("cluster with prefix %s does not exist", clusterNamePrefix) 492 return true 493 } 494 } 495 } else { 496 if !o.clusterExists(clusters, clusterName) { 497 // clusters that don't start with pr 498 log.Logger().Debugf("cluster %s does not exist", clusterName) 499 return true 500 } 501 } 502 log.Logger().Debugf("cluster %s exists, excluding service account %s", clusterName, saDisplayName) 503 return false 504 } 505 506 func (o *GCGKEOptions) getServiceAccounts() ([]serviceAccount, error) { 507 cmd := "gcloud iam service-accounts list --format=json --project " + o.Flags.ProjectID 508 data, err := o.GetCommandOutput("", "bash", "-c", cmd) 509 if err != nil { 510 return nil, err 511 } 512 513 var serviceAccounts []serviceAccount 514 err = json.Unmarshal([]byte(data), &serviceAccounts) 515 if err != nil { 516 return nil, err 517 } 518 519 var modifiedSAs []serviceAccount 520 for _, sa := range serviceAccounts { 521 // If there isn't a display name or the display name contains spaces, which is the case for Terraform-created 522 // SAs, just split the email instead. 523 if sa.DisplayName == "" || strings.Contains(sa.DisplayName, " ") { 524 sa.DisplayName = sa.Email[:strings.IndexByte(sa.Email, '@')] 525 } 526 modifiedSAs = append(modifiedSAs, sa) 527 } 528 return modifiedSAs, nil 529 } 530 531 func (o *GCGKEOptions) clusterExistsWithPrefix(clusters []cluster, clusterNamePrefix string) bool { 532 for _, cluster := range clusters { 533 if strings.HasPrefix(cluster.Name, clusterNamePrefix) { 534 return true 535 } 536 } 537 return false 538 } 539 540 func (o *GCGKEOptions) clusterExists(clusters []cluster, clusterName string) bool { 541 for _, cluster := range clusters { 542 if cluster.Name == clusterName { 543 return true 544 } 545 } 546 return false 547 } 548 549 func (o *GCGKEOptions) getCurrentGoogleProjectId() (string, error) { 550 cmd := "gcloud config get-value core/project" 551 data, err := o.GetCommandOutput("", "bash", "-c", cmd) 552 if err != nil { 553 return "", err 554 } 555 return data, nil 556 } 557 558 func (o *GCGKEOptions) getIamPolicy() (iamPolicy, error) { 559 cmd := fmt.Sprintf("gcloud projects get-iam-policy %s --format=json", o.Flags.ProjectID) 560 data, err := o.GetCommandOutput("", "bash", "-c", cmd) 561 if err != nil { 562 return iamPolicy{}, err 563 } 564 565 var policy iamPolicy 566 err = json.Unmarshal([]byte(data), &policy) 567 if err != nil { 568 return iamPolicy{}, err 569 } 570 571 return policy, nil 572 } 573 574 func (o *GCGKEOptions) determineUnusedIamBindings(policy iamPolicy) ([]string, error) { 575 var line []string 576 577 clusters, err := o.getClusters() 578 if err != nil { 579 return line, err 580 } 581 for _, b := range policy.Bindings { 582 for _, m := range b.Members { 583 if strings.HasPrefix(m, "deleted:serviceAccount:") || strings.HasPrefix(m, "serviceAccount:") { 584 saName := strings.TrimPrefix(m, "deleted:") 585 saName = strings.TrimPrefix(saName, "serviceAccount:") 586 displayName := saName[:strings.IndexByte(saName, '@')] 587 588 if isServiceAccount(displayName) { 589 if o.shouldRemoveServiceAccount(displayName, clusters) { 590 cmd := fmt.Sprintf("gcloud projects remove-iam-policy-binding %s --member=%s --role=%s --quiet", o.Flags.ProjectID, m, b.Role) 591 line = append(line, cmd) 592 } 593 } 594 } 595 } 596 } 597 return line, nil 598 } 599 600 func isServiceAccount(sa string) bool { 601 for _, suffix := range ServiceAccountSuffixes { 602 if strings.HasSuffix(sa, suffix) { 603 return true 604 } 605 } 606 return false 607 }