github.com/cycloidio/terraform@v1.1.10-0.20220513142504-76d5c768dc63/cloud/backend_common.go (about) 1 package cloud 2 3 import ( 4 "bufio" 5 "context" 6 "errors" 7 "fmt" 8 "io" 9 "math" 10 "strconv" 11 "strings" 12 "time" 13 14 tfe "github.com/hashicorp/go-tfe" 15 "github.com/cycloidio/terraform/backend" 16 "github.com/cycloidio/terraform/plans" 17 "github.com/cycloidio/terraform/terraform" 18 ) 19 20 var ( 21 errApplyDiscarded = errors.New("Apply discarded.") 22 errDestroyDiscarded = errors.New("Destroy discarded.") 23 errRunApproved = errors.New("approved using the UI or API") 24 errRunDiscarded = errors.New("discarded using the UI or API") 25 errRunOverridden = errors.New("overridden using the UI or API") 26 ) 27 28 var ( 29 backoffMin = 1000.0 30 backoffMax = 3000.0 31 32 runPollInterval = 3 * time.Second 33 ) 34 35 // backoff will perform exponential backoff based on the iteration and 36 // limited by the provided min and max (in milliseconds) durations. 37 func backoff(min, max float64, iter int) time.Duration { 38 backoff := math.Pow(2, float64(iter)/5) * min 39 if backoff > max { 40 backoff = max 41 } 42 return time.Duration(backoff) * time.Millisecond 43 } 44 45 func (b *Cloud) waitForRun(stopCtx, cancelCtx context.Context, op *backend.Operation, opType string, r *tfe.Run, w *tfe.Workspace) (*tfe.Run, error) { 46 started := time.Now() 47 updated := started 48 for i := 0; ; i++ { 49 select { 50 case <-stopCtx.Done(): 51 return r, stopCtx.Err() 52 case <-cancelCtx.Done(): 53 return r, cancelCtx.Err() 54 case <-time.After(backoff(backoffMin, backoffMax, i)): 55 // Timer up, show status 56 } 57 58 // Retrieve the run to get its current status. 59 r, err := b.client.Runs.Read(stopCtx, r.ID) 60 if err != nil { 61 return r, generalError("Failed to retrieve run", err) 62 } 63 64 // Return if the run is no longer pending. 65 if r.Status != tfe.RunPending && r.Status != tfe.RunConfirmed { 66 if i == 0 && opType == "plan" && b.CLI != nil { 67 b.CLI.Output(b.Colorize().Color(fmt.Sprintf("Waiting for the %s to start...\n", opType))) 68 } 69 if i > 0 && b.CLI != nil { 70 // Insert a blank line to separate the ouputs. 71 b.CLI.Output("") 72 } 73 return r, nil 74 } 75 76 // Check if 30 seconds have passed since the last update. 77 current := time.Now() 78 if b.CLI != nil && (i == 0 || current.Sub(updated).Seconds() > 30) { 79 updated = current 80 position := 0 81 elapsed := "" 82 83 // Calculate and set the elapsed time. 84 if i > 0 { 85 elapsed = fmt.Sprintf( 86 " (%s elapsed)", current.Sub(started).Truncate(30*time.Second)) 87 } 88 89 // Retrieve the workspace used to run this operation in. 90 w, err = b.client.Workspaces.Read(stopCtx, b.organization, w.Name) 91 if err != nil { 92 return nil, generalError("Failed to retrieve workspace", err) 93 } 94 95 // If the workspace is locked the run will not be queued and we can 96 // update the status without making any expensive calls. 97 if w.Locked && w.CurrentRun != nil { 98 cr, err := b.client.Runs.Read(stopCtx, w.CurrentRun.ID) 99 if err != nil { 100 return r, generalError("Failed to retrieve current run", err) 101 } 102 if cr.Status == tfe.RunPending { 103 b.CLI.Output(b.Colorize().Color( 104 "Waiting for the manually locked workspace to be unlocked..." + elapsed)) 105 continue 106 } 107 } 108 109 // Skip checking the workspace queue when we are the current run. 110 if w.CurrentRun == nil || w.CurrentRun.ID != r.ID { 111 found := false 112 options := tfe.RunListOptions{} 113 runlist: 114 for { 115 rl, err := b.client.Runs.List(stopCtx, w.ID, options) 116 if err != nil { 117 return r, generalError("Failed to retrieve run list", err) 118 } 119 120 // Loop through all runs to calculate the workspace queue position. 121 for _, item := range rl.Items { 122 if !found { 123 if r.ID == item.ID { 124 found = true 125 } 126 continue 127 } 128 129 // If the run is in a final state, ignore it and continue. 130 switch item.Status { 131 case tfe.RunApplied, tfe.RunCanceled, tfe.RunDiscarded, tfe.RunErrored: 132 continue 133 case tfe.RunPlanned: 134 if op.Type == backend.OperationTypePlan { 135 continue 136 } 137 } 138 139 // Increase the workspace queue position. 140 position++ 141 142 // Stop searching when we reached the current run. 143 if w.CurrentRun != nil && w.CurrentRun.ID == item.ID { 144 break runlist 145 } 146 } 147 148 // Exit the loop when we've seen all pages. 149 if rl.CurrentPage >= rl.TotalPages { 150 break 151 } 152 153 // Update the page number to get the next page. 154 options.PageNumber = rl.NextPage 155 } 156 157 if position > 0 { 158 b.CLI.Output(b.Colorize().Color(fmt.Sprintf( 159 "Waiting for %d run(s) to finish before being queued...%s", 160 position, 161 elapsed, 162 ))) 163 continue 164 } 165 } 166 167 options := tfe.RunQueueOptions{} 168 search: 169 for { 170 rq, err := b.client.Organizations.RunQueue(stopCtx, b.organization, options) 171 if err != nil { 172 return r, generalError("Failed to retrieve queue", err) 173 } 174 175 // Search through all queued items to find our run. 176 for _, item := range rq.Items { 177 if r.ID == item.ID { 178 position = item.PositionInQueue 179 break search 180 } 181 } 182 183 // Exit the loop when we've seen all pages. 184 if rq.CurrentPage >= rq.TotalPages { 185 break 186 } 187 188 // Update the page number to get the next page. 189 options.PageNumber = rq.NextPage 190 } 191 192 if position > 0 { 193 c, err := b.client.Organizations.Capacity(stopCtx, b.organization) 194 if err != nil { 195 return r, generalError("Failed to retrieve capacity", err) 196 } 197 b.CLI.Output(b.Colorize().Color(fmt.Sprintf( 198 "Waiting for %d queued run(s) to finish before starting...%s", 199 position-c.Running, 200 elapsed, 201 ))) 202 continue 203 } 204 205 b.CLI.Output(b.Colorize().Color(fmt.Sprintf( 206 "Waiting for the %s to start...%s", opType, elapsed))) 207 } 208 } 209 } 210 211 func (b *Cloud) costEstimate(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error { 212 if r.CostEstimate == nil { 213 return nil 214 } 215 216 msgPrefix := "Cost Estimation" 217 started := time.Now() 218 updated := started 219 for i := 0; ; i++ { 220 select { 221 case <-stopCtx.Done(): 222 return stopCtx.Err() 223 case <-cancelCtx.Done(): 224 return cancelCtx.Err() 225 case <-time.After(backoff(backoffMin, backoffMax, i)): 226 } 227 228 // Retrieve the cost estimate to get its current status. 229 ce, err := b.client.CostEstimates.Read(stopCtx, r.CostEstimate.ID) 230 if err != nil { 231 return generalError("Failed to retrieve cost estimate", err) 232 } 233 234 // If the run is canceled or errored, but the cost-estimate still has 235 // no result, there is nothing further to render. 236 if ce.Status != tfe.CostEstimateFinished { 237 if r.Status == tfe.RunCanceled || r.Status == tfe.RunErrored { 238 return nil 239 } 240 } 241 242 // checking if i == 0 so as to avoid printing this starting horizontal-rule 243 // every retry, and that it only prints it on the first (i=0) attempt. 244 if b.CLI != nil && i == 0 { 245 b.CLI.Output("\n------------------------------------------------------------------------\n") 246 } 247 248 switch ce.Status { 249 case tfe.CostEstimateFinished: 250 delta, err := strconv.ParseFloat(ce.DeltaMonthlyCost, 64) 251 if err != nil { 252 return generalError("Unexpected error", err) 253 } 254 255 sign := "+" 256 if delta < 0 { 257 sign = "-" 258 } 259 260 deltaRepr := strings.Replace(ce.DeltaMonthlyCost, "-", "", 1) 261 262 if b.CLI != nil { 263 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 264 b.CLI.Output(b.Colorize().Color(fmt.Sprintf("Resources: %d of %d estimated", ce.MatchedResourcesCount, ce.ResourcesCount))) 265 b.CLI.Output(b.Colorize().Color(fmt.Sprintf(" $%s/mo %s$%s", ce.ProposedMonthlyCost, sign, deltaRepr))) 266 267 if len(r.PolicyChecks) == 0 && r.HasChanges && op.Type == backend.OperationTypeApply { 268 b.CLI.Output("\n------------------------------------------------------------------------") 269 } 270 } 271 272 return nil 273 case tfe.CostEstimatePending, tfe.CostEstimateQueued: 274 // Check if 30 seconds have passed since the last update. 275 current := time.Now() 276 if b.CLI != nil && (i == 0 || current.Sub(updated).Seconds() > 30) { 277 updated = current 278 elapsed := "" 279 280 // Calculate and set the elapsed time. 281 if i > 0 { 282 elapsed = fmt.Sprintf( 283 " (%s elapsed)", current.Sub(started).Truncate(30*time.Second)) 284 } 285 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 286 b.CLI.Output(b.Colorize().Color("Waiting for cost estimate to complete..." + elapsed + "\n")) 287 } 288 continue 289 case tfe.CostEstimateSkippedDueToTargeting: 290 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 291 b.CLI.Output("Not available for this plan, because it was created with the -target option.") 292 b.CLI.Output("\n------------------------------------------------------------------------") 293 return nil 294 case tfe.CostEstimateErrored: 295 b.CLI.Output(msgPrefix + " errored.\n") 296 b.CLI.Output("\n------------------------------------------------------------------------") 297 return nil 298 case tfe.CostEstimateCanceled: 299 return fmt.Errorf(msgPrefix + " canceled.") 300 default: 301 return fmt.Errorf("Unknown or unexpected cost estimate state: %s", ce.Status) 302 } 303 } 304 } 305 306 func (b *Cloud) checkPolicy(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error { 307 if b.CLI != nil { 308 b.CLI.Output("\n------------------------------------------------------------------------\n") 309 } 310 for i, pc := range r.PolicyChecks { 311 // Read the policy check logs. This is a blocking call that will only 312 // return once the policy check is complete. 313 logs, err := b.client.PolicyChecks.Logs(stopCtx, pc.ID) 314 if err != nil { 315 return generalError("Failed to retrieve policy check logs", err) 316 } 317 reader := bufio.NewReaderSize(logs, 64*1024) 318 319 // Retrieve the policy check to get its current status. 320 pc, err := b.client.PolicyChecks.Read(stopCtx, pc.ID) 321 if err != nil { 322 return generalError("Failed to retrieve policy check", err) 323 } 324 325 // If the run is canceled or errored, but the policy check still has 326 // no result, there is nothing further to render. 327 if r.Status == tfe.RunCanceled || r.Status == tfe.RunErrored { 328 switch pc.Status { 329 case tfe.PolicyPending, tfe.PolicyQueued, tfe.PolicyUnreachable: 330 continue 331 } 332 } 333 334 var msgPrefix string 335 switch pc.Scope { 336 case tfe.PolicyScopeOrganization: 337 msgPrefix = "Organization Policy Check" 338 case tfe.PolicyScopeWorkspace: 339 msgPrefix = "Workspace Policy Check" 340 default: 341 msgPrefix = fmt.Sprintf("Unknown policy check (%s)", pc.Scope) 342 } 343 344 if b.CLI != nil { 345 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 346 } 347 348 if b.CLI != nil { 349 for next := true; next; { 350 var l, line []byte 351 352 for isPrefix := true; isPrefix; { 353 l, isPrefix, err = reader.ReadLine() 354 if err != nil { 355 if err != io.EOF { 356 return generalError("Failed to read logs", err) 357 } 358 next = false 359 } 360 line = append(line, l...) 361 } 362 363 if next || len(line) > 0 { 364 b.CLI.Output(b.Colorize().Color(string(line))) 365 } 366 } 367 } 368 369 switch pc.Status { 370 case tfe.PolicyPasses: 371 if (r.HasChanges && op.Type == backend.OperationTypeApply || i < len(r.PolicyChecks)-1) && b.CLI != nil { 372 b.CLI.Output("\n------------------------------------------------------------------------") 373 } 374 continue 375 case tfe.PolicyErrored: 376 return fmt.Errorf(msgPrefix + " errored.") 377 case tfe.PolicyHardFailed: 378 return fmt.Errorf(msgPrefix + " hard failed.") 379 case tfe.PolicySoftFailed: 380 runUrl := fmt.Sprintf(runHeader, b.hostname, b.organization, op.Workspace, r.ID) 381 382 if op.Type == backend.OperationTypePlan || op.UIOut == nil || op.UIIn == nil || 383 !pc.Actions.IsOverridable || !pc.Permissions.CanOverride { 384 return fmt.Errorf(msgPrefix + " soft failed.\n" + runUrl) 385 } 386 387 if op.AutoApprove { 388 if _, err = b.client.PolicyChecks.Override(stopCtx, pc.ID); err != nil { 389 return generalError(fmt.Sprintf("Failed to override policy check.\n%s", runUrl), err) 390 } 391 } else { 392 opts := &terraform.InputOpts{ 393 Id: "override", 394 Query: "\nDo you want to override the soft failed policy check?", 395 Description: "Only 'override' will be accepted to override.", 396 } 397 err = b.confirm(stopCtx, op, opts, r, "override") 398 if err != nil && err != errRunOverridden { 399 return fmt.Errorf( 400 fmt.Sprintf("Failed to override: %s\n%s\n", err.Error(), runUrl), 401 ) 402 } 403 404 if err != errRunOverridden { 405 if _, err = b.client.PolicyChecks.Override(stopCtx, pc.ID); err != nil { 406 return generalError(fmt.Sprintf("Failed to override policy check.\n%s", runUrl), err) 407 } 408 } else { 409 b.CLI.Output(fmt.Sprintf("The run needs to be manually overridden or discarded.\n%s\n", runUrl)) 410 } 411 } 412 413 if b.CLI != nil { 414 b.CLI.Output("------------------------------------------------------------------------") 415 } 416 default: 417 return fmt.Errorf("Unknown or unexpected policy state: %s", pc.Status) 418 } 419 } 420 421 return nil 422 } 423 424 func (b *Cloud) confirm(stopCtx context.Context, op *backend.Operation, opts *terraform.InputOpts, r *tfe.Run, keyword string) error { 425 doneCtx, cancel := context.WithCancel(stopCtx) 426 result := make(chan error, 2) 427 428 go func() { 429 // Make sure we cancel doneCtx before we return 430 // so the input command is also canceled. 431 defer cancel() 432 433 for { 434 select { 435 case <-doneCtx.Done(): 436 return 437 case <-stopCtx.Done(): 438 return 439 case <-time.After(runPollInterval): 440 // Retrieve the run again to get its current status. 441 r, err := b.client.Runs.Read(stopCtx, r.ID) 442 if err != nil { 443 result <- generalError("Failed to retrieve run", err) 444 return 445 } 446 447 switch keyword { 448 case "override": 449 if r.Status != tfe.RunPolicyOverride { 450 if r.Status == tfe.RunDiscarded { 451 err = errRunDiscarded 452 } else { 453 err = errRunOverridden 454 } 455 } 456 case "yes": 457 if !r.Actions.IsConfirmable { 458 if r.Status == tfe.RunDiscarded { 459 err = errRunDiscarded 460 } else { 461 err = errRunApproved 462 } 463 } 464 } 465 466 if err != nil { 467 if b.CLI != nil { 468 b.CLI.Output(b.Colorize().Color( 469 fmt.Sprintf("[reset][yellow]%s[reset]", err.Error()))) 470 } 471 472 if err == errRunDiscarded { 473 err = errApplyDiscarded 474 if op.PlanMode == plans.DestroyMode { 475 err = errDestroyDiscarded 476 } 477 } 478 479 result <- err 480 return 481 } 482 } 483 } 484 }() 485 486 result <- func() error { 487 v, err := op.UIIn.Input(doneCtx, opts) 488 if err != nil && err != context.Canceled && stopCtx.Err() != context.Canceled { 489 return fmt.Errorf("Error asking %s: %v", opts.Id, err) 490 } 491 492 // We return the error of our parent channel as we don't 493 // care about the error of the doneCtx which is only used 494 // within this function. So if the doneCtx was canceled 495 // because stopCtx was canceled, this will properly return 496 // a context.Canceled error and otherwise it returns nil. 497 if doneCtx.Err() == context.Canceled || stopCtx.Err() == context.Canceled { 498 return stopCtx.Err() 499 } 500 501 // Make sure we cancel the context here so the loop that 502 // checks for external changes to the run is ended before 503 // we start to make changes ourselves. 504 cancel() 505 506 if v != keyword { 507 // Retrieve the run again to get its current status. 508 r, err = b.client.Runs.Read(stopCtx, r.ID) 509 if err != nil { 510 return generalError("Failed to retrieve run", err) 511 } 512 513 // Make sure we discard the run if possible. 514 if r.Actions.IsDiscardable { 515 err = b.client.Runs.Discard(stopCtx, r.ID, tfe.RunDiscardOptions{}) 516 if err != nil { 517 if op.PlanMode == plans.DestroyMode { 518 return generalError("Failed to discard destroy", err) 519 } 520 return generalError("Failed to discard apply", err) 521 } 522 } 523 524 // Even if the run was discarded successfully, we still 525 // return an error as the apply command was canceled. 526 if op.PlanMode == plans.DestroyMode { 527 return errDestroyDiscarded 528 } 529 return errApplyDiscarded 530 } 531 532 return nil 533 }() 534 535 return <-result 536 }