github.com/kanishk98/terraform@v1.3.0-dev.0.20220917174235-661ca8088a6a/internal/cloud/backend_common.go (about) 1 package cloud 2 3 import ( 4 "bufio" 5 "context" 6 "fmt" 7 "io" 8 "math" 9 "strconv" 10 "strings" 11 "time" 12 13 tfe "github.com/hashicorp/go-tfe" 14 "github.com/hashicorp/terraform/internal/backend" 15 "github.com/hashicorp/terraform/internal/plans" 16 "github.com/hashicorp/terraform/internal/terraform" 17 ) 18 19 var ( 20 backoffMin = 1000.0 21 backoffMax = 3000.0 22 23 runPollInterval = 3 * time.Second 24 ) 25 26 // backoff will perform exponential backoff based on the iteration and 27 // limited by the provided min and max (in milliseconds) durations. 28 func backoff(min, max float64, iter int) time.Duration { 29 backoff := math.Pow(2, float64(iter)/5) * min 30 if backoff > max { 31 backoff = max 32 } 33 return time.Duration(backoff) * time.Millisecond 34 } 35 36 func (b *Cloud) waitForRun(stopCtx, cancelCtx context.Context, op *backend.Operation, opType string, r *tfe.Run, w *tfe.Workspace) (*tfe.Run, error) { 37 started := time.Now() 38 updated := started 39 for i := 0; ; i++ { 40 select { 41 case <-stopCtx.Done(): 42 return r, stopCtx.Err() 43 case <-cancelCtx.Done(): 44 return r, cancelCtx.Err() 45 case <-time.After(backoff(backoffMin, backoffMax, i)): 46 // Timer up, show status 47 } 48 49 // Retrieve the run to get its current status. 50 r, err := b.client.Runs.Read(stopCtx, r.ID) 51 if err != nil { 52 return r, generalError("Failed to retrieve run", err) 53 } 54 55 // Return if the run is no longer pending. 56 if r.Status != tfe.RunPending && r.Status != tfe.RunConfirmed { 57 if i == 0 && opType == "plan" && b.CLI != nil { 58 b.CLI.Output(b.Colorize().Color(fmt.Sprintf("Waiting for the %s to start...\n", opType))) 59 } 60 if i > 0 && b.CLI != nil { 61 // Insert a blank line to separate the ouputs. 62 b.CLI.Output("") 63 } 64 return r, nil 65 } 66 67 // Check if 30 seconds have passed since the last update. 68 current := time.Now() 69 if b.CLI != nil && (i == 0 || current.Sub(updated).Seconds() > 30) { 70 updated = current 71 position := 0 72 elapsed := "" 73 74 // Calculate and set the elapsed time. 75 if i > 0 { 76 elapsed = fmt.Sprintf( 77 " (%s elapsed)", current.Sub(started).Truncate(30*time.Second)) 78 } 79 80 // Retrieve the workspace used to run this operation in. 81 w, err = b.client.Workspaces.Read(stopCtx, b.organization, w.Name) 82 if err != nil { 83 return nil, generalError("Failed to retrieve workspace", err) 84 } 85 86 // If the workspace is locked the run will not be queued and we can 87 // update the status without making any expensive calls. 88 if w.Locked && w.CurrentRun != nil { 89 cr, err := b.client.Runs.Read(stopCtx, w.CurrentRun.ID) 90 if err != nil { 91 return r, generalError("Failed to retrieve current run", err) 92 } 93 if cr.Status == tfe.RunPending { 94 b.CLI.Output(b.Colorize().Color( 95 "Waiting for the manually locked workspace to be unlocked..." + elapsed)) 96 continue 97 } 98 } 99 100 // Skip checking the workspace queue when we are the current run. 101 if w.CurrentRun == nil || w.CurrentRun.ID != r.ID { 102 found := false 103 options := &tfe.RunListOptions{} 104 runlist: 105 for { 106 rl, err := b.client.Runs.List(stopCtx, w.ID, options) 107 if err != nil { 108 return r, generalError("Failed to retrieve run list", err) 109 } 110 111 // Loop through all runs to calculate the workspace queue position. 112 for _, item := range rl.Items { 113 if !found { 114 if r.ID == item.ID { 115 found = true 116 } 117 continue 118 } 119 120 // If the run is in a final state, ignore it and continue. 121 switch item.Status { 122 case tfe.RunApplied, tfe.RunCanceled, tfe.RunDiscarded, tfe.RunErrored: 123 continue 124 case tfe.RunPlanned: 125 if op.Type == backend.OperationTypePlan { 126 continue 127 } 128 } 129 130 // Increase the workspace queue position. 131 position++ 132 133 // Stop searching when we reached the current run. 134 if w.CurrentRun != nil && w.CurrentRun.ID == item.ID { 135 break runlist 136 } 137 } 138 139 // Exit the loop when we've seen all pages. 140 if rl.CurrentPage >= rl.TotalPages { 141 break 142 } 143 144 // Update the page number to get the next page. 145 options.PageNumber = rl.NextPage 146 } 147 148 if position > 0 { 149 b.CLI.Output(b.Colorize().Color(fmt.Sprintf( 150 "Waiting for %d run(s) to finish before being queued...%s", 151 position, 152 elapsed, 153 ))) 154 continue 155 } 156 } 157 158 options := tfe.ReadRunQueueOptions{} 159 search: 160 for { 161 rq, err := b.client.Organizations.ReadRunQueue(stopCtx, b.organization, options) 162 if err != nil { 163 return r, generalError("Failed to retrieve queue", err) 164 } 165 166 // Search through all queued items to find our run. 167 for _, item := range rq.Items { 168 if r.ID == item.ID { 169 position = item.PositionInQueue 170 break search 171 } 172 } 173 174 // Exit the loop when we've seen all pages. 175 if rq.CurrentPage >= rq.TotalPages { 176 break 177 } 178 179 // Update the page number to get the next page. 180 options.PageNumber = rq.NextPage 181 } 182 183 if position > 0 { 184 c, err := b.client.Organizations.ReadCapacity(stopCtx, b.organization) 185 if err != nil { 186 return r, generalError("Failed to retrieve capacity", err) 187 } 188 b.CLI.Output(b.Colorize().Color(fmt.Sprintf( 189 "Waiting for %d queued run(s) to finish before starting...%s", 190 position-c.Running, 191 elapsed, 192 ))) 193 continue 194 } 195 196 b.CLI.Output(b.Colorize().Color(fmt.Sprintf( 197 "Waiting for the %s to start...%s", opType, elapsed))) 198 } 199 } 200 } 201 202 func (b *Cloud) waitTaskStage(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run, stageID string, outputTitle string) error { 203 integration := &IntegrationContext{ 204 B: b, 205 StopContext: stopCtx, 206 CancelContext: cancelCtx, 207 Op: op, 208 Run: r, 209 } 210 return b.runTasks(integration, integration.BeginOutput(outputTitle), stageID) 211 } 212 213 func (b *Cloud) costEstimate(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error { 214 if r.CostEstimate == nil { 215 return nil 216 } 217 218 msgPrefix := "Cost Estimation" 219 started := time.Now() 220 updated := started 221 for i := 0; ; i++ { 222 select { 223 case <-stopCtx.Done(): 224 return stopCtx.Err() 225 case <-cancelCtx.Done(): 226 return cancelCtx.Err() 227 case <-time.After(backoff(backoffMin, backoffMax, i)): 228 } 229 230 // Retrieve the cost estimate to get its current status. 231 ce, err := b.client.CostEstimates.Read(stopCtx, r.CostEstimate.ID) 232 if err != nil { 233 return generalError("Failed to retrieve cost estimate", err) 234 } 235 236 // If the run is canceled or errored, but the cost-estimate still has 237 // no result, there is nothing further to render. 238 if ce.Status != tfe.CostEstimateFinished { 239 if r.Status == tfe.RunCanceled || r.Status == tfe.RunErrored { 240 return nil 241 } 242 } 243 244 // checking if i == 0 so as to avoid printing this starting horizontal-rule 245 // every retry, and that it only prints it on the first (i=0) attempt. 246 if b.CLI != nil && i == 0 { 247 b.CLI.Output("\n------------------------------------------------------------------------\n") 248 } 249 250 switch ce.Status { 251 case tfe.CostEstimateFinished: 252 delta, err := strconv.ParseFloat(ce.DeltaMonthlyCost, 64) 253 if err != nil { 254 return generalError("Unexpected error", err) 255 } 256 257 sign := "+" 258 if delta < 0 { 259 sign = "-" 260 } 261 262 deltaRepr := strings.Replace(ce.DeltaMonthlyCost, "-", "", 1) 263 264 if b.CLI != nil { 265 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 266 b.CLI.Output(b.Colorize().Color(fmt.Sprintf("Resources: %d of %d estimated", ce.MatchedResourcesCount, ce.ResourcesCount))) 267 b.CLI.Output(b.Colorize().Color(fmt.Sprintf(" $%s/mo %s$%s", ce.ProposedMonthlyCost, sign, deltaRepr))) 268 269 if len(r.PolicyChecks) == 0 && r.HasChanges && op.Type == backend.OperationTypeApply { 270 b.CLI.Output("\n------------------------------------------------------------------------") 271 } 272 } 273 274 return nil 275 case tfe.CostEstimatePending, tfe.CostEstimateQueued: 276 // Check if 30 seconds have passed since the last update. 277 current := time.Now() 278 if b.CLI != nil && (i == 0 || current.Sub(updated).Seconds() > 30) { 279 updated = current 280 elapsed := "" 281 282 // Calculate and set the elapsed time. 283 if i > 0 { 284 elapsed = fmt.Sprintf( 285 " (%s elapsed)", current.Sub(started).Truncate(30*time.Second)) 286 } 287 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 288 b.CLI.Output(b.Colorize().Color("Waiting for cost estimate to complete..." + elapsed + "\n")) 289 } 290 continue 291 case tfe.CostEstimateSkippedDueToTargeting: 292 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 293 b.CLI.Output("Not available for this plan, because it was created with the -target option.") 294 b.CLI.Output("\n------------------------------------------------------------------------") 295 return nil 296 case tfe.CostEstimateErrored: 297 b.CLI.Output(msgPrefix + " errored.\n") 298 b.CLI.Output("\n------------------------------------------------------------------------") 299 return nil 300 case tfe.CostEstimateCanceled: 301 return fmt.Errorf(msgPrefix + " canceled.") 302 default: 303 return fmt.Errorf("Unknown or unexpected cost estimate state: %s", ce.Status) 304 } 305 } 306 } 307 308 func (b *Cloud) checkPolicy(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error { 309 if b.CLI != nil { 310 b.CLI.Output("\n------------------------------------------------------------------------\n") 311 } 312 for i, pc := range r.PolicyChecks { 313 // Read the policy check logs. This is a blocking call that will only 314 // return once the policy check is complete. 315 logs, err := b.client.PolicyChecks.Logs(stopCtx, pc.ID) 316 if err != nil { 317 return generalError("Failed to retrieve policy check logs", err) 318 } 319 reader := bufio.NewReaderSize(logs, 64*1024) 320 321 // Retrieve the policy check to get its current status. 322 pc, err := b.client.PolicyChecks.Read(stopCtx, pc.ID) 323 if err != nil { 324 return generalError("Failed to retrieve policy check", err) 325 } 326 327 // If the run is canceled or errored, but the policy check still has 328 // no result, there is nothing further to render. 329 if r.Status == tfe.RunCanceled || r.Status == tfe.RunErrored { 330 switch pc.Status { 331 case tfe.PolicyPending, tfe.PolicyQueued, tfe.PolicyUnreachable: 332 continue 333 } 334 } 335 336 var msgPrefix string 337 switch pc.Scope { 338 case tfe.PolicyScopeOrganization: 339 msgPrefix = "Organization Policy Check" 340 case tfe.PolicyScopeWorkspace: 341 msgPrefix = "Workspace Policy Check" 342 default: 343 msgPrefix = fmt.Sprintf("Unknown policy check (%s)", pc.Scope) 344 } 345 346 if b.CLI != nil { 347 b.CLI.Output(b.Colorize().Color("[bold]" + msgPrefix + ":\n")) 348 } 349 350 if b.CLI != nil { 351 for next := true; next; { 352 var l, line []byte 353 354 for isPrefix := true; isPrefix; { 355 l, isPrefix, err = reader.ReadLine() 356 if err != nil { 357 if err != io.EOF { 358 return generalError("Failed to read logs", err) 359 } 360 next = false 361 } 362 line = append(line, l...) 363 } 364 365 if next || len(line) > 0 { 366 b.CLI.Output(b.Colorize().Color(string(line))) 367 } 368 } 369 } 370 371 switch pc.Status { 372 case tfe.PolicyPasses: 373 if (r.HasChanges && op.Type == backend.OperationTypeApply || i < len(r.PolicyChecks)-1) && b.CLI != nil { 374 b.CLI.Output("\n------------------------------------------------------------------------") 375 } 376 continue 377 case tfe.PolicyErrored: 378 return fmt.Errorf(msgPrefix + " errored.") 379 case tfe.PolicyHardFailed: 380 return fmt.Errorf(msgPrefix + " hard failed.") 381 case tfe.PolicySoftFailed: 382 runUrl := fmt.Sprintf(runHeader, b.hostname, b.organization, op.Workspace, r.ID) 383 384 if op.Type == backend.OperationTypePlan || op.UIOut == nil || op.UIIn == nil || 385 !pc.Actions.IsOverridable || !pc.Permissions.CanOverride { 386 return fmt.Errorf(msgPrefix + " soft failed.\n" + runUrl) 387 } 388 389 if op.AutoApprove { 390 if _, err = b.client.PolicyChecks.Override(stopCtx, pc.ID); err != nil { 391 return generalError(fmt.Sprintf("Failed to override policy check.\n%s", runUrl), err) 392 } 393 } else if !b.input { 394 return errPolicyOverrideNeedsUIConfirmation 395 } else { 396 opts := &terraform.InputOpts{ 397 Id: "override", 398 Query: "\nDo you want to override the soft failed policy check?", 399 Description: "Only 'override' will be accepted to override.", 400 } 401 err = b.confirm(stopCtx, op, opts, r, "override") 402 if err != nil && err != errRunOverridden { 403 return fmt.Errorf( 404 fmt.Sprintf("Failed to override: %s\n%s\n", err.Error(), runUrl), 405 ) 406 } 407 408 if err != errRunOverridden { 409 if _, err = b.client.PolicyChecks.Override(stopCtx, pc.ID); err != nil { 410 return generalError(fmt.Sprintf("Failed to override policy check.\n%s", runUrl), err) 411 } 412 } else { 413 b.CLI.Output(fmt.Sprintf("The run needs to be manually overridden or discarded.\n%s\n", runUrl)) 414 } 415 } 416 417 if b.CLI != nil { 418 b.CLI.Output("------------------------------------------------------------------------") 419 } 420 default: 421 return fmt.Errorf("Unknown or unexpected policy state: %s", pc.Status) 422 } 423 } 424 425 return nil 426 } 427 428 func (b *Cloud) confirm(stopCtx context.Context, op *backend.Operation, opts *terraform.InputOpts, r *tfe.Run, keyword string) error { 429 doneCtx, cancel := context.WithCancel(stopCtx) 430 result := make(chan error, 2) 431 432 go func() { 433 // Make sure we cancel doneCtx before we return 434 // so the input command is also canceled. 435 defer cancel() 436 437 for { 438 select { 439 case <-doneCtx.Done(): 440 return 441 case <-stopCtx.Done(): 442 return 443 case <-time.After(runPollInterval): 444 // Retrieve the run again to get its current status. 445 r, err := b.client.Runs.Read(stopCtx, r.ID) 446 if err != nil { 447 result <- generalError("Failed to retrieve run", err) 448 return 449 } 450 451 switch keyword { 452 case "override": 453 if r.Status != tfe.RunPolicyOverride { 454 if r.Status == tfe.RunDiscarded { 455 err = errRunDiscarded 456 } else { 457 err = errRunOverridden 458 } 459 } 460 case "yes": 461 if !r.Actions.IsConfirmable { 462 if r.Status == tfe.RunDiscarded { 463 err = errRunDiscarded 464 } else { 465 err = errRunApproved 466 } 467 } 468 } 469 470 if err != nil { 471 if b.CLI != nil { 472 b.CLI.Output(b.Colorize().Color( 473 fmt.Sprintf("[reset][yellow]%s[reset]", err.Error()))) 474 } 475 476 if err == errRunDiscarded { 477 err = errApplyDiscarded 478 if op.PlanMode == plans.DestroyMode { 479 err = errDestroyDiscarded 480 } 481 } 482 483 result <- err 484 return 485 } 486 } 487 } 488 }() 489 490 result <- func() error { 491 v, err := op.UIIn.Input(doneCtx, opts) 492 if err != nil && err != context.Canceled && stopCtx.Err() != context.Canceled { 493 return fmt.Errorf("Error asking %s: %v", opts.Id, err) 494 } 495 496 // We return the error of our parent channel as we don't 497 // care about the error of the doneCtx which is only used 498 // within this function. So if the doneCtx was canceled 499 // because stopCtx was canceled, this will properly return 500 // a context.Canceled error and otherwise it returns nil. 501 if doneCtx.Err() == context.Canceled || stopCtx.Err() == context.Canceled { 502 return stopCtx.Err() 503 } 504 505 // Make sure we cancel the context here so the loop that 506 // checks for external changes to the run is ended before 507 // we start to make changes ourselves. 508 cancel() 509 510 if v != keyword { 511 // Retrieve the run again to get its current status. 512 r, err = b.client.Runs.Read(stopCtx, r.ID) 513 if err != nil { 514 return generalError("Failed to retrieve run", err) 515 } 516 517 // Make sure we discard the run if possible. 518 if r.Actions.IsDiscardable { 519 err = b.client.Runs.Discard(stopCtx, r.ID, tfe.RunDiscardOptions{}) 520 if err != nil { 521 if op.PlanMode == plans.DestroyMode { 522 return generalError("Failed to discard destroy", err) 523 } 524 return generalError("Failed to discard apply", err) 525 } 526 } 527 528 // Even if the run was discarded successfully, we still 529 // return an error as the apply command was canceled. 530 if op.PlanMode == plans.DestroyMode { 531 return errDestroyDiscarded 532 } 533 return errApplyDiscarded 534 } 535 536 return nil 537 }() 538 539 return <-result 540 }