github.com/ctrox/terraform@v0.11.12-beta1/backend/remote/backend_common.go (about)

     1  package remote
     2  
     3  import (
     4  	"bufio"
     5  	"context"
     6  	"errors"
     7  	"fmt"
     8  	"math"
     9  	"time"
    10  
    11  	tfe "github.com/hashicorp/go-tfe"
    12  	"github.com/hashicorp/terraform/backend"
    13  	"github.com/hashicorp/terraform/terraform"
    14  )
    15  
    16  // backoff will perform exponential backoff based on the iteration and
    17  // limited by the provided min and max (in milliseconds) durations.
    18  func backoff(min, max float64, iter int) time.Duration {
    19  	backoff := math.Pow(2, float64(iter)/5) * min
    20  	if backoff > max {
    21  		backoff = max
    22  	}
    23  	return time.Duration(backoff) * time.Millisecond
    24  }
    25  
    26  func (b *Remote) waitForRun(stopCtx, cancelCtx context.Context, op *backend.Operation, opType string, r *tfe.Run, w *tfe.Workspace) (*tfe.Run, error) {
    27  	started := time.Now()
    28  	updated := started
    29  	for i := 0; ; i++ {
    30  		select {
    31  		case <-stopCtx.Done():
    32  			return r, stopCtx.Err()
    33  		case <-cancelCtx.Done():
    34  			return r, cancelCtx.Err()
    35  		case <-time.After(backoff(1000, 3000, i)):
    36  			// Timer up, show status
    37  		}
    38  
    39  		// Retrieve the run to get its current status.
    40  		r, err := b.client.Runs.Read(stopCtx, r.ID)
    41  		if err != nil {
    42  			return r, generalError("error retrieving run", err)
    43  		}
    44  
    45  		// Return if the run is no longer pending.
    46  		if r.Status != tfe.RunPending && r.Status != tfe.RunConfirmed {
    47  			if i == 0 && opType == "plan" && b.CLI != nil {
    48  				b.CLI.Output(b.Colorize().Color(fmt.Sprintf("Waiting for the %s to start...\n", opType)))
    49  			}
    50  			if i > 0 && b.CLI != nil {
    51  				// Insert a blank line to separate the ouputs.
    52  				b.CLI.Output("")
    53  			}
    54  			return r, nil
    55  		}
    56  
    57  		// Check if 30 seconds have passed since the last update.
    58  		current := time.Now()
    59  		if b.CLI != nil && (i == 0 || current.Sub(updated).Seconds() > 30) {
    60  			updated = current
    61  			position := 0
    62  			elapsed := ""
    63  
    64  			// Calculate and set the elapsed time.
    65  			if i > 0 {
    66  				elapsed = fmt.Sprintf(
    67  					" (%s elapsed)", current.Sub(started).Truncate(30*time.Second))
    68  			}
    69  
    70  			// Retrieve the workspace used to run this operation in.
    71  			w, err = b.client.Workspaces.Read(stopCtx, b.organization, w.Name)
    72  			if err != nil {
    73  				return nil, generalError("error retrieving workspace", err)
    74  			}
    75  
    76  			// If the workspace is locked the run will not be queued and we can
    77  			// update the status without making any expensive calls.
    78  			if w.Locked && w.CurrentRun != nil {
    79  				cr, err := b.client.Runs.Read(stopCtx, w.CurrentRun.ID)
    80  				if err != nil {
    81  					return r, generalError("error retrieving current run", err)
    82  				}
    83  				if cr.Status == tfe.RunPending {
    84  					b.CLI.Output(b.Colorize().Color(
    85  						"Waiting for the manually locked workspace to be unlocked..." + elapsed))
    86  					continue
    87  				}
    88  			}
    89  
    90  			// Skip checking the workspace queue when we are the current run.
    91  			if w.CurrentRun == nil || w.CurrentRun.ID != r.ID {
    92  				found := false
    93  				options := tfe.RunListOptions{}
    94  			runlist:
    95  				for {
    96  					rl, err := b.client.Runs.List(stopCtx, w.ID, options)
    97  					if err != nil {
    98  						return r, generalError("error retrieving run list", err)
    99  					}
   100  
   101  					// Loop through all runs to calculate the workspace queue position.
   102  					for _, item := range rl.Items {
   103  						if !found {
   104  							if r.ID == item.ID {
   105  								found = true
   106  							}
   107  							continue
   108  						}
   109  
   110  						// If the run is in a final state, ignore it and continue.
   111  						switch item.Status {
   112  						case tfe.RunApplied, tfe.RunCanceled, tfe.RunDiscarded, tfe.RunErrored:
   113  							continue
   114  						case tfe.RunPlanned:
   115  							if op.Type == backend.OperationTypePlan {
   116  								continue
   117  							}
   118  						}
   119  
   120  						// Increase the workspace queue position.
   121  						position++
   122  
   123  						// Stop searching when we reached the current run.
   124  						if w.CurrentRun != nil && w.CurrentRun.ID == item.ID {
   125  							break runlist
   126  						}
   127  					}
   128  
   129  					// Exit the loop when we've seen all pages.
   130  					if rl.CurrentPage >= rl.TotalPages {
   131  						break
   132  					}
   133  
   134  					// Update the page number to get the next page.
   135  					options.PageNumber = rl.NextPage
   136  				}
   137  
   138  				if position > 0 {
   139  					b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
   140  						"Waiting for %d run(s) to finish before being queued...%s",
   141  						position,
   142  						elapsed,
   143  					)))
   144  					continue
   145  				}
   146  			}
   147  
   148  			options := tfe.RunQueueOptions{}
   149  		search:
   150  			for {
   151  				rq, err := b.client.Organizations.RunQueue(stopCtx, b.organization, options)
   152  				if err != nil {
   153  					return r, generalError("error retrieving queue", err)
   154  				}
   155  
   156  				// Search through all queued items to find our run.
   157  				for _, item := range rq.Items {
   158  					if r.ID == item.ID {
   159  						position = item.PositionInQueue
   160  						break search
   161  					}
   162  				}
   163  
   164  				// Exit the loop when we've seen all pages.
   165  				if rq.CurrentPage >= rq.TotalPages {
   166  					break
   167  				}
   168  
   169  				// Update the page number to get the next page.
   170  				options.PageNumber = rq.NextPage
   171  			}
   172  
   173  			if position > 0 {
   174  				c, err := b.client.Organizations.Capacity(stopCtx, b.organization)
   175  				if err != nil {
   176  					return r, generalError("error retrieving capacity", err)
   177  				}
   178  				b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
   179  					"Waiting for %d queued run(s) to finish before starting...%s",
   180  					position-c.Running,
   181  					elapsed,
   182  				)))
   183  				continue
   184  			}
   185  
   186  			b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
   187  				"Waiting for the %s to start...%s", opType, elapsed)))
   188  		}
   189  	}
   190  }
   191  
   192  func (b *Remote) checkPolicy(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error {
   193  	if b.CLI != nil {
   194  		b.CLI.Output("\n------------------------------------------------------------------------\n")
   195  	}
   196  	for i, pc := range r.PolicyChecks {
   197  		logs, err := b.client.PolicyChecks.Logs(stopCtx, pc.ID)
   198  		if err != nil {
   199  			return generalError("error retrieving policy check logs", err)
   200  		}
   201  		scanner := bufio.NewScanner(logs)
   202  
   203  		// Retrieve the policy check to get its current status.
   204  		pc, err := b.client.PolicyChecks.Read(stopCtx, pc.ID)
   205  		if err != nil {
   206  			return generalError("error retrieving policy check", err)
   207  		}
   208  
   209  		var msgPrefix string
   210  		switch pc.Scope {
   211  		case tfe.PolicyScopeOrganization:
   212  			msgPrefix = "Organization policy check"
   213  		case tfe.PolicyScopeWorkspace:
   214  			msgPrefix = "Workspace policy check"
   215  		default:
   216  			msgPrefix = fmt.Sprintf("Unknown policy check (%s)", pc.Scope)
   217  		}
   218  
   219  		if b.CLI != nil {
   220  			b.CLI.Output(b.Colorize().Color(msgPrefix + ":\n"))
   221  		}
   222  
   223  		for scanner.Scan() {
   224  			if b.CLI != nil {
   225  				b.CLI.Output(b.Colorize().Color(scanner.Text()))
   226  			}
   227  		}
   228  		if err := scanner.Err(); err != nil {
   229  			return generalError("error reading logs", err)
   230  		}
   231  
   232  		switch pc.Status {
   233  		case tfe.PolicyPasses:
   234  			if (op.Type == backend.OperationTypeApply || i < len(r.PolicyChecks)-1) && b.CLI != nil {
   235  				b.CLI.Output("\n------------------------------------------------------------------------")
   236  			}
   237  			continue
   238  		case tfe.PolicyErrored:
   239  			return fmt.Errorf(msgPrefix + " errored.")
   240  		case tfe.PolicyHardFailed:
   241  			return fmt.Errorf(msgPrefix + " hard failed.")
   242  		case tfe.PolicySoftFailed:
   243  			if op.Type == backend.OperationTypePlan || op.UIOut == nil || op.UIIn == nil ||
   244  				op.AutoApprove || !pc.Actions.IsOverridable || !pc.Permissions.CanOverride {
   245  				return fmt.Errorf(msgPrefix + " soft failed.")
   246  			}
   247  		default:
   248  			return fmt.Errorf("Unknown or unexpected policy state: %s", pc.Status)
   249  		}
   250  
   251  		opts := &terraform.InputOpts{
   252  			Id:          "override",
   253  			Query:       "\nDo you want to override the soft failed policy check?",
   254  			Description: "Only 'override' will be accepted to override.",
   255  		}
   256  
   257  		if err = b.confirm(stopCtx, op, opts, r, "override"); err != nil {
   258  			return err
   259  		}
   260  
   261  		if _, err = b.client.PolicyChecks.Override(stopCtx, pc.ID); err != nil {
   262  			return generalError("error overriding policy check", err)
   263  		}
   264  
   265  		if b.CLI != nil {
   266  			b.CLI.Output("------------------------------------------------------------------------")
   267  		}
   268  	}
   269  
   270  	return nil
   271  }
   272  
   273  func (b *Remote) confirm(stopCtx context.Context, op *backend.Operation, opts *terraform.InputOpts, r *tfe.Run, keyword string) error {
   274  	v, err := op.UIIn.Input(opts)
   275  	if err != nil {
   276  		return fmt.Errorf("Error asking %s: %v", opts.Id, err)
   277  	}
   278  	if v != keyword {
   279  		// Retrieve the run again to get its current status.
   280  		r, err = b.client.Runs.Read(stopCtx, r.ID)
   281  		if err != nil {
   282  			return generalError("error retrieving run", err)
   283  		}
   284  
   285  		// Make sure we discard the run if possible.
   286  		if r.Actions.IsDiscardable {
   287  			err = b.client.Runs.Discard(stopCtx, r.ID, tfe.RunDiscardOptions{})
   288  			if err != nil {
   289  				if op.Destroy {
   290  					return generalError("error disarding destroy", err)
   291  				}
   292  				return generalError("error disarding apply", err)
   293  			}
   294  		}
   295  
   296  		// Even if the run was disarding successfully, we still
   297  		// return an error as the apply command was cancelled.
   298  		if op.Destroy {
   299  			return errors.New("Destroy discarded.")
   300  		}
   301  		return errors.New("Apply discarded.")
   302  	}
   303  
   304  	return nil
   305  }