github.com/rstandt/terraform@v0.12.32-0.20230710220336-b1063613405c/backend/remote/backend.go (about)

     1  package remote
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"log"
     7  	"net/http"
     8  	"net/url"
     9  	"os"
    10  	"sort"
    11  	"strings"
    12  	"sync"
    13  	"time"
    14  
    15  	tfe "github.com/hashicorp/go-tfe"
    16  	version "github.com/hashicorp/go-version"
    17  	svchost "github.com/hashicorp/terraform-svchost"
    18  	"github.com/hashicorp/terraform-svchost/disco"
    19  	"github.com/hashicorp/terraform/backend"
    20  	"github.com/hashicorp/terraform/configs/configschema"
    21  	"github.com/hashicorp/terraform/state"
    22  	"github.com/hashicorp/terraform/state/remote"
    23  	"github.com/hashicorp/terraform/terraform"
    24  	"github.com/hashicorp/terraform/tfdiags"
    25  	tfversion "github.com/hashicorp/terraform/version"
    26  	"github.com/mitchellh/cli"
    27  	"github.com/mitchellh/colorstring"
    28  	"github.com/zclconf/go-cty/cty"
    29  
    30  	backendLocal "github.com/hashicorp/terraform/backend/local"
    31  )
    32  
    33  const (
    34  	defaultHostname    = "app.terraform.io"
    35  	defaultParallelism = 10
    36  	stateServiceID     = "state.v2"
    37  	tfeServiceID       = "tfe.v2.1"
    38  )
    39  
    40  // Remote is an implementation of EnhancedBackend that performs all
    41  // operations in a remote backend.
    42  type Remote struct {
    43  	// CLI and Colorize control the CLI output. If CLI is nil then no CLI
    44  	// output will be done. If CLIColor is nil then no coloring will be done.
    45  	CLI      cli.Ui
    46  	CLIColor *colorstring.Colorize
    47  
    48  	// ShowDiagnostics prints diagnostic messages to the UI.
    49  	ShowDiagnostics func(vals ...interface{})
    50  
    51  	// ContextOpts are the base context options to set when initializing a
    52  	// new Terraform context. Many of these will be overridden or merged by
    53  	// Operation. See Operation for more details.
    54  	ContextOpts *terraform.ContextOpts
    55  
    56  	// client is the remote backend API client.
    57  	client *tfe.Client
    58  
    59  	// lastRetry is set to the last time a request was retried.
    60  	lastRetry time.Time
    61  
    62  	// hostname of the remote backend server.
    63  	hostname string
    64  
    65  	// organization is the organization that contains the target workspaces.
    66  	organization string
    67  
    68  	// workspace is used to map the default workspace to a remote workspace.
    69  	workspace string
    70  
    71  	// prefix is used to filter down a set of workspaces that use a single
    72  	// configuration.
    73  	prefix string
    74  
    75  	// services is used for service discovery
    76  	services *disco.Disco
    77  
    78  	// local, if non-nil, will be used for all enhanced behavior. This
    79  	// allows local behavior with the remote backend functioning as remote
    80  	// state storage backend.
    81  	local backend.Enhanced
    82  
    83  	// forceLocal, if true, will force the use of the local backend.
    84  	forceLocal bool
    85  
    86  	// opLock locks operations
    87  	opLock sync.Mutex
    88  }
    89  
    90  var _ backend.Backend = (*Remote)(nil)
    91  
    92  // New creates a new initialized remote backend.
    93  func New(services *disco.Disco) *Remote {
    94  	return &Remote{
    95  		services: services,
    96  	}
    97  }
    98  
    99  // ConfigSchema implements backend.Enhanced.
   100  func (b *Remote) ConfigSchema() *configschema.Block {
   101  	return &configschema.Block{
   102  		Attributes: map[string]*configschema.Attribute{
   103  			"hostname": {
   104  				Type:        cty.String,
   105  				Optional:    true,
   106  				Description: schemaDescriptions["hostname"],
   107  			},
   108  			"organization": {
   109  				Type:        cty.String,
   110  				Required:    true,
   111  				Description: schemaDescriptions["organization"],
   112  			},
   113  			"token": {
   114  				Type:        cty.String,
   115  				Optional:    true,
   116  				Description: schemaDescriptions["token"],
   117  			},
   118  		},
   119  
   120  		BlockTypes: map[string]*configschema.NestedBlock{
   121  			"workspaces": {
   122  				Block: configschema.Block{
   123  					Attributes: map[string]*configschema.Attribute{
   124  						"name": {
   125  							Type:        cty.String,
   126  							Optional:    true,
   127  							Description: schemaDescriptions["name"],
   128  						},
   129  						"prefix": {
   130  							Type:        cty.String,
   131  							Optional:    true,
   132  							Description: schemaDescriptions["prefix"],
   133  						},
   134  					},
   135  				},
   136  				Nesting: configschema.NestingSingle,
   137  			},
   138  		},
   139  	}
   140  }
   141  
   142  // PrepareConfig implements backend.Backend.
   143  func (b *Remote) PrepareConfig(obj cty.Value) (cty.Value, tfdiags.Diagnostics) {
   144  	var diags tfdiags.Diagnostics
   145  
   146  	if val := obj.GetAttr("organization"); val.IsNull() || val.AsString() == "" {
   147  		diags = diags.Append(tfdiags.AttributeValue(
   148  			tfdiags.Error,
   149  			"Invalid organization value",
   150  			`The "organization" attribute value must not be empty.`,
   151  			cty.Path{cty.GetAttrStep{Name: "organization"}},
   152  		))
   153  	}
   154  
   155  	var name, prefix string
   156  	if workspaces := obj.GetAttr("workspaces"); !workspaces.IsNull() {
   157  		if val := workspaces.GetAttr("name"); !val.IsNull() {
   158  			name = val.AsString()
   159  		}
   160  		if val := workspaces.GetAttr("prefix"); !val.IsNull() {
   161  			prefix = val.AsString()
   162  		}
   163  	}
   164  
   165  	// Make sure that we have either a workspace name or a prefix.
   166  	if name == "" && prefix == "" {
   167  		diags = diags.Append(tfdiags.AttributeValue(
   168  			tfdiags.Error,
   169  			"Invalid workspaces configuration",
   170  			`Either workspace "name" or "prefix" is required.`,
   171  			cty.Path{cty.GetAttrStep{Name: "workspaces"}},
   172  		))
   173  	}
   174  
   175  	// Make sure that only one of workspace name or a prefix is configured.
   176  	if name != "" && prefix != "" {
   177  		diags = diags.Append(tfdiags.AttributeValue(
   178  			tfdiags.Error,
   179  			"Invalid workspaces configuration",
   180  			`Only one of workspace "name" or "prefix" is allowed.`,
   181  			cty.Path{cty.GetAttrStep{Name: "workspaces"}},
   182  		))
   183  	}
   184  
   185  	return obj, diags
   186  }
   187  
   188  // Configure implements backend.Enhanced.
   189  func (b *Remote) Configure(obj cty.Value) tfdiags.Diagnostics {
   190  	var diags tfdiags.Diagnostics
   191  
   192  	// Get the hostname.
   193  	if val := obj.GetAttr("hostname"); !val.IsNull() && val.AsString() != "" {
   194  		b.hostname = val.AsString()
   195  	} else {
   196  		b.hostname = defaultHostname
   197  	}
   198  
   199  	// Get the organization.
   200  	if val := obj.GetAttr("organization"); !val.IsNull() {
   201  		b.organization = val.AsString()
   202  	}
   203  
   204  	// Get the workspaces configuration block and retrieve the
   205  	// default workspace name and prefix.
   206  	if workspaces := obj.GetAttr("workspaces"); !workspaces.IsNull() {
   207  		if val := workspaces.GetAttr("name"); !val.IsNull() {
   208  			b.workspace = val.AsString()
   209  		}
   210  		if val := workspaces.GetAttr("prefix"); !val.IsNull() {
   211  			b.prefix = val.AsString()
   212  		}
   213  	}
   214  
   215  	// Determine if we are forced to use the local backend.
   216  	b.forceLocal = os.Getenv("TF_FORCE_LOCAL_BACKEND") != ""
   217  
   218  	serviceID := tfeServiceID
   219  	if b.forceLocal {
   220  		serviceID = stateServiceID
   221  	}
   222  
   223  	// Discover the service URL for this host to confirm that it provides
   224  	// a remote backend API and to get the version constraints.
   225  	service, constraints, err := b.discover(serviceID)
   226  
   227  	// First check any contraints we might have received.
   228  	if constraints != nil {
   229  		diags = diags.Append(b.checkConstraints(constraints))
   230  		if diags.HasErrors() {
   231  			return diags
   232  		}
   233  	}
   234  
   235  	// When we don't have any constraints errors, also check for discovery
   236  	// errors before we continue.
   237  	if err != nil {
   238  		diags = diags.Append(tfdiags.AttributeValue(
   239  			tfdiags.Error,
   240  			strings.ToUpper(err.Error()[:1])+err.Error()[1:],
   241  			"", // no description is needed here, the error is clear
   242  			cty.Path{cty.GetAttrStep{Name: "hostname"}},
   243  		))
   244  		return diags
   245  	}
   246  
   247  	// Retrieve the token for this host as configured in the credentials
   248  	// section of the CLI Config File.
   249  	token, err := b.token()
   250  	if err != nil {
   251  		diags = diags.Append(tfdiags.AttributeValue(
   252  			tfdiags.Error,
   253  			strings.ToUpper(err.Error()[:1])+err.Error()[1:],
   254  			"", // no description is needed here, the error is clear
   255  			cty.Path{cty.GetAttrStep{Name: "hostname"}},
   256  		))
   257  		return diags
   258  	}
   259  
   260  	// Get the token from the config if no token was configured for this
   261  	// host in credentials section of the CLI Config File.
   262  	if token == "" {
   263  		if val := obj.GetAttr("token"); !val.IsNull() {
   264  			token = val.AsString()
   265  		}
   266  	}
   267  
   268  	// Return an error if we still don't have a token at this point.
   269  	if token == "" {
   270  		loginCommand := "terraform login"
   271  		if b.hostname != defaultHostname {
   272  			loginCommand = loginCommand + " " + b.hostname
   273  		}
   274  		diags = diags.Append(tfdiags.Sourceless(
   275  			tfdiags.Error,
   276  			"Required token could not be found",
   277  			fmt.Sprintf(
   278  				"Run the following command to generate a token for %s:\n    %s",
   279  				b.hostname,
   280  				loginCommand,
   281  			),
   282  		))
   283  		return diags
   284  	}
   285  
   286  	cfg := &tfe.Config{
   287  		Address:      service.String(),
   288  		BasePath:     service.Path,
   289  		Token:        token,
   290  		Headers:      make(http.Header),
   291  		RetryLogHook: b.retryLogHook,
   292  	}
   293  
   294  	// Set the version header to the current version.
   295  	cfg.Headers.Set(tfversion.Header, tfversion.Version)
   296  
   297  	// Create the remote backend API client.
   298  	b.client, err = tfe.NewClient(cfg)
   299  	if err != nil {
   300  		diags = diags.Append(tfdiags.Sourceless(
   301  			tfdiags.Error,
   302  			"Failed to create the Terraform Enterprise client",
   303  			fmt.Sprintf(
   304  				`The "remote" backend encountered an unexpected error while creating the `+
   305  					`Terraform Enterprise client: %s.`, err,
   306  			),
   307  		))
   308  		return diags
   309  	}
   310  
   311  	// Check if the organization exists by reading its entitlements.
   312  	entitlements, err := b.client.Organizations.Entitlements(context.Background(), b.organization)
   313  	if err != nil {
   314  		if err == tfe.ErrResourceNotFound {
   315  			err = fmt.Errorf("organization %s does not exist", b.organization)
   316  		}
   317  		diags = diags.Append(tfdiags.AttributeValue(
   318  			tfdiags.Error,
   319  			"Failed to read organization entitlements",
   320  			fmt.Sprintf(
   321  				`The "remote" backend encountered an unexpected error while reading the `+
   322  					`organization settings: %s.`, err,
   323  			),
   324  			cty.Path{cty.GetAttrStep{Name: "organization"}},
   325  		))
   326  		return diags
   327  	}
   328  
   329  	// Configure a local backend for when we need to run operations locally.
   330  	b.local = backendLocal.NewWithBackend(b)
   331  	b.forceLocal = b.forceLocal || !entitlements.Operations
   332  
   333  	// Enable retries for server errors as the backend is now fully configured.
   334  	b.client.RetryServerErrors(true)
   335  
   336  	return diags
   337  }
   338  
   339  // discover the remote backend API service URL and version constraints.
   340  func (b *Remote) discover(serviceID string) (*url.URL, *disco.Constraints, error) {
   341  	hostname, err := svchost.ForComparison(b.hostname)
   342  	if err != nil {
   343  		return nil, nil, err
   344  	}
   345  
   346  	host, err := b.services.Discover(hostname)
   347  	if err != nil {
   348  		return nil, nil, err
   349  	}
   350  
   351  	service, err := host.ServiceURL(serviceID)
   352  	// Return the error, unless its a disco.ErrVersionNotSupported error.
   353  	if _, ok := err.(*disco.ErrVersionNotSupported); !ok && err != nil {
   354  		return nil, nil, err
   355  	}
   356  
   357  	// We purposefully ignore the error and return the previous error, as
   358  	// checking for version constraints is considered optional.
   359  	constraints, _ := host.VersionConstraints(serviceID, "terraform")
   360  
   361  	return service, constraints, err
   362  }
   363  
   364  // checkConstraints checks service version constrains against our own
   365  // version and returns rich and informational diagnostics in case any
   366  // incompatibilities are detected.
   367  func (b *Remote) checkConstraints(c *disco.Constraints) tfdiags.Diagnostics {
   368  	var diags tfdiags.Diagnostics
   369  
   370  	if c == nil || c.Minimum == "" || c.Maximum == "" {
   371  		return diags
   372  	}
   373  
   374  	// Generate a parsable constraints string.
   375  	excluding := ""
   376  	if len(c.Excluding) > 0 {
   377  		excluding = fmt.Sprintf(", != %s", strings.Join(c.Excluding, ", != "))
   378  	}
   379  	constStr := fmt.Sprintf(">= %s%s, <= %s", c.Minimum, excluding, c.Maximum)
   380  
   381  	// Create the constraints to check against.
   382  	constraints, err := version.NewConstraint(constStr)
   383  	if err != nil {
   384  		return diags.Append(checkConstraintsWarning(err))
   385  	}
   386  
   387  	// Create the version to check.
   388  	v, err := version.NewVersion(tfversion.Version)
   389  	if err != nil {
   390  		return diags.Append(checkConstraintsWarning(err))
   391  	}
   392  
   393  	// Return if we satisfy all constraints.
   394  	if constraints.Check(v) {
   395  		return diags
   396  	}
   397  
   398  	// Find out what action (upgrade/downgrade) we should advice.
   399  	minimum, err := version.NewVersion(c.Minimum)
   400  	if err != nil {
   401  		return diags.Append(checkConstraintsWarning(err))
   402  	}
   403  
   404  	maximum, err := version.NewVersion(c.Maximum)
   405  	if err != nil {
   406  		return diags.Append(checkConstraintsWarning(err))
   407  	}
   408  
   409  	var excludes []*version.Version
   410  	for _, exclude := range c.Excluding {
   411  		v, err := version.NewVersion(exclude)
   412  		if err != nil {
   413  			return diags.Append(checkConstraintsWarning(err))
   414  		}
   415  		excludes = append(excludes, v)
   416  	}
   417  
   418  	// Sort all the excludes.
   419  	sort.Sort(version.Collection(excludes))
   420  
   421  	var action, toVersion string
   422  	switch {
   423  	case minimum.GreaterThan(v):
   424  		action = "upgrade"
   425  		toVersion = ">= " + minimum.String()
   426  	case maximum.LessThan(v):
   427  		action = "downgrade"
   428  		toVersion = "<= " + maximum.String()
   429  	case len(excludes) > 0:
   430  		// Get the latest excluded version.
   431  		action = "upgrade"
   432  		toVersion = "> " + excludes[len(excludes)-1].String()
   433  	}
   434  
   435  	switch {
   436  	case len(excludes) == 1:
   437  		excluding = fmt.Sprintf(", excluding version %s", excludes[0].String())
   438  	case len(excludes) > 1:
   439  		var vs []string
   440  		for _, v := range excludes {
   441  			vs = append(vs, v.String())
   442  		}
   443  		excluding = fmt.Sprintf(", excluding versions %s", strings.Join(vs, ", "))
   444  	default:
   445  		excluding = ""
   446  	}
   447  
   448  	summary := fmt.Sprintf("Incompatible Terraform version v%s", v.String())
   449  	details := fmt.Sprintf(
   450  		"The configured Terraform Enterprise backend is compatible with Terraform "+
   451  			"versions >= %s, <= %s%s.", c.Minimum, c.Maximum, excluding,
   452  	)
   453  
   454  	if action != "" && toVersion != "" {
   455  		summary = fmt.Sprintf("Please %s Terraform to %s", action, toVersion)
   456  		details += fmt.Sprintf(" Please %s to a supported version and try again.", action)
   457  	}
   458  
   459  	// Return the customized and informational error message.
   460  	return diags.Append(tfdiags.Sourceless(tfdiags.Error, summary, details))
   461  }
   462  
   463  // token returns the token for this host as configured in the credentials
   464  // section of the CLI Config File. If no token was configured, an empty
   465  // string will be returned instead.
   466  func (b *Remote) token() (string, error) {
   467  	hostname, err := svchost.ForComparison(b.hostname)
   468  	if err != nil {
   469  		return "", err
   470  	}
   471  	creds, err := b.services.CredentialsForHost(hostname)
   472  	if err != nil {
   473  		log.Printf("[WARN] Failed to get credentials for %s: %s (ignoring)", b.hostname, err)
   474  		return "", nil
   475  	}
   476  	if creds != nil {
   477  		return creds.Token(), nil
   478  	}
   479  	return "", nil
   480  }
   481  
   482  // retryLogHook is invoked each time a request is retried allowing the
   483  // backend to log any connection issues to prevent data loss.
   484  func (b *Remote) retryLogHook(attemptNum int, resp *http.Response) {
   485  	if b.CLI != nil {
   486  		// Ignore the first retry to make sure any delayed output will
   487  		// be written to the console before we start logging retries.
   488  		//
   489  		// The retry logic in the TFE client will retry both rate limited
   490  		// requests and server errors, but in the remote backend we only
   491  		// care about server errors so we ignore rate limit (429) errors.
   492  		if attemptNum == 0 || (resp != nil && resp.StatusCode == 429) {
   493  			// Reset the last retry time.
   494  			b.lastRetry = time.Now()
   495  			return
   496  		}
   497  
   498  		if attemptNum == 1 {
   499  			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(initialRetryError)))
   500  		} else {
   501  			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(
   502  				fmt.Sprintf(repeatedRetryError, time.Since(b.lastRetry).Round(time.Second)))))
   503  		}
   504  	}
   505  }
   506  
   507  // Workspaces implements backend.Enhanced.
   508  func (b *Remote) Workspaces() ([]string, error) {
   509  	if b.prefix == "" {
   510  		return nil, backend.ErrWorkspacesNotSupported
   511  	}
   512  	return b.workspaces()
   513  }
   514  
   515  // workspaces returns a filtered list of remote workspace names.
   516  func (b *Remote) workspaces() ([]string, error) {
   517  	options := tfe.WorkspaceListOptions{}
   518  	switch {
   519  	case b.workspace != "":
   520  		options.Search = tfe.String(b.workspace)
   521  	case b.prefix != "":
   522  		options.Search = tfe.String(b.prefix)
   523  	}
   524  
   525  	// Create a slice to contain all the names.
   526  	var names []string
   527  
   528  	for {
   529  		wl, err := b.client.Workspaces.List(context.Background(), b.organization, options)
   530  		if err != nil {
   531  			return nil, err
   532  		}
   533  
   534  		for _, w := range wl.Items {
   535  			if b.workspace != "" && w.Name == b.workspace {
   536  				names = append(names, backend.DefaultStateName)
   537  				continue
   538  			}
   539  			if b.prefix != "" && strings.HasPrefix(w.Name, b.prefix) {
   540  				names = append(names, strings.TrimPrefix(w.Name, b.prefix))
   541  			}
   542  		}
   543  
   544  		// Exit the loop when we've seen all pages.
   545  		if wl.CurrentPage >= wl.TotalPages {
   546  			break
   547  		}
   548  
   549  		// Update the page number to get the next page.
   550  		options.PageNumber = wl.NextPage
   551  	}
   552  
   553  	// Sort the result so we have consistent output.
   554  	sort.StringSlice(names).Sort()
   555  
   556  	return names, nil
   557  }
   558  
   559  // DeleteWorkspace implements backend.Enhanced.
   560  func (b *Remote) DeleteWorkspace(name string) error {
   561  	if b.workspace == "" && name == backend.DefaultStateName {
   562  		return backend.ErrDefaultWorkspaceNotSupported
   563  	}
   564  	if b.prefix == "" && name != backend.DefaultStateName {
   565  		return backend.ErrWorkspacesNotSupported
   566  	}
   567  
   568  	// Configure the remote workspace name.
   569  	switch {
   570  	case name == backend.DefaultStateName:
   571  		name = b.workspace
   572  	case b.prefix != "" && !strings.HasPrefix(name, b.prefix):
   573  		name = b.prefix + name
   574  	}
   575  
   576  	client := &remoteClient{
   577  		client:       b.client,
   578  		organization: b.organization,
   579  		workspace: &tfe.Workspace{
   580  			Name: name,
   581  		},
   582  	}
   583  
   584  	return client.Delete()
   585  }
   586  
   587  // StateMgr implements backend.Enhanced.
   588  func (b *Remote) StateMgr(name string) (state.State, error) {
   589  	if b.workspace == "" && name == backend.DefaultStateName {
   590  		return nil, backend.ErrDefaultWorkspaceNotSupported
   591  	}
   592  	if b.prefix == "" && name != backend.DefaultStateName {
   593  		return nil, backend.ErrWorkspacesNotSupported
   594  	}
   595  
   596  	// Configure the remote workspace name.
   597  	switch {
   598  	case name == backend.DefaultStateName:
   599  		name = b.workspace
   600  	case b.prefix != "" && !strings.HasPrefix(name, b.prefix):
   601  		name = b.prefix + name
   602  	}
   603  
   604  	workspace, err := b.client.Workspaces.Read(context.Background(), b.organization, name)
   605  	if err != nil && err != tfe.ErrResourceNotFound {
   606  		return nil, fmt.Errorf("Failed to retrieve workspace %s: %v", name, err)
   607  	}
   608  
   609  	if err == tfe.ErrResourceNotFound {
   610  		options := tfe.WorkspaceCreateOptions{
   611  			Name: tfe.String(name),
   612  		}
   613  
   614  		// We only set the Terraform Version for the new workspace if this is
   615  		// a release candidate or a final release.
   616  		if tfversion.Prerelease == "" || strings.HasPrefix(tfversion.Prerelease, "rc") {
   617  			options.TerraformVersion = tfe.String(tfversion.String())
   618  		}
   619  
   620  		workspace, err = b.client.Workspaces.Create(context.Background(), b.organization, options)
   621  		if err != nil {
   622  			return nil, fmt.Errorf("Error creating workspace %s: %v", name, err)
   623  		}
   624  	}
   625  
   626  	client := &remoteClient{
   627  		client:       b.client,
   628  		organization: b.organization,
   629  		workspace:    workspace,
   630  
   631  		// This is optionally set during Terraform Enterprise runs.
   632  		runID: os.Getenv("TFE_RUN_ID"),
   633  	}
   634  
   635  	return &remote.State{Client: client}, nil
   636  }
   637  
   638  func (b *Remote) StateMgrWithoutCheckVersion(name string) (state.State, error) {
   639  	return b.StateMgr(name)
   640  }
   641  
   642  // Operation implements backend.Enhanced.
   643  func (b *Remote) Operation(ctx context.Context, op *backend.Operation) (*backend.RunningOperation, error) {
   644  	// Get the remote workspace name.
   645  	name := op.Workspace
   646  	switch {
   647  	case op.Workspace == backend.DefaultStateName:
   648  		name = b.workspace
   649  	case b.prefix != "" && !strings.HasPrefix(op.Workspace, b.prefix):
   650  		name = b.prefix + op.Workspace
   651  	}
   652  
   653  	// Retrieve the workspace for this operation.
   654  	w, err := b.client.Workspaces.Read(ctx, b.organization, name)
   655  	if err != nil {
   656  		switch err {
   657  		case context.Canceled:
   658  			return nil, err
   659  		case tfe.ErrResourceNotFound:
   660  			return nil, fmt.Errorf(
   661  				"workspace %s not found\n\n"+
   662  					"The configured \"remote\" backend returns '404 Not Found' errors for resources\n"+
   663  					"that do not exist, as well as for resources that a user doesn't have access\n"+
   664  					"to. When the resource does exists, please check the rights for the used token.",
   665  				name,
   666  			)
   667  		default:
   668  			return nil, fmt.Errorf(
   669  				"The configured \"remote\" backend encountered an unexpected error:\n\n%s",
   670  				err,
   671  			)
   672  		}
   673  	}
   674  
   675  	// Check if we need to use the local backend to run the operation.
   676  	if b.forceLocal || !w.Operations {
   677  		return b.local.Operation(ctx, op)
   678  	}
   679  
   680  	// Set the remote workspace name.
   681  	op.Workspace = w.Name
   682  
   683  	// Determine the function to call for our operation
   684  	var f func(context.Context, context.Context, *backend.Operation, *tfe.Workspace) (*tfe.Run, error)
   685  	switch op.Type {
   686  	case backend.OperationTypePlan:
   687  		f = b.opPlan
   688  	case backend.OperationTypeApply:
   689  		f = b.opApply
   690  	default:
   691  		return nil, fmt.Errorf(
   692  			"\n\nThe \"remote\" backend does not support the %q operation.", op.Type)
   693  	}
   694  
   695  	// Lock
   696  	b.opLock.Lock()
   697  
   698  	// Build our running operation
   699  	// the runninCtx is only used to block until the operation returns.
   700  	runningCtx, done := context.WithCancel(context.Background())
   701  	runningOp := &backend.RunningOperation{
   702  		Context:   runningCtx,
   703  		PlanEmpty: true,
   704  	}
   705  
   706  	// stopCtx wraps the context passed in, and is used to signal a graceful Stop.
   707  	stopCtx, stop := context.WithCancel(ctx)
   708  	runningOp.Stop = stop
   709  
   710  	// cancelCtx is used to cancel the operation immediately, usually
   711  	// indicating that the process is exiting.
   712  	cancelCtx, cancel := context.WithCancel(context.Background())
   713  	runningOp.Cancel = cancel
   714  
   715  	// Do it.
   716  	go func() {
   717  		defer done()
   718  		defer stop()
   719  		defer cancel()
   720  
   721  		defer b.opLock.Unlock()
   722  
   723  		r, opErr := f(stopCtx, cancelCtx, op, w)
   724  		if opErr != nil && opErr != context.Canceled {
   725  			b.ReportResult(runningOp, opErr)
   726  			return
   727  		}
   728  
   729  		if r == nil && opErr == context.Canceled {
   730  			runningOp.Result = backend.OperationFailure
   731  			return
   732  		}
   733  
   734  		if r != nil {
   735  			// Retrieve the run to get its current status.
   736  			r, err := b.client.Runs.Read(cancelCtx, r.ID)
   737  			if err != nil {
   738  				b.ReportResult(runningOp, generalError("Failed to retrieve run", err))
   739  				return
   740  			}
   741  
   742  			// Record if there are any changes.
   743  			runningOp.PlanEmpty = !r.HasChanges
   744  
   745  			if opErr == context.Canceled {
   746  				if err := b.cancel(cancelCtx, op, r); err != nil {
   747  					b.ReportResult(runningOp, generalError("Failed to retrieve run", err))
   748  					return
   749  				}
   750  			}
   751  
   752  			if r.Status == tfe.RunCanceled || r.Status == tfe.RunErrored {
   753  				runningOp.Result = backend.OperationFailure
   754  			}
   755  		}
   756  	}()
   757  
   758  	// Return the running operation.
   759  	return runningOp, nil
   760  }
   761  
   762  func (b *Remote) cancel(cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error {
   763  	if r.Actions.IsCancelable {
   764  		// Only ask if the remote operation should be canceled
   765  		// if the auto approve flag is not set.
   766  		if !op.AutoApprove {
   767  			v, err := op.UIIn.Input(cancelCtx, &terraform.InputOpts{
   768  				Id:          "cancel",
   769  				Query:       "\nDo you want to cancel the remote operation?",
   770  				Description: "Only 'yes' will be accepted to cancel.",
   771  			})
   772  			if err != nil {
   773  				return generalError("Failed asking to cancel", err)
   774  			}
   775  			if v != "yes" {
   776  				if b.CLI != nil {
   777  					b.CLI.Output(b.Colorize().Color(strings.TrimSpace(operationNotCanceled)))
   778  				}
   779  				return nil
   780  			}
   781  		} else {
   782  			if b.CLI != nil {
   783  				// Insert a blank line to separate the ouputs.
   784  				b.CLI.Output("")
   785  			}
   786  		}
   787  
   788  		// Try to cancel the remote operation.
   789  		err := b.client.Runs.Cancel(cancelCtx, r.ID, tfe.RunCancelOptions{})
   790  		if err != nil {
   791  			return generalError("Failed to cancel run", err)
   792  		}
   793  		if b.CLI != nil {
   794  			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(operationCanceled)))
   795  		}
   796  	}
   797  
   798  	return nil
   799  }
   800  
   801  // ReportResult is a helper for the common chore of setting the status of
   802  // a running operation and showing any diagnostics produced during that
   803  // operation.
   804  //
   805  // If the given diagnostics contains errors then the operation's result
   806  // will be set to backend.OperationFailure. It will be set to
   807  // backend.OperationSuccess otherwise. It will then use b.ShowDiagnostics
   808  // to show the given diagnostics before returning.
   809  //
   810  // Callers should feel free to do each of these operations separately in
   811  // more complex cases where e.g. diagnostics are interleaved with other
   812  // output, but terminating immediately after reporting error diagnostics is
   813  // common and can be expressed concisely via this method.
   814  func (b *Remote) ReportResult(op *backend.RunningOperation, err error) {
   815  	var diags tfdiags.Diagnostics
   816  
   817  	diags = diags.Append(err)
   818  	if diags.HasErrors() {
   819  		op.Result = backend.OperationFailure
   820  	} else {
   821  		op.Result = backend.OperationSuccess
   822  	}
   823  
   824  	if b.ShowDiagnostics != nil {
   825  		b.ShowDiagnostics(diags)
   826  	} else {
   827  		// Shouldn't generally happen, but if it does then we'll at least
   828  		// make some noise in the logs to help us spot it.
   829  		if len(diags) != 0 {
   830  			log.Printf(
   831  				"[ERROR] Remote backend needs to report diagnostics but ShowDiagnostics is not set:\n%s",
   832  				diags.ErrWithWarnings(),
   833  			)
   834  		}
   835  	}
   836  }
   837  
   838  // Colorize returns the Colorize structure that can be used for colorizing
   839  // output. This is guaranteed to always return a non-nil value and so useful
   840  // as a helper to wrap any potentially colored strings.
   841  //
   842  // TODO SvH: Rename this back to Colorize as soon as we can pass -no-color.
   843  func (b *Remote) cliColorize() *colorstring.Colorize {
   844  	if b.CLIColor != nil {
   845  		return b.CLIColor
   846  	}
   847  
   848  	return &colorstring.Colorize{
   849  		Colors:  colorstring.DefaultColors,
   850  		Disable: true,
   851  	}
   852  }
   853  
   854  func generalError(msg string, err error) error {
   855  	var diags tfdiags.Diagnostics
   856  
   857  	if urlErr, ok := err.(*url.Error); ok {
   858  		err = urlErr.Err
   859  	}
   860  
   861  	switch err {
   862  	case context.Canceled:
   863  		return err
   864  	case tfe.ErrResourceNotFound:
   865  		diags = diags.Append(tfdiags.Sourceless(
   866  			tfdiags.Error,
   867  			fmt.Sprintf("%s: %v", msg, err),
   868  			`The configured "remote" backend returns '404 Not Found' errors for resources `+
   869  				`that do not exist, as well as for resources that a user doesn't have access `+
   870  				`to. If the resource does exists, please check the rights for the used token.`,
   871  		))
   872  		return diags.Err()
   873  	default:
   874  		diags = diags.Append(tfdiags.Sourceless(
   875  			tfdiags.Error,
   876  			fmt.Sprintf("%s: %v", msg, err),
   877  			`The configured "remote" backend encountered an unexpected error. Sometimes `+
   878  				`this is caused by network connection problems, in which case you could retry `+
   879  				`the command. If the issue persists please open a support ticket to get help `+
   880  				`resolving the problem.`,
   881  		))
   882  		return diags.Err()
   883  	}
   884  }
   885  
   886  func checkConstraintsWarning(err error) tfdiags.Diagnostic {
   887  	return tfdiags.Sourceless(
   888  		tfdiags.Warning,
   889  		fmt.Sprintf("Failed to check version constraints: %v", err),
   890  		"Checking version constraints is considered optional, but this is an"+
   891  			"unexpected error which should be reported.",
   892  	)
   893  }
   894  
   895  // The newline in this error is to make it look good in the CLI!
   896  const initialRetryError = `
   897  [reset][yellow]There was an error connecting to the remote backend. Please do not exit
   898  Terraform to prevent data loss! Trying to restore the connection...
   899  [reset]
   900  `
   901  
   902  const repeatedRetryError = `
   903  [reset][yellow]Still trying to restore the connection... (%s elapsed)[reset]
   904  `
   905  
   906  const operationCanceled = `
   907  [reset][red]The remote operation was successfully cancelled.[reset]
   908  `
   909  
   910  const operationNotCanceled = `
   911  [reset][red]The remote operation was not cancelled.[reset]
   912  `
   913  
   914  var schemaDescriptions = map[string]string{
   915  	"hostname":     "The remote backend hostname to connect to (defaults to app.terraform.io).",
   916  	"organization": "The name of the organization containing the targeted workspace(s).",
   917  	"token": "The token used to authenticate with the remote backend. If credentials for the\n" +
   918  		"host are configured in the CLI Config File, then those will be used instead.",
   919  	"name": "A workspace name used to map the default workspace to a named remote workspace.\n" +
   920  		"When configured only the default workspace can be used. This option conflicts\n" +
   921  		"with \"prefix\"",
   922  	"prefix": "A prefix used to filter workspaces using a single configuration. New workspaces\n" +
   923  		"will automatically be prefixed with this prefix. If omitted only the default\n" +
   924  		"workspace can be used. This option conflicts with \"name\"",
   925  }