github.com/kevinklinger/open_terraform@v1.3.6/noninternal/moduletest/provider.go (about) 1 package moduletest 2 3 import ( 4 "fmt" 5 "log" 6 "sync" 7 8 "github.com/zclconf/go-cty/cty" 9 "github.com/zclconf/go-cty/cty/gocty" 10 ctyjson "github.com/zclconf/go-cty/cty/json" 11 12 "github.com/hashicorp/hcl/v2/hclsyntax" 13 "github.com/kevinklinger/open_terraform/noninternal/configs/configschema" 14 "github.com/kevinklinger/open_terraform/noninternal/providers" 15 "github.com/kevinklinger/open_terraform/noninternal/repl" 16 "github.com/kevinklinger/open_terraform/noninternal/tfdiags" 17 ) 18 19 // Provider is an implementation of providers.Interface which we're 20 // using as a likely-only-temporary vehicle for research on an opinionated 21 // module testing workflow in Terraform. 22 // 23 // We expose this to configuration as "terraform.io/builtin/test", but 24 // any attempt to configure it will emit a warning that it is experimental 25 // and likely to change or be removed entirely in future Terraform CLI 26 // releases. 27 // 28 // The testing provider exists to gather up test results during a Terraform 29 // apply operation. Its "test_results" managed resource type doesn't have any 30 // user-visible effect on its own, but when used in conjunction with the 31 // "terraform test" experimental command it is the intermediary that holds 32 // the test results while the test runs, so that the test command can then 33 // report them. 34 // 35 // For correct behavior of the assertion tracking, the "terraform test" 36 // command must be sure to use the same instance of Provider for both the 37 // plan and apply steps, so that the assertions that were planned can still 38 // be tracked during apply. For other commands that don't explicitly support 39 // test assertions, the provider will still succeed but the assertions data 40 // may not be complete if the apply step fails. 41 type Provider struct { 42 // components tracks all of the "component" names that have been 43 // used in test assertions resources so far. Each resource must have 44 // a unique component name. 45 components map[string]*Component 46 47 // Must lock mutex in order to interact with the components map, because 48 // test assertions can potentially run concurrently. 49 mutex sync.RWMutex 50 } 51 52 var _ providers.Interface = (*Provider)(nil) 53 54 // NewProvider returns a new instance of the test provider. 55 func NewProvider() *Provider { 56 return &Provider{ 57 components: make(map[string]*Component), 58 } 59 } 60 61 // TestResults returns the current record of test results tracked inside the 62 // provider. 63 // 64 // The result is a direct reference to the internal state of the provider, 65 // so the caller mustn't modify it nor store it across calls to provider 66 // operations. 67 func (p *Provider) TestResults() map[string]*Component { 68 return p.components 69 } 70 71 // Reset returns the recieving provider back to its original state, with no 72 // recorded test results. 73 // 74 // It additionally detaches the instance from any data structure previously 75 // returned by method TestResults, freeing the caller from the constraints 76 // in its documentation about mutability and storage. 77 // 78 // For convenience in the presumed common case of resetting as part of 79 // capturing the results for storage, this method also returns the result 80 // that method TestResults would've returned if called prior to the call 81 // to Reset. 82 func (p *Provider) Reset() map[string]*Component { 83 p.mutex.Lock() 84 log.Print("[TRACE] moduletest.Provider: Reset") 85 ret := p.components 86 p.components = make(map[string]*Component) 87 p.mutex.Unlock() 88 return ret 89 } 90 91 // GetProviderSchema returns the complete schema for the provider. 92 func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse { 93 return providers.GetProviderSchemaResponse{ 94 ResourceTypes: map[string]providers.Schema{ 95 "test_assertions": testAssertionsSchema, 96 }, 97 } 98 } 99 100 // ValidateProviderConfig validates the provider configuration. 101 func (p *Provider) ValidateProviderConfig(req providers.ValidateProviderConfigRequest) providers.ValidateProviderConfigResponse { 102 // This provider has no configurable settings, so nothing to validate. 103 var res providers.ValidateProviderConfigResponse 104 return res 105 } 106 107 // ConfigureProvider configures and initializes the provider. 108 func (p *Provider) ConfigureProvider(providers.ConfigureProviderRequest) providers.ConfigureProviderResponse { 109 // This provider has no configurable settings, but we use the configure 110 // request as an opportunity to generate a warning about it being 111 // experimental. 112 var res providers.ConfigureProviderResponse 113 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 114 tfdiags.Warning, 115 "The test provider is experimental", 116 "The Terraform team is using the test provider (terraform.io/builtin/test) as part of ongoing research about declarative testing of Terraform modules.\n\nThe availability and behavior of this provider is expected to change significantly even in patch releases, so we recommend using this provider only in test configurations and constraining your test configurations to an exact Terraform version.", 117 nil, 118 )) 119 return res 120 } 121 122 // ValidateResourceConfig is used to validate configuration values for a resource. 123 func (p *Provider) ValidateResourceConfig(req providers.ValidateResourceConfigRequest) providers.ValidateResourceConfigResponse { 124 log.Print("[TRACE] moduletest.Provider: ValidateResourceConfig") 125 126 var res providers.ValidateResourceConfigResponse 127 if req.TypeName != "test_assertions" { // we only have one resource type 128 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 129 return res 130 } 131 132 config := req.Config 133 if !config.GetAttr("component").IsKnown() { 134 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 135 tfdiags.Error, 136 "Invalid component expression", 137 "The component name must be a static value given in the configuration, and may not be derived from a resource type attribute that will only be known during the apply step.", 138 cty.GetAttrPath("component"), 139 )) 140 } 141 if !hclsyntax.ValidIdentifier(config.GetAttr("component").AsString()) { 142 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 143 tfdiags.Error, 144 "Invalid component name", 145 "The component name must be a valid identifier, starting with a letter followed by zero or more letters, digits, and underscores.", 146 cty.GetAttrPath("component"), 147 )) 148 } 149 for it := config.GetAttr("equal").ElementIterator(); it.Next(); { 150 k, obj := it.Element() 151 if !hclsyntax.ValidIdentifier(k.AsString()) { 152 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 153 tfdiags.Error, 154 "Invalid assertion name", 155 "An assertion name must be a valid identifier, starting with a letter followed by zero or more letters, digits, and underscores.", 156 cty.GetAttrPath("equal").Index(k), 157 )) 158 } 159 if !obj.GetAttr("description").IsKnown() { 160 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 161 tfdiags.Error, 162 "Invalid description expression", 163 "The description must be a static value given in the configuration, and may not be derived from a resource type attribute that will only be known during the apply step.", 164 cty.GetAttrPath("equal").Index(k).GetAttr("description"), 165 )) 166 } 167 } 168 for it := config.GetAttr("check").ElementIterator(); it.Next(); { 169 k, obj := it.Element() 170 if !hclsyntax.ValidIdentifier(k.AsString()) { 171 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 172 tfdiags.Error, 173 "Invalid assertion name", 174 "An assertion name must be a valid identifier, starting with a letter followed by zero or more letters, digits, and underscores.", 175 cty.GetAttrPath("check").Index(k), 176 )) 177 } 178 if !obj.GetAttr("description").IsKnown() { 179 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 180 tfdiags.Error, 181 "Invalid description expression", 182 "The description must be a static value given in the configuration, and may not be derived from a resource type attribute that will only be known during the apply step.", 183 cty.GetAttrPath("equal").Index(k).GetAttr("description"), 184 )) 185 } 186 } 187 188 return res 189 } 190 191 // ReadResource refreshes a resource and returns its current state. 192 func (p *Provider) ReadResource(req providers.ReadResourceRequest) providers.ReadResourceResponse { 193 log.Print("[TRACE] moduletest.Provider: ReadResource") 194 195 var res providers.ReadResourceResponse 196 if req.TypeName != "test_assertions" { // we only have one resource type 197 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 198 return res 199 } 200 // Test assertions are not a real remote object, so there isn't actually 201 // anything to refresh here. 202 res.NewState = req.PriorState 203 return res 204 } 205 206 // UpgradeResourceState is called to allow the provider to adapt the raw value 207 // stored in the state in case the schema has changed since it was originally 208 // written. 209 func (p *Provider) UpgradeResourceState(req providers.UpgradeResourceStateRequest) providers.UpgradeResourceStateResponse { 210 log.Print("[TRACE] moduletest.Provider: UpgradeResourceState") 211 212 var res providers.UpgradeResourceStateResponse 213 if req.TypeName != "test_assertions" { // we only have one resource type 214 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 215 return res 216 } 217 218 // We assume here that there can never be a flatmap version of this 219 // resource type's data, because this provider was never included in a 220 // version of Terraform that used flatmap and this provider's schema 221 // contains attributes that are not flatmap-compatible anyway. 222 if len(req.RawStateFlatmap) != 0 { 223 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("can't upgrade a flatmap state for %q", req.TypeName)) 224 return res 225 } 226 if req.Version != 0 { 227 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("the state for this %s was created by a newer version of the provider", req.TypeName)) 228 return res 229 } 230 231 v, err := ctyjson.Unmarshal(req.RawStateJSON, testAssertionsSchema.Block.ImpliedType()) 232 if err != nil { 233 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("failed to decode state for %s: %s", req.TypeName, err)) 234 return res 235 } 236 237 res.UpgradedState = v 238 return res 239 } 240 241 // PlanResourceChange takes the current state and proposed state of a 242 // resource, and returns the planned final state. 243 func (p *Provider) PlanResourceChange(req providers.PlanResourceChangeRequest) (resp providers.PlanResourceChangeResponse) { 244 log.Print("[TRACE] moduletest.Provider: PlanResourceChange") 245 246 // this is a destroy plan, 247 if req.ProposedNewState.IsNull() { 248 resp.PlannedState = req.ProposedNewState 249 resp.PlannedPrivate = req.PriorPrivate 250 return resp 251 } 252 253 var res providers.PlanResourceChangeResponse 254 if req.TypeName != "test_assertions" { // we only have one resource type 255 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 256 return res 257 } 258 259 // During planning, our job is to gather up all of the planned test 260 // assertions marked as pending, which will then allow us to include 261 // all of them in test results even if there's a failure during apply 262 // that prevents the full completion of the graph walk. 263 // 264 // In a sense our plan phase is similar to the compile step for a 265 // test program written in another language. Planning itself can fail, 266 // which means we won't be able to form a complete test plan at all, 267 // but if we succeed in planning then subsequent problems can be treated 268 // as test failures at "runtime", while still keeping a full manifest 269 // of all of the tests that ought to have run if the apply had run to 270 // completion. 271 272 proposed := req.ProposedNewState 273 res.PlannedState = proposed 274 componentName := proposed.GetAttr("component").AsString() // proven known during validate 275 p.mutex.Lock() 276 defer p.mutex.Unlock() 277 // NOTE: Ideally we'd do something here to verify if two assertions 278 // resources in the configuration attempt to declare the same component, 279 // but we can't actually do that because Terraform calls PlanResourceChange 280 // during both plan and apply, and so the second one would always fail. 281 // Since this is just providing a temporary pseudo-syntax for writing tests 282 // anyway, we'll live with this for now and aim to solve it with a future 283 // iteration of testing that's better integrated into the Terraform 284 // language. 285 /* 286 if _, exists := p.components[componentName]; exists { 287 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 288 tfdiags.Error, 289 "Duplicate test component", 290 fmt.Sprintf("Another test_assertions resource already declared assertions for the component name %q.", componentName), 291 cty.GetAttrPath("component"), 292 )) 293 return res 294 } 295 */ 296 297 component := Component{ 298 Assertions: make(map[string]*Assertion), 299 } 300 301 for it := proposed.GetAttr("equal").ElementIterator(); it.Next(); { 302 k, obj := it.Element() 303 name := k.AsString() 304 if _, exists := component.Assertions[name]; exists { 305 // We can't actually get here in practice because so far we've 306 // only been pulling keys from one map, and so any duplicates 307 // would've been caught during config decoding, but this is here 308 // just to make these two blocks symmetrical to avoid mishaps in 309 // future refactoring/reorganization. 310 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 311 tfdiags.Error, 312 "Duplicate test assertion", 313 fmt.Sprintf("Another assertion block in this resource already declared an assertion named %q.", name), 314 cty.GetAttrPath("equal").Index(k), 315 )) 316 continue 317 } 318 319 var desc string 320 descVal := obj.GetAttr("description") 321 if descVal.IsNull() { 322 descVal = cty.StringVal("") 323 } 324 err := gocty.FromCtyValue(descVal, &desc) 325 if err != nil { 326 // We shouldn't get here because we've already validated everything 327 // that would make FromCtyValue fail above and during validate. 328 res.Diagnostics = res.Diagnostics.Append(err) 329 } 330 331 component.Assertions[name] = &Assertion{ 332 Outcome: Pending, 333 Description: desc, 334 } 335 } 336 337 for it := proposed.GetAttr("check").ElementIterator(); it.Next(); { 338 k, obj := it.Element() 339 name := k.AsString() 340 if _, exists := component.Assertions[name]; exists { 341 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 342 tfdiags.Error, 343 "Duplicate test assertion", 344 fmt.Sprintf("Another assertion block in this resource already declared an assertion named %q.", name), 345 cty.GetAttrPath("check").Index(k), 346 )) 347 continue 348 } 349 350 var desc string 351 descVal := obj.GetAttr("description") 352 if descVal.IsNull() { 353 descVal = cty.StringVal("") 354 } 355 err := gocty.FromCtyValue(descVal, &desc) 356 if err != nil { 357 // We shouldn't get here because we've already validated everything 358 // that would make FromCtyValue fail above and during validate. 359 res.Diagnostics = res.Diagnostics.Append(err) 360 } 361 362 component.Assertions[name] = &Assertion{ 363 Outcome: Pending, 364 Description: desc, 365 } 366 } 367 368 p.components[componentName] = &component 369 return res 370 } 371 372 // ApplyResourceChange takes the planned state for a resource, which may 373 // yet contain unknown computed values, and applies the changes returning 374 // the final state. 375 func (p *Provider) ApplyResourceChange(req providers.ApplyResourceChangeRequest) providers.ApplyResourceChangeResponse { 376 log.Print("[TRACE] moduletest.Provider: ApplyResourceChange") 377 378 var res providers.ApplyResourceChangeResponse 379 if req.TypeName != "test_assertions" { // we only have one resource type 380 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 381 return res 382 } 383 384 // During apply we actually check the assertions and record the results. 385 // An assertion failure isn't reflected as an error from the apply call 386 // because if possible we'd like to continue exercising other objects 387 // downstream in case that allows us to gather more information to report. 388 // (If something downstream returns an error then that could prevent us 389 // from completing other assertions, though.) 390 391 planned := req.PlannedState 392 res.NewState = planned 393 if res.NewState.IsNull() { 394 // If we're destroying then we'll just quickly return success to 395 // allow the test process to clean up after itself. 396 return res 397 } 398 componentName := planned.GetAttr("component").AsString() // proven known during validate 399 400 p.mutex.Lock() 401 defer p.mutex.Unlock() 402 component := p.components[componentName] 403 if component == nil { 404 // We might get here when using this provider outside of the 405 // "terraform test" command, where there won't be any mechanism to 406 // preserve the test provider instance between the plan and apply 407 // phases. In that case, we assume that nobody will come looking to 408 // collect the results anyway, and so we can just silently skip 409 // checking. 410 return res 411 } 412 413 for it := planned.GetAttr("equal").ElementIterator(); it.Next(); { 414 k, obj := it.Element() 415 name := k.AsString() 416 var desc string 417 if plan, exists := component.Assertions[name]; exists { 418 desc = plan.Description 419 } 420 assert := &Assertion{ 421 Outcome: Pending, 422 Description: desc, 423 } 424 425 gotVal := obj.GetAttr("got") 426 wantVal := obj.GetAttr("want") 427 switch { 428 case wantVal.RawEquals(gotVal): 429 assert.Outcome = Passed 430 gotStr := repl.FormatValue(gotVal, 4) 431 assert.Message = fmt.Sprintf("correct value\n got: %s\n", gotStr) 432 default: 433 assert.Outcome = Failed 434 gotStr := repl.FormatValue(gotVal, 4) 435 wantStr := repl.FormatValue(wantVal, 4) 436 assert.Message = fmt.Sprintf("wrong value\n got: %s\n want: %s\n", gotStr, wantStr) 437 } 438 439 component.Assertions[name] = assert 440 } 441 442 for it := planned.GetAttr("check").ElementIterator(); it.Next(); { 443 k, obj := it.Element() 444 name := k.AsString() 445 var desc string 446 if plan, exists := component.Assertions[name]; exists { 447 desc = plan.Description 448 } 449 assert := &Assertion{ 450 Outcome: Pending, 451 Description: desc, 452 } 453 454 condVal := obj.GetAttr("condition") 455 switch { 456 case condVal.IsNull(): 457 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 458 tfdiags.Error, 459 "Invalid check condition", 460 "The condition value must be a boolean expression, not null.", 461 cty.GetAttrPath("check").Index(k).GetAttr("condition"), 462 )) 463 continue 464 case condVal.True(): 465 assert.Outcome = Passed 466 assert.Message = "condition passed" 467 default: 468 assert.Outcome = Failed 469 // For "check" we can't really return a decent error message 470 // because we've lost all of the context by the time we get here. 471 // "equal" will be better for most tests for that reason, and also 472 // this is one reason why in the long run it would be better for 473 // test assertions to be a first-class language feature rather than 474 // just a provider-based concept. 475 assert.Message = "condition failed" 476 } 477 478 component.Assertions[name] = assert 479 } 480 481 return res 482 } 483 484 // ImportResourceState requests that the given resource be imported. 485 func (p *Provider) ImportResourceState(req providers.ImportResourceStateRequest) providers.ImportResourceStateResponse { 486 var res providers.ImportResourceStateResponse 487 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("%s is not importable", req.TypeName)) 488 return res 489 } 490 491 // ValidateDataResourceConfig is used to to validate the resource configuration values. 492 func (p *Provider) ValidateDataResourceConfig(req providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse { 493 // This provider has no data resouce types at all. 494 var res providers.ValidateDataResourceConfigResponse 495 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported data source %s", req.TypeName)) 496 return res 497 } 498 499 // ReadDataSource returns the data source's current state. 500 func (p *Provider) ReadDataSource(req providers.ReadDataSourceRequest) providers.ReadDataSourceResponse { 501 // This provider has no data resouce types at all. 502 var res providers.ReadDataSourceResponse 503 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported data source %s", req.TypeName)) 504 return res 505 } 506 507 // Stop is called when the provider should halt any in-flight actions. 508 func (p *Provider) Stop() error { 509 // This provider doesn't do anything that can be cancelled. 510 return nil 511 } 512 513 // Close is a noop for this provider, since it's run in-process. 514 func (p *Provider) Close() error { 515 return nil 516 } 517 518 var testAssertionsSchema = providers.Schema{ 519 Block: &configschema.Block{ 520 Attributes: map[string]*configschema.Attribute{ 521 "component": { 522 Type: cty.String, 523 Description: "The name of the component being tested. This is just for namespacing assertions in a result report.", 524 DescriptionKind: configschema.StringPlain, 525 Required: true, 526 }, 527 }, 528 BlockTypes: map[string]*configschema.NestedBlock{ 529 "equal": { 530 Nesting: configschema.NestingMap, 531 Block: configschema.Block{ 532 Attributes: map[string]*configschema.Attribute{ 533 "description": { 534 Type: cty.String, 535 Description: "An optional human-readable description of what's being tested by this assertion.", 536 DescriptionKind: configschema.StringPlain, 537 Required: true, 538 }, 539 "got": { 540 Type: cty.DynamicPseudoType, 541 Description: "The actual result value generated by the relevant component.", 542 DescriptionKind: configschema.StringPlain, 543 Required: true, 544 }, 545 "want": { 546 Type: cty.DynamicPseudoType, 547 Description: "The value that the component is expected to have generated.", 548 DescriptionKind: configschema.StringPlain, 549 Required: true, 550 }, 551 }, 552 }, 553 }, 554 "check": { 555 Nesting: configschema.NestingMap, 556 Block: configschema.Block{ 557 Attributes: map[string]*configschema.Attribute{ 558 "description": { 559 Type: cty.String, 560 Description: "An optional (but strongly recommended) human-readable description of what's being tested by this assertion.", 561 DescriptionKind: configschema.StringPlain, 562 Required: true, 563 }, 564 "condition": { 565 Type: cty.Bool, 566 Description: "An expression that must be true in order for the test to pass.", 567 DescriptionKind: configschema.StringPlain, 568 Required: true, 569 }, 570 }, 571 }, 572 }, 573 }, 574 }, 575 }