github.com/myhau/pulumi/pkg/v3@v3.70.2-0.20221116134521-f2775972e587/testing/integration/program.go (about) 1 // Copyright 2016-2018, Pulumi Corporation. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 cryptorand "crypto/rand" 20 "encoding/hex" 21 "encoding/json" 22 "errors" 23 "flag" 24 "fmt" 25 "io" 26 "io/ioutil" 27 "os" 28 "os/exec" 29 "path/filepath" 30 "regexp" 31 "runtime" 32 "strconv" 33 "strings" 34 "testing" 35 "time" 36 37 multierror "github.com/hashicorp/go-multierror" 38 "gopkg.in/yaml.v3" 39 40 "github.com/pulumi/pulumi/pkg/v3/backend/filestate" 41 "github.com/pulumi/pulumi/pkg/v3/engine" 42 "github.com/pulumi/pulumi/pkg/v3/operations" 43 "github.com/pulumi/pulumi/pkg/v3/resource/stack" 44 "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" 45 "github.com/pulumi/pulumi/sdk/v3/go/common/resource" 46 "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" 47 pulumi_testing "github.com/pulumi/pulumi/sdk/v3/go/common/testing" 48 "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" 49 "github.com/pulumi/pulumi/sdk/v3/go/common/tools" 50 "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" 51 "github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil" 52 "github.com/pulumi/pulumi/sdk/v3/go/common/util/retry" 53 "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" 54 "github.com/stretchr/testify/assert" 55 user "github.com/tweekmonster/luser" 56 ) 57 58 const ( 59 PythonRuntime = "python" 60 NodeJSRuntime = "nodejs" 61 GoRuntime = "go" 62 DotNetRuntime = "dotnet" 63 YAMLRuntime = "yaml" 64 JavaRuntime = "java" 65 ) 66 67 const windowsOS = "windows" 68 69 // RuntimeValidationStackInfo contains details related to the stack that runtime validation logic may want to use. 70 type RuntimeValidationStackInfo struct { 71 StackName tokens.QName 72 Deployment *apitype.DeploymentV3 73 RootResource apitype.ResourceV3 74 Outputs map[string]interface{} 75 Events []apitype.EngineEvent 76 } 77 78 // EditDir is an optional edit to apply to the example, as subsequent deployments. 79 type EditDir struct { 80 Dir string 81 ExtraRuntimeValidation func(t *testing.T, stack RuntimeValidationStackInfo) 82 83 // Additive is true if Dir should be copied *on top* of the test directory. 84 // Otherwise Dir *replaces* the test directory, except we keep .pulumi/ and Pulumi.yaml and Pulumi.<stack>.yaml. 85 Additive bool 86 87 // ExpectFailure is true if we expect this test to fail. This is very coarse grained, and will essentially 88 // tolerate *any* failure in the program (IDEA: in the future, offer a way to narrow this down more). 89 ExpectFailure bool 90 91 // ExpectNoChanges is true if the edit is expected to not propose any changes. 92 ExpectNoChanges bool 93 94 // Stdout is the writer to use for all stdout messages. 95 Stdout io.Writer 96 // Stderr is the writer to use for all stderr messages. 97 Stderr io.Writer 98 // Verbose may be set to true to print messages as they occur, rather than buffering and showing upon failure. 99 Verbose bool 100 101 // Run program directory in query mode. 102 QueryMode bool 103 } 104 105 // TestCommandStats is a collection of data related to running a single command during a test. 106 type TestCommandStats struct { 107 // StartTime is the time at which the command was started 108 StartTime string `json:"startTime"` 109 // EndTime is the time at which the command exited 110 EndTime string `json:"endTime"` 111 // ElapsedSeconds is the time at which the command exited 112 ElapsedSeconds float64 `json:"elapsedSeconds"` 113 // StackName is the name of the stack 114 StackName string `json:"stackName"` 115 // TestId is the unique ID of the test run 116 TestID string `json:"testId"` 117 // StepName is the command line which was invoked 118 StepName string `json:"stepName"` 119 // CommandLine is the command line which was invoked 120 CommandLine string `json:"commandLine"` 121 // TestName is the name of the directory in which the test was executed 122 TestName string `json:"testName"` 123 // IsError is true if the command failed 124 IsError bool `json:"isError"` 125 // The Cloud that the test was run against, or empty for local deployments 126 CloudURL string `json:"cloudURL"` 127 } 128 129 // TestStatsReporter reports results and metadata from a test run. 130 type TestStatsReporter interface { 131 ReportCommand(stats TestCommandStats) 132 } 133 134 // ConfigValue is used to provide config values to a test program. 135 type ConfigValue struct { 136 // The config key to pass to `pulumi config`. 137 Key string 138 // The config value to pass to `pulumi config`. 139 Value string 140 // Secret indicates that the `--secret` flag should be specified when calling `pulumi config`. 141 Secret bool 142 // Path indicates that the `--path` flag should be specified when calling `pulumi config`. 143 Path bool 144 } 145 146 // ProgramTestOptions provides options for ProgramTest 147 type ProgramTestOptions struct { 148 // Dir is the program directory to test. 149 Dir string 150 // Array of NPM packages which must be `yarn linked` (e.g. {"pulumi", "@pulumi/aws"}) 151 Dependencies []string 152 // Map of package names to versions. The test will use the specified versions of these packages instead of what 153 // is declared in `package.json`. 154 Overrides map[string]string 155 // Map of config keys and values to set (e.g. {"aws:region": "us-east-2"}). 156 Config map[string]string 157 // Map of secure config keys and values to set (e.g. {"aws:region": "us-east-2"}). 158 Secrets map[string]string 159 // List of config keys and values to set in order, including Secret and Path options. 160 OrderedConfig []ConfigValue 161 // SecretsProvider is the optional custom secrets provider to use instead of the default. 162 SecretsProvider string 163 // EditDirs is an optional list of edits to apply to the example, as subsequent deployments. 164 EditDirs []EditDir 165 // ExtraRuntimeValidation is an optional callback for additional validation, called before applying edits. 166 ExtraRuntimeValidation func(t *testing.T, stack RuntimeValidationStackInfo) 167 // RelativeWorkDir is an optional path relative to `Dir` which should be used as working directory during tests. 168 RelativeWorkDir string 169 // AllowEmptyPreviewChanges is true if we expect that this test's no-op preview may propose changes (e.g. 170 // because the test is sensitive to the exact contents of its working directory and those contents change 171 // incidentally between the initial update and the empty update). 172 AllowEmptyPreviewChanges bool 173 // AllowEmptyUpdateChanges is true if we expect that this test's no-op update may perform changes (e.g. 174 // because the test is sensitive to the exact contents of its working directory and those contents change 175 // incidentally between the initial update and the empty update). 176 AllowEmptyUpdateChanges bool 177 // ExpectFailure is true if we expect this test to fail. This is very coarse grained, and will essentially 178 // tolerate *any* failure in the program (IDEA: in the future, offer a way to narrow this down more). 179 ExpectFailure bool 180 // ExpectRefreshChanges may be set to true if a test is expected to have changes yielded by an immediate refresh. 181 // This could occur, for example, is a resource's state is constantly changing outside of Pulumi (e.g., timestamps). 182 ExpectRefreshChanges bool 183 // RetryFailedSteps indicates that failed updates, refreshes, and destroys should be retried after a brief 184 // intermission. A maximum of 3 retries will be attempted. 185 RetryFailedSteps bool 186 // SkipRefresh indicates that the refresh step should be skipped entirely. 187 SkipRefresh bool 188 // SkipPreview indicates that the preview step should be skipped entirely. 189 SkipPreview bool 190 // SkipUpdate indicates that the update step should be skipped entirely. 191 SkipUpdate bool 192 // SkipExportImport skips testing that exporting and importing the stack works properly. 193 SkipExportImport bool 194 // SkipEmptyPreviewUpdate skips the no-change preview/update that is performed that validates 195 // that no changes happen. 196 SkipEmptyPreviewUpdate bool 197 // SkipStackRemoval indicates that the stack should not be removed. (And so the test's results could be inspected 198 // in the Pulumi Service after the test has completed.) 199 SkipStackRemoval bool 200 // Destroy on cleanup defers stack destruction until the test cleanup step, rather than after 201 // program test execution. This is useful for more realistic stack reference testing, allowing one 202 // project and stack to be stood up and a second to be run before the first is destroyed. 203 // 204 // Implies NoParallel because we expect that another caller to ProgramTest will set that 205 DestroyOnCleanup bool 206 // Quick implies SkipPreview, SkipExportImport and SkipEmptyPreviewUpdate 207 Quick bool 208 // RequireService indicates that the test must be run against the Pulumi Service 209 RequireService bool 210 // PreviewCommandlineFlags specifies flags to add to the `pulumi preview` command line (e.g. "--color=raw") 211 PreviewCommandlineFlags []string 212 // UpdateCommandlineFlags specifies flags to add to the `pulumi up` command line (e.g. "--color=raw") 213 UpdateCommandlineFlags []string 214 // QueryCommandlineFlags specifies flags to add to the `pulumi query` command line (e.g. "--color=raw") 215 QueryCommandlineFlags []string 216 // RunBuild indicates that the build step should be run (e.g. run `yarn build` for `nodejs` programs) 217 RunBuild bool 218 // RunUpdateTest will ensure that updates to the package version can test for spurious diffs 219 RunUpdateTest bool 220 // DecryptSecretsInOutput will ensure that stack output is passed `--show-secrets` parameter 221 // Used in conjunction with ExtraRuntimeValidation 222 DecryptSecretsInOutput bool 223 224 // CloudURL is an optional URL to override the default Pulumi Service API (https://api.pulumi-staging.io). The 225 // PULUMI_ACCESS_TOKEN environment variable must also be set to a valid access token for the target cloud. 226 CloudURL string 227 228 // StackName allows the stack name to be explicitly provided instead of computed from the 229 // environment during tests. 230 StackName string 231 232 // If non-empty, specifies the value of the `--tracing` flag to pass 233 // to Pulumi CLI, which may be a Zipkin endpoint or a 234 // `file:./local.trace` style url for AppDash tracing. 235 // 236 // Template `{command}` syntax will be expanded to the current 237 // command name such as `pulumi-stack-rm`. This is useful for 238 // file-based tracing since `ProgramTest` performs multiple 239 // CLI invocations that can inadvertently overwrite the trace 240 // file. 241 Tracing string 242 243 // If non-empty, specifies the value of the `--test.coverprofile` flag to pass to the Pulumi CLI. As with the 244 // Tracing field, the `{command}` template will expand to the current command name. 245 // 246 // If PULUMI_TEST_COVERAGE_PATH is set, this defaults to $PULUMI_TEST_COVERAGE_PATH/{command}-[random suffix].out 247 CoverProfile string 248 249 // NoParallel will opt the test out of being ran in parallel. 250 NoParallel bool 251 252 // PrePulumiCommand specifies a callback that will be executed before each `pulumi` invocation. This callback may 253 // optionally return another callback to be invoked after the `pulumi` invocation completes. 254 PrePulumiCommand func(verb string) (func(err error) error, error) 255 256 // ReportStats optionally specifies how to report results from the test for external collection. 257 ReportStats TestStatsReporter 258 259 // Stdout is the writer to use for all stdout messages. 260 Stdout io.Writer 261 // Stderr is the writer to use for all stderr messages. 262 Stderr io.Writer 263 // Verbose may be set to true to print messages as they occur, rather than buffering and showing upon failure. 264 Verbose bool 265 266 // DebugLogging may be set to anything >0 to enable excessively verbose debug logging from `pulumi`. This 267 // is equivalent to `--logflow --logtostderr -v=N`, where N is the value of DebugLogLevel. This may also 268 // be enabled by setting the environment variable PULUMI_TEST_DEBUG_LOG_LEVEL. 269 DebugLogLevel int 270 // DebugUpdates may be set to true to enable debug logging from `pulumi preview`, `pulumi up`, and 271 // `pulumi destroy`. This may also be enabled by setting the environment variable PULUMI_TEST_DEBUG_UPDATES. 272 DebugUpdates bool 273 274 // Bin is a location of a `pulumi` executable to be run. Taken from the $PATH if missing. 275 Bin string 276 // YarnBin is a location of a `yarn` executable to be run. Taken from the $PATH if missing. 277 YarnBin string 278 // GoBin is a location of a `go` executable to be run. Taken from the $PATH if missing. 279 GoBin string 280 // PythonBin is a location of a `python` executable to be run. Taken from the $PATH if missing. 281 PythonBin string 282 // PipenvBin is a location of a `pipenv` executable to run. Taken from the $PATH if missing. 283 PipenvBin string 284 // DotNetBin is a location of a `dotnet` executable to be run. Taken from the $PATH if missing. 285 DotNetBin string 286 287 // Additional environment variables to pass for each command we run. 288 Env []string 289 290 // Automatically create and use a virtual environment, rather than using the Pipenv tool. This is now the default 291 // behavior, so this option no longer has any affect. To go back to the old behavior use the `UsePipenv` option. 292 UseAutomaticVirtualEnv bool 293 // Use the Pipenv tool to manage the virtual environment. 294 UsePipenv bool 295 296 // If set, this hook is called after the `pulumi preview` command has completed. 297 PreviewCompletedHook func(dir string) error 298 299 // JSONOutput indicates that the `--json` flag should be passed to `up`, `preview`, 300 // `refresh` and `destroy` commands. 301 JSONOutput bool 302 303 // If set, this hook is called after `pulumi stack export` on the exported file. If `SkipExportImport` is set, this 304 // hook is ignored. 305 ExportStateValidator func(t *testing.T, stack []byte) 306 307 // If not nil, specifies the logic of preparing a project by 308 // ensuring dependencies. If left as nil, runs default 309 // preparation logic by dispatching on whether the project 310 // uses Node, Python, .NET or Go. 311 PrepareProject func(*engine.Projinfo) error 312 313 // Array of provider plugin dependencies which come from local packages. 314 LocalProviders []LocalDependency 315 } 316 317 type LocalDependency struct { 318 Package string 319 Path string 320 } 321 322 func (opts *ProgramTestOptions) GetDebugLogLevel() int { 323 if opts.DebugLogLevel > 0 { 324 return opts.DebugLogLevel 325 } 326 if du := os.Getenv("PULUMI_TEST_DEBUG_LOG_LEVEL"); du != "" { 327 if n, e := strconv.Atoi(du); e != nil { 328 panic(e) 329 } else if n > 0 { 330 return n 331 } 332 } 333 return 0 334 } 335 336 func (opts *ProgramTestOptions) GetDebugUpdates() bool { 337 return opts.DebugUpdates || os.Getenv("PULUMI_TEST_DEBUG_UPDATES") != "" 338 } 339 340 // GetStackName returns a stack name to use for this test. 341 func (opts *ProgramTestOptions) GetStackName() tokens.QName { 342 if opts.StackName == "" { 343 // Fetch the host and test dir names, cleaned so to contain just [a-zA-Z0-9-_] chars. 344 hostname, err := os.Hostname() 345 contract.AssertNoErrorf(err, "failure to fetch hostname for stack prefix") 346 var host string 347 for _, c := range hostname { 348 if len(host) >= 10 { 349 break 350 } 351 if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || 352 (c >= '0' && c <= '9') || c == '-' || c == '_' { 353 host += string(c) 354 } 355 } 356 357 var test string 358 for _, c := range filepath.Base(opts.Dir) { 359 if len(test) >= 10 { 360 break 361 } 362 if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || 363 (c >= '0' && c <= '9') || c == '-' || c == '_' { 364 test += string(c) 365 } 366 } 367 368 b := make([]byte, 4) 369 _, err = cryptorand.Read(b) 370 contract.AssertNoError(err) 371 372 opts.StackName = strings.ToLower("p-it-" + host + "-" + test + "-" + hex.EncodeToString(b)) 373 } 374 375 return tokens.QName(opts.StackName) 376 } 377 378 // GetStackNameWithOwner gets the name of the stack prepended with an owner, if PULUMI_TEST_OWNER is set. 379 // We use this in CI to create test stacks in an organization that all developers have access to, for debugging. 380 func (opts *ProgramTestOptions) GetStackNameWithOwner() tokens.QName { 381 owner := os.Getenv("PULUMI_TEST_OWNER") 382 383 if opts.RequireService && owner != "" { 384 return tokens.QName(fmt.Sprintf("%s/%s", owner, opts.GetStackName())) 385 } 386 387 return opts.GetStackName() 388 } 389 390 // With combines a source set of options with a set of overrides. 391 func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOptions { 392 if overrides.Dir != "" { 393 opts.Dir = overrides.Dir 394 } 395 if overrides.Dependencies != nil { 396 opts.Dependencies = overrides.Dependencies 397 } 398 if overrides.Overrides != nil { 399 opts.Overrides = overrides.Overrides 400 } 401 for k, v := range overrides.Config { 402 if opts.Config == nil { 403 opts.Config = make(map[string]string) 404 } 405 opts.Config[k] = v 406 } 407 for k, v := range overrides.Secrets { 408 if opts.Secrets == nil { 409 opts.Secrets = make(map[string]string) 410 } 411 opts.Secrets[k] = v 412 } 413 if overrides.OrderedConfig != nil { 414 for _, cv := range overrides.OrderedConfig { 415 opts.OrderedConfig = append(opts.OrderedConfig, cv) 416 } 417 } 418 if overrides.SecretsProvider != "" { 419 opts.SecretsProvider = overrides.SecretsProvider 420 } 421 if overrides.EditDirs != nil { 422 opts.EditDirs = overrides.EditDirs 423 } 424 if overrides.ExtraRuntimeValidation != nil { 425 opts.ExtraRuntimeValidation = overrides.ExtraRuntimeValidation 426 } 427 if overrides.RelativeWorkDir != "" { 428 opts.RelativeWorkDir = overrides.RelativeWorkDir 429 } 430 if overrides.AllowEmptyPreviewChanges { 431 opts.AllowEmptyPreviewChanges = overrides.AllowEmptyPreviewChanges 432 } 433 if overrides.AllowEmptyUpdateChanges { 434 opts.AllowEmptyUpdateChanges = overrides.AllowEmptyUpdateChanges 435 } 436 if overrides.ExpectFailure { 437 opts.ExpectFailure = overrides.ExpectFailure 438 } 439 if overrides.ExpectRefreshChanges { 440 opts.ExpectRefreshChanges = overrides.ExpectRefreshChanges 441 } 442 if overrides.RetryFailedSteps { 443 opts.RetryFailedSteps = overrides.RetryFailedSteps 444 } 445 if overrides.SkipRefresh { 446 opts.SkipRefresh = overrides.SkipRefresh 447 } 448 if overrides.SkipPreview { 449 opts.SkipPreview = overrides.SkipPreview 450 } 451 if overrides.SkipUpdate { 452 opts.SkipUpdate = overrides.SkipUpdate 453 } 454 if overrides.SkipExportImport { 455 opts.SkipExportImport = overrides.SkipExportImport 456 } 457 if overrides.SkipEmptyPreviewUpdate { 458 opts.SkipEmptyPreviewUpdate = overrides.SkipEmptyPreviewUpdate 459 } 460 if overrides.SkipStackRemoval { 461 opts.SkipStackRemoval = overrides.SkipStackRemoval 462 } 463 if overrides.DestroyOnCleanup { 464 opts.DestroyOnCleanup = overrides.DestroyOnCleanup 465 } 466 if overrides.Quick { 467 opts.Quick = overrides.Quick 468 } 469 if overrides.RequireService { 470 opts.RequireService = overrides.RequireService 471 } 472 if overrides.PreviewCommandlineFlags != nil { 473 opts.PreviewCommandlineFlags = append(opts.PreviewCommandlineFlags, overrides.PreviewCommandlineFlags...) 474 } 475 if overrides.UpdateCommandlineFlags != nil { 476 opts.UpdateCommandlineFlags = append(opts.UpdateCommandlineFlags, overrides.UpdateCommandlineFlags...) 477 } 478 if overrides.QueryCommandlineFlags != nil { 479 opts.QueryCommandlineFlags = append(opts.QueryCommandlineFlags, overrides.QueryCommandlineFlags...) 480 } 481 if overrides.RunBuild { 482 opts.RunBuild = overrides.RunBuild 483 } 484 if overrides.RunUpdateTest { 485 opts.RunUpdateTest = overrides.RunUpdateTest 486 } 487 if overrides.DecryptSecretsInOutput { 488 opts.DecryptSecretsInOutput = overrides.DecryptSecretsInOutput 489 } 490 if overrides.CloudURL != "" { 491 opts.CloudURL = overrides.CloudURL 492 } 493 if overrides.StackName != "" { 494 opts.StackName = overrides.StackName 495 } 496 if overrides.Tracing != "" { 497 opts.Tracing = overrides.Tracing 498 } 499 if overrides.CoverProfile != "" { 500 opts.CoverProfile = overrides.CoverProfile 501 } 502 if overrides.NoParallel { 503 opts.NoParallel = overrides.NoParallel 504 } 505 if overrides.PrePulumiCommand != nil { 506 opts.PrePulumiCommand = overrides.PrePulumiCommand 507 } 508 if overrides.ReportStats != nil { 509 opts.ReportStats = overrides.ReportStats 510 } 511 if overrides.Stdout != nil { 512 opts.Stdout = overrides.Stdout 513 } 514 if overrides.Stderr != nil { 515 opts.Stderr = overrides.Stderr 516 } 517 if overrides.Verbose { 518 opts.Verbose = overrides.Verbose 519 } 520 if overrides.DebugLogLevel != 0 { 521 opts.DebugLogLevel = overrides.DebugLogLevel 522 } 523 if overrides.DebugUpdates { 524 opts.DebugUpdates = overrides.DebugUpdates 525 } 526 if overrides.Bin != "" { 527 opts.Bin = overrides.Bin 528 } 529 if overrides.YarnBin != "" { 530 opts.YarnBin = overrides.YarnBin 531 } 532 if overrides.GoBin != "" { 533 opts.GoBin = overrides.GoBin 534 } 535 if overrides.PipenvBin != "" { 536 opts.PipenvBin = overrides.PipenvBin 537 } 538 if overrides.DotNetBin != "" { 539 opts.DotNetBin = overrides.DotNetBin 540 } 541 if overrides.Env != nil { 542 opts.Env = append(opts.Env, overrides.Env...) 543 } 544 if overrides.UseAutomaticVirtualEnv { 545 opts.UseAutomaticVirtualEnv = overrides.UseAutomaticVirtualEnv 546 } 547 if overrides.UsePipenv { 548 opts.UsePipenv = overrides.UsePipenv 549 } 550 if overrides.PreviewCompletedHook != nil { 551 opts.PreviewCompletedHook = overrides.PreviewCompletedHook 552 } 553 if overrides.JSONOutput { 554 opts.JSONOutput = overrides.JSONOutput 555 } 556 if overrides.ExportStateValidator != nil { 557 opts.ExportStateValidator = overrides.ExportStateValidator 558 } 559 if overrides.PrepareProject != nil { 560 opts.PrepareProject = overrides.PrepareProject 561 } 562 if overrides.LocalProviders != nil { 563 opts.LocalProviders = append(opts.LocalProviders, overrides.LocalProviders...) 564 } 565 return opts 566 } 567 568 type regexFlag struct { 569 re *regexp.Regexp 570 } 571 572 func (rf *regexFlag) String() string { 573 if rf.re == nil { 574 return "" 575 } 576 return rf.re.String() 577 } 578 579 func (rf *regexFlag) Set(v string) error { 580 r, err := regexp.Compile(v) 581 if err != nil { 582 return err 583 } 584 rf.re = r 585 return nil 586 } 587 588 var directoryMatcher regexFlag 589 var listDirs bool 590 var pipMutex *fsutil.FileMutex 591 592 func init() { 593 flag.Var(&directoryMatcher, "dirs", "optional list of regexes to use to select integration tests to run") 594 flag.BoolVar(&listDirs, "list-dirs", false, "list available integration tests without running them") 595 596 mutexPath := filepath.Join(os.TempDir(), "pip-mutex.lock") 597 pipMutex = fsutil.NewFileMutex(mutexPath) 598 } 599 600 // GetLogs retrieves the logs for a given stack in a particular region making the query provided. 601 // 602 // [provider] should be one of "aws" or "azure" 603 func GetLogs( 604 t *testing.T, 605 provider, region string, 606 stackInfo RuntimeValidationStackInfo, 607 query operations.LogQuery) *[]operations.LogEntry { 608 609 snap, err := stack.DeserializeDeploymentV3( 610 context.Background(), 611 *stackInfo.Deployment, 612 stack.DefaultSecretsProvider) 613 assert.NoError(t, err) 614 615 tree := operations.NewResourceTree(snap.Resources) 616 if !assert.NotNil(t, tree) { 617 return nil 618 } 619 620 cfg := map[config.Key]string{ 621 config.MustMakeKey(provider, "region"): region, 622 } 623 ops := tree.OperationsProvider(cfg) 624 625 // Validate logs from example 626 logs, err := ops.GetLogs(query) 627 if !assert.NoError(t, err) { 628 return nil 629 } 630 631 return logs 632 } 633 634 func prepareProgram(t *testing.T, opts *ProgramTestOptions) { 635 // If we're just listing tests, simply print this test's directory. 636 if listDirs { 637 fmt.Printf("%s\n", opts.Dir) 638 } 639 640 // If we have a matcher, ensure that this test matches its pattern. 641 if directoryMatcher.re != nil && !directoryMatcher.re.Match([]byte(opts.Dir)) { 642 t.Skip(fmt.Sprintf("Skipping: '%v' does not match '%v'", opts.Dir, directoryMatcher.re)) 643 } 644 645 // Disable stack backups for tests to avoid filling up ~/.pulumi/backups with unnecessary 646 // backups of test stacks. 647 opts.Env = append(opts.Env, fmt.Sprintf("%s=1", filestate.DisableCheckpointBackupsEnvVar)) 648 649 // We want tests to default into being ran in parallel, hence the odd double negative. 650 if !opts.NoParallel && !opts.DestroyOnCleanup { 651 t.Parallel() 652 } 653 654 if os.Getenv("PULUMI_TEST_USE_SERVICE") == "true" { 655 opts.RequireService = true 656 } 657 if opts.RequireService { 658 // This token is set in CI jobs, so this escape hatch is here to enable a smooth local dev 659 // experience, i.e.: running "make" and not seeing many failures due to a missing token. 660 if os.Getenv("PULUMI_ACCESS_TOKEN") == "" { 661 t.Skipf("Skipping: PULUMI_ACCESS_TOKEN is not set") 662 } 663 } else if opts.CloudURL == "" { 664 opts.CloudURL = MakeTempBackend(t) 665 } 666 667 // If the test panics, recover and log instead of letting the panic escape the test. Even though *this* test will 668 // have run deferred functions and cleaned up, if the panic reaches toplevel it will kill the process and prevent 669 // other tests running in parallel from cleaning up. 670 defer func() { 671 if failure := recover(); failure != nil { 672 t.Errorf("panic testing %v: %v", opts.Dir, failure) 673 } 674 }() 675 676 // Set up some default values for sending test reports and tracing data. We use environment varaiables to 677 // control these globally and set reasonable values for our own use in CI. 678 if opts.ReportStats == nil { 679 if v := os.Getenv("PULUMI_TEST_REPORT_CONFIG"); v != "" { 680 splits := strings.Split(v, ":") 681 if len(splits) != 3 { 682 t.Errorf("report config should be set to a value of the form: <aws-region>:<bucket-name>:<keyPrefix>") 683 } 684 685 opts.ReportStats = NewS3Reporter(splits[0], splits[1], splits[2]) 686 } 687 } 688 689 if opts.Tracing == "" { 690 opts.Tracing = os.Getenv("PULUMI_TEST_TRACE_ENDPOINT") 691 } 692 693 if opts.CoverProfile == "" { 694 if cov := os.Getenv("PULUMI_TEST_COVERAGE_PATH"); cov != "" { 695 var b [4]byte 696 if _, err := cryptorand.Read(b[:]); err != nil { 697 t.Errorf("could not read random bytes: %v", err) 698 } 699 opts.CoverProfile = filepath.Join(cov, "{command}-"+hex.EncodeToString(b[:])+".cov") 700 } 701 } 702 703 if opts.Quick { 704 opts.SkipPreview = true 705 opts.SkipExportImport = true 706 opts.SkipEmptyPreviewUpdate = true 707 } 708 } 709 710 // ProgramTest runs a lifecycle of Pulumi commands in a program working directory, using the `pulumi` and `yarn` 711 // binaries available on PATH. It essentially executes the following workflow: 712 // 713 // yarn install 714 // yarn link <each opts.Depencies> 715 // (+) yarn run build 716 // pulumi init 717 // (*) pulumi login 718 // pulumi stack init integrationtesting 719 // pulumi config set <each opts.Config> 720 // pulumi config set --secret <each opts.Secrets> 721 // pulumi preview 722 // pulumi up 723 // pulumi stack export --file stack.json 724 // pulumi stack import --file stack.json 725 // pulumi preview (expected to be empty) 726 // pulumi up (expected to be empty) 727 // pulumi destroy --yes 728 // pulumi stack rm --yes integrationtesting 729 // 730 // (*) Only if PULUMI_ACCESS_TOKEN is set. 731 // (+) Only if `opts.RunBuild` is true. 732 // 733 // All commands must return success return codes for the test to succeed, unless ExpectFailure is true. 734 func ProgramTest(t *testing.T, opts *ProgramTestOptions) { 735 prepareProgram(t, opts) 736 pt := newProgramTester(t, opts) 737 err := pt.TestLifeCycleInitAndDestroy() 738 assert.NoError(t, err) 739 } 740 741 // ProgramTestManualLifeCycle returns a ProgramTester than must be manually controlled in terms of its lifecycle 742 func ProgramTestManualLifeCycle(t *testing.T, opts *ProgramTestOptions) *ProgramTester { 743 prepareProgram(t, opts) 744 pt := newProgramTester(t, opts) 745 return pt 746 } 747 748 // ProgramTester contains state associated with running a single test pass. 749 type ProgramTester struct { 750 t *testing.T // the Go tester for this run. 751 opts *ProgramTestOptions // options that control this test run. 752 bin string // the `pulumi` binary we are using. 753 yarnBin string // the `yarn` binary we are using. 754 goBin string // the `go` binary we are using. 755 pythonBin string // the `python` binary we are using. 756 pipenvBin string // The `pipenv` binary we are using. 757 dotNetBin string // the `dotnet` binary we are using. 758 updateEventLog string // The path to the engine event log for `pulumi up` in this test. 759 maxStepTries int // The maximum number of times to retry a failed pulumi step. 760 tmpdir string // the temporary directory we use for our test environment 761 projdir string // the project directory we use for this run 762 TestFinished bool // whether or not the test if finished 763 } 764 765 func newProgramTester(t *testing.T, opts *ProgramTestOptions) *ProgramTester { 766 stackName := opts.GetStackName() 767 maxStepTries := 1 768 if opts.RetryFailedSteps { 769 maxStepTries = 3 770 } 771 return &ProgramTester{ 772 t: t, 773 opts: opts, 774 updateEventLog: filepath.Join(os.TempDir(), string(stackName)+"-events.json"), 775 maxStepTries: maxStepTries, 776 } 777 } 778 779 // MakeTempBackend creates a temporary backend directory which will clean up on test exit. 780 func MakeTempBackend(t *testing.T) string { 781 tempDir, err := os.MkdirTemp("", "") 782 if err != nil { 783 t.Fatalf("Failed to create temporary directory: %v", err) 784 } 785 t.Cleanup(func() { os.RemoveAll(tempDir) }) 786 return fmt.Sprintf("file://%s", filepath.ToSlash(tempDir)) 787 } 788 789 func (pt *ProgramTester) getBin() (string, error) { 790 return getCmdBin(&pt.bin, "pulumi", pt.opts.Bin) 791 } 792 793 func (pt *ProgramTester) getYarnBin() (string, error) { 794 return getCmdBin(&pt.yarnBin, "yarn", pt.opts.YarnBin) 795 } 796 797 func (pt *ProgramTester) getGoBin() (string, error) { 798 return getCmdBin(&pt.goBin, "go", pt.opts.GoBin) 799 } 800 801 // getPythonBin returns a path to the currently-installed `python` binary, or an error if it could not be found. 802 func (pt *ProgramTester) getPythonBin() (string, error) { 803 if pt.pythonBin == "" { 804 pt.pythonBin = pt.opts.PythonBin 805 if pt.opts.PythonBin == "" { 806 var err error 807 // Look for `python3` by default, but fallback to `python` if not found, except on Windows 808 // where we look for these in the reverse order because the default python.org Windows 809 // installation does not include a `python3` binary, and the existence of a `python3.exe` 810 // symlink to `python.exe` on some systems does not work correctly with the Python `venv` 811 // module. 812 pythonCmds := []string{"python3", "python"} 813 if runtime.GOOS == windowsOS { 814 pythonCmds = []string{"python", "python3"} 815 } 816 for _, bin := range pythonCmds { 817 pt.pythonBin, err = exec.LookPath(bin) 818 // Break on the first cmd we find on the path (if any). 819 if err == nil { 820 break 821 } 822 } 823 if err != nil { 824 return "", fmt.Errorf("Expected to find one of %q on $PATH: %w", pythonCmds, err) 825 } 826 } 827 } 828 return pt.pythonBin, nil 829 } 830 831 // getPipenvBin returns a path to the currently-installed Pipenv tool, or an error if the tool could not be found. 832 func (pt *ProgramTester) getPipenvBin() (string, error) { 833 return getCmdBin(&pt.pipenvBin, "pipenv", pt.opts.PipenvBin) 834 } 835 836 func (pt *ProgramTester) getDotNetBin() (string, error) { 837 return getCmdBin(&pt.dotNetBin, "dotnet", pt.opts.DotNetBin) 838 } 839 840 func (pt *ProgramTester) pulumiCmd(name string, args []string) ([]string, error) { 841 bin, err := pt.getBin() 842 if err != nil { 843 return nil, err 844 } 845 cmd := []string{bin} 846 if du := pt.opts.GetDebugLogLevel(); du > 0 { 847 cmd = append(cmd, "--logflow", "--logtostderr", "-v="+strconv.Itoa(du)) 848 } 849 cmd = append(cmd, args...) 850 if tracing := pt.opts.Tracing; tracing != "" { 851 cmd = append(cmd, "--tracing", strings.ReplaceAll(tracing, "{command}", name)) 852 } 853 if cov := pt.opts.CoverProfile; cov != "" { 854 cmd = append(cmd, "--test.coverprofile", strings.ReplaceAll(cov, "{command}", name)) 855 } 856 return cmd, nil 857 } 858 859 func (pt *ProgramTester) yarnCmd(args []string) ([]string, error) { 860 bin, err := pt.getYarnBin() 861 if err != nil { 862 return nil, err 863 } 864 result := []string{bin} 865 result = append(result, args...) 866 return withOptionalYarnFlags(result), nil 867 } 868 869 func (pt *ProgramTester) pythonCmd(args []string) ([]string, error) { 870 bin, err := pt.getPythonBin() 871 if err != nil { 872 return nil, err 873 } 874 875 cmd := []string{bin} 876 return append(cmd, args...), nil 877 } 878 879 func (pt *ProgramTester) pipenvCmd(args []string) ([]string, error) { 880 bin, err := pt.getPipenvBin() 881 if err != nil { 882 return nil, err 883 } 884 885 cmd := []string{bin} 886 return append(cmd, args...), nil 887 } 888 889 func (pt *ProgramTester) runCommand(name string, args []string, wd string) error { 890 return RunCommand(pt.t, name, args, wd, pt.opts) 891 } 892 893 func (pt *ProgramTester) runPulumiCommand(name string, args []string, wd string, expectFailure bool) error { 894 cmd, err := pt.pulumiCmd(name, args) 895 if err != nil { 896 return err 897 } 898 899 var postFn func(error) error 900 if pt.opts.PrePulumiCommand != nil { 901 postFn, err = pt.opts.PrePulumiCommand(args[0]) 902 if err != nil { 903 return err 904 } 905 } 906 907 isUpdate := args[0] == "preview" || args[0] == "up" || args[0] == "destroy" || args[0] == "refresh" 908 909 // If we're doing a preview or an update and this project is a Python project, we need to run 910 // the command in the context of the virtual environment that Pipenv created in order to pick up 911 // the correct version of Python. We also need to do this for destroy and refresh so that 912 // dynamic providers are run in the right virtual environment. 913 // This is only necessary when not using automatic virtual environment support. 914 if pt.opts.UsePipenv && isUpdate { 915 projinfo, err := pt.getProjinfo(wd) 916 if err != nil { 917 return nil 918 } 919 920 if projinfo.Proj.Runtime.Name() == "python" { 921 pipenvBin, err := pt.getPipenvBin() 922 if err != nil { 923 return err 924 } 925 926 // "pipenv run" activates the current virtual environment and runs the remainder of the arguments as if it 927 // were a command. 928 cmd = append([]string{pipenvBin, "run"}, cmd...) 929 } 930 } 931 932 _, _, err = retry.Until(context.Background(), retry.Acceptor{ 933 Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) { 934 runerr := pt.runCommand(name, cmd, wd) 935 if runerr == nil { 936 return true, nil, nil 937 } else if _, ok := runerr.(*exec.ExitError); ok && isUpdate && !expectFailure { 938 // the update command failed, let's try again, assuming we haven't failed a few times. 939 if try+1 >= pt.maxStepTries { 940 return false, nil, fmt.Errorf("%v did not succeed after %v tries", cmd, try+1) 941 } 942 943 pt.t.Logf("%v failed: %v; retrying...", cmd, runerr) 944 return false, nil, nil 945 } 946 947 // some other error, fail 948 return false, nil, runerr 949 }, 950 }) 951 if postFn != nil { 952 if postErr := postFn(err); postErr != nil { 953 return multierror.Append(err, postErr) 954 } 955 } 956 return err 957 } 958 959 func (pt *ProgramTester) runYarnCommand(name string, args []string, wd string) error { 960 // Yarn will time out if multiple processes are trying to install packages at the same time. 961 pulumi_testing.YarnInstallMutex.Lock() 962 defer pulumi_testing.YarnInstallMutex.Unlock() 963 pt.t.Log("acquired yarn install lock") 964 defer pt.t.Log("released yarn install lock") 965 966 cmd, err := pt.yarnCmd(args) 967 if err != nil { 968 return err 969 } 970 971 _, _, err = retry.Until(context.Background(), retry.Acceptor{ 972 Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) { 973 runerr := pt.runCommand(name, cmd, wd) 974 if runerr == nil { 975 return true, nil, nil 976 } else if _, ok := runerr.(*exec.ExitError); ok { 977 // yarn failed, let's try again, assuming we haven't failed a few times. 978 if try+1 >= 3 { 979 return false, nil, fmt.Errorf("%v did not complete after %v tries", cmd, try+1) 980 } 981 982 return false, nil, nil 983 } 984 985 // someother error, fail 986 return false, nil, runerr 987 }, 988 }) 989 return err 990 } 991 992 func (pt *ProgramTester) runPythonCommand(name string, args []string, wd string) error { 993 cmd, err := pt.pythonCmd(args) 994 if err != nil { 995 return err 996 } 997 998 return pt.runCommand(name, cmd, wd) 999 } 1000 1001 func (pt *ProgramTester) runVirtualEnvCommand(name string, args []string, wd string) error { 1002 // When installing with `pip install -e`, a PKG-INFO file is created. If two packages are being installed 1003 // this way simultaneously (which happens often, when running tests), both installations will be writing the 1004 // same file simultaneously. If one process catches "PKG-INFO" in a half-written state, the one process that 1005 // observed the torn write will fail to install the package. 1006 // 1007 // To avoid this problem, we use pipMutex to explicitly serialize installation operations. Doing so avoids 1008 // the problem of multiple processes stomping on the same files in the source tree. Note that pipMutex is a 1009 // file mutex, so this strategy works even if the go test runner chooses to split up text execution across 1010 // multiple processes. (Furthermore, each test gets an instance of ProgramTester and thus the mutex, so we'd 1011 // need to be sharing the mutex globally in each test process if we weren't using the file system to lock.) 1012 if name == "virtualenv-pip-install-package" { 1013 if err := pipMutex.Lock(); err != nil { 1014 panic(err) 1015 } 1016 1017 if pt.opts.Verbose { 1018 pt.t.Log("acquired pip install lock") 1019 defer pt.t.Log("released pip install lock") 1020 } 1021 defer func() { 1022 if err := pipMutex.Unlock(); err != nil { 1023 panic(err) 1024 } 1025 }() 1026 } 1027 1028 virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0]) 1029 if err != nil { 1030 return err 1031 } 1032 1033 cmd := append([]string{virtualenvBinPath}, args[1:]...) 1034 return pt.runCommand(name, cmd, wd) 1035 } 1036 1037 func (pt *ProgramTester) runPipenvCommand(name string, args []string, wd string) error { 1038 // Pipenv uses setuptools to install and uninstall packages. Setuptools has an installation mode called "develop" 1039 // that we use to install the package being tested, since it is 1) lightweight and 2) not doing so has its own set 1040 // of annoying problems. 1041 // 1042 // Setuptools develop does three things: 1043 // 1. It invokes the "egg_info" command in the target package, 1044 // 2. It creates a special `.egg-link` sentinel file in the current site-packages folder, pointing to the package 1045 // being installed's path on disk 1046 // 3. It updates easy-install.pth in site-packages so that pip understand that this package has been installed. 1047 // 1048 // Steps 2 and 3 operate entirely within the context of a virtualenv. The state that they mutate is fully contained 1049 // within the current virtualenv. However, step 1 operates in the context of the package's source tree. Egg info 1050 // is responsible for producing a minimal "egg" for a particular package, and its largest responsibility is creating 1051 // a PKG-INFO file for a package. PKG-INFO contains, among other things, the version of the package being installed. 1052 // 1053 // If two packages are being installed in "develop" mode simultaneously (which happens often, when running tests), 1054 // both installations will run "egg_info" on the source tree and both processes will be writing the same files 1055 // simultaneously. If one process catches "PKG-INFO" in a half-written state, the one process that observed the 1056 // torn write will fail to install the package (setuptools crashes). 1057 // 1058 // To avoid this problem, we use pipMutex to explicitly serialize installation operations. Doing so avoids the 1059 // problem of multiple processes stomping on the same files in the source tree. Note that pipMutex is a file 1060 // mutex, so this strategy works even if the go test runner chooses to split up text execution across multiple 1061 // processes. (Furthermore, each test gets an instance of ProgramTester and thus the mutex, so we'd need to be 1062 // sharing the mutex globally in each test process if we weren't using the file system to lock.) 1063 if name == "pipenv-install-package" { 1064 if err := pipMutex.Lock(); err != nil { 1065 panic(err) 1066 } 1067 1068 if pt.opts.Verbose { 1069 pt.t.Log("acquired pip install lock") 1070 defer pt.t.Log("released pip install lock") 1071 } 1072 defer func() { 1073 if err := pipMutex.Unlock(); err != nil { 1074 panic(err) 1075 } 1076 }() 1077 } 1078 1079 cmd, err := pt.pipenvCmd(args) 1080 if err != nil { 1081 return err 1082 } 1083 1084 return pt.runCommand(name, cmd, wd) 1085 } 1086 1087 // TestLifeCyclePrepare prepares a test by creating a temporary directory 1088 func (pt *ProgramTester) TestLifeCyclePrepare() error { 1089 tmpdir, projdir, err := pt.copyTestToTemporaryDirectory() 1090 pt.tmpdir = tmpdir 1091 pt.projdir = projdir 1092 return err 1093 } 1094 1095 // TestCleanUp cleans up the temporary directory that a test used 1096 func (pt *ProgramTester) TestCleanUp() { 1097 testFinished := pt.TestFinished 1098 if pt.tmpdir != "" { 1099 if !testFinished || pt.t.Failed() { 1100 // Test aborted or failed. Maybe copy to "failed tests" directory. 1101 failedTestsDir := os.Getenv("PULUMI_FAILED_TESTS_DIR") 1102 if failedTestsDir != "" { 1103 dest := filepath.Join(failedTestsDir, pt.t.Name()+uniqueSuffix()) 1104 contract.IgnoreError(fsutil.CopyFile(dest, pt.tmpdir, nil)) 1105 } 1106 } else { 1107 contract.IgnoreError(os.RemoveAll(pt.tmpdir)) 1108 } 1109 } else { 1110 // When tmpdir is empty, we ran "in tree", which means we wrote output 1111 // to the "command-output" folder in the projdir, and we should clean 1112 // it up if the test passed 1113 if testFinished && !pt.t.Failed() { 1114 contract.IgnoreError(os.RemoveAll(filepath.Join(pt.projdir, commandOutputFolderName))) 1115 } 1116 } 1117 } 1118 1119 // TestLifeCycleInitAndDestroy executes the test and cleans up 1120 func (pt *ProgramTester) TestLifeCycleInitAndDestroy() error { 1121 err := pt.TestLifeCyclePrepare() 1122 if err != nil { 1123 return fmt.Errorf("copying test to temp dir %s: %w", pt.tmpdir, err) 1124 } 1125 1126 pt.TestFinished = false 1127 if pt.opts.DestroyOnCleanup { 1128 pt.t.Cleanup(pt.TestCleanUp) 1129 } else { 1130 defer pt.TestCleanUp() 1131 } 1132 1133 err = pt.TestLifeCycleInitialize() 1134 if err != nil { 1135 return fmt.Errorf("initializing test project: %w", err) 1136 } 1137 1138 destroyStack := func() { 1139 destroyErr := pt.TestLifeCycleDestroy() 1140 assert.NoError(pt.t, destroyErr) 1141 } 1142 if pt.opts.DestroyOnCleanup { 1143 // Allow other tests to refer to this stack until the test is complete. 1144 pt.t.Cleanup(destroyStack) 1145 } else { 1146 // Ensure that before we exit, we attempt to destroy and remove the stack. 1147 defer destroyStack() 1148 } 1149 1150 if err = pt.TestPreviewUpdateAndEdits(); err != nil { 1151 return fmt.Errorf("running test preview, update, and edits: %w", err) 1152 } 1153 1154 if pt.opts.RunUpdateTest { 1155 err = upgradeProjectDeps(pt.projdir, pt) 1156 if err != nil { 1157 return fmt.Errorf("upgrading project dependencies: %w", err) 1158 } 1159 1160 if err = pt.TestPreviewUpdateAndEdits(); err != nil { 1161 return fmt.Errorf("running test preview, update, and edits (updateTest): %w", err) 1162 } 1163 } 1164 1165 pt.TestFinished = true 1166 return nil 1167 } 1168 1169 func upgradeProjectDeps(projectDir string, pt *ProgramTester) error { 1170 projInfo, err := pt.getProjinfo(projectDir) 1171 if err != nil { 1172 return fmt.Errorf("getting project info: %w", err) 1173 } 1174 1175 switch rt := projInfo.Proj.Runtime.Name(); rt { 1176 case NodeJSRuntime: 1177 if err = pt.yarnLinkPackageDeps(projectDir); err != nil { 1178 return err 1179 } 1180 case PythonRuntime: 1181 if err = pt.installPipPackageDeps(projectDir); err != nil { 1182 return err 1183 } 1184 default: 1185 return fmt.Errorf("unrecognized project runtime: %s", rt) 1186 } 1187 1188 return nil 1189 } 1190 1191 // TestLifeCycleInitialize initializes the project directory and stack along with any configuration 1192 func (pt *ProgramTester) TestLifeCycleInitialize() error { 1193 dir := pt.projdir 1194 stackName := pt.opts.GetStackName() 1195 1196 // If RelativeWorkDir is specified, apply that relative to the temp folder for use as working directory during tests. 1197 if pt.opts.RelativeWorkDir != "" { 1198 dir = filepath.Join(dir, pt.opts.RelativeWorkDir) 1199 } 1200 1201 // Set the default target Pulumi API if not overridden in options. 1202 if pt.opts.CloudURL == "" { 1203 pulumiAPI := os.Getenv("PULUMI_API") 1204 if pulumiAPI != "" { 1205 pt.opts.CloudURL = pulumiAPI 1206 } 1207 } 1208 1209 // Ensure all links are present, the stack is created, and all configs are applied. 1210 pt.t.Logf("Initializing project (dir %s; stack %s)", dir, stackName) 1211 1212 // Login as needed. 1213 stackInitName := string(pt.opts.GetStackNameWithOwner()) 1214 1215 if os.Getenv("PULUMI_ACCESS_TOKEN") == "" && pt.opts.CloudURL == "" { 1216 fmt.Printf("Using existing logged in user for tests. Set PULUMI_ACCESS_TOKEN and/or PULUMI_API to override.\n") 1217 } else { 1218 // Set PulumiCredentialsPathEnvVar to our CWD, so we use credentials specific to just this 1219 // test. 1220 pt.opts.Env = append(pt.opts.Env, fmt.Sprintf("%s=%s", workspace.PulumiCredentialsPathEnvVar, dir)) 1221 1222 loginArgs := []string{"login"} 1223 loginArgs = addFlagIfNonNil(loginArgs, "--cloud-url", pt.opts.CloudURL) 1224 1225 // If this is a local OR cloud login, then don't attach the owner to the stack-name. 1226 if pt.opts.CloudURL != "" { 1227 stackInitName = string(pt.opts.GetStackName()) 1228 } 1229 1230 if err := pt.runPulumiCommand("pulumi-login", loginArgs, dir, false); err != nil { 1231 return err 1232 } 1233 } 1234 1235 // Stack init 1236 stackInitArgs := []string{"stack", "init", stackInitName} 1237 if pt.opts.SecretsProvider != "" { 1238 stackInitArgs = append(stackInitArgs, "--secrets-provider", pt.opts.SecretsProvider) 1239 } 1240 if err := pt.runPulumiCommand("pulumi-stack-init", stackInitArgs, dir, false); err != nil { 1241 return err 1242 } 1243 1244 if len(pt.opts.Config)+len(pt.opts.Secrets) > 0 { 1245 setAllArgs := []string{"config", "set-all"} 1246 1247 for key, value := range pt.opts.Config { 1248 setAllArgs = append(setAllArgs, "--plaintext", fmt.Sprintf("%s=%s", key, value)) 1249 } 1250 for key, value := range pt.opts.Secrets { 1251 setAllArgs = append(setAllArgs, "--secret", fmt.Sprintf("%s=%s", key, value)) 1252 } 1253 1254 if err := pt.runPulumiCommand("pulumi-config", setAllArgs, dir, false); err != nil { 1255 return err 1256 } 1257 } 1258 1259 for _, cv := range pt.opts.OrderedConfig { 1260 configArgs := []string{"config", "set", cv.Key, cv.Value} 1261 if cv.Secret { 1262 configArgs = append(configArgs, "--secret") 1263 } 1264 if cv.Path { 1265 configArgs = append(configArgs, "--path") 1266 } 1267 if err := pt.runPulumiCommand("pulumi-config", configArgs, dir, false); err != nil { 1268 return err 1269 } 1270 } 1271 1272 return nil 1273 } 1274 1275 // TestLifeCycleDestroy destroys a stack and removes it 1276 func (pt *ProgramTester) TestLifeCycleDestroy() error { 1277 if pt.projdir != "" { 1278 // Destroy and remove the stack. 1279 pt.t.Log("Destroying stack") 1280 destroy := []string{"destroy", "--non-interactive", "--yes", "--skip-preview"} 1281 if pt.opts.GetDebugUpdates() { 1282 destroy = append(destroy, "-d") 1283 } 1284 if pt.opts.JSONOutput { 1285 destroy = append(destroy, "--json") 1286 } 1287 if err := pt.runPulumiCommand("pulumi-destroy", destroy, pt.projdir, false); err != nil { 1288 return err 1289 } 1290 1291 if pt.t.Failed() { 1292 pt.t.Logf("Test failed, retaining stack '%s'", pt.opts.GetStackNameWithOwner()) 1293 return nil 1294 } 1295 1296 if !pt.opts.SkipStackRemoval { 1297 return pt.runPulumiCommand("pulumi-stack-rm", []string{"stack", "rm", "--yes"}, pt.projdir, false) 1298 } 1299 } 1300 return nil 1301 } 1302 1303 // TestPreviewUpdateAndEdits runs the preview, update, and any relevant edits 1304 func (pt *ProgramTester) TestPreviewUpdateAndEdits() error { 1305 dir := pt.projdir 1306 // Now preview and update the real changes. 1307 pt.t.Log("Performing primary preview and update") 1308 initErr := pt.PreviewAndUpdate(dir, "initial", pt.opts.ExpectFailure, false, false) 1309 1310 // If the initial preview/update failed, just exit without trying the rest (but make sure to destroy). 1311 if initErr != nil { 1312 return fmt.Errorf("initial failure: %w", initErr) 1313 } 1314 1315 // Perform an empty preview and update; nothing is expected to happen here. 1316 if !pt.opts.SkipExportImport { 1317 pt.t.Log("Roundtripping checkpoint via stack export and stack import") 1318 1319 if err := pt.exportImport(dir); err != nil { 1320 return fmt.Errorf("empty preview + update: %w", err) 1321 } 1322 } 1323 1324 if !pt.opts.SkipEmptyPreviewUpdate { 1325 msg := "" 1326 if !pt.opts.AllowEmptyUpdateChanges { 1327 msg = "(no changes expected)" 1328 } 1329 pt.t.Logf("Performing empty preview and update%s", msg) 1330 if err := pt.PreviewAndUpdate(dir, "empty", pt.opts.ExpectFailure, 1331 !pt.opts.AllowEmptyPreviewChanges, !pt.opts.AllowEmptyUpdateChanges); err != nil { 1332 1333 return fmt.Errorf("empty preview: %w", err) 1334 } 1335 } 1336 1337 // Run additional validation provided by the test options, passing in the checkpoint info. 1338 if err := pt.performExtraRuntimeValidation(pt.opts.ExtraRuntimeValidation, dir); err != nil { 1339 return err 1340 } 1341 1342 if !pt.opts.SkipRefresh { 1343 // Perform a refresh and ensure it doesn't yield changes. 1344 refresh := []string{"refresh", "--non-interactive", "--yes", "--skip-preview"} 1345 if pt.opts.GetDebugUpdates() { 1346 refresh = append(refresh, "-d") 1347 } 1348 if pt.opts.JSONOutput { 1349 refresh = append(refresh, "--json") 1350 } 1351 if !pt.opts.ExpectRefreshChanges { 1352 refresh = append(refresh, "--expect-no-changes") 1353 } 1354 if err := pt.runPulumiCommand("pulumi-refresh", refresh, dir, false); err != nil { 1355 return err 1356 } 1357 } 1358 1359 // If there are any edits, apply them and run a preview and update for each one. 1360 return pt.testEdits(dir) 1361 } 1362 1363 func (pt *ProgramTester) exportImport(dir string) error { 1364 exportCmd := []string{"stack", "export", "--file", "stack.json"} 1365 importCmd := []string{"stack", "import", "--file", "stack.json"} 1366 1367 defer func() { 1368 contract.IgnoreError(os.Remove(filepath.Join(dir, "stack.json"))) 1369 }() 1370 1371 if err := pt.runPulumiCommand("pulumi-stack-export", exportCmd, dir, false); err != nil { 1372 return err 1373 } 1374 1375 if f := pt.opts.ExportStateValidator; f != nil { 1376 bytes, err := ioutil.ReadFile(filepath.Join(dir, "stack.json")) 1377 if err != nil { 1378 pt.t.Logf("Failed to read stack.json: %s", err.Error()) 1379 return err 1380 } 1381 pt.t.Logf("Calling ExportStateValidator") 1382 f(pt.t, bytes) 1383 } 1384 1385 return pt.runPulumiCommand("pulumi-stack-import", importCmd, dir, false) 1386 } 1387 1388 // PreviewAndUpdate runs pulumi preview followed by pulumi up 1389 func (pt *ProgramTester) PreviewAndUpdate(dir string, name string, shouldFail, expectNopPreview, 1390 expectNopUpdate bool) error { 1391 1392 preview := []string{"preview", "--non-interactive", "--diff"} 1393 update := []string{"up", "--non-interactive", "--yes", "--skip-preview", "--event-log", pt.updateEventLog} 1394 if pt.opts.GetDebugUpdates() { 1395 preview = append(preview, "-d") 1396 update = append(update, "-d") 1397 } 1398 if pt.opts.JSONOutput { 1399 preview = append(preview, "--json") 1400 update = append(update, "--json") 1401 } 1402 if expectNopPreview { 1403 preview = append(preview, "--expect-no-changes") 1404 } 1405 if expectNopUpdate { 1406 update = append(update, "--expect-no-changes") 1407 } 1408 if pt.opts.PreviewCommandlineFlags != nil { 1409 preview = append(preview, pt.opts.PreviewCommandlineFlags...) 1410 } 1411 if pt.opts.UpdateCommandlineFlags != nil { 1412 update = append(update, pt.opts.UpdateCommandlineFlags...) 1413 } 1414 1415 // If not in quick mode, run an explicit preview. 1416 if !pt.opts.SkipPreview { 1417 if err := pt.runPulumiCommand("pulumi-preview-"+name, preview, dir, shouldFail); err != nil { 1418 if shouldFail { 1419 pt.t.Log("Permitting failure (ExpectFailure=true for this preview)") 1420 return nil 1421 } 1422 return err 1423 } 1424 if pt.opts.PreviewCompletedHook != nil { 1425 if err := pt.opts.PreviewCompletedHook(dir); err != nil { 1426 return err 1427 } 1428 } 1429 } 1430 1431 // Now run an update. 1432 if !pt.opts.SkipUpdate { 1433 if err := pt.runPulumiCommand("pulumi-update-"+name, update, dir, shouldFail); err != nil { 1434 if shouldFail { 1435 pt.t.Log("Permitting failure (ExpectFailure=true for this update)") 1436 return nil 1437 } 1438 return err 1439 } 1440 } 1441 1442 // If we expected a failure, but none occurred, return an error. 1443 if shouldFail { 1444 return errors.New("expected this step to fail, but it succeeded") 1445 } 1446 1447 return nil 1448 } 1449 1450 func (pt *ProgramTester) query(dir string, name string, shouldFail bool) error { 1451 1452 query := []string{"query", "--non-interactive"} 1453 if pt.opts.GetDebugUpdates() { 1454 query = append(query, "-d") 1455 } 1456 if pt.opts.QueryCommandlineFlags != nil { 1457 query = append(query, pt.opts.QueryCommandlineFlags...) 1458 } 1459 1460 // Now run a query. 1461 if err := pt.runPulumiCommand("pulumi-query-"+name, query, dir, shouldFail); err != nil { 1462 if shouldFail { 1463 pt.t.Log("Permitting failure (ExpectFailure=true for this update)") 1464 return nil 1465 } 1466 return err 1467 } 1468 1469 // If we expected a failure, but none occurred, return an error. 1470 if shouldFail { 1471 return errors.New("expected this step to fail, but it succeeded") 1472 } 1473 1474 return nil 1475 } 1476 1477 func (pt *ProgramTester) testEdits(dir string) error { 1478 for i, edit := range pt.opts.EditDirs { 1479 var err error 1480 if err = pt.testEdit(dir, i, edit); err != nil { 1481 return err 1482 } 1483 } 1484 return nil 1485 } 1486 1487 func (pt *ProgramTester) testEdit(dir string, i int, edit EditDir) error { 1488 pt.t.Logf("Applying edit '%v' and rerunning preview and update", edit.Dir) 1489 1490 if edit.Additive { 1491 // Just copy new files into dir 1492 if err := fsutil.CopyFile(dir, edit.Dir, nil); err != nil { 1493 return fmt.Errorf("Couldn't copy %v into %v: %w", edit.Dir, dir, err) 1494 } 1495 } else { 1496 // Create a new temporary directory 1497 newDir, err := ioutil.TempDir("", pt.opts.StackName+"-") 1498 if err != nil { 1499 return fmt.Errorf("Couldn't create new temporary directory: %w", err) 1500 } 1501 1502 // Delete whichever copy of the test is unused when we return 1503 dirToDelete := newDir 1504 defer func() { 1505 contract.IgnoreError(os.RemoveAll(dirToDelete)) 1506 }() 1507 1508 // Copy everything except Pulumi.yaml, Pulumi.<stack-name>.yaml, and .pulumi from source into new directory 1509 exclusions := make(map[string]bool) 1510 projectYaml := workspace.ProjectFile + ".yaml" 1511 configYaml := workspace.ProjectFile + "." + pt.opts.StackName + ".yaml" 1512 exclusions[workspace.BookkeepingDir] = true 1513 exclusions[projectYaml] = true 1514 exclusions[configYaml] = true 1515 1516 if err := fsutil.CopyFile(newDir, edit.Dir, exclusions); err != nil { 1517 return fmt.Errorf("Couldn't copy %v into %v: %w", edit.Dir, newDir, err) 1518 } 1519 1520 // Copy Pulumi.yaml, Pulumi.<stack-name>.yaml, and .pulumi from old directory to new directory 1521 oldProjectYaml := filepath.Join(dir, projectYaml) 1522 newProjectYaml := filepath.Join(newDir, projectYaml) 1523 1524 oldConfigYaml := filepath.Join(dir, configYaml) 1525 newConfigYaml := filepath.Join(newDir, configYaml) 1526 1527 oldProjectDir := filepath.Join(dir, workspace.BookkeepingDir) 1528 newProjectDir := filepath.Join(newDir, workspace.BookkeepingDir) 1529 1530 if err := fsutil.CopyFile(newProjectYaml, oldProjectYaml, nil); err != nil { 1531 return fmt.Errorf("Couldn't copy Pulumi.yaml: %w", err) 1532 } 1533 if err := fsutil.CopyFile(newConfigYaml, oldConfigYaml, nil); err != nil { 1534 return fmt.Errorf("Couldn't copy Pulumi.%s.yaml: %w", pt.opts.StackName, err) 1535 } 1536 if err := fsutil.CopyFile(newProjectDir, oldProjectDir, nil); err != nil { 1537 return fmt.Errorf("Couldn't copy .pulumi: %w", err) 1538 } 1539 1540 // Finally, replace our current temp directory with the new one. 1541 dirOld := dir + ".old" 1542 if err := os.Rename(dir, dirOld); err != nil { 1543 return fmt.Errorf("Couldn't rename %v to %v: %w", dir, dirOld, err) 1544 } 1545 1546 // There's a brief window here where the old temp dir name could be taken from us. 1547 1548 if err := os.Rename(newDir, dir); err != nil { 1549 return fmt.Errorf("Couldn't rename %v to %v: %w", newDir, dir, err) 1550 } 1551 1552 // Keep dir, delete oldDir 1553 dirToDelete = dirOld 1554 } 1555 1556 err := pt.prepareProjectDir(dir) 1557 if err != nil { 1558 return fmt.Errorf("Couldn't prepare project in %v: %w", dir, err) 1559 } 1560 1561 oldStdOut := pt.opts.Stdout 1562 oldStderr := pt.opts.Stderr 1563 oldVerbose := pt.opts.Verbose 1564 if edit.Stdout != nil { 1565 pt.opts.Stdout = edit.Stdout 1566 } 1567 if edit.Stderr != nil { 1568 pt.opts.Stderr = edit.Stderr 1569 } 1570 if edit.Verbose { 1571 pt.opts.Verbose = true 1572 } 1573 1574 defer func() { 1575 pt.opts.Stdout = oldStdOut 1576 pt.opts.Stderr = oldStderr 1577 pt.opts.Verbose = oldVerbose 1578 }() 1579 1580 if !edit.QueryMode { 1581 if err = pt.PreviewAndUpdate(dir, fmt.Sprintf("edit-%d", i), 1582 edit.ExpectFailure, edit.ExpectNoChanges, edit.ExpectNoChanges); err != nil { 1583 return err 1584 } 1585 } else { 1586 if err = pt.query(dir, fmt.Sprintf("query-%d", i), edit.ExpectFailure); err != nil { 1587 return err 1588 } 1589 } 1590 return pt.performExtraRuntimeValidation(edit.ExtraRuntimeValidation, dir) 1591 } 1592 1593 func (pt *ProgramTester) performExtraRuntimeValidation( 1594 extraRuntimeValidation func(t *testing.T, stack RuntimeValidationStackInfo), dir string) error { 1595 1596 if extraRuntimeValidation == nil { 1597 return nil 1598 } 1599 1600 stackName := pt.opts.GetStackName() 1601 1602 // Create a temporary file name for the stack export 1603 tempDir, err := ioutil.TempDir("", string(stackName)) 1604 if err != nil { 1605 return err 1606 } 1607 fileName := filepath.Join(tempDir, "stack.json") 1608 1609 // Invoke `pulumi stack export` 1610 // There are situations where we want to get access to the secrets in the validation 1611 // this will allow us to get access to them as part of running ExtraRuntimeValidation 1612 var pulumiCommand []string 1613 if pt.opts.DecryptSecretsInOutput { 1614 pulumiCommand = append(pulumiCommand, "stack", "export", "--show-secrets", "--file", fileName) 1615 } else { 1616 pulumiCommand = append(pulumiCommand, "stack", "export", "--file", fileName) 1617 } 1618 if err = pt.runPulumiCommand("pulumi-export", 1619 pulumiCommand, dir, false); err != nil { 1620 return fmt.Errorf("expected to export stack to file: %s: %w", fileName, err) 1621 } 1622 1623 // Open the exported JSON file 1624 f, err := os.Open(fileName) 1625 if err != nil { 1626 return fmt.Errorf("expected to be able to open file with stack exports: %s: %w", fileName, err) 1627 } 1628 defer func() { 1629 contract.IgnoreClose(f) 1630 contract.IgnoreError(os.RemoveAll(tempDir)) 1631 }() 1632 1633 // Unmarshal the Deployment 1634 var untypedDeployment apitype.UntypedDeployment 1635 if err = json.NewDecoder(f).Decode(&untypedDeployment); err != nil { 1636 return err 1637 } 1638 var deployment apitype.DeploymentV3 1639 if err = json.Unmarshal(untypedDeployment.Deployment, &deployment); err != nil { 1640 return err 1641 } 1642 1643 // Get the root resource and outputs from the deployment 1644 var rootResource apitype.ResourceV3 1645 var outputs map[string]interface{} 1646 for _, res := range deployment.Resources { 1647 if res.Type == resource.RootStackType { 1648 rootResource = res 1649 outputs = res.Outputs 1650 } 1651 } 1652 1653 events, err := pt.readUpdateEventLog() 1654 if err != nil { 1655 return err 1656 } 1657 1658 // Populate stack info object with all of this data to pass to the validation function 1659 stackInfo := RuntimeValidationStackInfo{ 1660 StackName: pt.opts.GetStackName(), 1661 Deployment: &deployment, 1662 RootResource: rootResource, 1663 Outputs: outputs, 1664 Events: events, 1665 } 1666 1667 pt.t.Log("Performing extra runtime validation.") 1668 extraRuntimeValidation(pt.t, stackInfo) 1669 pt.t.Log("Extra runtime validation complete.") 1670 return nil 1671 } 1672 1673 func (pt *ProgramTester) readUpdateEventLog() ([]apitype.EngineEvent, error) { 1674 events := []apitype.EngineEvent{} 1675 eventsFile, err := os.Open(pt.updateEventLog) 1676 if err != nil { 1677 if os.IsNotExist(err) { 1678 return events, nil 1679 } 1680 return events, fmt.Errorf("expected to be able to open event log file %s: %w", 1681 pt.updateEventLog, err) 1682 } 1683 1684 defer contract.IgnoreClose(eventsFile) 1685 1686 decoder := json.NewDecoder(eventsFile) 1687 for { 1688 var event apitype.EngineEvent 1689 if err = decoder.Decode(&event); err != nil { 1690 if err == io.EOF { 1691 break 1692 } 1693 return events, fmt.Errorf("failed decoding engine event from log file %s: %w", 1694 pt.updateEventLog, err) 1695 } 1696 events = append(events, event) 1697 } 1698 1699 return events, nil 1700 } 1701 1702 // copyTestToTemporaryDirectory creates a temporary directory to run the test in and copies the test to it. 1703 func (pt *ProgramTester) copyTestToTemporaryDirectory() (string, string, error) { 1704 // Get the source dir and project info. 1705 sourceDir := pt.opts.Dir 1706 projinfo, err := pt.getProjinfo(sourceDir) 1707 if err != nil { 1708 return "", "", err 1709 } 1710 1711 if pt.opts.Stdout == nil { 1712 pt.opts.Stdout = os.Stdout 1713 } 1714 if pt.opts.Stderr == nil { 1715 pt.opts.Stderr = os.Stderr 1716 } 1717 1718 pt.t.Logf("sample: %v", sourceDir) 1719 bin, err := pt.getBin() 1720 if err != nil { 1721 return "", "", err 1722 } 1723 pt.t.Logf("pulumi: %v\n", bin) 1724 1725 stackName := string(pt.opts.GetStackName()) 1726 1727 // For most projects, we will copy to a temporary directory. For Go projects, however, we must create 1728 // a folder structure that adheres to GOPATH requirements 1729 var tmpdir, projdir string 1730 if projinfo.Proj.Runtime.Name() == "go" { 1731 targetDir, err := tools.CreateTemporaryGoFolder("stackName") 1732 if err != nil { 1733 return "", "", fmt.Errorf("Couldn't create temporary directory: %w", err) 1734 } 1735 tmpdir = targetDir 1736 projdir = targetDir 1737 } else { 1738 targetDir, tempErr := ioutil.TempDir("", stackName+"-") 1739 if tempErr != nil { 1740 return "", "", fmt.Errorf("Couldn't create temporary directory: %w", tempErr) 1741 } 1742 tmpdir = targetDir 1743 projdir = targetDir 1744 } 1745 // Copy the source project. 1746 if copyErr := fsutil.CopyFile(tmpdir, sourceDir, nil); copyErr != nil { 1747 return "", "", copyErr 1748 } 1749 // Reload the projinfo before making mutating changes (workspace.LoadProject caches the in-memory Project by path) 1750 projinfo, err = pt.getProjinfo(projdir) 1751 if err != nil { 1752 return "", "", err 1753 } 1754 1755 // Add dynamic plugin paths from ProgramTester 1756 if (projinfo.Proj.Plugins == nil || projinfo.Proj.Plugins.Providers == nil) && pt.opts.LocalProviders != nil { 1757 projinfo.Proj.Plugins = &workspace.Plugins{ 1758 Providers: make([]workspace.PluginOptions, 0), 1759 } 1760 } 1761 1762 if pt.opts.LocalProviders != nil { 1763 for _, provider := range pt.opts.LocalProviders { 1764 projinfo.Proj.Plugins.Providers = append(projinfo.Proj.Plugins.Providers, workspace.PluginOptions{ 1765 Name: provider.Package, 1766 Path: provider.Path, 1767 }) 1768 } 1769 } 1770 1771 if projinfo.Proj.Plugins != nil { 1772 for i, provider := range projinfo.Proj.Plugins.Providers { 1773 if !filepath.IsAbs(provider.Path) { 1774 path, err := filepath.Abs(provider.Path) 1775 if err != nil { 1776 return "", "", fmt.Errorf("could not get absolute path for plugin %s: %w", provider.Path, err) 1777 } 1778 projinfo.Proj.Plugins.Providers[i].Path = path 1779 } 1780 } 1781 for i, language := range projinfo.Proj.Plugins.Languages { 1782 if !filepath.IsAbs(language.Path) { 1783 path, err := filepath.Abs(language.Path) 1784 if err != nil { 1785 return "", "", fmt.Errorf("could not get absolute path for plugin %s: %w", language.Path, err) 1786 } 1787 projinfo.Proj.Plugins.Languages[i].Path = path 1788 } 1789 } 1790 for i, analyzer := range projinfo.Proj.Plugins.Analyzers { 1791 if !filepath.IsAbs(analyzer.Path) { 1792 path, err := filepath.Abs(analyzer.Path) 1793 if err != nil { 1794 return "", "", fmt.Errorf("could not get absolute path for plugin %s: %w", analyzer.Path, err) 1795 } 1796 projinfo.Proj.Plugins.Analyzers[i].Path = path 1797 } 1798 } 1799 } 1800 projfile := filepath.Join(projdir, workspace.ProjectFile+".yaml") 1801 bytes, err := yaml.Marshal(projinfo.Proj) 1802 if err != nil { 1803 return "", "", fmt.Errorf("error marshalling project %q: %w", projfile, err) 1804 } 1805 1806 if err := ioutil.WriteFile(projfile, bytes, 0600); err != nil { 1807 return "", "", fmt.Errorf("error writing project: %w", err) 1808 } 1809 1810 err = pt.prepareProject(projinfo) 1811 if err != nil { 1812 return "", "", fmt.Errorf("Failed to prepare %v: %w", projdir, err) 1813 } 1814 1815 // TODO[pulumi/pulumi#5455]: Dynamic providers fail to load when used from multi-lang components. 1816 // Until that's been fixed, this environment variable can be set by a test, which results in 1817 // a package.json being emitted in the project directory and `yarn install && yarn link @pulumi/pulumi` 1818 // being run. 1819 // When the underlying issue has been fixed, the use of this environment variable should be removed. 1820 var yarnLinkPulumi bool 1821 for _, env := range pt.opts.Env { 1822 if env == "PULUMI_TEST_YARN_LINK_PULUMI=true" { 1823 yarnLinkPulumi = true 1824 break 1825 } 1826 } 1827 if yarnLinkPulumi { 1828 const packageJSON = `{ 1829 "name": "test", 1830 "peerDependencies": { 1831 "@pulumi/pulumi": "latest" 1832 } 1833 }` 1834 if err := ioutil.WriteFile(filepath.Join(projdir, "package.json"), []byte(packageJSON), 0600); err != nil { 1835 return "", "", err 1836 } 1837 if err := pt.runYarnCommand("yarn-link", []string{"link", "@pulumi/pulumi"}, projdir); err != nil { 1838 return "", "", err 1839 } 1840 if err = pt.runYarnCommand("yarn-install", []string{"install"}, projdir); err != nil { 1841 return "", "", err 1842 } 1843 } 1844 1845 pt.t.Logf("projdir: %v", projdir) 1846 return tmpdir, projdir, nil 1847 } 1848 1849 func (pt *ProgramTester) getProjinfo(projectDir string) (*engine.Projinfo, error) { 1850 // Load up the package so we know things like what language the project is. 1851 projfile := filepath.Join(projectDir, workspace.ProjectFile+".yaml") 1852 proj, err := workspace.LoadProject(projfile) 1853 if err != nil { 1854 return nil, err 1855 } 1856 return &engine.Projinfo{Proj: proj, Root: projectDir}, nil 1857 } 1858 1859 // prepareProject runs setup necessary to get the project ready for `pulumi` commands. 1860 func (pt *ProgramTester) prepareProject(projinfo *engine.Projinfo) error { 1861 if pt.opts.PrepareProject != nil { 1862 return pt.opts.PrepareProject(projinfo) 1863 } 1864 return pt.defaultPrepareProject(projinfo) 1865 } 1866 1867 // prepareProjectDir runs setup necessary to get the project ready for `pulumi` commands. 1868 func (pt *ProgramTester) prepareProjectDir(projectDir string) error { 1869 projinfo, err := pt.getProjinfo(projectDir) 1870 if err != nil { 1871 return err 1872 } 1873 return pt.prepareProject(projinfo) 1874 } 1875 1876 // prepareNodeJSProject runs setup necessary to get a Node.js project ready for `pulumi` commands. 1877 func (pt *ProgramTester) prepareNodeJSProject(projinfo *engine.Projinfo) error { 1878 if err := pulumi_testing.WriteYarnRCForTest(projinfo.Root); err != nil { 1879 return err 1880 } 1881 1882 // Get the correct pwd to run Yarn in. 1883 cwd, _, err := projinfo.GetPwdMain() 1884 if err != nil { 1885 return err 1886 } 1887 1888 // If the test requested some packages to be overridden, we do two things. First, if the package is listed as a 1889 // direct dependency of the project, we change the version constraint in the package.json. For transitive 1890 // dependeices, we use yarn's "resolutions" feature to force them to a specific version. 1891 if len(pt.opts.Overrides) > 0 { 1892 packageJSON, err := readPackageJSON(cwd) 1893 if err != nil { 1894 return err 1895 } 1896 1897 resolutions := make(map[string]interface{}) 1898 1899 for packageName, packageVersion := range pt.opts.Overrides { 1900 for _, section := range []string{"dependencies", "devDependencies"} { 1901 if _, has := packageJSON[section]; has { 1902 entry := packageJSON[section].(map[string]interface{}) 1903 1904 if _, has := entry[packageName]; has { 1905 entry[packageName] = packageVersion 1906 } 1907 1908 } 1909 } 1910 1911 pt.t.Logf("adding resolution for %s to version %s", packageName, packageVersion) 1912 resolutions["**/"+packageName] = packageVersion 1913 } 1914 1915 // Wack any existing resolutions section with our newly computed one. 1916 packageJSON["resolutions"] = resolutions 1917 1918 if err := writePackageJSON(cwd, packageJSON); err != nil { 1919 return err 1920 } 1921 } 1922 1923 // Now ensure dependencies are present. 1924 if err = pt.runYarnCommand("yarn-install", []string{"install"}, cwd); err != nil { 1925 return err 1926 } 1927 1928 if !pt.opts.RunUpdateTest { 1929 if err = pt.yarnLinkPackageDeps(cwd); err != nil { 1930 return err 1931 } 1932 } 1933 1934 if pt.opts.RunBuild { 1935 // And finally compile it using whatever build steps are in the package.json file. 1936 if err = pt.runYarnCommand("yarn-build", []string{"run", "build"}, cwd); err != nil { 1937 return err 1938 } 1939 } 1940 1941 return nil 1942 1943 } 1944 1945 // readPackageJSON unmarshals the package.json file located in pathToPackage. 1946 func readPackageJSON(pathToPackage string) (map[string]interface{}, error) { 1947 f, err := os.Open(filepath.Join(pathToPackage, "package.json")) 1948 if err != nil { 1949 return nil, fmt.Errorf("opening package.json: %w", err) 1950 } 1951 defer contract.IgnoreClose(f) 1952 1953 var ret map[string]interface{} 1954 if err := json.NewDecoder(f).Decode(&ret); err != nil { 1955 return nil, fmt.Errorf("decoding package.json: %w", err) 1956 } 1957 1958 return ret, nil 1959 } 1960 1961 func writePackageJSON(pathToPackage string, metadata map[string]interface{}) error { 1962 // os.Create truncates the already existing file. 1963 f, err := os.Create(filepath.Join(pathToPackage, "package.json")) 1964 if err != nil { 1965 return fmt.Errorf("opening package.json: %w", err) 1966 } 1967 defer contract.IgnoreClose(f) 1968 1969 encoder := json.NewEncoder(f) 1970 encoder.SetEscapeHTML(false) 1971 encoder.SetIndent("", " ") 1972 1973 return fmt.Errorf("writing package.json: %w", encoder.Encode(metadata)) 1974 } 1975 1976 // preparePythonProject runs setup necessary to get a Python project ready for `pulumi` commands. 1977 func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error { 1978 cwd, _, err := projinfo.GetPwdMain() 1979 if err != nil { 1980 return err 1981 } 1982 1983 if pt.opts.UsePipenv { 1984 if err = pt.preparePythonProjectWithPipenv(cwd); err != nil { 1985 return err 1986 } 1987 } else { 1988 if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", "venv"}, cwd); err != nil { 1989 return err 1990 } 1991 1992 projinfo.Proj.Runtime.SetOption("virtualenv", "venv") 1993 projfile := filepath.Join(projinfo.Root, workspace.ProjectFile+".yaml") 1994 if err = projinfo.Proj.Save(projfile); err != nil { 1995 return fmt.Errorf("saving project: %w", err) 1996 } 1997 1998 if err := pt.runVirtualEnvCommand("virtualenv-pip-install", 1999 []string{"python", "-m", "pip", "install", "-r", "requirements.txt"}, cwd); err != nil { 2000 return err 2001 } 2002 } 2003 2004 if !pt.opts.RunUpdateTest { 2005 if err = pt.installPipPackageDeps(cwd); err != nil { 2006 return err 2007 } 2008 } 2009 2010 return nil 2011 } 2012 2013 func (pt *ProgramTester) preparePythonProjectWithPipenv(cwd string) error { 2014 // Allow ENV var based overload of desired Python version for 2015 // the Pipenv environment. This is useful in CI scenarios that 2016 // need to pin a specific version such as 3.9.x vs 3.10.x. 2017 pythonVersion := os.Getenv("PYTHON_VERSION") 2018 if pythonVersion == "" { 2019 pythonVersion = "3" 2020 } 2021 2022 // Create a new Pipenv environment. This bootstraps a new virtual environment containing the version of Python that 2023 // we requested. Note that this version of Python is sourced from the machine, so you must first install the version 2024 // of Python that you are requesting on the host machine before building a virtualenv for it. 2025 2026 if err := pt.runPipenvCommand("pipenv-new", []string{"--python", pythonVersion}, cwd); err != nil { 2027 return err 2028 } 2029 2030 // Install the package's dependencies. We do this by running `pip` inside the virtualenv that `pipenv` has created. 2031 // We don't use `pipenv install` because we don't want a lock file and prefer the similar model of `pip install` 2032 // which matches what our customers do 2033 err := pt.runPipenvCommand("pipenv-install", []string{"run", "pip", "install", "-r", "requirements.txt"}, cwd) 2034 if err != nil { 2035 return err 2036 } 2037 return nil 2038 } 2039 2040 // YarnLinkPackageDeps bring in package dependencies via yarn 2041 func (pt *ProgramTester) yarnLinkPackageDeps(cwd string) error { 2042 for _, dependency := range pt.opts.Dependencies { 2043 if err := pt.runYarnCommand("yarn-link", []string{"link", dependency}, cwd); err != nil { 2044 return err 2045 } 2046 } 2047 2048 return nil 2049 } 2050 2051 // InstallPipPackageDeps brings in package dependencies via pip install 2052 func (pt *ProgramTester) installPipPackageDeps(cwd string) error { 2053 var err error 2054 for _, dep := range pt.opts.Dependencies { 2055 // If the given filepath isn't absolute, make it absolute. We're about to pass it to pipenv and pipenv is 2056 // operating inside of a random folder in /tmp. 2057 if !filepath.IsAbs(dep) { 2058 dep, err = filepath.Abs(dep) 2059 if err != nil { 2060 return err 2061 } 2062 } 2063 2064 if pt.opts.UsePipenv { 2065 if err := pt.runPipenvCommand("pipenv-install-package", 2066 []string{"run", "pip", "install", "-e", dep}, cwd); err != nil { 2067 return err 2068 } 2069 } else { 2070 if err := pt.runVirtualEnvCommand("virtualenv-pip-install-package", 2071 []string{"python", "-m", "pip", "install", "-e", dep}, cwd); err != nil { 2072 return err 2073 } 2074 } 2075 } 2076 2077 return nil 2078 } 2079 2080 func getVirtualenvBinPath(cwd, bin string) (string, error) { 2081 virtualenvBinPath := filepath.Join(cwd, "venv", "bin", bin) 2082 if runtime.GOOS == windowsOS { 2083 virtualenvBinPath = filepath.Join(cwd, "venv", "Scripts", fmt.Sprintf("%s.exe", bin)) 2084 } 2085 if info, err := os.Stat(virtualenvBinPath); err != nil || info.IsDir() { 2086 return "", fmt.Errorf("Expected %s to exist in virtual environment at %q", bin, virtualenvBinPath) 2087 } 2088 return virtualenvBinPath, nil 2089 } 2090 2091 // getSanitizedPkg strips the version string from a go dep 2092 // Note: most of the pulumi modules don't use major version subdirectories for modules 2093 func getSanitizedModulePath(pkg string) string { 2094 re := regexp.MustCompile(`v\d`) 2095 v := re.FindString(pkg) 2096 if v != "" { 2097 return strings.TrimSuffix(strings.Replace(pkg, v, "", -1), "/") 2098 } 2099 return pkg 2100 2101 } 2102 2103 func getRewritePath(pkg string, gopath string, depRoot string) string { 2104 2105 var depParts []string 2106 sanitizedPkg := getSanitizedModulePath(pkg) 2107 2108 splitPkg := strings.Split(sanitizedPkg, "/") 2109 2110 if depRoot != "" { 2111 // Get the package name 2112 // This is the value after "github.com/foo/bar" 2113 repoName := splitPkg[2] 2114 basePath := splitPkg[len(splitPkg)-1] 2115 if basePath == repoName { 2116 depParts = append([]string{depRoot, repoName}) 2117 } else { 2118 depParts = append([]string{depRoot, repoName, basePath}) 2119 } 2120 return filepath.Join(depParts...) 2121 } 2122 depParts = append([]string{gopath, "src"}, splitPkg...) 2123 return filepath.Join(depParts...) 2124 2125 } 2126 2127 // Fetchs the GOPATH 2128 func GoPath() (string, error) { 2129 gopath := os.Getenv("GOPATH") 2130 if gopath == "" { 2131 usr, userErr := user.Current() 2132 if userErr != nil { 2133 return "", userErr 2134 } 2135 gopath = filepath.Join(usr.HomeDir, "go") 2136 } 2137 return gopath, nil 2138 } 2139 2140 // prepareGoProject runs setup necessary to get a Go project ready for `pulumi` commands. 2141 func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error { 2142 // Go programs are compiled, so we will compile the project first. 2143 goBin, err := pt.getGoBin() 2144 if err != nil { 2145 return fmt.Errorf("locating `go` binary: %w", err) 2146 } 2147 2148 depRoot := os.Getenv("PULUMI_GO_DEP_ROOT") 2149 gopath, userError := GoPath() 2150 if userError != nil { 2151 return userError 2152 } 2153 2154 cwd, _, err := projinfo.GetPwdMain() 2155 if err != nil { 2156 return err 2157 } 2158 2159 // initialize a go.mod for dependency resolution if one doesn't exist 2160 _, err = os.Stat(filepath.Join(cwd, "go.mod")) 2161 if err != nil { 2162 err = pt.runCommand("go-mod-init", []string{goBin, "mod", "init"}, cwd) 2163 if err != nil { 2164 return err 2165 } 2166 } 2167 2168 // link local dependencies 2169 for _, pkg := range pt.opts.Dependencies { 2170 var editStr string 2171 if strings.ContainsRune(pkg, '=') { 2172 // Use a literal replacement path. 2173 editStr = pkg 2174 } else { 2175 dep := getRewritePath(pkg, gopath, depRoot) 2176 editStr = fmt.Sprintf("%s=%s", pkg, dep) 2177 } 2178 err = pt.runCommand("go-mod-edit", []string{goBin, "mod", "edit", "-replace", editStr}, cwd) 2179 if err != nil { 2180 return err 2181 } 2182 } 2183 2184 // tidy to resolve all transitive dependencies including from local dependencies above. 2185 err = pt.runCommand("go-mod-tidy", []string{goBin, "mod", "tidy"}, cwd) 2186 if err != nil { 2187 return err 2188 } 2189 2190 if pt.opts.RunBuild { 2191 outBin := filepath.Join(gopath, "bin", string(projinfo.Proj.Name)) 2192 if runtime.GOOS == windowsOS { 2193 outBin = fmt.Sprintf("%s.exe", outBin) 2194 } 2195 err = pt.runCommand("go-build", []string{goBin, "build", "-o", outBin, "."}, cwd) 2196 if err != nil { 2197 return err 2198 } 2199 2200 _, err = os.Stat(outBin) 2201 if err != nil { 2202 return fmt.Errorf("error finding built application artifact: %w", err) 2203 } 2204 } 2205 2206 return nil 2207 } 2208 2209 // prepareDotNetProject runs setup necessary to get a .NET project ready for `pulumi` commands. 2210 func (pt *ProgramTester) prepareDotNetProject(projinfo *engine.Projinfo) error { 2211 dotNetBin, err := pt.getDotNetBin() 2212 if err != nil { 2213 return fmt.Errorf("locating `dotnet` binary: %w", err) 2214 } 2215 2216 cwd, _, err := projinfo.GetPwdMain() 2217 if err != nil { 2218 return err 2219 } 2220 2221 localNuget := os.Getenv("PULUMI_LOCAL_NUGET") 2222 if localNuget == "" { 2223 home := os.Getenv("HOME") 2224 localNuget = filepath.Join(home, ".pulumi-dev", "nuget") 2225 } 2226 2227 for _, dep := range pt.opts.Dependencies { 2228 2229 // dotnet add package requires a specific version in case of a pre-release, so we have to look it up. 2230 globPattern := filepath.Join(localNuget, dep+".?.*.nupkg") 2231 matches, err := filepath.Glob(globPattern) 2232 if err != nil { 2233 return fmt.Errorf("failed to find a local Pulumi NuGet package: %w", err) 2234 } 2235 if len(matches) != 1 { 2236 return fmt.Errorf("attempting to find a local NuGet package %s by searching %s yielded %d results: %v", 2237 dep, 2238 globPattern, 2239 len(matches), 2240 matches) 2241 } 2242 file := filepath.Base(matches[0]) 2243 r := strings.NewReplacer(dep+".", "", ".nupkg", "") 2244 version := r.Replace(file) 2245 2246 // We don't restore because the program might depend on external 2247 // packages which cannot be found in our local nuget source. A restore 2248 // will happen automatically as part of the `pulumi up`. 2249 err = pt.runCommand("dotnet-add-package", 2250 []string{dotNetBin, "add", "package", dep, 2251 "-v", version, 2252 "-s", localNuget, 2253 "--no-restore"}, 2254 cwd) 2255 if err != nil { 2256 return fmt.Errorf("failed to add dependency on %s: %w", dep, err) 2257 } 2258 } 2259 2260 return nil 2261 } 2262 2263 func (pt *ProgramTester) prepareYAMLProject(projinfo *engine.Projinfo) error { 2264 // YAML doesn't need any system setup, and should auto-install required plugins 2265 return nil 2266 } 2267 2268 func (pt *ProgramTester) prepareJavaProject(projinfo *engine.Projinfo) error { 2269 // Java doesn't need any system setup, and should auto-install required plugins 2270 return nil 2271 } 2272 2273 func (pt *ProgramTester) defaultPrepareProject(projinfo *engine.Projinfo) error { 2274 // Based on the language, invoke the right routine to prepare the target directory. 2275 switch rt := projinfo.Proj.Runtime.Name(); rt { 2276 case NodeJSRuntime: 2277 return pt.prepareNodeJSProject(projinfo) 2278 case PythonRuntime: 2279 return pt.preparePythonProject(projinfo) 2280 case GoRuntime: 2281 return pt.prepareGoProject(projinfo) 2282 case DotNetRuntime: 2283 return pt.prepareDotNetProject(projinfo) 2284 case YAMLRuntime: 2285 return pt.prepareYAMLProject(projinfo) 2286 case JavaRuntime: 2287 return pt.prepareJavaProject(projinfo) 2288 default: 2289 return fmt.Errorf("unrecognized project runtime: %s", rt) 2290 } 2291 }