github.com/facebookincubator/ttpforge@v1.0.13-0.20240405153150-5ae801628835/cmd/test.go (about) 1 /* 2 Copyright © 2023-present, Meta Platforms, Inc. and affiliates 3 Permission is hereby granted, free of charge, to any person obtaining a copy 4 of this software and associated documentation files (the "Software"), to deal 5 in the Software without restriction, including without limitation the rights 6 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 copies of the Software, and to permit persons to whom the Software is 8 furnished to do so, subject to the following conditions: 9 The above copyright notice and this permission notice shall be included in 10 all copies or substantial portions of the Software. 11 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 THE SOFTWARE. 18 */ 19 20 package cmd 21 22 import ( 23 "context" 24 "fmt" 25 "os" 26 "os/exec" 27 "time" 28 29 "github.com/facebookincubator/ttpforge/pkg/blocks" 30 "github.com/facebookincubator/ttpforge/pkg/logging" 31 "github.com/facebookincubator/ttpforge/pkg/preprocess" 32 33 "github.com/spf13/afero" 34 "github.com/spf13/cobra" 35 "gopkg.in/yaml.v3" 36 ) 37 38 type testCase struct { 39 Name string `yaml:"name"` 40 Description string `yaml:"description"` 41 Args map[string]string `yaml:"args"` 42 DryRun bool `yaml:"dry_run"` 43 } 44 45 // We want to verify everything but the steps themselves against 46 // our schema. The steps will, in turn, be validated 47 // by the subsequent invocation of the `ttpforge run` command. 48 type ttpNonStepFields struct { 49 blocks.PreambleFields `yaml:",inline"` 50 Cases []testCase `yaml:"tests"` 51 } 52 53 // Note - this command cannot be unit tested 54 // because it calls os.Executable() and actually re-executes 55 // the same binary ("itself", though with a different command) 56 // as a subprocess 57 func buildTestCommand(cfg *Config) *cobra.Command { 58 var timeoutSeconds int 59 runCmd := &cobra.Command{ 60 Use: "test [repo_name//path/to/ttp]", 61 Short: "Test the TTP found in the specified YAML file.", 62 Args: cobra.MinimumNArgs(1), 63 RunE: func(cmd *cobra.Command, args []string) error { 64 // don't want confusing usage display for errors past this point 65 cmd.SilenceUsage = true 66 67 for _, ttpRef := range args { 68 // find the TTP file 69 _, ttpAbsPath, err := cfg.repoCollection.ResolveTTPRef(ttpRef) 70 if err != nil { 71 return fmt.Errorf("failed to resolve TTP reference %v: %w", ttpRef, err) 72 } 73 if err := runTestsForTTP(ttpAbsPath, timeoutSeconds); err != nil { 74 return fmt.Errorf("test(s) for TTP %v failed: %w", ttpRef, err) 75 } 76 } 77 return nil 78 }, 79 } 80 runCmd.PersistentFlags().IntVar(&timeoutSeconds, "time-out-seconds", 10, "Timeout allowed for each test case") 81 82 return runCmd 83 } 84 85 func runTestsForTTP(ttpAbsPath string, timeoutSeconds int) error { 86 logging.DividerThick() 87 logging.L().Infof("TESTING TTP FILE:") 88 logging.L().Info(ttpAbsPath) 89 // preprocess to separate out the `tests:` section from the `steps:` 90 // section and avoid YAML parsing errors associated with template syntax 91 contents, err := afero.ReadFile(afero.NewOsFs(), ttpAbsPath) 92 if err != nil { 93 return fmt.Errorf("failed to read TTP file %v: %w", ttpAbsPath, err) 94 } 95 preprocessResult, err := preprocess.Parse(contents) 96 if err != nil { 97 return err 98 } 99 100 // load the test cases and preamble fields - we don't load 101 // the steps themselves because that process will 102 // be tested when we call `ttpforge run` 103 var ttpf ttpNonStepFields 104 err = yaml.Unmarshal(preprocessResult.PreambleBytes, &ttpf) 105 if err != nil { 106 return fmt.Errorf("failed to parse TTP file %v: %w", ttpAbsPath, err) 107 } 108 109 // validate as many fields as we can prior to actually 110 // invoking `ttpforge run` 111 if err := ttpf.PreambleFields.Validate(true); err != nil { 112 return fmt.Errorf("invalid TTP file %v: %w", ttpAbsPath, err) 113 } 114 115 // look up the path of this binary (ttpforge) 116 selfPath, err := os.Executable() 117 if err != nil { 118 return fmt.Errorf("could not resolve self path (path to current ttpforge binary): %w", err) 119 } 120 121 var testCases []testCase 122 if len(ttpf.Cases) == 0 { 123 if len(ttpf.ArgSpecs) == 0 { 124 // since this TTP doesn't accept arguments, it doesn't need a test case 125 // to validate its steps - so we can add an implicit dry run case 126 testCases = append(testCases, testCase{ 127 Name: "auto_generated_dry_run", 128 Description: "Auto-generated dry run test case", 129 DryRun: true, 130 }) 131 } else { 132 logging.L().Warnf("No tests defined in TTP file %v; exiting...", ttpAbsPath) 133 return nil 134 } 135 } else { 136 testCases = append(testCases, ttpf.Cases...) 137 } 138 139 // run all cases 140 logging.DividerThick() 141 logging.L().Infof("EXECUTING %v TEST CASE(S)", len(testCases)) 142 for tcIdx, tc := range testCases { 143 logging.DividerThin() 144 logging.L().Infof("RUNNING TEST CASE #%d: %q", tcIdx+1, tc.Name) 145 logging.DividerThin() 146 ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second) 147 defer cancel() 148 cmd := exec.CommandContext(ctx, selfPath) 149 cmd.Args = append(cmd.Args, "run", ttpAbsPath) 150 for argName, argVal := range tc.Args { 151 cmd.Args = append(cmd.Args, "--arg") 152 cmd.Args = append(cmd.Args, argName+"="+argVal) 153 } 154 if tc.DryRun { 155 cmd.Args = append(cmd.Args, "--dry-run") 156 } 157 cmd.Stdout = os.Stdout 158 cmd.Stderr = os.Stderr 159 160 err = cmd.Run() 161 if err != nil { 162 return fmt.Errorf("test case %q failed: %w", tc.Name, err) 163 } 164 } 165 logging.DividerThin() 166 logging.L().Info("ALL TESTS COMPLETED SUCCESSFULLY!") 167 return nil 168 }