github.com/bshelton229/agent@v3.5.4+incompatible/clicommand/pipeline_upload.go (about) 1 package clicommand 2 3 import ( 4 "encoding/json" 5 "io/ioutil" 6 "os" 7 "path" 8 "path/filepath" 9 "strings" 10 "time" 11 12 "github.com/buildkite/agent/agent" 13 "github.com/buildkite/agent/api" 14 "github.com/buildkite/agent/cliconfig" 15 "github.com/buildkite/agent/logger" 16 "github.com/buildkite/agent/retry" 17 "github.com/buildkite/agent/stdin" 18 "github.com/urfave/cli" 19 ) 20 21 var PipelineUploadHelpDescription = `Usage: 22 23 buildkite-agent pipeline upload <file> [arguments...] 24 25 Description: 26 27 Allows you to change the pipeline of a running build by uploading either a 28 YAML (recommended) or JSON configuration file. If no configuration file is 29 provided, the command looks for the file in the following locations: 30 31 - buildkite.yml 32 - buildkite.yaml 33 - buildkite.json 34 - .buildkite/pipeline.yml 35 - .buildkite/pipeline.yaml 36 - .buildkite/pipeline.json 37 38 You can also pipe build pipelines to the command allowing you to create 39 scripts that generate dynamic pipelines. 40 41 Example: 42 43 $ buildkite-agent pipeline upload 44 $ buildkite-agent pipeline upload my-custom-pipeline.yml 45 $ ./script/dynamic_step_generator | buildkite-agent pipeline upload` 46 47 type PipelineUploadConfig struct { 48 FilePath string `cli:"arg:0" label:"upload paths"` 49 Replace bool `cli:"replace"` 50 Job string `cli:"job"` 51 AgentAccessToken string `cli:"agent-access-token"` 52 Endpoint string `cli:"endpoint" validate:"required"` 53 DryRun bool `cli:"dry-run"` 54 NoColor bool `cli:"no-color"` 55 NoInterpolation bool `cli:"no-interpolation"` 56 Debug bool `cli:"debug"` 57 DebugHTTP bool `cli:"debug-http"` 58 } 59 60 var PipelineUploadCommand = cli.Command{ 61 Name: "upload", 62 Usage: "Uploads a description of a build pipeline adds it to the currently running build after the current job.", 63 Description: PipelineUploadHelpDescription, 64 Flags: []cli.Flag{ 65 cli.BoolFlag{ 66 Name: "replace", 67 Usage: "Replace the rest of the existing pipeline with the steps uploaded. Jobs that are already running are not removed.", 68 EnvVar: "BUILDKITE_PIPELINE_REPLACE", 69 }, 70 cli.StringFlag{ 71 Name: "job", 72 Value: "", 73 Usage: "The job that is making the changes to it's build", 74 EnvVar: "BUILDKITE_JOB_ID", 75 }, 76 cli.BoolFlag{ 77 Name: "dry-run", 78 Usage: "Rather than uploading the pipeline, it will be echoed to stdout", 79 EnvVar: "BUILDKITE_PIPELINE_UPLOAD_DRY_RUN", 80 }, 81 cli.BoolFlag{ 82 Name: "no-interpolation", 83 Usage: "Skip variable interpolation the pipeline when uploaded", 84 EnvVar: "BUILDKITE_PIPELINE_NO_INTERPOLATION", 85 }, 86 AgentAccessTokenFlag, 87 EndpointFlag, 88 NoColorFlag, 89 DebugFlag, 90 DebugHTTPFlag, 91 }, 92 Action: func(c *cli.Context) { 93 // The configuration will be loaded into this struct 94 cfg := PipelineUploadConfig{} 95 96 // Load the configuration 97 loader := cliconfig.Loader{CLI: c, Config: &cfg} 98 if err := loader.Load(); err != nil { 99 logger.Fatal("%s", err) 100 } 101 102 // Setup the any global configuration options 103 HandleGlobalFlags(cfg) 104 105 // Find the pipeline file either from STDIN or the first 106 // argument 107 var input []byte 108 var err error 109 var filename string 110 111 if cfg.FilePath != "" { 112 logger.Info("Reading pipeline config from \"%s\"", cfg.FilePath) 113 114 filename = filepath.Base(cfg.FilePath) 115 input, err = ioutil.ReadFile(cfg.FilePath) 116 if err != nil { 117 logger.Fatal("Failed to read file: %s", err) 118 } 119 } else if stdin.IsReadable() { 120 logger.Info("Reading pipeline config from STDIN") 121 122 // Actually read the file from STDIN 123 input, err = ioutil.ReadAll(os.Stdin) 124 if err != nil { 125 logger.Fatal("Failed to read from STDIN: %s", err) 126 } 127 } else { 128 logger.Info("Searching for pipeline config...") 129 130 paths := []string{ 131 "buildkite.yml", 132 "buildkite.yaml", 133 "buildkite.json", 134 filepath.FromSlash(".buildkite/pipeline.yml"), 135 filepath.FromSlash(".buildkite/pipeline.yaml"), 136 filepath.FromSlash(".buildkite/pipeline.json"), 137 } 138 139 // Collect all the files that exist 140 exists := []string{} 141 for _, path := range paths { 142 if _, err := os.Stat(path); err == nil { 143 exists = append(exists, path) 144 } 145 } 146 147 // If more than 1 of the config files exist, throw an 148 // error. There can only be one!! 149 if len(exists) > 1 { 150 logger.Fatal("Found multiple configuration files: %s. Please only have 1 configuration file present.", strings.Join(exists, ", ")) 151 } else if len(exists) == 0 { 152 logger.Fatal("Could not find a default pipeline configuration file. See `buildkite-agent pipeline upload --help` for more information.") 153 } 154 155 found := exists[0] 156 157 logger.Info("Found config file \"%s\"", found) 158 159 // Read the default file 160 filename = path.Base(found) 161 input, err = ioutil.ReadFile(found) 162 if err != nil { 163 logger.Fatal("Failed to read file \"%s\" (%s)", found, err) 164 } 165 } 166 167 // Make sure the file actually has something in it 168 if len(input) == 0 { 169 logger.Fatal("Config file is empty") 170 } 171 172 // Parse the pipeline 173 result, err := agent.PipelineParser{ 174 Filename: filename, 175 Pipeline: input, 176 NoInterpolation: cfg.NoInterpolation, 177 }.Parse() 178 if err != nil { 179 logger.Fatal("Pipeline parsing of \"%s\" failed (%s)", filename, err) 180 } 181 182 // In dry-run mode we just output the generated pipeline to stdout 183 if cfg.DryRun { 184 enc := json.NewEncoder(os.Stdout) 185 enc.SetIndent("", " ") 186 187 // Dump json indented to stdout. All logging happens to stderr 188 // this can be used with other tools to get interpolated json 189 if err := enc.Encode(result); err != nil { 190 logger.Fatal("%#v", err) 191 } 192 193 os.Exit(0) 194 } 195 196 // Check we have a job id set if not in dry run 197 if cfg.Job == "" { 198 logger.Fatal("Missing job parameter. Usually this is set in the environment for a Buildkite job via BUILDKITE_JOB_ID.") 199 } 200 201 // Check we have an agent access token if not in dry run 202 if cfg.AgentAccessToken == "" { 203 logger.Fatal("Missing agent-access-token parameter. Usually this is set in the environment for a Buildkite job via BUILDKITE_AGENT_ACCESS_TOKEN.") 204 } 205 206 // Create the API client 207 client := agent.APIClient{ 208 Endpoint: cfg.Endpoint, 209 Token: cfg.AgentAccessToken, 210 }.Create() 211 212 // Generate a UUID that will identifiy this pipeline change. We 213 // do this outside of the retry loop because we want this UUID 214 // to be the same for each attempt at updating the pipeline. 215 uuid := api.NewUUID() 216 217 // Retry the pipeline upload a few times before giving up 218 err = retry.Do(func(s *retry.Stats) error { 219 _, err = client.Pipelines.Upload(cfg.Job, &api.Pipeline{UUID: uuid, Pipeline: result, Replace: cfg.Replace}) 220 if err != nil { 221 logger.Warn("%s (%s)", err, s) 222 223 // 422 responses will always fail no need to retry 224 if apierr, ok := err.(*api.ErrorResponse); ok && apierr.Response.StatusCode == 422 { 225 logger.Error("Unrecoverable error, skipping retries") 226 s.Break() 227 } 228 } 229 230 return err 231 }, &retry.Config{Maximum: 5, Interval: 1 * time.Second}) 232 if err != nil { 233 logger.Fatal("Failed to upload and process pipeline: %s", err) 234 } 235 236 logger.Info("Successfully uploaded and parsed pipeline config") 237 }, 238 }