github.com/jaylevin/jenkins-library@v1.230.4/pkg/command/command.go (about) 1 package command 2 3 import ( 4 "bufio" 5 "bytes" 6 "fmt" 7 "io" 8 "os" 9 "os/exec" 10 "regexp" 11 "strings" 12 "syscall" 13 14 "github.com/SAP/jenkins-library/pkg/log" 15 "github.com/SAP/jenkins-library/pkg/piperutils" 16 "github.com/pkg/errors" 17 ) 18 19 const ( 20 RelaxedURLRegEx = `(?:\b)((http(s?):\/\/)?(((www\.)?[a-zA-Z0-9\.\-\_]+(\.[a-zA-Z]{2,3})+)|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))(\/[a-zA-Z0-9\_\-\.\/\?\%\#\&\=]*)?)(?:\s|\b)` 21 ) 22 23 // Command defines the information required for executing a call to any executable 24 type Command struct { 25 ErrorCategoryMapping map[string][]string 26 URLReportFileName string 27 dir string 28 stdin io.Reader 29 stdout io.Writer 30 stderr io.Writer 31 env []string 32 exitCode int 33 } 34 35 type runner interface { 36 SetDir(dir string) 37 SetEnv(env []string) 38 AppendEnv(env []string) 39 Stdin(in io.Reader) 40 Stdout(out io.Writer) 41 Stderr(err io.Writer) 42 GetStdout() io.Writer 43 GetStderr() io.Writer 44 } 45 46 // ExecRunner mock for intercepting calls to executables 47 type ExecRunner interface { 48 runner 49 RunExecutable(executable string, params ...string) error 50 RunExecutableInBackground(executable string, params ...string) (Execution, error) 51 } 52 53 // ShellRunner mock for intercepting shell calls 54 type ShellRunner interface { 55 runner 56 RunShell(shell string, command string) error 57 } 58 59 // SetDir sets the working directory for the execution 60 func (c *Command) SetDir(dir string) { 61 c.dir = dir 62 } 63 64 // SetEnv sets explicit environment variables to be used for execution 65 func (c *Command) SetEnv(env []string) { 66 c.env = env 67 } 68 69 // AppendEnv appends environment variables to be used for execution 70 func (c *Command) AppendEnv(env []string) { 71 c.env = append(c.env, env...) 72 } 73 74 func (c *Command) GetOsEnv() []string { 75 return os.Environ() 76 } 77 78 // Stdin .. 79 func (c *Command) Stdin(stdin io.Reader) { 80 c.stdin = stdin 81 } 82 83 // Stdout .. 84 func (c *Command) Stdout(stdout io.Writer) { 85 c.stdout = stdout 86 } 87 88 // Stderr .. 89 func (c *Command) Stderr(stderr io.Writer) { 90 c.stderr = stderr 91 } 92 93 // GetStdout Returns the writer for stdout 94 func (c *Command) GetStdout() io.Writer { 95 return c.stdout 96 } 97 98 //GetStderr Retursn the writer for stderr 99 func (c *Command) GetStderr() io.Writer { 100 return c.stderr 101 } 102 103 // ExecCommand defines how to execute os commands 104 var ExecCommand = exec.Command 105 106 // RunShell runs the specified command on the shell 107 func (c *Command) RunShell(shell, script string) error { 108 109 c.prepareOut() 110 111 cmd := ExecCommand(shell) 112 113 if len(c.dir) > 0 { 114 cmd.Dir = c.dir 115 } 116 117 appendEnvironment(cmd, c.env) 118 119 in := bytes.Buffer{} 120 in.Write([]byte(script)) 121 cmd.Stdin = &in 122 123 log.Entry().Infof("running shell script: %v %v", shell, script) 124 125 if err := c.runCmd(cmd); err != nil { 126 return errors.Wrapf(err, "running shell script failed with %v", shell) 127 } 128 return nil 129 } 130 131 // RunExecutable runs the specified executable with parameters 132 // !! While the cmd.Env is applied during command execution, it is NOT involved when the actual executable is resolved. 133 // Thus the executable needs to be on the PATH of the current process and it is not sufficient to alter the PATH on cmd.Env. 134 func (c *Command) RunExecutable(executable string, params ...string) error { 135 136 c.prepareOut() 137 138 cmd := ExecCommand(executable, params...) 139 140 if len(c.dir) > 0 { 141 cmd.Dir = c.dir 142 } 143 144 log.Entry().Infof("running command: %v %v", executable, strings.Join(params, (" "))) 145 146 appendEnvironment(cmd, c.env) 147 148 if c.stdin != nil { 149 cmd.Stdin = c.stdin 150 } 151 152 if err := c.runCmd(cmd); err != nil { 153 return errors.Wrapf(err, "running command '%v' failed", executable) 154 } 155 return nil 156 } 157 158 // RunExecutableInBackground runs the specified executable with parameters in the background non blocking 159 // !! While the cmd.Env is applied during command execution, it is NOT involved when the actual executable is resolved. 160 // Thus the executable needs to be on the PATH of the current process and it is not sufficient to alter the PATH on cmd.Env. 161 func (c *Command) RunExecutableInBackground(executable string, params ...string) (Execution, error) { 162 163 c.prepareOut() 164 165 cmd := ExecCommand(executable, params...) 166 167 if len(c.dir) > 0 { 168 cmd.Dir = c.dir 169 } 170 171 log.Entry().Infof("running command: %v %v", executable, strings.Join(params, (" "))) 172 173 appendEnvironment(cmd, c.env) 174 175 if c.stdin != nil { 176 cmd.Stdin = c.stdin 177 } 178 179 execution, err := c.startCmd(cmd) 180 181 if err != nil { 182 return nil, errors.Wrapf(err, "starting command '%v' failed", executable) 183 } 184 185 return execution, nil 186 } 187 188 // GetExitCode allows to retrieve the exit code of a command execution 189 func (c *Command) GetExitCode() int { 190 return c.exitCode 191 } 192 193 func appendEnvironment(cmd *exec.Cmd, env []string) { 194 195 if len(env) > 0 { 196 197 // When cmd.Env is nil the environment variables from the current 198 // process are also used by the forked process. Our environment variables 199 // should not replace the existing environment, but they should be appended. 200 // Hence we populate cmd.Env first with the current environment in case we 201 // find it empty. In case there is already something, we append to that environment. 202 // In that case we assume the current values of `cmd.Env` has either been setup based 203 // on `os.Environ()` or that was initialized in another way for a good reason. 204 // 205 // In case we have the same environment variable as in the current environment (`os.Environ()`) 206 // and in `env`, the environment variable from `env` is effectively used since this is the 207 // later one. There is no merging between both environment variables. 208 // 209 // cf. https://golang.org/pkg/os/exec/#Command 210 // If Env contains duplicate environment keys, only the last 211 // value in the slice for each duplicate key is used. 212 213 if len(cmd.Env) == 0 { 214 cmd.Env = os.Environ() 215 } 216 cmd.Env = append(cmd.Env, env...) 217 } 218 } 219 220 func (c *Command) startCmd(cmd *exec.Cmd) (*execution, error) { 221 222 stdout, stderr, err := cmdPipes(cmd) 223 224 if err != nil { 225 return nil, errors.Wrap(err, "getting command pipes failed") 226 } 227 228 err = cmd.Start() 229 if err != nil { 230 return nil, errors.Wrap(err, "starting command failed") 231 } 232 233 execution := execution{cmd: cmd} 234 execution.wg.Add(2) 235 236 srcOut := stdout 237 srcErr := stderr 238 239 if c.ErrorCategoryMapping != nil { 240 prOut, pwOut := io.Pipe() 241 trOut := io.TeeReader(stdout, pwOut) 242 srcOut = prOut 243 244 prErr, pwErr := io.Pipe() 245 trErr := io.TeeReader(stderr, pwErr) 246 srcErr = prErr 247 248 execution.wg.Add(2) 249 250 go func() { 251 defer execution.wg.Done() 252 defer pwOut.Close() 253 c.scanLog(trOut) 254 }() 255 256 go func() { 257 defer execution.wg.Done() 258 defer pwErr.Close() 259 c.scanLog(trErr) 260 }() 261 } 262 263 go func() { 264 if c.URLReportFileName != "" { 265 var buf bytes.Buffer 266 br := bufio.NewWriter(&buf) 267 _, execution.errCopyStdout = piperutils.CopyData(io.MultiWriter(c.stdout, br), srcOut) 268 br.Flush() 269 handleURLs(buf.String(), c.URLReportFileName) 270 } else { 271 _, execution.errCopyStdout = piperutils.CopyData(c.stdout, srcOut) 272 } 273 execution.wg.Done() 274 }() 275 276 go func() { 277 if c.URLReportFileName != "" { 278 var buf bytes.Buffer 279 bw := bufio.NewWriter(&buf) 280 _, execution.errCopyStderr = piperutils.CopyData(io.MultiWriter(c.stderr, bw), srcErr) 281 bw.Flush() 282 handleURLs(buf.String(), c.URLReportFileName) 283 } else { 284 _, execution.errCopyStderr = piperutils.CopyData(c.stderr, srcErr) 285 } 286 execution.wg.Done() 287 }() 288 289 return &execution, nil 290 } 291 292 func handleURLs(s, file string) { 293 reg := regexp.MustCompile(RelaxedURLRegEx) 294 matches := reg.FindAllStringSubmatch(s, -1) 295 f, err := os.Create(file) 296 if err != nil { 297 log.Entry().WithError(err).Info("failed to create url report file") 298 } 299 defer f.Close() 300 for _, match := range matches { 301 _, err = f.WriteString(match[1] + "\n") 302 if err != nil { 303 log.Entry().WithError(err).Info("failed to write record to url report") 304 } 305 } 306 } 307 308 func (c *Command) scanLog(in io.Reader) { 309 scanner := bufio.NewScanner(in) 310 scanner.Split(scanShortLines) 311 for scanner.Scan() { 312 line := scanner.Text() 313 c.parseConsoleErrors(line) 314 } 315 if err := scanner.Err(); err != nil { 316 log.Entry().WithError(err).Info("failed to scan log file") 317 } 318 } 319 320 func scanShortLines(data []byte, atEOF bool) (advance int, token []byte, err error) { 321 lenData := len(data) 322 if atEOF && lenData == 0 { 323 return 0, nil, nil 324 } 325 if lenData > 32767 && !bytes.Contains(data[0:lenData], []byte("\n")) { 326 // we will neglect long output 327 // no use cases known where this would be relevant 328 // current accepted implication: error pattern would not be found 329 // -> resulting in wrong error categorization 330 return lenData, nil, nil 331 } 332 if i := bytes.IndexByte(data, '\n'); i >= 0 && i < 32767 { 333 // We have a full newline-terminated line with a size limit 334 // Size limit is required since otherwise scanner would stall 335 return i + 1, data[0:i], nil 336 } 337 // If we're at EOF, we have a final, non-terminated line. Return it. 338 if atEOF { 339 return len(data), data, nil 340 } 341 // Request more data. 342 return 0, nil, nil 343 } 344 345 func (c *Command) parseConsoleErrors(logLine string) { 346 for category, categoryErrors := range c.ErrorCategoryMapping { 347 for _, errorPart := range categoryErrors { 348 if matchPattern(logLine, errorPart) { 349 log.SetErrorCategory(log.ErrorCategoryByString(category)) 350 return 351 } 352 } 353 } 354 } 355 356 func matchPattern(text, pattern string) bool { 357 if len(pattern) == 0 && len(text) != 0 { 358 return false 359 } 360 parts := strings.Split(pattern, "*") 361 for _, part := range parts { 362 if !strings.Contains(text, part) { 363 return false 364 } 365 } 366 return true 367 } 368 369 func (c *Command) runCmd(cmd *exec.Cmd) error { 370 371 execution, err := c.startCmd(cmd) 372 if err != nil { 373 return err 374 } 375 376 err = execution.Wait() 377 378 if execution.errCopyStdout != nil || execution.errCopyStderr != nil { 379 return fmt.Errorf("failed to capture stdout/stderr: '%v'/'%v'", execution.errCopyStdout, execution.errCopyStderr) 380 } 381 382 if err != nil { 383 // provide fallback to ensure a non 0 exit code in case of an error 384 c.exitCode = 1 385 // try to identify the detailed error code 386 if exitErr, ok := err.(*exec.ExitError); ok { 387 if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { 388 c.exitCode = status.ExitStatus() 389 } 390 } 391 return errors.Wrap(err, "cmd.Run() failed") 392 } 393 c.exitCode = 0 394 return nil 395 } 396 397 func (c *Command) prepareOut() { 398 399 //ToDo: check use of multiwriter instead to always write into os.Stdout and os.Stdin? 400 //stdout := io.MultiWriter(os.Stdout, &stdoutBuf) 401 //stderr := io.MultiWriter(os.Stderr, &stderrBuf) 402 403 if c.stdout == nil { 404 c.stdout = os.Stdout 405 } 406 if c.stderr == nil { 407 c.stderr = os.Stderr 408 } 409 } 410 411 func cmdPipes(cmd *exec.Cmd) (io.ReadCloser, io.ReadCloser, error) { 412 stdout, err := cmd.StdoutPipe() 413 if err != nil { 414 return nil, nil, errors.Wrap(err, "getting Stdout pipe failed") 415 } 416 417 stderr, err := cmd.StderrPipe() 418 if err != nil { 419 return nil, nil, errors.Wrap(err, "getting Stderr pipe failed") 420 } 421 422 return stdout, stderr, nil 423 }