go.fuchsia.dev/jiri@v0.0.0-20240502161911-b66513b29486/cmd/jiri/runp.go (about) 1 // Copyright 2015 The Vanadium Authors, All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bufio" 9 "fmt" 10 "io" 11 "os" 12 "os/exec" 13 "os/signal" 14 "regexp" 15 "sort" 16 "strings" 17 "sync" 18 19 "go.fuchsia.dev/jiri" 20 "go.fuchsia.dev/jiri/cmdline" 21 "go.fuchsia.dev/jiri/envvar" 22 "go.fuchsia.dev/jiri/project" 23 "go.fuchsia.dev/jiri/simplemr" 24 "go.fuchsia.dev/jiri/tool" 25 ) 26 27 var runpFlags struct { 28 projectKeys string 29 verbose bool 30 interactive bool 31 uncommitted bool 32 noUncommitted bool 33 untracked bool 34 noUntracked bool 35 showNamePrefix bool 36 showPathPrefix bool 37 showKeyPrefix bool 38 exitOnError bool 39 collateOutput bool 40 branch string 41 remote string 42 } 43 44 var cmdRunP = &cmdline.Command{ 45 Runner: jiri.RunnerFunc(runRunp), 46 Name: "runp", 47 Short: "Run a command in parallel across jiri projects", 48 Long: `Run a command in parallel across one or more jiri projects. Commands are run 49 using the shell specified by the users $SHELL environment variable, or "sh" 50 if that's not set. Thus commands are run as $SHELL -c "args..." 51 `, 52 ArgsName: "<command line>", 53 ArgsLong: `A command line to be run in each project specified by the supplied command 54 line flags. Any environment variables intended to be evaluated when the 55 command line is run must be quoted to avoid expansion before being passed to 56 runp by the shell. 57 `, 58 } 59 60 func init() { 61 cmdRunP.Flags.BoolVar(&runpFlags.verbose, "v", false, "Print verbose logging information") 62 cmdRunP.Flags.StringVar(&runpFlags.projectKeys, "projects", "", "A Regular expression specifying project keys to run commands in. By default, runp will use projects that have the same branch checked as the current project unless it is run from outside of a project in which case it will default to using all projects.") 63 cmdRunP.Flags.BoolVar(&runpFlags.uncommitted, "uncommitted", false, "Match projects that have uncommitted changes") 64 cmdRunP.Flags.BoolVar(&runpFlags.noUncommitted, "no-uncommitted", false, "Match projects that have no uncommitted changes") 65 cmdRunP.Flags.BoolVar(&runpFlags.untracked, "untracked", false, "Match projects that have untracked files") 66 cmdRunP.Flags.BoolVar(&runpFlags.noUntracked, "no-untracked", false, "Match projects that have no untracked files") 67 cmdRunP.Flags.BoolVar(&runpFlags.interactive, "interactive", false, "If set, the command to be run is interactive and should not have its stdout/stderr manipulated. This flag cannot be used with -show-name-prefix, -show-key-prefix or -collate-stdout.") 68 cmdRunP.Flags.BoolVar(&runpFlags.showNamePrefix, "show-name-prefix", false, "If set, each line of output from each project will begin with the name of the project followed by a colon. This is intended for use with long running commands where the output needs to be streamed. Stdout and stderr are spliced apart. This flag cannot be used with -interactive, -show-path-prefix, -show-key-prefix or -collate-stdout.") 69 cmdRunP.Flags.BoolVar(&runpFlags.showPathPrefix, "show-path-prefix", false, "If set, each line of output from each project will begin with the path of the project followed by a colon. This is intended for use with long running commands where the output needs to be streamed. Stdout and stderr are spliced apart. This flag cannot be used with -interactive, -show-name-prefix, -show-key-prefix or -collate-stdout.") 70 cmdRunP.Flags.BoolVar(&runpFlags.showKeyPrefix, "show-key-prefix", false, "If set, each line of output from each project will begin with the key of the project followed by a colon. This is intended for use with long running commands where the output needs to be streamed. Stdout and stderr are spliced apart. This flag cannot be used with -interactive, -show-name-prefix, -show-path-prefix or -collate-stdout") 71 cmdRunP.Flags.BoolVar(&runpFlags.collateOutput, "collate-stdout", true, "Collate all stdout output from each parallel invocation and display it as if had been generated sequentially. This flag cannot be used with -show-name-prefix, -show-key-prefix or -interactive.") 72 cmdRunP.Flags.BoolVar(&runpFlags.exitOnError, "exit-on-error", false, "If set, all commands will killed as soon as one reports an error, otherwise, each will run to completion.") 73 cmdRunP.Flags.StringVar(&runpFlags.branch, "branch", "", "A regular expression specifying branch names to use in matching projects. A project will match if the specified branch exists, even if it is not checked out.") 74 cmdRunP.Flags.StringVar(&runpFlags.remote, "remote", "", "A Regular expression specifying projects to run commands in by matching against their remote URLs.") 75 } 76 77 type mapInput struct { 78 project.Project 79 key project.ProjectKey 80 jirix *jiri.X 81 index, total int 82 result error 83 } 84 85 func newmapInput(jirix *jiri.X, project project.Project, key project.ProjectKey, index, total int) *mapInput { 86 return &mapInput{ 87 Project: project, 88 key: key, 89 jirix: jirix.Clone(tool.ContextOpts{}), 90 index: index, 91 total: total, 92 } 93 } 94 95 func projectNames(mapInputs map[project.ProjectKey]*mapInput) []string { 96 n := []string{} 97 for _, mi := range mapInputs { 98 n = append(n, mi.Project.Name) 99 } 100 sort.Strings(n) 101 return n 102 } 103 104 func projectKeys(mapInputs map[project.ProjectKey]*mapInput) []string { 105 n := []string{} 106 for key := range mapInputs { 107 n = append(n, key.String()) 108 } 109 sort.Strings(n) 110 return n 111 } 112 113 type runner struct { 114 args []string 115 serializedWriterLock sync.Mutex 116 collatedOutputLock sync.Mutex 117 } 118 119 func (r *runner) serializedWriter(w io.Writer) io.Writer { 120 return &sharedLockWriter{&r.serializedWriterLock, w} 121 } 122 123 type sharedLockWriter struct { 124 mu *sync.Mutex 125 f io.Writer 126 } 127 128 func (lw *sharedLockWriter) Write(d []byte) (int, error) { 129 lw.mu.Lock() 130 defer lw.mu.Unlock() 131 return lw.f.Write(d) 132 } 133 134 func copyWithPrefix(prefix string, w io.Writer, r io.Reader) { 135 reader := bufio.NewReader(r) 136 for { 137 line, err := reader.ReadString('\n') 138 if err != nil { 139 if line != "" { 140 fmt.Fprintf(w, "%v: %v\n", prefix, line) 141 } 142 break 143 } 144 fmt.Fprintf(w, "%v: %v", prefix, line) 145 } 146 } 147 148 type mapOutput struct { 149 mi *mapInput 150 outputFilename string 151 key string 152 err error 153 } 154 155 func (r *runner) Map(mr *simplemr.MR, key string, val interface{}) error { 156 mi := val.(*mapInput) 157 output := &mapOutput{ 158 key: key, 159 mi: mi} 160 jirix := mi.jirix 161 path := os.Getenv("SHELL") 162 if path == "" { 163 path = "sh" 164 } 165 var wg sync.WaitGroup 166 cmd := exec.Command(path, "-c", strings.Join(r.args, " ")) 167 cmd.Env = envvar.MapToSlice(jirix.Env()) 168 cmd.Dir = mi.Project.Path 169 cmd.Stdin = mi.jirix.Stdin() 170 var stdoutCloser, stderrCloser io.Closer 171 if runpFlags.interactive { 172 cmd.Stdout = os.Stdout 173 cmd.Stderr = os.Stderr 174 } else { 175 var stdout io.Writer 176 stderr := r.serializedWriter(jirix.Stderr()) 177 var cleanup func() 178 if runpFlags.collateOutput { 179 // Write standard output to a file, stderr 180 // is not collated. 181 f, err := os.CreateTemp("", "jiri-runp-") 182 if err != nil { 183 return err 184 } 185 stdout = f 186 output.outputFilename = f.Name() 187 cleanup = func() { 188 os.Remove(output.outputFilename) 189 } 190 // The child process will have exited by the 191 // time this method returns so it's safe to close the file 192 // here. 193 defer f.Close() 194 } else { 195 stdout = r.serializedWriter(os.Stdout) 196 cleanup = func() {} 197 } 198 if !runpFlags.showNamePrefix && !runpFlags.showKeyPrefix && !runpFlags.showPathPrefix { 199 // write directly to stdout, stderr if there's no prefix 200 cmd.Stdout = stdout 201 cmd.Stderr = stderr 202 } else { 203 stdoutReader, stdoutWriter, err := os.Pipe() 204 if err != nil { 205 cleanup() 206 return err 207 } 208 stderrReader, stderrWriter, err := os.Pipe() 209 if err != nil { 210 cleanup() 211 stdoutReader.Close() 212 stdoutWriter.Close() 213 return err 214 } 215 cmd.Stdout = stdoutWriter 216 cmd.Stderr = stderrWriter 217 // Record the write end of the pipe so that it can be closed 218 // after the child has exited, this ensures that all goroutines 219 // will finish. 220 stdoutCloser = stdoutWriter 221 stderrCloser = stderrWriter 222 prefix := key 223 if runpFlags.showNamePrefix { 224 prefix = mi.Project.Name 225 } 226 if runpFlags.showPathPrefix { 227 prefix = mi.Project.Path 228 } 229 wg.Add(2) 230 go func() { copyWithPrefix(prefix, stdout, stdoutReader); wg.Done() }() 231 go func() { copyWithPrefix(prefix, stderr, stderrReader); wg.Done() }() 232 233 } 234 } 235 if err := cmd.Start(); err != nil { 236 mi.result = err 237 } 238 done := make(chan error) 239 go func() { 240 done <- cmd.Wait() 241 }() 242 select { 243 case output.err = <-done: 244 if output.err != nil && runpFlags.exitOnError { 245 mr.Cancel() 246 } 247 case <-mr.CancelCh(): 248 output.err = cmd.Process.Kill() 249 } 250 for _, closer := range []io.Closer{stdoutCloser, stderrCloser} { 251 if closer != nil { 252 closer.Close() 253 } 254 } 255 wg.Wait() 256 mr.MapOut(key, output) 257 return nil 258 } 259 260 func (r *runner) Reduce(mr *simplemr.MR, key string, values []interface{}) error { 261 for _, v := range values { 262 mo := v.(*mapOutput) 263 if mo.err != nil { 264 fmt.Fprintf(os.Stdout, "FAILED: %v: %s %v\n", mo.key, strings.Join(r.args, " "), mo.err) 265 return nil 266 } else { 267 if runpFlags.collateOutput { 268 r.collatedOutputLock.Lock() 269 defer r.collatedOutputLock.Unlock() 270 defer os.Remove(mo.outputFilename) 271 if fi, err := os.Open(mo.outputFilename); err == nil { 272 io.Copy(os.Stdout, fi) 273 fi.Close() 274 } else { 275 return err 276 } 277 } 278 } 279 } 280 return nil 281 } 282 283 func runRunp(jirix *jiri.X, args []string) error { 284 if runpFlags.interactive { 285 runpFlags.collateOutput = false 286 } 287 288 var keysRE, branchRE, remoteRE *regexp.Regexp 289 var err error 290 291 if runpFlags.projectKeys != "" { 292 re := "" 293 for _, pre := range strings.Split(runpFlags.projectKeys, ",") { 294 re += pre + "|" 295 } 296 re = strings.TrimRight(re, "|") 297 keysRE, err = regexp.Compile(re) 298 if err != nil { 299 return fmt.Errorf("failed to compile projects regexp: %q: %v", runpFlags.projectKeys, err) 300 } 301 } 302 303 if runpFlags.branch != "" { 304 branchRE, err = regexp.Compile(runpFlags.branch) 305 if err != nil { 306 return fmt.Errorf("failed to compile has-branch regexp: %q: %v", runpFlags.branch, err) 307 } 308 } 309 310 if runpFlags.remote != "" { 311 remoteRE, err = regexp.Compile(runpFlags.remote) 312 if err != nil { 313 return fmt.Errorf("failed to compile remotes regexp: %q: %v", runpFlags.remote, err) 314 } 315 } 316 317 if (runpFlags.showKeyPrefix || runpFlags.showNamePrefix || runpFlags.showPathPrefix) && runpFlags.interactive { 318 fmt.Fprintf(jirix.Stderr(), "WARNING: interactive mode being disabled because show-key-prefix or show-name-prefix or show-path-prefix was set\n") 319 runpFlags.interactive = false 320 runpFlags.collateOutput = true 321 } 322 323 dir, err := os.Getwd() 324 if err != nil { 325 return fmt.Errorf("os.Getwd() failed: %v", err) 326 } 327 if dir == jirix.Root || err != nil { 328 // jiri was run from outside of a project. Let's assume we'll 329 // use all projects if none have been specified via the projects flag. 330 if keysRE == nil { 331 keysRE = regexp.MustCompile(".*") 332 } 333 } 334 projects, err := project.LocalProjects(jirix, project.FastScan) 335 if err != nil { 336 return err 337 } 338 339 projectStateRequired := branchRE != nil || runpFlags.untracked || runpFlags.noUntracked || runpFlags.uncommitted || runpFlags.noUncommitted 340 var states map[project.ProjectKey]*project.ProjectState 341 if projectStateRequired { 342 var err error 343 states, err = project.GetProjectStates(jirix, projects, runpFlags.untracked || runpFlags.noUntracked || runpFlags.uncommitted || runpFlags.noUncommitted) 344 if err != nil { 345 return err 346 } 347 } 348 mapInputs := map[project.ProjectKey]*mapInput{} 349 var keys project.ProjectKeys 350 for _, localProject := range projects { 351 key := localProject.Key() 352 if keysRE != nil { 353 if !keysRE.MatchString(key.String()) { 354 continue 355 } 356 } 357 state := states[key] 358 if branchRE != nil { 359 found := false 360 for _, br := range state.Branches { 361 if branchRE.MatchString(br.Name) { 362 found = true 363 break 364 } 365 } 366 if !found { 367 continue 368 } 369 } 370 if remoteRE != nil && !remoteRE.MatchString(localProject.Remote) { 371 continue 372 } 373 if (runpFlags.untracked && !state.HasUntracked) || (runpFlags.noUntracked && state.HasUntracked) { 374 continue 375 } 376 if (runpFlags.uncommitted && !state.HasUncommitted) || (runpFlags.noUncommitted && state.HasUncommitted) { 377 continue 378 } 379 mapInputs[key] = &mapInput{ 380 Project: localProject, 381 jirix: jirix, 382 key: key, 383 } 384 keys = append(keys, key) 385 } 386 387 total := len(mapInputs) 388 index := 1 389 for _, mi := range mapInputs { 390 mi.index = index 391 mi.total = total 392 index++ 393 } 394 395 if runpFlags.verbose { 396 fmt.Fprintf(os.Stdout, "Project Names: %s\n", strings.Join(projectNames(mapInputs), " ")) 397 fmt.Fprintf(os.Stdout, "Project Keys: %s\n", strings.Join(projectKeys(mapInputs), " ")) 398 } 399 400 runner := &runner{ 401 args: args, 402 } 403 mr := simplemr.MR{} 404 if runpFlags.interactive { 405 // Run one mapper at a time. 406 mr.NumMappers = 1 407 sort.Sort(keys) 408 } else { 409 mr.NumMappers = int(jirix.Jobs) 410 } 411 in, out := make(chan *simplemr.Record, len(mapInputs)), make(chan *simplemr.Record, len(mapInputs)) 412 sigch := make(chan os.Signal) 413 signal.Notify(sigch, os.Interrupt) 414 jirix.TimerPush("Map and Reduce") 415 go func() { <-sigch; mr.Cancel() }() 416 go mr.Run(in, out, runner, runner) 417 for _, key := range keys { 418 in <- &simplemr.Record{key.String(), []interface{}{mapInputs[key]}} 419 } 420 close(in) 421 <-out 422 jirix.TimerPop() 423 return mr.Error() 424 }