github.com/jd-ly/tools@v0.5.7/internal/gocommand/invoke.go (about) 1 // Copyright 2020 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package gocommand is a helper for calling the go command. 6 package gocommand 7 8 import ( 9 "bytes" 10 "context" 11 "fmt" 12 "io" 13 "os" 14 "os/exec" 15 "regexp" 16 "strconv" 17 "strings" 18 "sync" 19 "time" 20 21 "github.com/jd-ly/tools/internal/event" 22 ) 23 24 // An Runner will run go command invocations and serialize 25 // them if it sees a concurrency error. 26 type Runner struct { 27 // once guards the runner initialization. 28 once sync.Once 29 30 // inFlight tracks available workers. 31 inFlight chan struct{} 32 33 // serialized guards the ability to run a go command serially, 34 // to avoid deadlocks when claiming workers. 35 serialized chan struct{} 36 } 37 38 const maxInFlight = 10 39 40 func (runner *Runner) initialize() { 41 runner.once.Do(func() { 42 runner.inFlight = make(chan struct{}, maxInFlight) 43 runner.serialized = make(chan struct{}, 1) 44 }) 45 } 46 47 // 1.13: go: updates to go.mod needed, but contents have changed 48 // 1.14: go: updating go.mod: existing contents have changed since last read 49 var modConcurrencyError = regexp.MustCompile(`go:.*go.mod.*contents have changed`) 50 51 // Run is a convenience wrapper around RunRaw. 52 // It returns only stdout and a "friendly" error. 53 func (runner *Runner) Run(ctx context.Context, inv Invocation) (*bytes.Buffer, error) { 54 stdout, _, friendly, _ := runner.RunRaw(ctx, inv) 55 return stdout, friendly 56 } 57 58 // RunPiped runs the invocation serially, always waiting for any concurrent 59 // invocations to complete first. 60 func (runner *Runner) RunPiped(ctx context.Context, inv Invocation, stdout, stderr io.Writer) error { 61 _, err := runner.runPiped(ctx, inv, stdout, stderr) 62 return err 63 } 64 65 // RunRaw runs the invocation, serializing requests only if they fight over 66 // go.mod changes. 67 func (runner *Runner) RunRaw(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { 68 // Make sure the runner is always initialized. 69 runner.initialize() 70 71 // First, try to run the go command concurrently. 72 stdout, stderr, friendlyErr, err := runner.runConcurrent(ctx, inv) 73 74 // If we encounter a load concurrency error, we need to retry serially. 75 if friendlyErr == nil || !modConcurrencyError.MatchString(friendlyErr.Error()) { 76 return stdout, stderr, friendlyErr, err 77 } 78 event.Error(ctx, "Load concurrency error, will retry serially", err) 79 80 // Run serially by calling runPiped. 81 stdout.Reset() 82 stderr.Reset() 83 friendlyErr, err = runner.runPiped(ctx, inv, stdout, stderr) 84 return stdout, stderr, friendlyErr, err 85 } 86 87 func (runner *Runner) runConcurrent(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { 88 // Wait for 1 worker to become available. 89 select { 90 case <-ctx.Done(): 91 return nil, nil, nil, ctx.Err() 92 case runner.inFlight <- struct{}{}: 93 defer func() { <-runner.inFlight }() 94 } 95 96 stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} 97 friendlyErr, err := inv.runWithFriendlyError(ctx, stdout, stderr) 98 return stdout, stderr, friendlyErr, err 99 } 100 101 func (runner *Runner) runPiped(ctx context.Context, inv Invocation, stdout, stderr io.Writer) (error, error) { 102 // Make sure the runner is always initialized. 103 runner.initialize() 104 105 // Acquire the serialization lock. This avoids deadlocks between two 106 // runPiped commands. 107 select { 108 case <-ctx.Done(): 109 return nil, ctx.Err() 110 case runner.serialized <- struct{}{}: 111 defer func() { <-runner.serialized }() 112 } 113 114 // Wait for all in-progress go commands to return before proceeding, 115 // to avoid load concurrency errors. 116 for i := 0; i < maxInFlight; i++ { 117 select { 118 case <-ctx.Done(): 119 return nil, ctx.Err() 120 case runner.inFlight <- struct{}{}: 121 // Make sure we always "return" any workers we took. 122 defer func() { <-runner.inFlight }() 123 } 124 } 125 126 return inv.runWithFriendlyError(ctx, stdout, stderr) 127 } 128 129 // An Invocation represents a call to the go command. 130 type Invocation struct { 131 Verb string 132 Args []string 133 BuildFlags []string 134 ModFlag string 135 ModFile string 136 Overlay string 137 // If CleanEnv is set, the invocation will run only with the environment 138 // in Env, not starting with os.Environ. 139 CleanEnv bool 140 Env []string 141 WorkingDir string 142 Logf func(format string, args ...interface{}) 143 } 144 145 func (i *Invocation) runWithFriendlyError(ctx context.Context, stdout, stderr io.Writer) (friendlyError error, rawError error) { 146 rawError = i.run(ctx, stdout, stderr) 147 if rawError != nil { 148 friendlyError = rawError 149 // Check for 'go' executable not being found. 150 if ee, ok := rawError.(*exec.Error); ok && ee.Err == exec.ErrNotFound { 151 friendlyError = fmt.Errorf("go command required, not found: %v", ee) 152 } 153 if ctx.Err() != nil { 154 friendlyError = ctx.Err() 155 } 156 friendlyError = fmt.Errorf("err: %v: stderr: %s", friendlyError, stderr) 157 } 158 return 159 } 160 161 func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error { 162 log := i.Logf 163 if log == nil { 164 log = func(string, ...interface{}) {} 165 } 166 167 goArgs := []string{i.Verb} 168 169 appendModFile := func() { 170 if i.ModFile != "" { 171 goArgs = append(goArgs, "-modfile="+i.ModFile) 172 } 173 } 174 appendModFlag := func() { 175 if i.ModFlag != "" { 176 goArgs = append(goArgs, "-mod="+i.ModFlag) 177 } 178 } 179 appendOverlayFlag := func() { 180 if i.Overlay != "" { 181 goArgs = append(goArgs, "-overlay="+i.Overlay) 182 } 183 } 184 185 switch i.Verb { 186 case "env", "version": 187 goArgs = append(goArgs, i.Args...) 188 case "mod": 189 // mod needs the sub-verb before flags. 190 goArgs = append(goArgs, i.Args[0]) 191 appendModFile() 192 goArgs = append(goArgs, i.Args[1:]...) 193 case "get": 194 goArgs = append(goArgs, i.BuildFlags...) 195 appendModFile() 196 goArgs = append(goArgs, i.Args...) 197 198 default: // notably list and build. 199 goArgs = append(goArgs, i.BuildFlags...) 200 appendModFile() 201 appendModFlag() 202 appendOverlayFlag() 203 goArgs = append(goArgs, i.Args...) 204 } 205 cmd := exec.Command("go", goArgs...) 206 cmd.Stdout = stdout 207 cmd.Stderr = stderr 208 // On darwin the cwd gets resolved to the real path, which breaks anything that 209 // expects the working directory to keep the original path, including the 210 // go command when dealing with modules. 211 // The Go stdlib has a special feature where if the cwd and the PWD are the 212 // same node then it trusts the PWD, so by setting it in the env for the child 213 // process we fix up all the paths returned by the go command. 214 if !i.CleanEnv { 215 cmd.Env = os.Environ() 216 } 217 cmd.Env = append(cmd.Env, i.Env...) 218 if i.WorkingDir != "" { 219 cmd.Env = append(cmd.Env, "PWD="+i.WorkingDir) 220 cmd.Dir = i.WorkingDir 221 } 222 defer func(start time.Time) { log("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now()) 223 224 return runCmdContext(ctx, cmd) 225 } 226 227 // runCmdContext is like exec.CommandContext except it sends os.Interrupt 228 // before os.Kill. 229 func runCmdContext(ctx context.Context, cmd *exec.Cmd) error { 230 if err := cmd.Start(); err != nil { 231 return err 232 } 233 resChan := make(chan error, 1) 234 go func() { 235 resChan <- cmd.Wait() 236 }() 237 238 select { 239 case err := <-resChan: 240 return err 241 case <-ctx.Done(): 242 } 243 // Cancelled. Interrupt and see if it ends voluntarily. 244 cmd.Process.Signal(os.Interrupt) 245 select { 246 case err := <-resChan: 247 return err 248 case <-time.After(time.Second): 249 } 250 // Didn't shut down in response to interrupt. Kill it hard. 251 cmd.Process.Kill() 252 return <-resChan 253 } 254 255 func cmdDebugStr(cmd *exec.Cmd) string { 256 env := make(map[string]string) 257 for _, kv := range cmd.Env { 258 split := strings.SplitN(kv, "=", 2) 259 k, v := split[0], split[1] 260 env[k] = v 261 } 262 263 var args []string 264 for _, arg := range cmd.Args { 265 quoted := strconv.Quote(arg) 266 if quoted[1:len(quoted)-1] != arg || strings.Contains(arg, " ") { 267 args = append(args, quoted) 268 } else { 269 args = append(args, arg) 270 } 271 } 272 return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], strings.Join(args, " ")) 273 }