github.com/0xKiwi/rules_go@v0.24.3/go/tools/builders/env.go (about) 1 // Copyright 2017 The Bazel Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "bytes" 19 "errors" 20 "flag" 21 "fmt" 22 "io" 23 "io/ioutil" 24 "log" 25 "os" 26 "os/exec" 27 "path/filepath" 28 "runtime" 29 "strconv" 30 "strings" 31 ) 32 33 var ( 34 // cgoEnvVars is the list of all cgo environment variable 35 cgoEnvVars = []string{"CGO_CFLAGS", "CGO_CXXFLAGS", "CGO_CPPFLAGS", "CGO_LDFLAGS"} 36 // cgoAbsEnvFlags are all the flags that need absolute path in cgoEnvVars 37 cgoAbsEnvFlags = []string{"-I", "-L", "-isysroot", "-isystem", "-iquote", "-include", "-gcc-toolchain", "--sysroot"} 38 ) 39 40 // env holds a small amount of Go environment and toolchain information 41 // which is common to multiple builders. Most Bazel-agnostic build information 42 // is collected in go/build.Default though. 43 // 44 // See ./README.rst for more information about handling arguments and 45 // environment variables. 46 type env struct { 47 // sdk is the path to the Go SDK, which contains tools for the host 48 // platform. This may be different than GOROOT. 49 sdk string 50 51 // installSuffix is the name of the directory below GOROOT/pkg that contains 52 // the .a files for the standard library we should build against. 53 // For example, linux_amd64_race. 54 installSuffix string 55 56 // verbose indicates whether subprocess command lines should be printed. 57 verbose bool 58 59 // workDirPath is a temporary work directory. It is created lazily. 60 workDirPath string 61 62 shouldPreserveWorkDir bool 63 } 64 65 // envFlags registers flags common to multiple builders and returns an env 66 // configured with those flags. 67 func envFlags(flags *flag.FlagSet) *env { 68 env := &env{} 69 flags.StringVar(&env.sdk, "sdk", "", "Path to the Go SDK.") 70 flags.Var(&tagFlag{}, "tags", "List of build tags considered true.") 71 flags.StringVar(&env.installSuffix, "installsuffix", "", "Standard library under GOROOT/pkg") 72 flags.BoolVar(&env.verbose, "v", false, "Whether subprocess command lines should be printed") 73 flags.BoolVar(&env.shouldPreserveWorkDir, "work", false, "if true, the temporary work directory will be preserved") 74 return env 75 } 76 77 // checkFlags checks whether env flags were set to valid values. checkFlags 78 // should be called after parsing flags. 79 func (e *env) checkFlags() error { 80 if e.sdk == "" { 81 return errors.New("-sdk was not set") 82 } 83 return nil 84 } 85 86 // workDir returns a path to a temporary work directory. The same directory 87 // is returned on multiple calls. The caller is responsible for cleaning 88 // up the work directory by calling cleanup. 89 func (e *env) workDir() (path string, cleanup func(), err error) { 90 if e.workDirPath != "" { 91 return e.workDirPath, func() {}, nil 92 } 93 // Keep the stem "rules_go_work" in sync with reproducible_binary_test.go. 94 e.workDirPath, err = ioutil.TempDir("", "rules_go_work-") 95 if err != nil { 96 return "", func() {}, err 97 } 98 if e.verbose { 99 log.Printf("WORK=%s\n", e.workDirPath) 100 } 101 if e.shouldPreserveWorkDir { 102 cleanup = func() {} 103 } else { 104 cleanup = func() { os.RemoveAll(e.workDirPath) } 105 } 106 return e.workDirPath, cleanup, nil 107 } 108 109 // goTool returns a slice containing the path to an executable at 110 // $GOROOT/pkg/$GOOS_$GOARCH/$tool and additional arguments. 111 func (e *env) goTool(tool string, args ...string) []string { 112 platform := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) 113 toolPath := filepath.Join(e.sdk, "pkg", "tool", platform, tool) 114 if runtime.GOOS == "windows" { 115 toolPath += ".exe" 116 } 117 return append([]string{toolPath}, args...) 118 } 119 120 // goCmd returns a slice containing the path to the go executable 121 // and additional arguments. 122 func (e *env) goCmd(cmd string, args ...string) []string { 123 exe := filepath.Join(e.sdk, "bin", "go") 124 if runtime.GOOS == "windows" { 125 exe += ".exe" 126 } 127 return append([]string{exe, cmd}, args...) 128 } 129 130 // runCommand executes a subprocess that inherits stdout, stderr, and the 131 // environment from this process. 132 func (e *env) runCommand(args []string) error { 133 cmd := exec.Command(args[0], args[1:]...) 134 // Redirecting stdout to stderr. This mirrors behavior in the go command: 135 // https://go.googlesource.com/go/+/refs/tags/go1.15.2/src/cmd/go/internal/work/exec.go#1958 136 cmd.Stdout = os.Stderr 137 cmd.Stderr = os.Stderr 138 return runAndLogCommand(cmd, e.verbose) 139 } 140 141 // runCommandToFile executes a subprocess and writes the output to the given 142 // writer. 143 func (e *env) runCommandToFile(w io.Writer, args []string) error { 144 cmd := exec.Command(args[0], args[1:]...) 145 cmd.Stdout = w 146 cmd.Stderr = os.Stderr 147 return runAndLogCommand(cmd, e.verbose) 148 } 149 150 func absEnv(envNameList []string, argList []string) error { 151 for _, envName := range envNameList { 152 splitedEnv := strings.Fields(os.Getenv(envName)) 153 absArgs(splitedEnv, argList) 154 if err := os.Setenv(envName, strings.Join(splitedEnv, " ")); err != nil { 155 return err 156 } 157 } 158 return nil 159 } 160 161 func runAndLogCommand(cmd *exec.Cmd, verbose bool) error { 162 formattedCmd := formatCommand(cmd) 163 if verbose { 164 os.Stderr.WriteString(formattedCmd) 165 } 166 cleanup := passLongArgsInResponseFiles(cmd) 167 defer cleanup() 168 if err := cmd.Run(); err != nil { 169 return fmt.Errorf("error running the following subcommand: %v\n%s", err, formattedCmd) 170 } 171 return nil 172 } 173 174 // expandParamsFiles looks for arguments in args of the form 175 // "-param=filename". When it finds these arguments it reads the file "filename" 176 // and replaces the argument with its content. 177 func expandParamsFiles(args []string) ([]string, error) { 178 var paramsIndices []int 179 for i, arg := range args { 180 if strings.HasPrefix(arg, "-param=") { 181 paramsIndices = append(paramsIndices, i) 182 } 183 } 184 if len(paramsIndices) == 0 { 185 return args, nil 186 } 187 var expandedArgs []string 188 last := 0 189 for _, pi := range paramsIndices { 190 expandedArgs = append(expandedArgs, args[last:pi]...) 191 last = pi + 1 192 193 fileName := args[pi][len("-param="):] 194 fileArgs, err := readParamsFile(fileName) 195 if err != nil { 196 return nil, err 197 } 198 expandedArgs = append(expandedArgs, fileArgs...) 199 } 200 expandedArgs = append(expandedArgs, args[last:]...) 201 return expandedArgs, nil 202 } 203 204 // readParamsFiles parses a Bazel params file in "shell" format. The file 205 // should contain one argument per line. Arguments may be quoted with single 206 // quotes. All characters within quoted strings are interpreted literally 207 // including newlines and excepting single quotes. Characters outside quoted 208 // strings may be escaped with a backslash. 209 func readParamsFile(name string) ([]string, error) { 210 data, err := ioutil.ReadFile(name) 211 if err != nil { 212 return nil, err 213 } 214 215 var args []string 216 var arg []byte 217 quote := false 218 escape := false 219 for p := 0; p < len(data); p++ { 220 b := data[p] 221 switch { 222 case escape: 223 arg = append(arg, b) 224 escape = false 225 226 case b == '\'': 227 quote = !quote 228 229 case !quote && b == '\\': 230 escape = true 231 232 case !quote && b == '\n': 233 args = append(args, string(arg)) 234 arg = arg[:0] 235 236 default: 237 arg = append(arg, b) 238 } 239 } 240 if quote { 241 return nil, fmt.Errorf("unterminated quote") 242 } 243 if escape { 244 return nil, fmt.Errorf("unterminated escape") 245 } 246 if len(arg) > 0 { 247 args = append(args, string(arg)) 248 } 249 return args, nil 250 } 251 252 // writeParamsFile formats a list of arguments in Bazel's "shell" format and writes 253 // it to a file. 254 func writeParamsFile(path string, args []string) error { 255 buf := new(bytes.Buffer) 256 for _, arg := range args { 257 if !strings.ContainsAny(arg, "'\n\\") { 258 fmt.Fprintln(buf, arg) 259 continue 260 } 261 buf.WriteByte('\'') 262 for _, r := range arg { 263 if r == '\'' { 264 buf.WriteString(`'\''`) 265 } else { 266 buf.WriteRune(r) 267 } 268 } 269 buf.WriteString("'\n") 270 } 271 return ioutil.WriteFile(path, buf.Bytes(), 0666) 272 } 273 274 // splitArgs splits a list of command line arguments into two parts: arguments 275 // that should be interpreted by the builder (before "--"), and arguments 276 // that should be passed through to the underlying tool (after "--"). 277 func splitArgs(args []string) (builderArgs []string, toolArgs []string) { 278 for i, arg := range args { 279 if arg == "--" { 280 return args[:i], args[i+1:] 281 } 282 } 283 return args, nil 284 } 285 286 // abs returns the absolute representation of path. Some tools/APIs require 287 // absolute paths to work correctly. Most notably, golang on Windows cannot 288 // handle relative paths to files whose absolute path is > ~250 chars, while 289 // it can handle absolute paths. See http://goo.gl/eqeWjm. 290 // 291 // Note that strings that begin with "__BAZEL_" are not absolutized. These are 292 // used on macOS for paths that the compiler wrapper (wrapped_clang) is 293 // supposed to know about. 294 func abs(path string) string { 295 if strings.HasPrefix(path, "__BAZEL_") { 296 return path 297 } 298 299 if abs, err := filepath.Abs(path); err != nil { 300 return path 301 } else { 302 return abs 303 } 304 } 305 306 // absArgs applies abs to strings that appear in args. Only paths that are 307 // part of options named by flags are modified. 308 func absArgs(args []string, flags []string) { 309 absNext := false 310 for i := range args { 311 if absNext { 312 args[i] = abs(args[i]) 313 absNext = false 314 continue 315 } 316 for _, f := range flags { 317 if !strings.HasPrefix(args[i], f) { 318 continue 319 } 320 possibleValue := args[i][len(f):] 321 if len(possibleValue) == 0 { 322 absNext = true 323 break 324 } 325 separator := "" 326 if possibleValue[0] == '=' { 327 possibleValue = possibleValue[1:] 328 separator = "=" 329 } 330 args[i] = fmt.Sprintf("%s%s%s", f, separator, abs(possibleValue)) 331 break 332 } 333 } 334 } 335 336 // formatCommand writes cmd to w in a format where it can be pasted into a 337 // shell. Spaces in environment variables and arguments are escaped as needed. 338 func formatCommand(cmd *exec.Cmd) string { 339 quoteIfNeeded := func(s string) string { 340 if strings.IndexByte(s, ' ') < 0 { 341 return s 342 } 343 return strconv.Quote(s) 344 } 345 quoteEnvIfNeeded := func(s string) string { 346 eq := strings.IndexByte(s, '=') 347 if eq < 0 { 348 return s 349 } 350 key, value := s[:eq], s[eq+1:] 351 if strings.IndexByte(value, ' ') < 0 { 352 return s 353 } 354 return fmt.Sprintf("%s=%s", key, strconv.Quote(value)) 355 } 356 var w bytes.Buffer 357 environ := cmd.Env 358 if environ == nil { 359 environ = os.Environ() 360 } 361 for _, e := range environ { 362 fmt.Fprintf(&w, "%s \\\n", quoteEnvIfNeeded(e)) 363 } 364 365 sep := "" 366 for _, arg := range cmd.Args { 367 fmt.Fprintf(&w, "%s%s", sep, quoteIfNeeded(arg)) 368 sep = " " 369 } 370 fmt.Fprint(&w, "\n") 371 return w.String() 372 } 373 374 // passLongArgsInResponseFiles modifies cmd such that, for 375 // certain programs, long arguments are passed in "response files", a 376 // file on disk with the arguments, with one arg per line. An actual 377 // argument starting with '@' means that the rest of the argument is 378 // a filename of arguments to expand. 379 // 380 // See https://github.com/golang/go/issues/18468 (Windows) and 381 // https://github.com/golang/go/issues/37768 (Darwin). 382 func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { 383 cleanup = func() {} // no cleanup by default 384 var argLen int 385 for _, arg := range cmd.Args { 386 argLen += len(arg) 387 } 388 // If we're not approaching 32KB of args, just pass args normally. 389 // (use 30KB instead to be conservative; not sure how accounting is done) 390 if !useResponseFile(cmd.Path, argLen) { 391 return 392 } 393 tf, err := ioutil.TempFile("", "args") 394 if err != nil { 395 log.Fatalf("error writing long arguments to response file: %v", err) 396 } 397 cleanup = func() { os.Remove(tf.Name()) } 398 var buf bytes.Buffer 399 for _, arg := range cmd.Args[1:] { 400 fmt.Fprintf(&buf, "%s\n", arg) 401 } 402 if _, err := tf.Write(buf.Bytes()); err != nil { 403 tf.Close() 404 cleanup() 405 log.Fatalf("error writing long arguments to response file: %v", err) 406 } 407 if err := tf.Close(); err != nil { 408 cleanup() 409 log.Fatalf("error writing long arguments to response file: %v", err) 410 } 411 cmd.Args = []string{cmd.Args[0], "@" + tf.Name()} 412 return cleanup 413 } 414 415 func useResponseFile(path string, argLen int) bool { 416 // Unless the program uses objabi.Flagparse, which understands 417 // response files, don't use response files. 418 // TODO: do we need more commands? asm? cgo? For now, no. 419 prog := strings.TrimSuffix(filepath.Base(path), ".exe") 420 switch prog { 421 case "compile", "link": 422 default: 423 return false 424 } 425 // Windows has a limit of 32 KB arguments. To be conservative and not 426 // worry about whether that includes spaces or not, just use 30 KB. 427 // Darwin's limit is less clear. The OS claims 256KB, but we've seen 428 // failures with arglen as small as 50KB. 429 if argLen > (30 << 10) { 430 return true 431 } 432 return false 433 }