github.com/0xKiwi/rules_go@v0.24.3/go/tools/builders/env.go (about)

     1  // Copyright 2017 The Bazel Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"bytes"
    19  	"errors"
    20  	"flag"
    21  	"fmt"
    22  	"io"
    23  	"io/ioutil"
    24  	"log"
    25  	"os"
    26  	"os/exec"
    27  	"path/filepath"
    28  	"runtime"
    29  	"strconv"
    30  	"strings"
    31  )
    32  
    33  var (
    34  	// cgoEnvVars is the list of all cgo environment variable
    35  	cgoEnvVars = []string{"CGO_CFLAGS", "CGO_CXXFLAGS", "CGO_CPPFLAGS", "CGO_LDFLAGS"}
    36  	// cgoAbsEnvFlags are all the flags that need absolute path in cgoEnvVars
    37  	cgoAbsEnvFlags = []string{"-I", "-L", "-isysroot", "-isystem", "-iquote", "-include", "-gcc-toolchain", "--sysroot"}
    38  )
    39  
    40  // env holds a small amount of Go environment and toolchain information
    41  // which is common to multiple builders. Most Bazel-agnostic build information
    42  // is collected in go/build.Default though.
    43  //
    44  // See ./README.rst for more information about handling arguments and
    45  // environment variables.
    46  type env struct {
    47  	// sdk is the path to the Go SDK, which contains tools for the host
    48  	// platform. This may be different than GOROOT.
    49  	sdk string
    50  
    51  	// installSuffix is the name of the directory below GOROOT/pkg that contains
    52  	// the .a files for the standard library we should build against.
    53  	// For example, linux_amd64_race.
    54  	installSuffix string
    55  
    56  	// verbose indicates whether subprocess command lines should be printed.
    57  	verbose bool
    58  
    59  	// workDirPath is a temporary work directory. It is created lazily.
    60  	workDirPath string
    61  
    62  	shouldPreserveWorkDir bool
    63  }
    64  
    65  // envFlags registers flags common to multiple builders and returns an env
    66  // configured with those flags.
    67  func envFlags(flags *flag.FlagSet) *env {
    68  	env := &env{}
    69  	flags.StringVar(&env.sdk, "sdk", "", "Path to the Go SDK.")
    70  	flags.Var(&tagFlag{}, "tags", "List of build tags considered true.")
    71  	flags.StringVar(&env.installSuffix, "installsuffix", "", "Standard library under GOROOT/pkg")
    72  	flags.BoolVar(&env.verbose, "v", false, "Whether subprocess command lines should be printed")
    73  	flags.BoolVar(&env.shouldPreserveWorkDir, "work", false, "if true, the temporary work directory will be preserved")
    74  	return env
    75  }
    76  
    77  // checkFlags checks whether env flags were set to valid values. checkFlags
    78  // should be called after parsing flags.
    79  func (e *env) checkFlags() error {
    80  	if e.sdk == "" {
    81  		return errors.New("-sdk was not set")
    82  	}
    83  	return nil
    84  }
    85  
    86  // workDir returns a path to a temporary work directory. The same directory
    87  // is returned on multiple calls. The caller is responsible for cleaning
    88  // up the work directory by calling cleanup.
    89  func (e *env) workDir() (path string, cleanup func(), err error) {
    90  	if e.workDirPath != "" {
    91  		return e.workDirPath, func() {}, nil
    92  	}
    93  	// Keep the stem "rules_go_work" in sync with reproducible_binary_test.go.
    94  	e.workDirPath, err = ioutil.TempDir("", "rules_go_work-")
    95  	if err != nil {
    96  		return "", func() {}, err
    97  	}
    98  	if e.verbose {
    99  		log.Printf("WORK=%s\n", e.workDirPath)
   100  	}
   101  	if e.shouldPreserveWorkDir {
   102  		cleanup = func() {}
   103  	} else {
   104  		cleanup = func() { os.RemoveAll(e.workDirPath) }
   105  	}
   106  	return e.workDirPath, cleanup, nil
   107  }
   108  
   109  // goTool returns a slice containing the path to an executable at
   110  // $GOROOT/pkg/$GOOS_$GOARCH/$tool and additional arguments.
   111  func (e *env) goTool(tool string, args ...string) []string {
   112  	platform := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
   113  	toolPath := filepath.Join(e.sdk, "pkg", "tool", platform, tool)
   114  	if runtime.GOOS == "windows" {
   115  		toolPath += ".exe"
   116  	}
   117  	return append([]string{toolPath}, args...)
   118  }
   119  
   120  // goCmd returns a slice containing the path to the go executable
   121  // and additional arguments.
   122  func (e *env) goCmd(cmd string, args ...string) []string {
   123  	exe := filepath.Join(e.sdk, "bin", "go")
   124  	if runtime.GOOS == "windows" {
   125  		exe += ".exe"
   126  	}
   127  	return append([]string{exe, cmd}, args...)
   128  }
   129  
   130  // runCommand executes a subprocess that inherits stdout, stderr, and the
   131  // environment from this process.
   132  func (e *env) runCommand(args []string) error {
   133  	cmd := exec.Command(args[0], args[1:]...)
   134  	// Redirecting stdout to stderr. This mirrors behavior in the go command:
   135  	// https://go.googlesource.com/go/+/refs/tags/go1.15.2/src/cmd/go/internal/work/exec.go#1958
   136  	cmd.Stdout = os.Stderr
   137  	cmd.Stderr = os.Stderr
   138  	return runAndLogCommand(cmd, e.verbose)
   139  }
   140  
   141  // runCommandToFile executes a subprocess and writes the output to the given
   142  // writer.
   143  func (e *env) runCommandToFile(w io.Writer, args []string) error {
   144  	cmd := exec.Command(args[0], args[1:]...)
   145  	cmd.Stdout = w
   146  	cmd.Stderr = os.Stderr
   147  	return runAndLogCommand(cmd, e.verbose)
   148  }
   149  
   150  func absEnv(envNameList []string, argList []string) error {
   151  	for _, envName := range envNameList {
   152  		splitedEnv := strings.Fields(os.Getenv(envName))
   153  		absArgs(splitedEnv, argList)
   154  		if err := os.Setenv(envName, strings.Join(splitedEnv, " ")); err != nil {
   155  			return err
   156  		}
   157  	}
   158  	return nil
   159  }
   160  
   161  func runAndLogCommand(cmd *exec.Cmd, verbose bool) error {
   162  	formattedCmd := formatCommand(cmd)
   163  	if verbose {
   164  		os.Stderr.WriteString(formattedCmd)
   165  	}
   166  	cleanup := passLongArgsInResponseFiles(cmd)
   167  	defer cleanup()
   168  	if err := cmd.Run(); err != nil {
   169  		return fmt.Errorf("error running the following subcommand: %v\n%s", err, formattedCmd)
   170  	}
   171  	return nil
   172  }
   173  
   174  // expandParamsFiles looks for arguments in args of the form
   175  // "-param=filename". When it finds these arguments it reads the file "filename"
   176  // and replaces the argument with its content.
   177  func expandParamsFiles(args []string) ([]string, error) {
   178  	var paramsIndices []int
   179  	for i, arg := range args {
   180  		if strings.HasPrefix(arg, "-param=") {
   181  			paramsIndices = append(paramsIndices, i)
   182  		}
   183  	}
   184  	if len(paramsIndices) == 0 {
   185  		return args, nil
   186  	}
   187  	var expandedArgs []string
   188  	last := 0
   189  	for _, pi := range paramsIndices {
   190  		expandedArgs = append(expandedArgs, args[last:pi]...)
   191  		last = pi + 1
   192  
   193  		fileName := args[pi][len("-param="):]
   194  		fileArgs, err := readParamsFile(fileName)
   195  		if err != nil {
   196  			return nil, err
   197  		}
   198  		expandedArgs = append(expandedArgs, fileArgs...)
   199  	}
   200  	expandedArgs = append(expandedArgs, args[last:]...)
   201  	return expandedArgs, nil
   202  }
   203  
   204  // readParamsFiles parses a Bazel params file in "shell" format. The file
   205  // should contain one argument per line. Arguments may be quoted with single
   206  // quotes. All characters within quoted strings are interpreted literally
   207  // including newlines and excepting single quotes. Characters outside quoted
   208  // strings may be escaped with a backslash.
   209  func readParamsFile(name string) ([]string, error) {
   210  	data, err := ioutil.ReadFile(name)
   211  	if err != nil {
   212  		return nil, err
   213  	}
   214  
   215  	var args []string
   216  	var arg []byte
   217  	quote := false
   218  	escape := false
   219  	for p := 0; p < len(data); p++ {
   220  		b := data[p]
   221  		switch {
   222  		case escape:
   223  			arg = append(arg, b)
   224  			escape = false
   225  
   226  		case b == '\'':
   227  			quote = !quote
   228  
   229  		case !quote && b == '\\':
   230  			escape = true
   231  
   232  		case !quote && b == '\n':
   233  			args = append(args, string(arg))
   234  			arg = arg[:0]
   235  
   236  		default:
   237  			arg = append(arg, b)
   238  		}
   239  	}
   240  	if quote {
   241  		return nil, fmt.Errorf("unterminated quote")
   242  	}
   243  	if escape {
   244  		return nil, fmt.Errorf("unterminated escape")
   245  	}
   246  	if len(arg) > 0 {
   247  		args = append(args, string(arg))
   248  	}
   249  	return args, nil
   250  }
   251  
   252  // writeParamsFile formats a list of arguments in Bazel's "shell" format and writes
   253  // it to a file.
   254  func writeParamsFile(path string, args []string) error {
   255  	buf := new(bytes.Buffer)
   256  	for _, arg := range args {
   257  		if !strings.ContainsAny(arg, "'\n\\") {
   258  			fmt.Fprintln(buf, arg)
   259  			continue
   260  		}
   261  		buf.WriteByte('\'')
   262  		for _, r := range arg {
   263  			if r == '\'' {
   264  				buf.WriteString(`'\''`)
   265  			} else {
   266  				buf.WriteRune(r)
   267  			}
   268  		}
   269  		buf.WriteString("'\n")
   270  	}
   271  	return ioutil.WriteFile(path, buf.Bytes(), 0666)
   272  }
   273  
   274  // splitArgs splits a list of command line arguments into two parts: arguments
   275  // that should be interpreted by the builder (before "--"), and arguments
   276  // that should be passed through to the underlying tool (after "--").
   277  func splitArgs(args []string) (builderArgs []string, toolArgs []string) {
   278  	for i, arg := range args {
   279  		if arg == "--" {
   280  			return args[:i], args[i+1:]
   281  		}
   282  	}
   283  	return args, nil
   284  }
   285  
   286  // abs returns the absolute representation of path. Some tools/APIs require
   287  // absolute paths to work correctly. Most notably, golang on Windows cannot
   288  // handle relative paths to files whose absolute path is > ~250 chars, while
   289  // it can handle absolute paths. See http://goo.gl/eqeWjm.
   290  //
   291  // Note that strings that begin with "__BAZEL_" are not absolutized. These are
   292  // used on macOS for paths that the compiler wrapper (wrapped_clang) is
   293  // supposed to know about.
   294  func abs(path string) string {
   295  	if strings.HasPrefix(path, "__BAZEL_") {
   296  		return path
   297  	}
   298  
   299  	if abs, err := filepath.Abs(path); err != nil {
   300  		return path
   301  	} else {
   302  		return abs
   303  	}
   304  }
   305  
   306  // absArgs applies abs to strings that appear in args. Only paths that are
   307  // part of options named by flags are modified.
   308  func absArgs(args []string, flags []string) {
   309  	absNext := false
   310  	for i := range args {
   311  		if absNext {
   312  			args[i] = abs(args[i])
   313  			absNext = false
   314  			continue
   315  		}
   316  		for _, f := range flags {
   317  			if !strings.HasPrefix(args[i], f) {
   318  				continue
   319  			}
   320  			possibleValue := args[i][len(f):]
   321  			if len(possibleValue) == 0 {
   322  				absNext = true
   323  				break
   324  			}
   325  			separator := ""
   326  			if possibleValue[0] == '=' {
   327  				possibleValue = possibleValue[1:]
   328  				separator = "="
   329  			}
   330  			args[i] = fmt.Sprintf("%s%s%s", f, separator, abs(possibleValue))
   331  			break
   332  		}
   333  	}
   334  }
   335  
   336  // formatCommand writes cmd to w in a format where it can be pasted into a
   337  // shell. Spaces in environment variables and arguments are escaped as needed.
   338  func formatCommand(cmd *exec.Cmd) string {
   339  	quoteIfNeeded := func(s string) string {
   340  		if strings.IndexByte(s, ' ') < 0 {
   341  			return s
   342  		}
   343  		return strconv.Quote(s)
   344  	}
   345  	quoteEnvIfNeeded := func(s string) string {
   346  		eq := strings.IndexByte(s, '=')
   347  		if eq < 0 {
   348  			return s
   349  		}
   350  		key, value := s[:eq], s[eq+1:]
   351  		if strings.IndexByte(value, ' ') < 0 {
   352  			return s
   353  		}
   354  		return fmt.Sprintf("%s=%s", key, strconv.Quote(value))
   355  	}
   356  	var w bytes.Buffer
   357  	environ := cmd.Env
   358  	if environ == nil {
   359  		environ = os.Environ()
   360  	}
   361  	for _, e := range environ {
   362  		fmt.Fprintf(&w, "%s \\\n", quoteEnvIfNeeded(e))
   363  	}
   364  
   365  	sep := ""
   366  	for _, arg := range cmd.Args {
   367  		fmt.Fprintf(&w, "%s%s", sep, quoteIfNeeded(arg))
   368  		sep = " "
   369  	}
   370  	fmt.Fprint(&w, "\n")
   371  	return w.String()
   372  }
   373  
   374  // passLongArgsInResponseFiles modifies cmd such that, for
   375  // certain programs, long arguments are passed in "response files", a
   376  // file on disk with the arguments, with one arg per line. An actual
   377  // argument starting with '@' means that the rest of the argument is
   378  // a filename of arguments to expand.
   379  //
   380  // See https://github.com/golang/go/issues/18468 (Windows) and
   381  // https://github.com/golang/go/issues/37768 (Darwin).
   382  func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) {
   383  	cleanup = func() {} // no cleanup by default
   384  	var argLen int
   385  	for _, arg := range cmd.Args {
   386  		argLen += len(arg)
   387  	}
   388  	// If we're not approaching 32KB of args, just pass args normally.
   389  	// (use 30KB instead to be conservative; not sure how accounting is done)
   390  	if !useResponseFile(cmd.Path, argLen) {
   391  		return
   392  	}
   393  	tf, err := ioutil.TempFile("", "args")
   394  	if err != nil {
   395  		log.Fatalf("error writing long arguments to response file: %v", err)
   396  	}
   397  	cleanup = func() { os.Remove(tf.Name()) }
   398  	var buf bytes.Buffer
   399  	for _, arg := range cmd.Args[1:] {
   400  		fmt.Fprintf(&buf, "%s\n", arg)
   401  	}
   402  	if _, err := tf.Write(buf.Bytes()); err != nil {
   403  		tf.Close()
   404  		cleanup()
   405  		log.Fatalf("error writing long arguments to response file: %v", err)
   406  	}
   407  	if err := tf.Close(); err != nil {
   408  		cleanup()
   409  		log.Fatalf("error writing long arguments to response file: %v", err)
   410  	}
   411  	cmd.Args = []string{cmd.Args[0], "@" + tf.Name()}
   412  	return cleanup
   413  }
   414  
   415  func useResponseFile(path string, argLen int) bool {
   416  	// Unless the program uses objabi.Flagparse, which understands
   417  	// response files, don't use response files.
   418  	// TODO: do we need more commands? asm? cgo? For now, no.
   419  	prog := strings.TrimSuffix(filepath.Base(path), ".exe")
   420  	switch prog {
   421  	case "compile", "link":
   422  	default:
   423  		return false
   424  	}
   425  	// Windows has a limit of 32 KB arguments. To be conservative and not
   426  	// worry about whether that includes spaces or not, just use 30 KB.
   427  	// Darwin's limit is less clear. The OS claims 256KB, but we've seen
   428  	// failures with arglen as small as 50KB.
   429  	if argLen > (30 << 10) {
   430  		return true
   431  	}
   432  	return false
   433  }