github.com/distbuild/reclient@v0.0.0-20240401075343-3de72e395564/internal/pkg/inputprocessor/toolchain/inputfiles.go (about)

     1  // Copyright 2023 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package toolchain
    16  
    17  import (
    18  	"bufio"
    19  	"os"
    20  	"path/filepath"
    21  	"strings"
    22  
    23  	"github.com/bazelbuild/reclient/internal/pkg/pathtranslator"
    24  
    25  	"github.com/bazelbuild/remote-apis-sdks/go/pkg/cache"
    26  	"github.com/bazelbuild/remote-apis-sdks/go/pkg/command"
    27  	"github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata"
    28  	log "github.com/golang/glog"
    29  )
    30  
    31  var (
    32  	toolchainFilesCache cache.SingleFlight
    33  )
    34  
    35  const (
    36  	remoteToolchainInputs = "remote_toolchain_inputs"
    37  )
    38  
    39  func doesExist(fmc filemetadata.Cache, fp string) (bool, error) {
    40  	if fmc != nil {
    41  		md := fmc.Get(fp)
    42  		if md.Err == nil {
    43  			return true, nil
    44  		}
    45  		if e, ok := md.Err.(*filemetadata.FileError); ok && e.IsNotFound {
    46  			return false, nil
    47  		}
    48  		return false, md.Err
    49  	}
    50  	_, err := os.Stat(fp)
    51  	if err == nil {
    52  		return true, nil
    53  	}
    54  	if os.IsNotExist(err) {
    55  		return false, nil
    56  	}
    57  	return false, err
    58  }
    59  
    60  func loadOrStoreToolchainCache(rp string, baseDir string, execRoot string, fmc filemetadata.Cache) (interface{}, error) {
    61  	rf, err := os.Open(rp)
    62  	if err != nil {
    63  		return nil, nil
    64  	}
    65  	defer rf.Close()
    66  
    67  	scanner := bufio.NewScanner(rf)
    68  	fileList := []string{}
    69  	for scanner.Scan() {
    70  		line := scanner.Text()
    71  		line = strings.TrimLeft(line, " \t")
    72  		if strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") {
    73  			continue
    74  		}
    75  		fp := filepath.Join(baseDir, line)
    76  		exists, err := doesExist(fmc, fp)
    77  		if err != nil {
    78  			log.Warningf("Error while checking whether %q exists. err: %v", err)
    79  		}
    80  		if exists {
    81  			fileList = append(fileList, pathtranslator.RelToExecRoot(execRoot, "", fp))
    82  		} else {
    83  			log.V(2).Infof("File %q defined in %q does not exist", fp, rp)
    84  		}
    85  	}
    86  	return fileList, nil
    87  }
    88  
    89  // processToolchainInputFiles uses the "<toolchain>_remote_toolchain_inputs" file that is
    90  // checked-in alongside the executable used to run the command, to determine the list
    91  // of files that constitute the toolchain inputs. If that file doesn't exist, it looks
    92  // for "remote_toolchain_inputs". Contents of either file should be clean paths. Paths
    93  // in the toolchains list should be relative to the exec root.
    94  func (p *InputProcessor) processToolchainInputFiles(execRoot string, toolchains []string, fmc filemetadata.Cache) (*command.InputSpec, error) {
    95  	toolchainInputFiles := []string{}
    96  	seenFiles := make(map[string]bool)
    97  	for _, tc := range toolchains {
    98  		if tc == "" {
    99  			continue
   100  		}
   101  		tcRel := pathtranslator.RelToExecRoot(execRoot, "", tc)
   102  		toolchainInputFiles = append(toolchainInputFiles, tcRel)
   103  
   104  		baseDir := filepath.Join(execRoot, filepath.Dir(tcRel))
   105  		rp := filepath.Join(baseDir, filepath.Base(tc)+"_"+remoteToolchainInputs)
   106  		if _, ok := seenFiles[rp]; ok {
   107  			continue
   108  		}
   109  
   110  		cache, err := toolchainFilesCache.LoadOrStore(rp, func() (interface{}, error) {
   111  			return loadOrStoreToolchainCache(rp, baseDir, execRoot, fmc)
   112  		})
   113  		if err == nil && cache == nil {
   114  			rp = filepath.Join(baseDir, remoteToolchainInputs)
   115  			if _, ok := seenFiles[rp]; !ok {
   116  				cache, err = toolchainFilesCache.LoadOrStore(rp, func() (interface{}, error) {
   117  					return loadOrStoreToolchainCache(rp, baseDir, execRoot, fmc)
   118  				})
   119  			}
   120  		}
   121  		seenFiles[rp] = true
   122  
   123  		if err != nil {
   124  			log.Warningf("Can't find toolchain inputs file %v for %v, this may be expected: %v", rp, tc, err)
   125  		}
   126  		if cache != nil {
   127  			toolchainInputFiles = append(toolchainInputFiles, cache.([]string)...)
   128  		}
   129  	}
   130  	return &command.InputSpec{Inputs: toolchainInputFiles}, nil
   131  }