github.com/distbuild/reclient@v0.0.0-20240401075343-3de72e395564/internal/pkg/inputprocessor/toolchain/inputfiles.go (about) 1 // Copyright 2023 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package toolchain 16 17 import ( 18 "bufio" 19 "os" 20 "path/filepath" 21 "strings" 22 23 "github.com/bazelbuild/reclient/internal/pkg/pathtranslator" 24 25 "github.com/bazelbuild/remote-apis-sdks/go/pkg/cache" 26 "github.com/bazelbuild/remote-apis-sdks/go/pkg/command" 27 "github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata" 28 log "github.com/golang/glog" 29 ) 30 31 var ( 32 toolchainFilesCache cache.SingleFlight 33 ) 34 35 const ( 36 remoteToolchainInputs = "remote_toolchain_inputs" 37 ) 38 39 func doesExist(fmc filemetadata.Cache, fp string) (bool, error) { 40 if fmc != nil { 41 md := fmc.Get(fp) 42 if md.Err == nil { 43 return true, nil 44 } 45 if e, ok := md.Err.(*filemetadata.FileError); ok && e.IsNotFound { 46 return false, nil 47 } 48 return false, md.Err 49 } 50 _, err := os.Stat(fp) 51 if err == nil { 52 return true, nil 53 } 54 if os.IsNotExist(err) { 55 return false, nil 56 } 57 return false, err 58 } 59 60 func loadOrStoreToolchainCache(rp string, baseDir string, execRoot string, fmc filemetadata.Cache) (interface{}, error) { 61 rf, err := os.Open(rp) 62 if err != nil { 63 return nil, nil 64 } 65 defer rf.Close() 66 67 scanner := bufio.NewScanner(rf) 68 fileList := []string{} 69 for scanner.Scan() { 70 line := scanner.Text() 71 line = strings.TrimLeft(line, " \t") 72 if strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") { 73 continue 74 } 75 fp := filepath.Join(baseDir, line) 76 exists, err := doesExist(fmc, fp) 77 if err != nil { 78 log.Warningf("Error while checking whether %q exists. err: %v", err) 79 } 80 if exists { 81 fileList = append(fileList, pathtranslator.RelToExecRoot(execRoot, "", fp)) 82 } else { 83 log.V(2).Infof("File %q defined in %q does not exist", fp, rp) 84 } 85 } 86 return fileList, nil 87 } 88 89 // processToolchainInputFiles uses the "<toolchain>_remote_toolchain_inputs" file that is 90 // checked-in alongside the executable used to run the command, to determine the list 91 // of files that constitute the toolchain inputs. If that file doesn't exist, it looks 92 // for "remote_toolchain_inputs". Contents of either file should be clean paths. Paths 93 // in the toolchains list should be relative to the exec root. 94 func (p *InputProcessor) processToolchainInputFiles(execRoot string, toolchains []string, fmc filemetadata.Cache) (*command.InputSpec, error) { 95 toolchainInputFiles := []string{} 96 seenFiles := make(map[string]bool) 97 for _, tc := range toolchains { 98 if tc == "" { 99 continue 100 } 101 tcRel := pathtranslator.RelToExecRoot(execRoot, "", tc) 102 toolchainInputFiles = append(toolchainInputFiles, tcRel) 103 104 baseDir := filepath.Join(execRoot, filepath.Dir(tcRel)) 105 rp := filepath.Join(baseDir, filepath.Base(tc)+"_"+remoteToolchainInputs) 106 if _, ok := seenFiles[rp]; ok { 107 continue 108 } 109 110 cache, err := toolchainFilesCache.LoadOrStore(rp, func() (interface{}, error) { 111 return loadOrStoreToolchainCache(rp, baseDir, execRoot, fmc) 112 }) 113 if err == nil && cache == nil { 114 rp = filepath.Join(baseDir, remoteToolchainInputs) 115 if _, ok := seenFiles[rp]; !ok { 116 cache, err = toolchainFilesCache.LoadOrStore(rp, func() (interface{}, error) { 117 return loadOrStoreToolchainCache(rp, baseDir, execRoot, fmc) 118 }) 119 } 120 } 121 seenFiles[rp] = true 122 123 if err != nil { 124 log.Warningf("Can't find toolchain inputs file %v for %v, this may be expected: %v", rp, tc, err) 125 } 126 if cache != nil { 127 toolchainInputFiles = append(toolchainInputFiles, cache.([]string)...) 128 } 129 } 130 return &command.InputSpec{Inputs: toolchainInputFiles}, nil 131 }