github.com/joey-fossa/fossa-cli@v0.7.34-0.20190708193710-569f1e8679f0/buildtools/buck/buck.go (about) 1 package buck 2 3 import ( 4 "runtime" 5 "strings" 6 "sync" 7 8 "github.com/apex/log" 9 "github.com/remeh/sizedwaitgroup" 10 11 "github.com/fossas/fossa-cli/api/fossa" 12 "github.com/fossas/fossa-cli/errors" 13 "github.com/fossas/fossa-cli/graph" 14 "github.com/fossas/fossa-cli/pkg" 15 ) 16 17 // AuditOutput models the output from the `buck audit` command. 18 type AuditOutput struct { 19 OutputMapping map[string][]string 20 } 21 22 // Buck defines an interface for all Buck tool implementations. 23 type Buck interface { 24 Deps(bool) (graph.Deps, error) 25 } 26 27 // Cmd implements Buck and defines how to retrieve buck output. 28 type Setup struct { 29 Target string 30 Cmd func(string, ...string) (string, error) 31 } 32 33 // New creates a new Buck instance that calls the buck build tool directly. 34 func New(target, binary string) Buck { 35 return Setup{ 36 Target: target, 37 Cmd: NewCmd(binary), 38 } 39 } 40 41 // Deps finds and uploads the dependencies of a Buck target using the supplied command and 42 // returns the dependency graph. 43 func (b Setup) Deps(upload bool) (graph.Deps, error) { 44 locatorMap, err := uploadDeps(b, upload) 45 if err != nil { 46 return graph.Deps{}, nil 47 } 48 49 transDeps, err := depGraph(b, locatorMap) 50 if err != nil { 51 return graph.Deps{}, nil 52 } 53 54 imports, err := directDeps(b, locatorMap) 55 if err != nil { 56 return graph.Deps{}, nil 57 } 58 59 return graph.Deps{ 60 Direct: imports, 61 Transitive: transDeps, 62 }, nil 63 } 64 65 func uploadDeps(b Setup, upload bool) (map[string]fossa.Locator, error) { 66 locatorMap := make(map[string]fossa.Locator) 67 depList := AuditOutput{} 68 var err error 69 if targetIsWildcard(b.Target) { 70 depList, err = allSubprojectDeps(b) 71 if err != nil { 72 return locatorMap, err 73 } 74 } else { 75 depList, err = cmdAudit(b.Cmd, "input", b.Target) 76 if err != nil { 77 return locatorMap, err 78 } 79 } 80 81 rootDir, err := b.Cmd("root") 82 if err != nil { 83 return locatorMap, errors.Wrap(err, "Cannot get buck root") 84 } 85 rootDir = strings.TrimSpace(rootDir) 86 87 wg := sizedwaitgroup.New(runtime.GOMAXPROCS(0)) 88 lock := sync.RWMutex{} 89 // Upload individual dependencies and keep a reference to the generated locators. 90 for d, f := range depList.OutputMapping { 91 wg.Add() 92 go func(dep string, files []string) { 93 defer wg.Done() 94 locator, err := fossa.UploadTarballDependencyFiles(rootDir, files, sanitizeBuckTarget(dep), upload) 95 if err != nil { 96 log.Warnf("Cannot upload files for %v: %s", dep, err) 97 } 98 99 lock.Lock() 100 locatorMap[dep] = locator 101 lock.Unlock() 102 }(d, f) 103 } 104 wg.Wait() 105 106 return locatorMap, nil 107 } 108 109 // allSubprojectDeps takes a target such as //third-party/... and determines 110 // the full list of dependencies specified by its targets. 111 func allSubprojectDeps(b Setup) (AuditOutput, error) { 112 allInputs := AuditOutput{OutputMapping: make(map[string][]string)} 113 114 targets, err := cmdTargets(b.Cmd, b.Target) 115 if err != nil { 116 return allInputs, err 117 } 118 119 targetInputLists := make(chan AuditOutput, len(targets)) 120 wg := sizedwaitgroup.New(runtime.GOMAXPROCS(0)) 121 for _, target := range targets { 122 wg.Add() 123 go func(t string) { 124 defer wg.Done() 125 inputList, err := cmdAudit(b.Cmd, "input", t) 126 targetInputLists <- inputList 127 if err != nil { 128 log.Warnf("Cannot retrieve inputs for %v: %s", t, err) 129 } 130 }(target) 131 } 132 wg.Wait() 133 134 close(targetInputLists) 135 for targetInputList := range targetInputLists { 136 for name, values := range targetInputList.OutputMapping { 137 _, present := allInputs.OutputMapping[name] 138 if !present { 139 allInputs.OutputMapping[name] = values 140 } 141 } 142 } 143 144 return allInputs, err 145 } 146 147 func depGraph(b Setup, locatorMap map[string]fossa.Locator) (map[pkg.ID]pkg.Package, error) { 148 transitiveDeps := make(map[pkg.ID]pkg.Package) 149 150 var allDependencies []string 151 if targetIsWildcard(b.Target) { 152 // We do not need to check for the transitive graph because we assume the given 153 // target returns all build rules underneath it, flattening the dependency graph. 154 var err error 155 allDependencies, err = cmdTargets(b.Cmd, b.Target) 156 if err != nil { 157 return transitiveDeps, err 158 } 159 } else { 160 depList, err := cmdAudit(b.Cmd, "dependencies", b.Target, "--transitive") 161 if err != nil { 162 return transitiveDeps, err 163 } 164 allDependencies = depList.OutputMapping[b.Target] 165 } 166 167 mapLock := sync.RWMutex{} 168 wg := sizedwaitgroup.New(runtime.GOMAXPROCS(0)) 169 for _, d := range allDependencies { 170 wg.Add() 171 go func(dep string) { 172 defer wg.Done() 173 transDeps, err := cmdAudit(b.Cmd, "dependencies", dep) 174 if err != nil { 175 log.Warnf("Cannot retrieve dependency list for %v: %s", dep, err) 176 } 177 178 var imports []pkg.Import 179 for _, transDep := range transDeps.OutputMapping[dep] { 180 imports = append(imports, pkg.Import{ 181 Target: transDep, 182 Resolved: pkg.ID{ 183 Type: pkg.Raw, 184 Name: locatorMap[transDep].Project, 185 Revision: locatorMap[transDep].Revision, 186 }, 187 }) 188 } 189 id := pkg.ID{ 190 Type: pkg.Raw, 191 Name: locatorMap[dep].Project, 192 Revision: locatorMap[dep].Revision, 193 } 194 195 mapLock.Lock() 196 transitiveDeps[id] = pkg.Package{ 197 ID: id, 198 Imports: imports, 199 } 200 mapLock.Unlock() 201 }(d) 202 } 203 wg.Wait() 204 205 return transitiveDeps, nil 206 } 207 208 func directDeps(b Setup, locatorMap map[string]fossa.Locator) ([]pkg.Import, error) { 209 imports := []pkg.Import{} 210 var directDeps []string 211 if targetIsWildcard(b.Target) { 212 var err error 213 directDeps, err = cmdTargets(b.Cmd, b.Target) 214 if err != nil { 215 return imports, err 216 } 217 } else { 218 deps, err := cmdAudit(b.Cmd, "dependencies", b.Target) 219 if err != nil { 220 return imports, err 221 } 222 directDeps = deps.OutputMapping[b.Target] 223 } 224 225 for _, dep := range directDeps { 226 imports = append(imports, pkg.Import{ 227 Target: dep, 228 Resolved: pkg.ID{ 229 Type: pkg.Raw, 230 Name: locatorMap[dep].Project, 231 Revision: locatorMap[dep].Revision, 232 }, 233 }) 234 } 235 236 return imports, nil 237 } 238 239 // targetIsWildcard checks if the specified target specifies subprojects. 240 func targetIsWildcard(target string) bool { 241 return strings.HasSuffix(target, "...") 242 } 243 244 // Change buildtarget `//src/fossa/buildtools:buck` into `buildtools-buck` 245 // to appease core and simplify naming. 246 func sanitizeBuckTarget(target string) string { 247 depSplit := strings.Split(target, "/") 248 return strings.Replace(depSplit[len(depSplit)-1], ":", "-", 1) 249 }