github.com/joey-fossa/fossa-cli@v0.7.34-0.20190708193710-569f1e8679f0/buildtools/buck/buck.go (about)

     1  package buck
     2  
     3  import (
     4  	"runtime"
     5  	"strings"
     6  	"sync"
     7  
     8  	"github.com/apex/log"
     9  	"github.com/remeh/sizedwaitgroup"
    10  
    11  	"github.com/fossas/fossa-cli/api/fossa"
    12  	"github.com/fossas/fossa-cli/errors"
    13  	"github.com/fossas/fossa-cli/graph"
    14  	"github.com/fossas/fossa-cli/pkg"
    15  )
    16  
    17  // AuditOutput models the output from the `buck audit` command.
    18  type AuditOutput struct {
    19  	OutputMapping map[string][]string
    20  }
    21  
    22  // Buck defines an interface for all Buck tool implementations.
    23  type Buck interface {
    24  	Deps(bool) (graph.Deps, error)
    25  }
    26  
    27  // Cmd implements Buck and defines how to retrieve buck output.
    28  type Setup struct {
    29  	Target string
    30  	Cmd    func(string, ...string) (string, error)
    31  }
    32  
    33  // New creates a new Buck instance that calls the buck build tool directly.
    34  func New(target, binary string) Buck {
    35  	return Setup{
    36  		Target: target,
    37  		Cmd:    NewCmd(binary),
    38  	}
    39  }
    40  
    41  // Deps finds and uploads the dependencies of a Buck target using the supplied command and
    42  // returns the dependency graph.
    43  func (b Setup) Deps(upload bool) (graph.Deps, error) {
    44  	locatorMap, err := uploadDeps(b, upload)
    45  	if err != nil {
    46  		return graph.Deps{}, nil
    47  	}
    48  
    49  	transDeps, err := depGraph(b, locatorMap)
    50  	if err != nil {
    51  		return graph.Deps{}, nil
    52  	}
    53  
    54  	imports, err := directDeps(b, locatorMap)
    55  	if err != nil {
    56  		return graph.Deps{}, nil
    57  	}
    58  
    59  	return graph.Deps{
    60  		Direct:     imports,
    61  		Transitive: transDeps,
    62  	}, nil
    63  }
    64  
    65  func uploadDeps(b Setup, upload bool) (map[string]fossa.Locator, error) {
    66  	locatorMap := make(map[string]fossa.Locator)
    67  	depList := AuditOutput{}
    68  	var err error
    69  	if targetIsWildcard(b.Target) {
    70  		depList, err = allSubprojectDeps(b)
    71  		if err != nil {
    72  			return locatorMap, err
    73  		}
    74  	} else {
    75  		depList, err = cmdAudit(b.Cmd, "input", b.Target)
    76  		if err != nil {
    77  			return locatorMap, err
    78  		}
    79  	}
    80  
    81  	rootDir, err := b.Cmd("root")
    82  	if err != nil {
    83  		return locatorMap, errors.Wrap(err, "Cannot get buck root")
    84  	}
    85  	rootDir = strings.TrimSpace(rootDir)
    86  
    87  	wg := sizedwaitgroup.New(runtime.GOMAXPROCS(0))
    88  	lock := sync.RWMutex{}
    89  	// Upload individual dependencies and keep a reference to the generated locators.
    90  	for d, f := range depList.OutputMapping {
    91  		wg.Add()
    92  		go func(dep string, files []string) {
    93  			defer wg.Done()
    94  			locator, err := fossa.UploadTarballDependencyFiles(rootDir, files, sanitizeBuckTarget(dep), upload)
    95  			if err != nil {
    96  				log.Warnf("Cannot upload files for %v: %s", dep, err)
    97  			}
    98  
    99  			lock.Lock()
   100  			locatorMap[dep] = locator
   101  			lock.Unlock()
   102  		}(d, f)
   103  	}
   104  	wg.Wait()
   105  
   106  	return locatorMap, nil
   107  }
   108  
   109  // allSubprojectDeps takes a target such as //third-party/... and determines
   110  // the full list of dependencies specified by its targets.
   111  func allSubprojectDeps(b Setup) (AuditOutput, error) {
   112  	allInputs := AuditOutput{OutputMapping: make(map[string][]string)}
   113  
   114  	targets, err := cmdTargets(b.Cmd, b.Target)
   115  	if err != nil {
   116  		return allInputs, err
   117  	}
   118  
   119  	targetInputLists := make(chan AuditOutput, len(targets))
   120  	wg := sizedwaitgroup.New(runtime.GOMAXPROCS(0))
   121  	for _, target := range targets {
   122  		wg.Add()
   123  		go func(t string) {
   124  			defer wg.Done()
   125  			inputList, err := cmdAudit(b.Cmd, "input", t)
   126  			targetInputLists <- inputList
   127  			if err != nil {
   128  				log.Warnf("Cannot retrieve inputs for %v: %s", t, err)
   129  			}
   130  		}(target)
   131  	}
   132  	wg.Wait()
   133  
   134  	close(targetInputLists)
   135  	for targetInputList := range targetInputLists {
   136  		for name, values := range targetInputList.OutputMapping {
   137  			_, present := allInputs.OutputMapping[name]
   138  			if !present {
   139  				allInputs.OutputMapping[name] = values
   140  			}
   141  		}
   142  	}
   143  
   144  	return allInputs, err
   145  }
   146  
   147  func depGraph(b Setup, locatorMap map[string]fossa.Locator) (map[pkg.ID]pkg.Package, error) {
   148  	transitiveDeps := make(map[pkg.ID]pkg.Package)
   149  
   150  	var allDependencies []string
   151  	if targetIsWildcard(b.Target) {
   152  		// We do not need to check for the transitive graph because we assume the given
   153  		// target returns all build rules underneath it, flattening the dependency graph.
   154  		var err error
   155  		allDependencies, err = cmdTargets(b.Cmd, b.Target)
   156  		if err != nil {
   157  			return transitiveDeps, err
   158  		}
   159  	} else {
   160  		depList, err := cmdAudit(b.Cmd, "dependencies", b.Target, "--transitive")
   161  		if err != nil {
   162  			return transitiveDeps, err
   163  		}
   164  		allDependencies = depList.OutputMapping[b.Target]
   165  	}
   166  
   167  	mapLock := sync.RWMutex{}
   168  	wg := sizedwaitgroup.New(runtime.GOMAXPROCS(0))
   169  	for _, d := range allDependencies {
   170  		wg.Add()
   171  		go func(dep string) {
   172  			defer wg.Done()
   173  			transDeps, err := cmdAudit(b.Cmd, "dependencies", dep)
   174  			if err != nil {
   175  				log.Warnf("Cannot retrieve dependency list for %v: %s", dep, err)
   176  			}
   177  
   178  			var imports []pkg.Import
   179  			for _, transDep := range transDeps.OutputMapping[dep] {
   180  				imports = append(imports, pkg.Import{
   181  					Target: transDep,
   182  					Resolved: pkg.ID{
   183  						Type:     pkg.Raw,
   184  						Name:     locatorMap[transDep].Project,
   185  						Revision: locatorMap[transDep].Revision,
   186  					},
   187  				})
   188  			}
   189  			id := pkg.ID{
   190  				Type:     pkg.Raw,
   191  				Name:     locatorMap[dep].Project,
   192  				Revision: locatorMap[dep].Revision,
   193  			}
   194  
   195  			mapLock.Lock()
   196  			transitiveDeps[id] = pkg.Package{
   197  				ID:      id,
   198  				Imports: imports,
   199  			}
   200  			mapLock.Unlock()
   201  		}(d)
   202  	}
   203  	wg.Wait()
   204  
   205  	return transitiveDeps, nil
   206  }
   207  
   208  func directDeps(b Setup, locatorMap map[string]fossa.Locator) ([]pkg.Import, error) {
   209  	imports := []pkg.Import{}
   210  	var directDeps []string
   211  	if targetIsWildcard(b.Target) {
   212  		var err error
   213  		directDeps, err = cmdTargets(b.Cmd, b.Target)
   214  		if err != nil {
   215  			return imports, err
   216  		}
   217  	} else {
   218  		deps, err := cmdAudit(b.Cmd, "dependencies", b.Target)
   219  		if err != nil {
   220  			return imports, err
   221  		}
   222  		directDeps = deps.OutputMapping[b.Target]
   223  	}
   224  
   225  	for _, dep := range directDeps {
   226  		imports = append(imports, pkg.Import{
   227  			Target: dep,
   228  			Resolved: pkg.ID{
   229  				Type:     pkg.Raw,
   230  				Name:     locatorMap[dep].Project,
   231  				Revision: locatorMap[dep].Revision,
   232  			},
   233  		})
   234  	}
   235  
   236  	return imports, nil
   237  }
   238  
   239  // targetIsWildcard checks if the specified target specifies subprojects.
   240  func targetIsWildcard(target string) bool {
   241  	return strings.HasSuffix(target, "...")
   242  }
   243  
   244  // Change buildtarget `//src/fossa/buildtools:buck` into `buildtools-buck`
   245  // to appease core and simplify naming.
   246  func sanitizeBuckTarget(target string) string {
   247  	depSplit := strings.Split(target, "/")
   248  	return strings.Replace(depSplit[len(depSplit)-1], ":", "-", 1)
   249  }