github.com/tiagovtristao/plz@v13.4.0+incompatible/src/gc/gc.go (about)

     1  // +build !bootstrap
     2  
     3  // Package gc implements "garbage collection" logic for Please, which is an attempt to identify
     4  // targets in the repo that are no longer needed.
     5  // The definition of "needed" is a bit unclear; we define it as non-test binaries, but the
     6  // command accepts an argument to add extra ones just in case (for example, if you have a repo which
     7  // is primarily a library, you might have to tell it that).
     8  package gc
     9  
    10  import (
    11  	"bytes"
    12  	"fmt"
    13  	"io/ioutil"
    14  	"os"
    15  	"sort"
    16  	"strings"
    17  
    18  	"github.com/Songmu/prompter"
    19  	"gopkg.in/op/go-logging.v1"
    20  
    21  	"github.com/thought-machine/please/src/core"
    22  	"github.com/thought-machine/please/src/parse/asp"
    23  )
    24  
    25  var log = logging.MustGetLogger("gc")
    26  
    27  type targetMap map[*core.BuildTarget]bool
    28  
    29  // GarbageCollect initiates the garbage collection logic.
    30  func GarbageCollect(state *core.BuildState, filter, targets, keepTargets []core.BuildLabel, keepLabels []string, conservative, targetsOnly, srcsOnly, noPrompt, dryRun, git bool) {
    31  	if targets, srcs := targetsToRemove(state.Graph, filter, targets, keepTargets, keepLabels, conservative); len(targets) > 0 {
    32  		if !srcsOnly {
    33  			fmt.Fprintf(os.Stderr, "Targets to remove (total %d of %d):\n", len(targets), state.Graph.Len())
    34  			for _, target := range targets {
    35  				fmt.Printf("  %s\n", target)
    36  			}
    37  		}
    38  		if !targetsOnly && len(srcs) > 0 {
    39  			fmt.Fprintf(os.Stderr, "Corresponding source files to remove:\n")
    40  			for _, src := range srcs {
    41  				fmt.Printf("  %s\n", src)
    42  			}
    43  		}
    44  		if dryRun {
    45  			return
    46  		} else if !noPrompt && !prompter.YN("Remove these targets / files?", false) {
    47  			os.Exit(1)
    48  		}
    49  		if !srcsOnly {
    50  			if err := removeTargets(state, targets); err != nil {
    51  				log.Fatalf("%s\n", err)
    52  			}
    53  		}
    54  		if !targetsOnly {
    55  			if git {
    56  				log.Notice("Running git rm %s\n", strings.Join(srcs, " "))
    57  				srcs = append([]string{"rm", "-q"}, srcs...)
    58  				cmd := core.ExecCommand("git", srcs...)
    59  				cmd.Stdout = os.Stdout
    60  				cmd.Stderr = os.Stderr
    61  				if err := cmd.Run(); err != nil {
    62  					log.Fatalf("git rm failed: %s\n", err)
    63  				}
    64  			} else {
    65  				for _, src := range srcs {
    66  					log.Notice("Deleting %s...\n", src)
    67  					if err := os.Remove(src); err != nil {
    68  						log.Fatalf("Failed to remove %s: %s\n", src, err)
    69  					}
    70  				}
    71  			}
    72  		}
    73  		fmt.Fprintf(os.Stderr, "Garbage collected!\n")
    74  	} else {
    75  		fmt.Fprintf(os.Stderr, "Nothing to remove\n")
    76  	}
    77  }
    78  
    79  // targetsToRemove finds the set of targets that are no longer needed and any extraneous sources.
    80  func targetsToRemove(graph *core.BuildGraph, filter, targets, targetsToKeep []core.BuildLabel, keepLabels []string, includeTests bool) (core.BuildLabels, []string) {
    81  	keepTargets := targetMap{}
    82  	for _, target := range graph.AllTargets() {
    83  		if (target.IsBinary && (!target.IsTest || includeTests)) || target.HasAnyLabel(keepLabels) || anyInclude(targetsToKeep, target.Label) || target.Label.Subrepo != "" {
    84  			log.Debug("GC root: %s", target.Label)
    85  			addTarget(graph, keepTargets, target)
    86  		}
    87  	}
    88  	// Any registered subincludes also count.
    89  	for _, pkg := range graph.PackageMap() {
    90  		for _, subinclude := range pkg.Subincludes {
    91  			log.Debug("GC root: %s", subinclude)
    92  			addTarget(graph, keepTargets, graph.TargetOrDie(subinclude))
    93  		}
    94  	}
    95  	log.Notice("%d targets to keep from initial scan", len(keepTargets))
    96  	for _, target := range targets {
    97  		if target.IsAllSubpackages() {
    98  			// For slightly awkward reasons these can't be handled outside :(
    99  			for _, pkg := range graph.PackageMap() {
   100  				if pkg.IsIncludedIn(target) {
   101  					for _, target := range pkg.AllTargets() {
   102  						log.Debug("GC root: %s", target.Label)
   103  						addTarget(graph, keepTargets, target)
   104  					}
   105  				}
   106  			}
   107  		} else {
   108  			addTarget(graph, keepTargets, graph.Target(target))
   109  		}
   110  	}
   111  	log.Notice("%d targets to keep after configured GC roots", len(keepTargets))
   112  	if !includeTests {
   113  		// This is a bit complex - need to identify any tests that are tests "on" the set of things
   114  		// we've already decided to keep.
   115  		for _, target := range graph.AllTargets() {
   116  			if target.IsTest {
   117  				for _, dep := range publicDependencies(graph, target) {
   118  					if keepTargets[dep] && !dep.TestOnly {
   119  						log.Debug("Keeping test %s on %s", target.Label, dep.Label)
   120  						addTarget(graph, keepTargets, target)
   121  					} else if dep.TestOnly {
   122  						log.Debug("Keeping test-only target %s", dep.Label)
   123  						addTarget(graph, keepTargets, dep)
   124  					}
   125  				}
   126  			}
   127  		}
   128  		log.Notice("%d targets to keep after exploring tests", len(keepTargets))
   129  	}
   130  	// Now build the set of sources that we'll keep. This is important because other targets that
   131  	// we're not deleting could still use the sources of the targets that we are.
   132  	keepSrcs := map[string]bool{}
   133  	for target := range keepTargets {
   134  		for _, src := range target.AllLocalSources() {
   135  			keepSrcs[src] = true
   136  		}
   137  	}
   138  	ret := make(core.BuildLabels, 0, len(keepTargets))
   139  	retSrcs := []string{}
   140  	for _, target := range graph.AllTargets() {
   141  		if sibling := gcSibling(graph, target); !sibling.HasParent() && !keepTargets[sibling] && isIncluded(sibling, filter) {
   142  			ret = append(ret, target.Label)
   143  			for _, src := range target.AllLocalSources() {
   144  				if !keepSrcs[src] {
   145  					retSrcs = append(retSrcs, src)
   146  				}
   147  			}
   148  		}
   149  	}
   150  	sort.Sort(ret)
   151  	sort.Strings(retSrcs)
   152  	log.Notice("%d targets to remove", len(ret))
   153  	log.Notice("%d sources to remove", len(retSrcs))
   154  	return ret, retSrcs
   155  }
   156  
   157  // isIncluded returns true if the given target is included in a set of filtering labels.
   158  func isIncluded(target *core.BuildTarget, filter []core.BuildLabel) bool {
   159  	if len(filter) == 0 {
   160  		return true // if you don't specify anything, the filter has no effect.
   161  	}
   162  	for _, f := range filter {
   163  		if f.Includes(target.Label) {
   164  			return true
   165  		}
   166  	}
   167  	return false
   168  }
   169  
   170  // addTarget adds a target and all its transitive dependencies to the given map.
   171  func addTarget(graph *core.BuildGraph, m targetMap, target *core.BuildTarget) {
   172  	if m[target] || target == nil {
   173  		return
   174  	}
   175  	log.Debug("  %s", target.Label)
   176  	m[target] = true
   177  	for _, dep := range target.DeclaredDependencies() {
   178  		addTarget(graph, m, graph.Target(dep))
   179  	}
   180  	for _, dep := range target.Dependencies() {
   181  		addTarget(graph, m, dep)
   182  	}
   183  }
   184  
   185  // anyInclude returns true if any of the given labels include this one.
   186  func anyInclude(labels []core.BuildLabel, label core.BuildLabel) bool {
   187  	for _, l := range labels {
   188  		if l.Includes(label) {
   189  			return true
   190  		}
   191  	}
   192  	return false
   193  }
   194  
   195  // publicDependencies returns the public dependencies of a target, considering any
   196  // private targets it might have declared.
   197  // For example, if we have dependencies as follows:
   198  //   //src/test:container_test
   199  //   //src/test:_container_test#lib
   200  //   //src/test:test
   201  // it will return //src/test:test for //src/test:container_test.
   202  func publicDependencies(graph *core.BuildGraph, target *core.BuildTarget) []*core.BuildTarget {
   203  	ret := []*core.BuildTarget{}
   204  	for _, dep := range target.DeclaredDependencies() {
   205  		if depTarget := graph.Target(dep); depTarget != nil {
   206  			if depTarget.Label.Parent() == target.Label.Parent() {
   207  				ret = append(ret, publicDependencies(graph, depTarget)...)
   208  			} else {
   209  				ret = append(ret, depTarget)
   210  			}
   211  		}
   212  	}
   213  	return ret
   214  }
   215  
   216  // RewriteFile rewrites a BUILD file to exclude a set of targets.
   217  func RewriteFile(state *core.BuildState, filename string, targets []string) error {
   218  	p := asp.NewParser(nil)
   219  	stmts, err := p.ParseFileOnly(filename)
   220  	if err != nil {
   221  		return err
   222  	}
   223  	b, err := ioutil.ReadFile(filename)
   224  	if err != nil {
   225  		return err // This is very unlikely since we already read it once above, but y'know...
   226  	}
   227  	lines := bytes.Split(b, []byte{'\n'})
   228  	linesToDelete := map[int]bool{}
   229  	for _, target := range targets {
   230  		stmt := asp.FindTarget(stmts, target)
   231  		if stmt == nil {
   232  			return fmt.Errorf("Can't find target %s in %s", target, filename)
   233  		}
   234  		start, end := asp.GetExtents(stmts, stmt, len(lines))
   235  		for i := start; i <= end; i++ {
   236  			linesToDelete[i-1] = true // -1 because the extents are 1-indexed
   237  		}
   238  	}
   239  	// Now rewrite the actual file
   240  	lines2 := make([][]byte, 0, len(lines))
   241  	for i, line := range lines {
   242  		if !linesToDelete[i] {
   243  			lines2 = append(lines2, line)
   244  		}
   245  	}
   246  	return ioutil.WriteFile(filename, bytes.Join(lines2, []byte{'\n'}), 0664)
   247  }
   248  
   249  // removeTargets rewrites the given set of targets out of their BUILD files.
   250  func removeTargets(state *core.BuildState, labels core.BuildLabels) error {
   251  	byPackage := map[*core.Package][]string{}
   252  	for _, l := range labels {
   253  		pkg := state.Graph.PackageOrDie(l)
   254  		byPackage[pkg] = append(byPackage[pkg], l.Name)
   255  	}
   256  	for pkg, victims := range byPackage {
   257  		log.Notice("Rewriting %s to remove %s...\n", pkg.Filename, strings.Join(victims, ", "))
   258  		if err := RewriteFile(state, pkg.Filename, victims); err != nil {
   259  			return err
   260  		}
   261  	}
   262  	return nil
   263  }
   264  
   265  // gcSibling finds any labelled sibling of this target, i.e. if it says gc_sibling:target1
   266  // then it returns target1 in the same package.
   267  // This is for cases where multiple targets are generated by the same rule and should
   268  // therefore share the same GC fate.
   269  func gcSibling(graph *core.BuildGraph, t *core.BuildTarget) *core.BuildTarget {
   270  	for _, l := range t.PrefixedLabels("gc_sibling:") {
   271  		if t2 := graph.Target(core.NewBuildLabel(t.Label.PackageName, l)); t2 != nil {
   272  			return t2
   273  		}
   274  		log.Warning("Target %s declared a gc_sibling of %s, but %s doesn't exist", t.Label, l, l)
   275  	}
   276  	return t
   277  }