github.com/tiagovtristao/plz@v13.4.0+incompatible/src/gc/gc.go (about) 1 // +build !bootstrap 2 3 // Package gc implements "garbage collection" logic for Please, which is an attempt to identify 4 // targets in the repo that are no longer needed. 5 // The definition of "needed" is a bit unclear; we define it as non-test binaries, but the 6 // command accepts an argument to add extra ones just in case (for example, if you have a repo which 7 // is primarily a library, you might have to tell it that). 8 package gc 9 10 import ( 11 "bytes" 12 "fmt" 13 "io/ioutil" 14 "os" 15 "sort" 16 "strings" 17 18 "github.com/Songmu/prompter" 19 "gopkg.in/op/go-logging.v1" 20 21 "github.com/thought-machine/please/src/core" 22 "github.com/thought-machine/please/src/parse/asp" 23 ) 24 25 var log = logging.MustGetLogger("gc") 26 27 type targetMap map[*core.BuildTarget]bool 28 29 // GarbageCollect initiates the garbage collection logic. 30 func GarbageCollect(state *core.BuildState, filter, targets, keepTargets []core.BuildLabel, keepLabels []string, conservative, targetsOnly, srcsOnly, noPrompt, dryRun, git bool) { 31 if targets, srcs := targetsToRemove(state.Graph, filter, targets, keepTargets, keepLabels, conservative); len(targets) > 0 { 32 if !srcsOnly { 33 fmt.Fprintf(os.Stderr, "Targets to remove (total %d of %d):\n", len(targets), state.Graph.Len()) 34 for _, target := range targets { 35 fmt.Printf(" %s\n", target) 36 } 37 } 38 if !targetsOnly && len(srcs) > 0 { 39 fmt.Fprintf(os.Stderr, "Corresponding source files to remove:\n") 40 for _, src := range srcs { 41 fmt.Printf(" %s\n", src) 42 } 43 } 44 if dryRun { 45 return 46 } else if !noPrompt && !prompter.YN("Remove these targets / files?", false) { 47 os.Exit(1) 48 } 49 if !srcsOnly { 50 if err := removeTargets(state, targets); err != nil { 51 log.Fatalf("%s\n", err) 52 } 53 } 54 if !targetsOnly { 55 if git { 56 log.Notice("Running git rm %s\n", strings.Join(srcs, " ")) 57 srcs = append([]string{"rm", "-q"}, srcs...) 58 cmd := core.ExecCommand("git", srcs...) 59 cmd.Stdout = os.Stdout 60 cmd.Stderr = os.Stderr 61 if err := cmd.Run(); err != nil { 62 log.Fatalf("git rm failed: %s\n", err) 63 } 64 } else { 65 for _, src := range srcs { 66 log.Notice("Deleting %s...\n", src) 67 if err := os.Remove(src); err != nil { 68 log.Fatalf("Failed to remove %s: %s\n", src, err) 69 } 70 } 71 } 72 } 73 fmt.Fprintf(os.Stderr, "Garbage collected!\n") 74 } else { 75 fmt.Fprintf(os.Stderr, "Nothing to remove\n") 76 } 77 } 78 79 // targetsToRemove finds the set of targets that are no longer needed and any extraneous sources. 80 func targetsToRemove(graph *core.BuildGraph, filter, targets, targetsToKeep []core.BuildLabel, keepLabels []string, includeTests bool) (core.BuildLabels, []string) { 81 keepTargets := targetMap{} 82 for _, target := range graph.AllTargets() { 83 if (target.IsBinary && (!target.IsTest || includeTests)) || target.HasAnyLabel(keepLabels) || anyInclude(targetsToKeep, target.Label) || target.Label.Subrepo != "" { 84 log.Debug("GC root: %s", target.Label) 85 addTarget(graph, keepTargets, target) 86 } 87 } 88 // Any registered subincludes also count. 89 for _, pkg := range graph.PackageMap() { 90 for _, subinclude := range pkg.Subincludes { 91 log.Debug("GC root: %s", subinclude) 92 addTarget(graph, keepTargets, graph.TargetOrDie(subinclude)) 93 } 94 } 95 log.Notice("%d targets to keep from initial scan", len(keepTargets)) 96 for _, target := range targets { 97 if target.IsAllSubpackages() { 98 // For slightly awkward reasons these can't be handled outside :( 99 for _, pkg := range graph.PackageMap() { 100 if pkg.IsIncludedIn(target) { 101 for _, target := range pkg.AllTargets() { 102 log.Debug("GC root: %s", target.Label) 103 addTarget(graph, keepTargets, target) 104 } 105 } 106 } 107 } else { 108 addTarget(graph, keepTargets, graph.Target(target)) 109 } 110 } 111 log.Notice("%d targets to keep after configured GC roots", len(keepTargets)) 112 if !includeTests { 113 // This is a bit complex - need to identify any tests that are tests "on" the set of things 114 // we've already decided to keep. 115 for _, target := range graph.AllTargets() { 116 if target.IsTest { 117 for _, dep := range publicDependencies(graph, target) { 118 if keepTargets[dep] && !dep.TestOnly { 119 log.Debug("Keeping test %s on %s", target.Label, dep.Label) 120 addTarget(graph, keepTargets, target) 121 } else if dep.TestOnly { 122 log.Debug("Keeping test-only target %s", dep.Label) 123 addTarget(graph, keepTargets, dep) 124 } 125 } 126 } 127 } 128 log.Notice("%d targets to keep after exploring tests", len(keepTargets)) 129 } 130 // Now build the set of sources that we'll keep. This is important because other targets that 131 // we're not deleting could still use the sources of the targets that we are. 132 keepSrcs := map[string]bool{} 133 for target := range keepTargets { 134 for _, src := range target.AllLocalSources() { 135 keepSrcs[src] = true 136 } 137 } 138 ret := make(core.BuildLabels, 0, len(keepTargets)) 139 retSrcs := []string{} 140 for _, target := range graph.AllTargets() { 141 if sibling := gcSibling(graph, target); !sibling.HasParent() && !keepTargets[sibling] && isIncluded(sibling, filter) { 142 ret = append(ret, target.Label) 143 for _, src := range target.AllLocalSources() { 144 if !keepSrcs[src] { 145 retSrcs = append(retSrcs, src) 146 } 147 } 148 } 149 } 150 sort.Sort(ret) 151 sort.Strings(retSrcs) 152 log.Notice("%d targets to remove", len(ret)) 153 log.Notice("%d sources to remove", len(retSrcs)) 154 return ret, retSrcs 155 } 156 157 // isIncluded returns true if the given target is included in a set of filtering labels. 158 func isIncluded(target *core.BuildTarget, filter []core.BuildLabel) bool { 159 if len(filter) == 0 { 160 return true // if you don't specify anything, the filter has no effect. 161 } 162 for _, f := range filter { 163 if f.Includes(target.Label) { 164 return true 165 } 166 } 167 return false 168 } 169 170 // addTarget adds a target and all its transitive dependencies to the given map. 171 func addTarget(graph *core.BuildGraph, m targetMap, target *core.BuildTarget) { 172 if m[target] || target == nil { 173 return 174 } 175 log.Debug(" %s", target.Label) 176 m[target] = true 177 for _, dep := range target.DeclaredDependencies() { 178 addTarget(graph, m, graph.Target(dep)) 179 } 180 for _, dep := range target.Dependencies() { 181 addTarget(graph, m, dep) 182 } 183 } 184 185 // anyInclude returns true if any of the given labels include this one. 186 func anyInclude(labels []core.BuildLabel, label core.BuildLabel) bool { 187 for _, l := range labels { 188 if l.Includes(label) { 189 return true 190 } 191 } 192 return false 193 } 194 195 // publicDependencies returns the public dependencies of a target, considering any 196 // private targets it might have declared. 197 // For example, if we have dependencies as follows: 198 // //src/test:container_test 199 // //src/test:_container_test#lib 200 // //src/test:test 201 // it will return //src/test:test for //src/test:container_test. 202 func publicDependencies(graph *core.BuildGraph, target *core.BuildTarget) []*core.BuildTarget { 203 ret := []*core.BuildTarget{} 204 for _, dep := range target.DeclaredDependencies() { 205 if depTarget := graph.Target(dep); depTarget != nil { 206 if depTarget.Label.Parent() == target.Label.Parent() { 207 ret = append(ret, publicDependencies(graph, depTarget)...) 208 } else { 209 ret = append(ret, depTarget) 210 } 211 } 212 } 213 return ret 214 } 215 216 // RewriteFile rewrites a BUILD file to exclude a set of targets. 217 func RewriteFile(state *core.BuildState, filename string, targets []string) error { 218 p := asp.NewParser(nil) 219 stmts, err := p.ParseFileOnly(filename) 220 if err != nil { 221 return err 222 } 223 b, err := ioutil.ReadFile(filename) 224 if err != nil { 225 return err // This is very unlikely since we already read it once above, but y'know... 226 } 227 lines := bytes.Split(b, []byte{'\n'}) 228 linesToDelete := map[int]bool{} 229 for _, target := range targets { 230 stmt := asp.FindTarget(stmts, target) 231 if stmt == nil { 232 return fmt.Errorf("Can't find target %s in %s", target, filename) 233 } 234 start, end := asp.GetExtents(stmts, stmt, len(lines)) 235 for i := start; i <= end; i++ { 236 linesToDelete[i-1] = true // -1 because the extents are 1-indexed 237 } 238 } 239 // Now rewrite the actual file 240 lines2 := make([][]byte, 0, len(lines)) 241 for i, line := range lines { 242 if !linesToDelete[i] { 243 lines2 = append(lines2, line) 244 } 245 } 246 return ioutil.WriteFile(filename, bytes.Join(lines2, []byte{'\n'}), 0664) 247 } 248 249 // removeTargets rewrites the given set of targets out of their BUILD files. 250 func removeTargets(state *core.BuildState, labels core.BuildLabels) error { 251 byPackage := map[*core.Package][]string{} 252 for _, l := range labels { 253 pkg := state.Graph.PackageOrDie(l) 254 byPackage[pkg] = append(byPackage[pkg], l.Name) 255 } 256 for pkg, victims := range byPackage { 257 log.Notice("Rewriting %s to remove %s...\n", pkg.Filename, strings.Join(victims, ", ")) 258 if err := RewriteFile(state, pkg.Filename, victims); err != nil { 259 return err 260 } 261 } 262 return nil 263 } 264 265 // gcSibling finds any labelled sibling of this target, i.e. if it says gc_sibling:target1 266 // then it returns target1 in the same package. 267 // This is for cases where multiple targets are generated by the same rule and should 268 // therefore share the same GC fate. 269 func gcSibling(graph *core.BuildGraph, t *core.BuildTarget) *core.BuildTarget { 270 for _, l := range t.PrefixedLabels("gc_sibling:") { 271 if t2 := graph.Target(core.NewBuildLabel(t.Label.PackageName, l)); t2 != nil { 272 return t2 273 } 274 log.Warning("Target %s declared a gc_sibling of %s, but %s doesn't exist", t.Label, l, l) 275 } 276 return t 277 }