github.com/cockroachdb/tools@v0.0.0-20230222021103-a6d27438930d/godoc/dirtrees.go (about) 1 // Copyright 2010 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // This file contains the code dealing with package directory trees. 6 7 package godoc 8 9 import ( 10 "go/doc" 11 "go/parser" 12 "go/token" 13 "log" 14 "os" 15 pathpkg "path" 16 "runtime" 17 "sort" 18 "strings" 19 20 "golang.org/x/tools/godoc/vfs" 21 ) 22 23 // Conventional name for directories containing test data. 24 // Excluded from directory trees. 25 const testdataDirName = "testdata" 26 27 type Directory struct { 28 Depth int 29 Path string // directory path; includes Name 30 Name string // directory name 31 HasPkg bool // true if the directory contains at least one package 32 Synopsis string // package documentation, if any 33 RootType vfs.RootType // root type of the filesystem containing the directory 34 Dirs []*Directory // subdirectories 35 } 36 37 func isGoFile(fi os.FileInfo) bool { 38 name := fi.Name() 39 return !fi.IsDir() && 40 len(name) > 0 && name[0] != '.' && // ignore .files 41 pathpkg.Ext(name) == ".go" 42 } 43 44 func isPkgFile(fi os.FileInfo) bool { 45 return isGoFile(fi) && 46 !strings.HasSuffix(fi.Name(), "_test.go") // ignore test files 47 } 48 49 func isPkgDir(fi os.FileInfo) bool { 50 name := fi.Name() 51 return fi.IsDir() && len(name) > 0 && 52 name[0] != '_' && name[0] != '.' // ignore _files and .files 53 } 54 55 type treeBuilder struct { 56 c *Corpus 57 maxDepth int 58 } 59 60 // ioGate is a semaphore controlling VFS activity (ReadDir, parseFile, etc). 61 // Send before an operation and receive after. 62 var ioGate = make(chan struct{}, 20) 63 64 // workGate controls the number of concurrent workers. Too many concurrent 65 // workers and performance degrades and the race detector gets overwhelmed. If 66 // we cannot check out a concurrent worker, work is performed by the main thread 67 // instead of spinning up another goroutine. 68 var workGate = make(chan struct{}, runtime.NumCPU()*4) 69 70 func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth int) *Directory { 71 if name == testdataDirName { 72 return nil 73 } 74 75 if depth >= b.maxDepth { 76 // return a dummy directory so that the parent directory 77 // doesn't get discarded just because we reached the max 78 // directory depth 79 return &Directory{ 80 Depth: depth, 81 Path: path, 82 Name: name, 83 } 84 } 85 86 var synopses [3]string // prioritized package documentation (0 == highest priority) 87 88 show := true // show in package listing 89 hasPkgFiles := false 90 haveSummary := false 91 92 if hook := b.c.SummarizePackage; hook != nil { 93 if summary, show0, ok := hook(strings.TrimPrefix(path, "/src/")); ok { 94 hasPkgFiles = true 95 show = show0 96 synopses[0] = summary 97 haveSummary = true 98 } 99 } 100 101 ioGate <- struct{}{} 102 list, err := b.c.fs.ReadDir(path) 103 <-ioGate 104 if err != nil { 105 // TODO: propagate more. See golang.org/issue/14252. 106 // For now: 107 if b.c.Verbose { 108 log.Printf("newDirTree reading %s: %v", path, err) 109 } 110 } 111 112 // determine number of subdirectories and if there are package files 113 var dirchs []chan *Directory 114 var dirs []*Directory 115 116 for _, d := range list { 117 filename := pathpkg.Join(path, d.Name()) 118 switch { 119 case isPkgDir(d): 120 name := d.Name() 121 select { 122 case workGate <- struct{}{}: 123 ch := make(chan *Directory, 1) 124 dirchs = append(dirchs, ch) 125 go func() { 126 ch <- b.newDirTree(fset, filename, name, depth+1) 127 <-workGate 128 }() 129 default: 130 // no free workers, do work synchronously 131 dir := b.newDirTree(fset, filename, name, depth+1) 132 if dir != nil { 133 dirs = append(dirs, dir) 134 } 135 } 136 case !haveSummary && isPkgFile(d): 137 // looks like a package file, but may just be a file ending in ".go"; 138 // don't just count it yet (otherwise we may end up with hasPkgFiles even 139 // though the directory doesn't contain any real package files - was bug) 140 // no "optimal" package synopsis yet; continue to collect synopses 141 ioGate <- struct{}{} 142 const flags = parser.ParseComments | parser.PackageClauseOnly 143 file, err := b.c.parseFile(fset, filename, flags) 144 <-ioGate 145 if err != nil { 146 if b.c.Verbose { 147 log.Printf("Error parsing %v: %v", filename, err) 148 } 149 break 150 } 151 152 hasPkgFiles = true 153 if file.Doc != nil { 154 // prioritize documentation 155 i := -1 156 switch file.Name.Name { 157 case name: 158 i = 0 // normal case: directory name matches package name 159 case "main": 160 i = 1 // directory contains a main package 161 default: 162 i = 2 // none of the above 163 } 164 if 0 <= i && i < len(synopses) && synopses[i] == "" { 165 synopses[i] = doc.Synopsis(file.Doc.Text()) 166 } 167 } 168 haveSummary = synopses[0] != "" 169 } 170 } 171 172 // create subdirectory tree 173 for _, ch := range dirchs { 174 if d := <-ch; d != nil { 175 dirs = append(dirs, d) 176 } 177 } 178 179 // We need to sort the dirs slice because 180 // it is appended again after reading from dirchs. 181 sort.Slice(dirs, func(i, j int) bool { 182 return dirs[i].Name < dirs[j].Name 183 }) 184 185 // if there are no package files and no subdirectories 186 // containing package files, ignore the directory 187 if !hasPkgFiles && len(dirs) == 0 { 188 return nil 189 } 190 191 // select the highest-priority synopsis for the directory entry, if any 192 synopsis := "" 193 for _, synopsis = range synopses { 194 if synopsis != "" { 195 break 196 } 197 } 198 199 return &Directory{ 200 Depth: depth, 201 Path: path, 202 Name: name, 203 HasPkg: hasPkgFiles && show, // TODO(bradfitz): add proper Hide field? 204 Synopsis: synopsis, 205 RootType: b.c.fs.RootType(path), 206 Dirs: dirs, 207 } 208 } 209 210 // newDirectory creates a new package directory tree with at most maxDepth 211 // levels, anchored at root. The result tree is pruned such that it only 212 // contains directories that contain package files or that contain 213 // subdirectories containing package files (transitively). If a non-nil 214 // pathFilter is provided, directory paths additionally must be accepted 215 // by the filter (i.e., pathFilter(path) must be true). If a value >= 0 is 216 // provided for maxDepth, nodes at larger depths are pruned as well; they 217 // are assumed to contain package files even if their contents are not known 218 // (i.e., in this case the tree may contain directories w/o any package files). 219 func (c *Corpus) newDirectory(root string, maxDepth int) *Directory { 220 // The root could be a symbolic link so use Stat not Lstat. 221 d, err := c.fs.Stat(root) 222 // If we fail here, report detailed error messages; otherwise 223 // is is hard to see why a directory tree was not built. 224 switch { 225 case err != nil: 226 log.Printf("newDirectory(%s): %s", root, err) 227 return nil 228 case root != "/" && !isPkgDir(d): 229 log.Printf("newDirectory(%s): not a package directory", root) 230 return nil 231 case root == "/" && !d.IsDir(): 232 log.Printf("newDirectory(%s): not a directory", root) 233 return nil 234 } 235 if maxDepth < 0 { 236 maxDepth = 1e6 // "infinity" 237 } 238 b := treeBuilder{c, maxDepth} 239 // the file set provided is only for local parsing, no position 240 // information escapes and thus we don't need to save the set 241 return b.newDirTree(token.NewFileSet(), root, d.Name(), 0) 242 } 243 244 func (dir *Directory) walk(c chan<- *Directory, skipRoot bool) { 245 if dir != nil { 246 if !skipRoot { 247 c <- dir 248 } 249 for _, d := range dir.Dirs { 250 d.walk(c, false) 251 } 252 } 253 } 254 255 func (dir *Directory) iter(skipRoot bool) <-chan *Directory { 256 c := make(chan *Directory) 257 go func() { 258 dir.walk(c, skipRoot) 259 close(c) 260 }() 261 return c 262 } 263 264 func (dir *Directory) lookupLocal(name string) *Directory { 265 for _, d := range dir.Dirs { 266 if d.Name == name { 267 return d 268 } 269 } 270 return nil 271 } 272 273 func splitPath(p string) []string { 274 p = strings.TrimPrefix(p, "/") 275 if p == "" { 276 return nil 277 } 278 return strings.Split(p, "/") 279 } 280 281 // lookup looks for the *Directory for a given path, relative to dir. 282 func (dir *Directory) lookup(path string) *Directory { 283 d := splitPath(dir.Path) 284 p := splitPath(path) 285 i := 0 286 for i < len(d) { 287 if i >= len(p) || d[i] != p[i] { 288 return nil 289 } 290 i++ 291 } 292 for dir != nil && i < len(p) { 293 dir = dir.lookupLocal(p[i]) 294 i++ 295 } 296 return dir 297 } 298 299 // DirEntry describes a directory entry. The Depth and Height values 300 // are useful for presenting an entry in an indented fashion. 301 type DirEntry struct { 302 Depth int // >= 0 303 Height int // = DirList.MaxHeight - Depth, > 0 304 Path string // directory path; includes Name, relative to DirList root 305 Name string // directory name 306 HasPkg bool // true if the directory contains at least one package 307 Synopsis string // package documentation, if any 308 RootType vfs.RootType // root type of the filesystem containing the direntry 309 } 310 311 type DirList struct { 312 MaxHeight int // directory tree height, > 0 313 List []DirEntry 314 } 315 316 // hasThirdParty checks whether a list of directory entries has packages outside 317 // the standard library or not. 318 func hasThirdParty(list []DirEntry) bool { 319 for _, entry := range list { 320 if entry.RootType == vfs.RootTypeGoPath { 321 return true 322 } 323 } 324 return false 325 } 326 327 // listing creates a (linear) directory listing from a directory tree. 328 // If skipRoot is set, the root directory itself is excluded from the list. 329 // If filter is set, only the directory entries whose paths match the filter 330 // are included. 331 func (dir *Directory) listing(skipRoot bool, filter func(string) bool) *DirList { 332 if dir == nil { 333 return nil 334 } 335 336 // determine number of entries n and maximum height 337 n := 0 338 minDepth := 1 << 30 // infinity 339 maxDepth := 0 340 for d := range dir.iter(skipRoot) { 341 n++ 342 if minDepth > d.Depth { 343 minDepth = d.Depth 344 } 345 if maxDepth < d.Depth { 346 maxDepth = d.Depth 347 } 348 } 349 maxHeight := maxDepth - minDepth + 1 350 351 if n == 0 { 352 return nil 353 } 354 355 // create list 356 list := make([]DirEntry, 0, n) 357 for d := range dir.iter(skipRoot) { 358 if filter != nil && !filter(d.Path) { 359 continue 360 } 361 var p DirEntry 362 p.Depth = d.Depth - minDepth 363 p.Height = maxHeight - p.Depth 364 // the path is relative to root.Path - remove the root.Path 365 // prefix (the prefix should always be present but avoid 366 // crashes and check) 367 path := strings.TrimPrefix(d.Path, dir.Path) 368 // remove leading separator if any - path must be relative 369 path = strings.TrimPrefix(path, "/") 370 p.Path = path 371 p.Name = d.Name 372 p.HasPkg = d.HasPkg 373 p.Synopsis = d.Synopsis 374 p.RootType = d.RootType 375 list = append(list, p) 376 } 377 378 return &DirList{maxHeight, list} 379 }