github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/cue/load/search.go (about) 1 // Copyright 2018 The CUE Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package load 16 17 import ( 18 // TODO: remove this usage 19 20 "os" 21 "path" 22 "path/filepath" 23 "regexp" 24 "strings" 25 26 "github.com/joomcode/cue/cue/build" 27 "github.com/joomcode/cue/cue/errors" 28 "github.com/joomcode/cue/cue/token" 29 ) 30 31 // A match represents the result of matching a single package pattern. 32 type match struct { 33 Pattern string // the pattern itself 34 Literal bool // whether it is a literal (no wildcards) 35 Pkgs []*build.Instance 36 Err errors.Error 37 } 38 39 // TODO: should be matched from module file only. 40 // The pattern is either "all" (all packages), "std" (standard packages), 41 // "cmd" (standard commands), or a path including "...". 42 func (l *loader) matchPackages(pattern, pkgName string) *match { 43 // cfg := l.cfg 44 m := &match{ 45 Pattern: pattern, 46 Literal: false, 47 } 48 // match := func(string) bool { return true } 49 // treeCanMatch := func(string) bool { return true } 50 // if !isMetaPackage(pattern) { 51 // match = matchPattern(pattern) 52 // treeCanMatch = treeCanMatchPattern(pattern) 53 // } 54 55 // have := map[string]bool{ 56 // "builtin": true, // ignore pseudo-package that exists only for documentation 57 // } 58 59 // for _, src := range cfg.srcDirs() { 60 // if pattern == "std" || pattern == "cmd" { 61 // continue 62 // } 63 // src = filepath.Clean(src) + string(filepath.Separator) 64 // root := src 65 // if pattern == "cmd" { 66 // root += "cmd" + string(filepath.Separator) 67 // } 68 // filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { 69 // if err != nil || path == src { 70 // return nil 71 // } 72 73 // want := true 74 // // Avoid .foo, _foo, and testdata directory trees. 75 // _, elem := filepath.Split(path) 76 // if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { 77 // want = false 78 // } 79 80 // name := filepath.ToSlash(path[len(src):]) 81 // if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { 82 // // The name "std" is only the standard library. 83 // // If the name is cmd, it's the root of the command tree. 84 // want = false 85 // } 86 // if !treeCanMatch(name) { 87 // want = false 88 // } 89 90 // if !fi.IsDir() { 91 // if fi.Mode()&os.ModeSymlink != 0 && want { 92 // if target, err := os.Stat(path); err == nil && target.IsDir() { 93 // fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) 94 // } 95 // } 96 // return nil 97 // } 98 // if !want { 99 // return skipDir 100 // } 101 102 // if have[name] { 103 // return nil 104 // } 105 // have[name] = true 106 // if !match(name) { 107 // return nil 108 // } 109 // pkg := l.importPkg(".", path) 110 // if err := pkg.Error; err != nil { 111 // if _, noGo := err.(*noCUEError); noGo { 112 // return nil 113 // } 114 // } 115 116 // // If we are expanding "cmd", skip main 117 // // packages under cmd/vendor. At least as of 118 // // March, 2017, there is one there for the 119 // // vendored pprof tool. 120 // if pattern == "cmd" && strings.HasPrefix(pkg.DisplayPath, "cmd/vendor") && pkg.PkgName == "main" { 121 // return nil 122 // } 123 124 // m.Pkgs = append(m.Pkgs, pkg) 125 // return nil 126 // }) 127 // } 128 return m 129 } 130 131 // matchPackagesInFS is like allPackages but is passed a pattern 132 // beginning ./ or ../, meaning it should scan the tree rooted 133 // at the given directory. There are ... in the pattern too. 134 // (See go help packages for pattern syntax.) 135 func (l *loader) matchPackagesInFS(pattern, pkgName string) *match { 136 c := l.cfg 137 m := &match{ 138 Pattern: pattern, 139 Literal: false, 140 } 141 142 // Find directory to begin the scan. 143 // Could be smarter but this one optimization 144 // is enough for now, since ... is usually at the 145 // end of a path. 146 i := strings.Index(pattern, "...") 147 dir, _ := path.Split(pattern[:i]) 148 149 root := l.abs(dir) 150 151 // Find new module root from here or check there are no additional 152 // cue.mod files between here and the next module. 153 154 if !hasFilepathPrefix(root, c.ModuleRoot) { 155 m.Err = errors.Newf(token.NoPos, 156 "cue: pattern %s refers to dir %s, outside module root %s", 157 pattern, root, c.ModuleRoot) 158 return m 159 } 160 161 pkgDir := filepath.Join(root, modDir) 162 // TODO(legacy): remove 163 pkgDir2 := filepath.Join(root, "pkg") 164 165 _ = c.fileSystem.walk(root, func(path string, fi os.FileInfo, err errors.Error) errors.Error { 166 if err != nil || !fi.IsDir() { 167 return nil 168 } 169 if path == pkgDir || path == pkgDir2 { 170 return skipDir 171 } 172 173 top := path == root 174 175 // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". 176 _, elem := filepath.Split(path) 177 dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." 178 if dot || strings.HasPrefix(elem, "_") || (elem == "testdata" && !top) { 179 return skipDir 180 } 181 182 if !top { 183 // Ignore other modules found in subdirectories. 184 if _, err := c.fileSystem.stat(filepath.Join(path, modDir)); err == nil { 185 return skipDir 186 } 187 } 188 189 // name := prefix + filepath.ToSlash(path) 190 // if !match(name) { 191 // return nil 192 // } 193 194 // We keep the directory if we can import it, or if we can't import it 195 // due to invalid CUE source files. This means that directories 196 // containing parse errors will be built (and fail) instead of being 197 // silently skipped as not matching the pattern. 198 // Do not take root, as we want to stay relative 199 // to one dir only. 200 dir, e := filepath.Rel(c.Dir, path) 201 if e != nil { 202 panic(err) 203 } else { 204 dir = "./" + dir 205 } 206 // TODO: consider not doing these checks here. 207 inst := c.newRelInstance(token.NoPos, dir, pkgName) 208 pkgs := l.importPkg(token.NoPos, inst) 209 for _, p := range pkgs { 210 if err := p.Err; err != nil && (p == nil || len(p.InvalidFiles) == 0) { 211 switch err.(type) { 212 case nil: 213 break 214 case *NoFilesError: 215 if c.DataFiles && len(p.OrphanedFiles) > 0 { 216 break 217 } 218 return nil 219 default: 220 m.Err = errors.Append(m.Err, err) 221 } 222 } 223 } 224 225 m.Pkgs = append(m.Pkgs, pkgs...) 226 return nil 227 }) 228 return m 229 } 230 231 // treeCanMatchPattern(pattern)(name) reports whether 232 // name or children of name can possibly match pattern. 233 // Pattern is the same limited glob accepted by matchPattern. 234 func treeCanMatchPattern(pattern string) func(name string) bool { 235 wildCard := false 236 if i := strings.Index(pattern, "..."); i >= 0 { 237 wildCard = true 238 pattern = pattern[:i] 239 } 240 return func(name string) bool { 241 return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || 242 wildCard && strings.HasPrefix(name, pattern) 243 } 244 } 245 246 // matchPattern(pattern)(name) reports whether 247 // name matches pattern. Pattern is a limited glob 248 // pattern in which '...' means 'any string' and there 249 // is no other special syntax. 250 // Unfortunately, there are two special cases. Quoting "go help packages": 251 // 252 // First, /... at the end of the pattern can match an empty string, 253 // so that net/... matches both net and packages in its subdirectories, like net/http. 254 // Second, any slash-separted pattern element containing a wildcard never 255 // participates in a match of the "vendor" element in the path of a vendored 256 // package, so that ./... does not match packages in subdirectories of 257 // ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. 258 // Note, however, that a directory named vendor that itself contains code 259 // is not a vendored package: cmd/vendor would be a command named vendor, 260 // and the pattern cmd/... matches it. 261 func matchPattern(pattern string) func(name string) bool { 262 // Convert pattern to regular expression. 263 // The strategy for the trailing /... is to nest it in an explicit ? expression. 264 // The strategy for the vendor exclusion is to change the unmatchable 265 // vendor strings to a disallowed code point (vendorChar) and to use 266 // "(anything but that codepoint)*" as the implementation of the ... wildcard. 267 // This is a bit complicated but the obvious alternative, 268 // namely a hand-written search like in most shell glob matchers, 269 // is too easy to make accidentally exponential. 270 // Using package regexp guarantees linear-time matching. 271 272 const vendorChar = "\x00" 273 274 if strings.Contains(pattern, vendorChar) { 275 return func(name string) bool { return false } 276 } 277 278 re := regexp.QuoteMeta(pattern) 279 re = replaceVendor(re, vendorChar) 280 switch { 281 case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): 282 re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` 283 case re == vendorChar+`/\.\.\.`: 284 re = `(/vendor|/` + vendorChar + `/\.\.\.)` 285 case strings.HasSuffix(re, `/\.\.\.`): 286 re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` 287 } 288 re = strings.Replace(re, `\.\.\.`, `[^`+vendorChar+`]*`, -1) 289 290 reg := regexp.MustCompile(`^` + re + `$`) 291 292 return func(name string) bool { 293 if strings.Contains(name, vendorChar) { 294 return false 295 } 296 return reg.MatchString(replaceVendor(name, vendorChar)) 297 } 298 } 299 300 // replaceVendor returns the result of replacing 301 // non-trailing vendor path elements in x with repl. 302 func replaceVendor(x, repl string) string { 303 if !strings.Contains(x, "vendor") { 304 return x 305 } 306 elem := strings.Split(x, "/") 307 for i := 0; i < len(elem)-1; i++ { 308 if elem[i] == "vendor" { 309 elem[i] = repl 310 } 311 } 312 return strings.Join(elem, "/") 313 } 314 315 // warnUnmatched warns about patterns that didn't match any packages. 316 func warnUnmatched(matches []*match) { 317 for _, m := range matches { 318 if len(m.Pkgs) == 0 { 319 m.Err = 320 errors.Newf(token.NoPos, "cue: %q matched no packages\n", m.Pattern) 321 } 322 } 323 } 324 325 // importPaths returns the matching paths to use for the given command line. 326 // It calls ImportPathsQuiet and then WarnUnmatched. 327 func (l *loader) importPaths(patterns []string) []*match { 328 matches := l.importPathsQuiet(patterns) 329 warnUnmatched(matches) 330 return matches 331 } 332 333 // importPathsQuiet is like ImportPaths but does not warn about patterns with no matches. 334 func (l *loader) importPathsQuiet(patterns []string) []*match { 335 var out []*match 336 for _, a := range cleanPatterns(patterns) { 337 if isMetaPackage(a) { 338 out = append(out, l.matchPackages(a, l.cfg.Package)) 339 continue 340 } 341 342 orig := a 343 pkgName := l.cfg.Package 344 switch p := strings.IndexByte(a, ':'); { 345 case p < 0: 346 case p == 0: 347 pkgName = a[1:] 348 a = "." 349 default: 350 pkgName = a[p+1:] 351 a = a[:p] 352 } 353 if pkgName == "*" { 354 pkgName = "" 355 } 356 357 if strings.Contains(a, "...") { 358 if isLocalImport(a) { 359 out = append(out, l.matchPackagesInFS(a, pkgName)) 360 } else { 361 out = append(out, l.matchPackages(a, pkgName)) 362 } 363 continue 364 } 365 366 var p *build.Instance 367 if isLocalImport(a) { 368 p = l.cfg.newRelInstance(token.NoPos, a, pkgName) 369 } else { 370 p = l.cfg.newInstance(token.NoPos, importPath(orig)) 371 } 372 373 pkgs := l.importPkg(token.NoPos, p) 374 out = append(out, &match{Pattern: a, Literal: true, Pkgs: pkgs}) 375 } 376 return out 377 } 378 379 // cleanPatterns returns the patterns to use for the given 380 // command line. It canonicalizes the patterns but does not 381 // evaluate any matches. 382 func cleanPatterns(patterns []string) []string { 383 if len(patterns) == 0 { 384 return []string{"."} 385 } 386 var out []string 387 for _, a := range patterns { 388 // Arguments are supposed to be import paths, but 389 // as a courtesy to Windows developers, rewrite \ to / 390 // in command-line arguments. Handles .\... and so on. 391 if filepath.Separator == '\\' { 392 a = strings.Replace(a, `\`, `/`, -1) 393 } 394 395 // Put argument in canonical form, but preserve leading ./. 396 if strings.HasPrefix(a, "./") { 397 a = "./" + path.Clean(a) 398 if a == "./." { 399 a = "." 400 } 401 } else { 402 a = path.Clean(a) 403 } 404 out = append(out, a) 405 } 406 return out 407 } 408 409 // isMetaPackage checks if name is a reserved package name that expands to multiple packages. 410 func isMetaPackage(name string) bool { 411 return name == "std" || name == "cmd" || name == "all" 412 } 413 414 // hasPathPrefix reports whether the path s begins with the 415 // elements in prefix. 416 func hasPathPrefix(s, prefix string) bool { 417 switch { 418 default: 419 return false 420 case len(s) == len(prefix): 421 return s == prefix 422 case len(s) > len(prefix): 423 if prefix != "" && prefix[len(prefix)-1] == '/' { 424 return strings.HasPrefix(s, prefix) 425 } 426 return s[len(prefix)] == '/' && s[:len(prefix)] == prefix 427 } 428 } 429 430 // hasFilepathPrefix reports whether the path s begins with the 431 // elements in prefix. 432 func hasFilepathPrefix(s, prefix string) bool { 433 switch { 434 default: 435 return false 436 case len(s) == len(prefix): 437 return s == prefix 438 case len(s) > len(prefix): 439 if prefix != "" && prefix[len(prefix)-1] == filepath.Separator { 440 return strings.HasPrefix(s, prefix) 441 } 442 return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix 443 } 444 } 445 446 // isStandardImportPath reports whether $GOROOT/src/path should be considered 447 // part of the standard distribution. For historical reasons we allow people to add 448 // their own code to $GOROOT instead of using $GOPATH, but we assume that 449 // code will start with a domain name (dot in the first element). 450 // 451 // Note that this function is meant to evaluate whether a directory found in GOROOT 452 // should be treated as part of the standard library. It should not be used to decide 453 // that a directory found in GOPATH should be rejected: directories in GOPATH 454 // need not have dots in the first element, and they just take their chances 455 // with future collisions in the standard library. 456 func isStandardImportPath(path string) bool { 457 i := strings.Index(path, "/") 458 if i < 0 { 459 i = len(path) 460 } 461 elem := path[:i] 462 return !strings.Contains(elem, ".") 463 } 464 465 // isRelativePath reports whether pattern should be interpreted as a directory 466 // path relative to the current directory, as opposed to a pattern matching 467 // import paths. 468 func isRelativePath(pattern string) bool { 469 return strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == ".." 470 } 471 472 // inDir checks whether path is in the file tree rooted at dir. 473 // If so, inDir returns an equivalent path relative to dir. 474 // If not, inDir returns an empty string. 475 // inDir makes some effort to succeed even in the presence of symbolic links. 476 // TODO(rsc): Replace internal/test.inDir with a call to this function for Go 1.12. 477 func inDir(path, dir string) string { 478 if rel := inDirLex(path, dir); rel != "" { 479 return rel 480 } 481 xpath, err := filepath.EvalSymlinks(path) 482 if err != nil || xpath == path { 483 xpath = "" 484 } else { 485 if rel := inDirLex(xpath, dir); rel != "" { 486 return rel 487 } 488 } 489 490 xdir, err := filepath.EvalSymlinks(dir) 491 if err == nil && xdir != dir { 492 if rel := inDirLex(path, xdir); rel != "" { 493 return rel 494 } 495 if xpath != "" { 496 if rel := inDirLex(xpath, xdir); rel != "" { 497 return rel 498 } 499 } 500 } 501 return "" 502 } 503 504 // inDirLex is like inDir but only checks the lexical form of the file names. 505 // It does not consider symbolic links. 506 // TODO(rsc): This is a copy of str.HasFilePathPrefix, modified to 507 // return the suffix. Most uses of str.HasFilePathPrefix should probably 508 // be calling InDir instead. 509 func inDirLex(path, dir string) string { 510 pv := strings.ToUpper(filepath.VolumeName(path)) 511 dv := strings.ToUpper(filepath.VolumeName(dir)) 512 path = path[len(pv):] 513 dir = dir[len(dv):] 514 switch { 515 default: 516 return "" 517 case pv != dv: 518 return "" 519 case len(path) == len(dir): 520 if path == dir { 521 return "." 522 } 523 return "" 524 case dir == "": 525 return path 526 case len(path) > len(dir): 527 if dir[len(dir)-1] == filepath.Separator { 528 if path[:len(dir)] == dir { 529 return path[len(dir):] 530 } 531 return "" 532 } 533 if path[len(dir)] == filepath.Separator && path[:len(dir)] == dir { 534 if len(path) == len(dir)+1 { 535 return "." 536 } 537 return path[len(dir)+1:] 538 } 539 return "" 540 } 541 }