github.com/joomcode/cue@v0.4.4-0.20221111115225-539fe3512047/cue/load/search.go (about)

     1  // Copyright 2018 The CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package load
    16  
    17  import (
    18  	// TODO: remove this usage
    19  
    20  	"os"
    21  	"path"
    22  	"path/filepath"
    23  	"regexp"
    24  	"strings"
    25  
    26  	"github.com/joomcode/cue/cue/build"
    27  	"github.com/joomcode/cue/cue/errors"
    28  	"github.com/joomcode/cue/cue/token"
    29  )
    30  
    31  // A match represents the result of matching a single package pattern.
    32  type match struct {
    33  	Pattern string // the pattern itself
    34  	Literal bool   // whether it is a literal (no wildcards)
    35  	Pkgs    []*build.Instance
    36  	Err     errors.Error
    37  }
    38  
    39  // TODO: should be matched from module file only.
    40  // The pattern is either "all" (all packages), "std" (standard packages),
    41  // "cmd" (standard commands), or a path including "...".
    42  func (l *loader) matchPackages(pattern, pkgName string) *match {
    43  	// cfg := l.cfg
    44  	m := &match{
    45  		Pattern: pattern,
    46  		Literal: false,
    47  	}
    48  	// match := func(string) bool { return true }
    49  	// treeCanMatch := func(string) bool { return true }
    50  	// if !isMetaPackage(pattern) {
    51  	// 	match = matchPattern(pattern)
    52  	// 	treeCanMatch = treeCanMatchPattern(pattern)
    53  	// }
    54  
    55  	// have := map[string]bool{
    56  	// 	"builtin": true, // ignore pseudo-package that exists only for documentation
    57  	// }
    58  
    59  	// for _, src := range cfg.srcDirs() {
    60  	// 	if pattern == "std" || pattern == "cmd" {
    61  	// 		continue
    62  	// 	}
    63  	// 	src = filepath.Clean(src) + string(filepath.Separator)
    64  	// 	root := src
    65  	// 	if pattern == "cmd" {
    66  	// 		root += "cmd" + string(filepath.Separator)
    67  	// 	}
    68  	// 	filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
    69  	// 		if err != nil || path == src {
    70  	// 			return nil
    71  	// 		}
    72  
    73  	// 		want := true
    74  	// 		// Avoid .foo, _foo, and testdata directory trees.
    75  	// 		_, elem := filepath.Split(path)
    76  	// 		if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
    77  	// 			want = false
    78  	// 		}
    79  
    80  	// 		name := filepath.ToSlash(path[len(src):])
    81  	// 		if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") {
    82  	// 			// The name "std" is only the standard library.
    83  	// 			// If the name is cmd, it's the root of the command tree.
    84  	// 			want = false
    85  	// 		}
    86  	// 		if !treeCanMatch(name) {
    87  	// 			want = false
    88  	// 		}
    89  
    90  	// 		if !fi.IsDir() {
    91  	// 			if fi.Mode()&os.ModeSymlink != 0 && want {
    92  	// 				if target, err := os.Stat(path); err == nil && target.IsDir() {
    93  	// 					fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path)
    94  	// 				}
    95  	// 			}
    96  	// 			return nil
    97  	// 		}
    98  	// 		if !want {
    99  	// 			return skipDir
   100  	// 		}
   101  
   102  	// 		if have[name] {
   103  	// 			return nil
   104  	// 		}
   105  	// 		have[name] = true
   106  	// 		if !match(name) {
   107  	// 			return nil
   108  	// 		}
   109  	// 		pkg := l.importPkg(".", path)
   110  	// 		if err := pkg.Error; err != nil {
   111  	// 			if _, noGo := err.(*noCUEError); noGo {
   112  	// 				return nil
   113  	// 			}
   114  	// 		}
   115  
   116  	// 		// If we are expanding "cmd", skip main
   117  	// 		// packages under cmd/vendor. At least as of
   118  	// 		// March, 2017, there is one there for the
   119  	// 		// vendored pprof tool.
   120  	// 		if pattern == "cmd" && strings.HasPrefix(pkg.DisplayPath, "cmd/vendor") && pkg.PkgName == "main" {
   121  	// 			return nil
   122  	// 		}
   123  
   124  	// 		m.Pkgs = append(m.Pkgs, pkg)
   125  	// 		return nil
   126  	// 	})
   127  	// }
   128  	return m
   129  }
   130  
   131  // matchPackagesInFS is like allPackages but is passed a pattern
   132  // beginning ./ or ../, meaning it should scan the tree rooted
   133  // at the given directory. There are ... in the pattern too.
   134  // (See go help packages for pattern syntax.)
   135  func (l *loader) matchPackagesInFS(pattern, pkgName string) *match {
   136  	c := l.cfg
   137  	m := &match{
   138  		Pattern: pattern,
   139  		Literal: false,
   140  	}
   141  
   142  	// Find directory to begin the scan.
   143  	// Could be smarter but this one optimization
   144  	// is enough for now, since ... is usually at the
   145  	// end of a path.
   146  	i := strings.Index(pattern, "...")
   147  	dir, _ := path.Split(pattern[:i])
   148  
   149  	root := l.abs(dir)
   150  
   151  	// Find new module root from here or check there are no additional
   152  	// cue.mod files between here and the next module.
   153  
   154  	if !hasFilepathPrefix(root, c.ModuleRoot) {
   155  		m.Err = errors.Newf(token.NoPos,
   156  			"cue: pattern %s refers to dir %s, outside module root %s",
   157  			pattern, root, c.ModuleRoot)
   158  		return m
   159  	}
   160  
   161  	pkgDir := filepath.Join(root, modDir)
   162  	// TODO(legacy): remove
   163  	pkgDir2 := filepath.Join(root, "pkg")
   164  
   165  	_ = c.fileSystem.walk(root, func(path string, fi os.FileInfo, err errors.Error) errors.Error {
   166  		if err != nil || !fi.IsDir() {
   167  			return nil
   168  		}
   169  		if path == pkgDir || path == pkgDir2 {
   170  			return skipDir
   171  		}
   172  
   173  		top := path == root
   174  
   175  		// Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..".
   176  		_, elem := filepath.Split(path)
   177  		dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".."
   178  		if dot || strings.HasPrefix(elem, "_") || (elem == "testdata" && !top) {
   179  			return skipDir
   180  		}
   181  
   182  		if !top {
   183  			// Ignore other modules found in subdirectories.
   184  			if _, err := c.fileSystem.stat(filepath.Join(path, modDir)); err == nil {
   185  				return skipDir
   186  			}
   187  		}
   188  
   189  		// name := prefix + filepath.ToSlash(path)
   190  		// if !match(name) {
   191  		// 	return nil
   192  		// }
   193  
   194  		// We keep the directory if we can import it, or if we can't import it
   195  		// due to invalid CUE source files. This means that directories
   196  		// containing parse errors will be built (and fail) instead of being
   197  		// silently skipped as not matching the pattern.
   198  		// Do not take root, as we want to stay relative
   199  		// to one dir only.
   200  		dir, e := filepath.Rel(c.Dir, path)
   201  		if e != nil {
   202  			panic(err)
   203  		} else {
   204  			dir = "./" + dir
   205  		}
   206  		// TODO: consider not doing these checks here.
   207  		inst := c.newRelInstance(token.NoPos, dir, pkgName)
   208  		pkgs := l.importPkg(token.NoPos, inst)
   209  		for _, p := range pkgs {
   210  			if err := p.Err; err != nil && (p == nil || len(p.InvalidFiles) == 0) {
   211  				switch err.(type) {
   212  				case nil:
   213  					break
   214  				case *NoFilesError:
   215  					if c.DataFiles && len(p.OrphanedFiles) > 0 {
   216  						break
   217  					}
   218  					return nil
   219  				default:
   220  					m.Err = errors.Append(m.Err, err)
   221  				}
   222  			}
   223  		}
   224  
   225  		m.Pkgs = append(m.Pkgs, pkgs...)
   226  		return nil
   227  	})
   228  	return m
   229  }
   230  
   231  // treeCanMatchPattern(pattern)(name) reports whether
   232  // name or children of name can possibly match pattern.
   233  // Pattern is the same limited glob accepted by matchPattern.
   234  func treeCanMatchPattern(pattern string) func(name string) bool {
   235  	wildCard := false
   236  	if i := strings.Index(pattern, "..."); i >= 0 {
   237  		wildCard = true
   238  		pattern = pattern[:i]
   239  	}
   240  	return func(name string) bool {
   241  		return len(name) <= len(pattern) && hasPathPrefix(pattern, name) ||
   242  			wildCard && strings.HasPrefix(name, pattern)
   243  	}
   244  }
   245  
   246  // matchPattern(pattern)(name) reports whether
   247  // name matches pattern. Pattern is a limited glob
   248  // pattern in which '...' means 'any string' and there
   249  // is no other special syntax.
   250  // Unfortunately, there are two special cases. Quoting "go help packages":
   251  //
   252  // First, /... at the end of the pattern can match an empty string,
   253  // so that net/... matches both net and packages in its subdirectories, like net/http.
   254  // Second, any slash-separted pattern element containing a wildcard never
   255  // participates in a match of the "vendor" element in the path of a vendored
   256  // package, so that ./... does not match packages in subdirectories of
   257  // ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do.
   258  // Note, however, that a directory named vendor that itself contains code
   259  // is not a vendored package: cmd/vendor would be a command named vendor,
   260  // and the pattern cmd/... matches it.
   261  func matchPattern(pattern string) func(name string) bool {
   262  	// Convert pattern to regular expression.
   263  	// The strategy for the trailing /... is to nest it in an explicit ? expression.
   264  	// The strategy for the vendor exclusion is to change the unmatchable
   265  	// vendor strings to a disallowed code point (vendorChar) and to use
   266  	// "(anything but that codepoint)*" as the implementation of the ... wildcard.
   267  	// This is a bit complicated but the obvious alternative,
   268  	// namely a hand-written search like in most shell glob matchers,
   269  	// is too easy to make accidentally exponential.
   270  	// Using package regexp guarantees linear-time matching.
   271  
   272  	const vendorChar = "\x00"
   273  
   274  	if strings.Contains(pattern, vendorChar) {
   275  		return func(name string) bool { return false }
   276  	}
   277  
   278  	re := regexp.QuoteMeta(pattern)
   279  	re = replaceVendor(re, vendorChar)
   280  	switch {
   281  	case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`):
   282  		re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)`
   283  	case re == vendorChar+`/\.\.\.`:
   284  		re = `(/vendor|/` + vendorChar + `/\.\.\.)`
   285  	case strings.HasSuffix(re, `/\.\.\.`):
   286  		re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?`
   287  	}
   288  	re = strings.Replace(re, `\.\.\.`, `[^`+vendorChar+`]*`, -1)
   289  
   290  	reg := regexp.MustCompile(`^` + re + `$`)
   291  
   292  	return func(name string) bool {
   293  		if strings.Contains(name, vendorChar) {
   294  			return false
   295  		}
   296  		return reg.MatchString(replaceVendor(name, vendorChar))
   297  	}
   298  }
   299  
   300  // replaceVendor returns the result of replacing
   301  // non-trailing vendor path elements in x with repl.
   302  func replaceVendor(x, repl string) string {
   303  	if !strings.Contains(x, "vendor") {
   304  		return x
   305  	}
   306  	elem := strings.Split(x, "/")
   307  	for i := 0; i < len(elem)-1; i++ {
   308  		if elem[i] == "vendor" {
   309  			elem[i] = repl
   310  		}
   311  	}
   312  	return strings.Join(elem, "/")
   313  }
   314  
   315  // warnUnmatched warns about patterns that didn't match any packages.
   316  func warnUnmatched(matches []*match) {
   317  	for _, m := range matches {
   318  		if len(m.Pkgs) == 0 {
   319  			m.Err =
   320  				errors.Newf(token.NoPos, "cue: %q matched no packages\n", m.Pattern)
   321  		}
   322  	}
   323  }
   324  
   325  // importPaths returns the matching paths to use for the given command line.
   326  // It calls ImportPathsQuiet and then WarnUnmatched.
   327  func (l *loader) importPaths(patterns []string) []*match {
   328  	matches := l.importPathsQuiet(patterns)
   329  	warnUnmatched(matches)
   330  	return matches
   331  }
   332  
   333  // importPathsQuiet is like ImportPaths but does not warn about patterns with no matches.
   334  func (l *loader) importPathsQuiet(patterns []string) []*match {
   335  	var out []*match
   336  	for _, a := range cleanPatterns(patterns) {
   337  		if isMetaPackage(a) {
   338  			out = append(out, l.matchPackages(a, l.cfg.Package))
   339  			continue
   340  		}
   341  
   342  		orig := a
   343  		pkgName := l.cfg.Package
   344  		switch p := strings.IndexByte(a, ':'); {
   345  		case p < 0:
   346  		case p == 0:
   347  			pkgName = a[1:]
   348  			a = "."
   349  		default:
   350  			pkgName = a[p+1:]
   351  			a = a[:p]
   352  		}
   353  		if pkgName == "*" {
   354  			pkgName = ""
   355  		}
   356  
   357  		if strings.Contains(a, "...") {
   358  			if isLocalImport(a) {
   359  				out = append(out, l.matchPackagesInFS(a, pkgName))
   360  			} else {
   361  				out = append(out, l.matchPackages(a, pkgName))
   362  			}
   363  			continue
   364  		}
   365  
   366  		var p *build.Instance
   367  		if isLocalImport(a) {
   368  			p = l.cfg.newRelInstance(token.NoPos, a, pkgName)
   369  		} else {
   370  			p = l.cfg.newInstance(token.NoPos, importPath(orig))
   371  		}
   372  
   373  		pkgs := l.importPkg(token.NoPos, p)
   374  		out = append(out, &match{Pattern: a, Literal: true, Pkgs: pkgs})
   375  	}
   376  	return out
   377  }
   378  
   379  // cleanPatterns returns the patterns to use for the given
   380  // command line. It canonicalizes the patterns but does not
   381  // evaluate any matches.
   382  func cleanPatterns(patterns []string) []string {
   383  	if len(patterns) == 0 {
   384  		return []string{"."}
   385  	}
   386  	var out []string
   387  	for _, a := range patterns {
   388  		// Arguments are supposed to be import paths, but
   389  		// as a courtesy to Windows developers, rewrite \ to /
   390  		// in command-line arguments. Handles .\... and so on.
   391  		if filepath.Separator == '\\' {
   392  			a = strings.Replace(a, `\`, `/`, -1)
   393  		}
   394  
   395  		// Put argument in canonical form, but preserve leading ./.
   396  		if strings.HasPrefix(a, "./") {
   397  			a = "./" + path.Clean(a)
   398  			if a == "./." {
   399  				a = "."
   400  			}
   401  		} else {
   402  			a = path.Clean(a)
   403  		}
   404  		out = append(out, a)
   405  	}
   406  	return out
   407  }
   408  
   409  // isMetaPackage checks if name is a reserved package name that expands to multiple packages.
   410  func isMetaPackage(name string) bool {
   411  	return name == "std" || name == "cmd" || name == "all"
   412  }
   413  
   414  // hasPathPrefix reports whether the path s begins with the
   415  // elements in prefix.
   416  func hasPathPrefix(s, prefix string) bool {
   417  	switch {
   418  	default:
   419  		return false
   420  	case len(s) == len(prefix):
   421  		return s == prefix
   422  	case len(s) > len(prefix):
   423  		if prefix != "" && prefix[len(prefix)-1] == '/' {
   424  			return strings.HasPrefix(s, prefix)
   425  		}
   426  		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
   427  	}
   428  }
   429  
   430  // hasFilepathPrefix reports whether the path s begins with the
   431  // elements in prefix.
   432  func hasFilepathPrefix(s, prefix string) bool {
   433  	switch {
   434  	default:
   435  		return false
   436  	case len(s) == len(prefix):
   437  		return s == prefix
   438  	case len(s) > len(prefix):
   439  		if prefix != "" && prefix[len(prefix)-1] == filepath.Separator {
   440  			return strings.HasPrefix(s, prefix)
   441  		}
   442  		return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix
   443  	}
   444  }
   445  
   446  // isStandardImportPath reports whether $GOROOT/src/path should be considered
   447  // part of the standard distribution. For historical reasons we allow people to add
   448  // their own code to $GOROOT instead of using $GOPATH, but we assume that
   449  // code will start with a domain name (dot in the first element).
   450  //
   451  // Note that this function is meant to evaluate whether a directory found in GOROOT
   452  // should be treated as part of the standard library. It should not be used to decide
   453  // that a directory found in GOPATH should be rejected: directories in GOPATH
   454  // need not have dots in the first element, and they just take their chances
   455  // with future collisions in the standard library.
   456  func isStandardImportPath(path string) bool {
   457  	i := strings.Index(path, "/")
   458  	if i < 0 {
   459  		i = len(path)
   460  	}
   461  	elem := path[:i]
   462  	return !strings.Contains(elem, ".")
   463  }
   464  
   465  // isRelativePath reports whether pattern should be interpreted as a directory
   466  // path relative to the current directory, as opposed to a pattern matching
   467  // import paths.
   468  func isRelativePath(pattern string) bool {
   469  	return strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == ".."
   470  }
   471  
   472  // inDir checks whether path is in the file tree rooted at dir.
   473  // If so, inDir returns an equivalent path relative to dir.
   474  // If not, inDir returns an empty string.
   475  // inDir makes some effort to succeed even in the presence of symbolic links.
   476  // TODO(rsc): Replace internal/test.inDir with a call to this function for Go 1.12.
   477  func inDir(path, dir string) string {
   478  	if rel := inDirLex(path, dir); rel != "" {
   479  		return rel
   480  	}
   481  	xpath, err := filepath.EvalSymlinks(path)
   482  	if err != nil || xpath == path {
   483  		xpath = ""
   484  	} else {
   485  		if rel := inDirLex(xpath, dir); rel != "" {
   486  			return rel
   487  		}
   488  	}
   489  
   490  	xdir, err := filepath.EvalSymlinks(dir)
   491  	if err == nil && xdir != dir {
   492  		if rel := inDirLex(path, xdir); rel != "" {
   493  			return rel
   494  		}
   495  		if xpath != "" {
   496  			if rel := inDirLex(xpath, xdir); rel != "" {
   497  				return rel
   498  			}
   499  		}
   500  	}
   501  	return ""
   502  }
   503  
   504  // inDirLex is like inDir but only checks the lexical form of the file names.
   505  // It does not consider symbolic links.
   506  // TODO(rsc): This is a copy of str.HasFilePathPrefix, modified to
   507  // return the suffix. Most uses of str.HasFilePathPrefix should probably
   508  // be calling InDir instead.
   509  func inDirLex(path, dir string) string {
   510  	pv := strings.ToUpper(filepath.VolumeName(path))
   511  	dv := strings.ToUpper(filepath.VolumeName(dir))
   512  	path = path[len(pv):]
   513  	dir = dir[len(dv):]
   514  	switch {
   515  	default:
   516  		return ""
   517  	case pv != dv:
   518  		return ""
   519  	case len(path) == len(dir):
   520  		if path == dir {
   521  			return "."
   522  		}
   523  		return ""
   524  	case dir == "":
   525  		return path
   526  	case len(path) > len(dir):
   527  		if dir[len(dir)-1] == filepath.Separator {
   528  			if path[:len(dir)] == dir {
   529  				return path[len(dir):]
   530  			}
   531  			return ""
   532  		}
   533  		if path[len(dir)] == filepath.Separator && path[:len(dir)] == dir {
   534  			if len(path) == len(dir)+1 {
   535  				return "."
   536  			}
   537  			return path[len(dir)+1:]
   538  		}
   539  		return ""
   540  	}
   541  }