github.com/christarazi/controller-tools@v0.3.1-0.20210907042920-aa94049173f8/pkg/loader/loader.go (about) 1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package loader 18 19 import ( 20 "fmt" 21 "go/ast" 22 "go/parser" 23 "go/scanner" 24 "go/token" 25 "go/types" 26 "io/ioutil" 27 "os" 28 "sync" 29 30 "golang.org/x/tools/go/packages" 31 ) 32 33 // Much of this is strongly inspired by the contents of go/packages, 34 // except that it allows for lazy loading of syntax and type-checking 35 // information to speed up cases where full traversal isn't needed. 36 37 // PrintErrors print errors associated with all packages 38 // in the given package graph, starting at the given root 39 // packages and traversing through all imports. It will skip 40 // any errors of the kinds specified in filterKinds. It will 41 // return true if any errors were printed. 42 func PrintErrors(pkgs []*Package, filterKinds ...packages.ErrorKind) bool { 43 pkgsRaw := make([]*packages.Package, len(pkgs)) 44 for i, pkg := range pkgs { 45 pkgsRaw[i] = pkg.Package 46 } 47 toSkip := make(map[packages.ErrorKind]struct{}) 48 for _, errKind := range filterKinds { 49 toSkip[errKind] = struct{}{} 50 } 51 hadErrors := false 52 packages.Visit(pkgsRaw, nil, func(pkgRaw *packages.Package) { 53 for _, err := range pkgRaw.Errors { 54 if _, skip := toSkip[err.Kind]; skip { 55 continue 56 } 57 hadErrors = true 58 fmt.Fprintln(os.Stderr, err) 59 } 60 }) 61 return hadErrors 62 } 63 64 // Package is a single, unique Go package that can be 65 // lazily parsed and type-checked. Packages should not 66 // be constructed directly -- instead, use LoadRoots. 67 // For a given call to LoadRoots, only a single instance 68 // of each package exists, and thus they may be used as keys 69 // and for comparison. 70 type Package struct { 71 *packages.Package 72 73 imports map[string]*Package 74 75 loader *loader 76 sync.Mutex 77 } 78 79 // Imports returns the imports for the given package, indexed by 80 // package path (*not* name in any particular file). 81 func (p *Package) Imports() map[string]*Package { 82 if p.imports == nil { 83 p.imports = p.loader.packagesFor(p.Package.Imports) 84 } 85 86 return p.imports 87 } 88 89 // NeedTypesInfo indicates that type-checking information is needed for this package. 90 // Actual type-checking information can be accessed via the Types and TypesInfo fields. 91 func (p *Package) NeedTypesInfo() { 92 if p.TypesInfo != nil { 93 return 94 } 95 p.NeedSyntax() 96 p.loader.typeCheck(p) 97 } 98 99 // NeedSyntax indicates that a parsed AST is needed for this package. 100 // Actual ASTs can be accessed via the Syntax field. 101 func (p *Package) NeedSyntax() { 102 if p.Syntax != nil { 103 return 104 } 105 out := make([]*ast.File, len(p.CompiledGoFiles)) 106 var wg sync.WaitGroup 107 wg.Add(len(p.CompiledGoFiles)) 108 for i, filename := range p.CompiledGoFiles { 109 go func(i int, filename string) { 110 defer wg.Done() 111 src, err := ioutil.ReadFile(filename) 112 if err != nil { 113 p.AddError(err) 114 return 115 } 116 out[i], err = p.loader.parseFile(filename, src) 117 if err != nil { 118 p.AddError(err) 119 return 120 } 121 }(i, filename) 122 } 123 wg.Wait() 124 for _, file := range out { 125 if file == nil { 126 return 127 } 128 } 129 p.Syntax = out 130 } 131 132 // AddError adds an error to the errors associated with the given package. 133 func (p *Package) AddError(err error) { 134 switch typedErr := err.(type) { 135 case *os.PathError: 136 // file-reading errors 137 p.Errors = append(p.Errors, packages.Error{ 138 Pos: typedErr.Path + ":1", 139 Msg: typedErr.Err.Error(), 140 Kind: packages.ParseError, 141 }) 142 case scanner.ErrorList: 143 // parsing/scanning errors 144 for _, subErr := range typedErr { 145 p.Errors = append(p.Errors, packages.Error{ 146 Pos: subErr.Pos.String(), 147 Msg: subErr.Msg, 148 Kind: packages.ParseError, 149 }) 150 } 151 case types.Error: 152 // type-checking errors 153 p.Errors = append(p.Errors, packages.Error{ 154 Pos: typedErr.Fset.Position(typedErr.Pos).String(), 155 Msg: typedErr.Msg, 156 Kind: packages.TypeError, 157 }) 158 case ErrList: 159 for _, subErr := range typedErr { 160 p.AddError(subErr) 161 } 162 case PositionedError: 163 p.Errors = append(p.Errors, packages.Error{ 164 Pos: p.loader.cfg.Fset.Position(typedErr.Pos).String(), 165 Msg: typedErr.Error(), 166 Kind: packages.UnknownError, 167 }) 168 default: 169 // should only happen for external errors, like ref checking 170 p.Errors = append(p.Errors, packages.Error{ 171 Pos: p.ID + ":-", 172 Msg: err.Error(), 173 Kind: packages.UnknownError, 174 }) 175 } 176 } 177 178 // loader loads packages and their imports. Loaded packages will have 179 // type size, imports, and exports file information populated. Additional 180 // information, like ASTs and type-checking information, can be accessed 181 // via methods on individual packages. 182 type loader struct { 183 // Roots are the loaded "root" packages in the package graph loaded via 184 // LoadRoots. 185 Roots []*Package 186 187 // cfg contains the package loading config (initialized on demand) 188 cfg *packages.Config 189 // packages contains the cache of Packages indexed by the underlying 190 // package.Package, so that we don't ever produce two Packages with 191 // the same underlying packages.Package. 192 packages map[*packages.Package]*Package 193 packagesMu sync.Mutex 194 } 195 196 // packageFor returns a wrapped Package for the given packages.Package, 197 // ensuring that there's a one-to-one mapping between the two. 198 // It's *not* threadsafe -- use packagesFor for that. 199 func (l *loader) packageFor(pkgRaw *packages.Package) *Package { 200 if l.packages[pkgRaw] == nil { 201 l.packages[pkgRaw] = &Package{ 202 Package: pkgRaw, 203 loader: l, 204 } 205 } 206 return l.packages[pkgRaw] 207 } 208 209 // packagesFor returns a map of Package objects for each packages.Package in the input 210 // map, ensuring that there's a one-to-one mapping between package.Package and Package 211 // (as per packageFor). 212 func (l *loader) packagesFor(pkgsRaw map[string]*packages.Package) map[string]*Package { 213 l.packagesMu.Lock() 214 defer l.packagesMu.Unlock() 215 216 out := make(map[string]*Package, len(pkgsRaw)) 217 for name, rawPkg := range pkgsRaw { 218 out[name] = l.packageFor(rawPkg) 219 } 220 return out 221 } 222 223 // typeCheck type-checks the given package. 224 func (l *loader) typeCheck(pkg *Package) { 225 // don't conflict with typeCheckFromExportData 226 227 pkg.TypesInfo = &types.Info{ 228 Types: make(map[ast.Expr]types.TypeAndValue), 229 Defs: make(map[*ast.Ident]types.Object), 230 Uses: make(map[*ast.Ident]types.Object), 231 Implicits: make(map[ast.Node]types.Object), 232 Scopes: make(map[ast.Node]*types.Scope), 233 Selections: make(map[*ast.SelectorExpr]*types.Selection), 234 } 235 236 pkg.Fset = l.cfg.Fset 237 pkg.Types = types.NewPackage(pkg.PkgPath, pkg.Name) 238 239 importer := importerFunc(func(path string) (*types.Package, error) { 240 if path == "unsafe" { 241 return types.Unsafe, nil 242 } 243 244 // The imports map is keyed by import path. 245 importedPkg := pkg.Imports()[path] 246 if importedPkg == nil { 247 return nil, fmt.Errorf("package %q possibly creates an import loop", path) 248 } 249 250 // it's possible to have a call to check in parallel to a call to this 251 // if one package in the package graph gets its dependency filtered out, 252 // but another doesn't (so one wants a "dummy" package here, and another 253 // wants the full check). 254 // 255 // Thus, we need to lock here (at least for the time being) to avoid 256 // races between the above write to `pkg.Types` and this checking of 257 // importedPkg.Types. 258 importedPkg.Lock() 259 defer importedPkg.Unlock() 260 261 if importedPkg.Types != nil && importedPkg.Types.Complete() { 262 return importedPkg.Types, nil 263 } 264 265 // if we haven't already loaded typecheck data, we don't care about this package's types 266 return types.NewPackage(importedPkg.PkgPath, importedPkg.Name), nil 267 }) 268 269 var errs []error 270 271 // type-check 272 checkConfig := &types.Config{ 273 Importer: importer, 274 275 IgnoreFuncBodies: true, // we only need decl-level info 276 277 Error: func(err error) { 278 errs = append(errs, err) 279 }, 280 281 Sizes: pkg.TypesSizes, 282 } 283 if err := types.NewChecker(checkConfig, l.cfg.Fset, pkg.Types, pkg.TypesInfo).Files(pkg.Syntax); err != nil { 284 errs = append(errs, err) 285 } 286 287 // make sure that if a given sub-import is ill-typed, we mark this package as ill-typed as well. 288 illTyped := len(errs) > 0 289 if !illTyped { 290 for _, importedPkg := range pkg.Imports() { 291 if importedPkg.IllTyped { 292 illTyped = true 293 break 294 } 295 } 296 } 297 pkg.IllTyped = illTyped 298 299 // publish errors to the package error list. 300 for _, err := range errs { 301 pkg.AddError(err) 302 } 303 } 304 305 // parseFile parses the given file, including comments. 306 func (l *loader) parseFile(filename string, src []byte) (*ast.File, error) { 307 // skip function bodies 308 file, err := parser.ParseFile(l.cfg.Fset, filename, src, parser.AllErrors|parser.ParseComments) 309 if err != nil { 310 return nil, err 311 } 312 313 return file, nil 314 } 315 316 // LoadRoots loads the given "root" packages by path, transitively loading 317 // and all imports as well. 318 // 319 // Loaded packages will have type size, imports, and exports file information 320 // populated. Additional information, like ASTs and type-checking information, 321 // can be accessed via methods on individual packages. 322 func LoadRoots(roots ...string) ([]*Package, error) { 323 return LoadRootsWithConfig(&packages.Config{}, roots...) 324 } 325 326 // LoadRootsWithConfig functions like LoadRoots, except that it allows passing 327 // a custom loading config. The config will be modified to suit the needs of 328 // the loader. 329 // 330 // This is generally only useful for use in testing when you need to modify 331 // loading settings to load from a fake location. 332 func LoadRootsWithConfig(cfg *packages.Config, roots ...string) ([]*Package, error) { 333 l := &loader{ 334 cfg: cfg, 335 packages: make(map[*packages.Package]*Package), 336 } 337 l.cfg.Mode |= packages.LoadImports | packages.NeedTypesSizes 338 if l.cfg.Fset == nil { 339 l.cfg.Fset = token.NewFileSet() 340 } 341 // put our build flags first so that callers can override them 342 l.cfg.BuildFlags = append([]string{"-tags", "ignore_autogenerated"}, l.cfg.BuildFlags...) 343 344 rawPkgs, err := packages.Load(l.cfg, roots...) 345 if err != nil { 346 return nil, err 347 } 348 349 for _, rawPkg := range rawPkgs { 350 l.Roots = append(l.Roots, l.packageFor(rawPkg)) 351 } 352 353 return l.Roots, nil 354 } 355 356 // importFunc is an implementation of the single-method 357 // types.Importer interface based on a function value. 358 type importerFunc func(path string) (*types.Package, error) 359 360 func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }