k8s.io/kubernetes@v1.29.3/test/typecheck/main.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // do a fast type check of kubernetes code, for all platforms. 18 package main 19 20 import ( 21 "flag" 22 "fmt" 23 "io" 24 "log" 25 "os" 26 "path/filepath" 27 "sort" 28 "strings" 29 "sync" 30 "time" 31 32 "golang.org/x/tools/go/packages" 33 ) 34 35 var ( 36 verbose = flag.Bool("verbose", false, "print more information") 37 cross = flag.Bool("cross", true, "build for all platforms") 38 platforms = flag.String("platform", "", "comma-separated list of platforms to typecheck") 39 timings = flag.Bool("time", false, "output times taken for each phase") 40 defuses = flag.Bool("defuse", false, "output defs/uses") 41 serial = flag.Bool("serial", false, "don't type check platforms in parallel (equivalent to --parallel=1)") 42 parallel = flag.Int("parallel", 2, "limits how many platforms can be checked in parallel. 0 means no limit.") 43 skipTest = flag.Bool("skip-test", false, "don't type check test code") 44 tags = flag.String("tags", "", "comma-separated list of build tags to apply in addition to go's defaults") 45 ignoreDirs = flag.String("ignore-dirs", "", "comma-separated list of directories to ignore in addition to the default hardcoded list including staging, vendor, and hidden dirs") 46 47 // When processed in order, windows and darwin are early to make 48 // interesting OS-based errors happen earlier. 49 crossPlatforms = []string{ 50 "linux/amd64", "windows/386", 51 "darwin/amd64", "darwin/arm64", 52 "linux/arm", "linux/386", 53 "windows/amd64", "linux/arm64", 54 "linux/ppc64le", "linux/s390x", 55 "windows/arm64", 56 } 57 58 // directories we always ignore 59 standardIgnoreDirs = []string{ 60 // Staging code is symlinked from vendor/k8s.io, and uses import 61 // paths as if it were inside of vendor/. It fails typechecking 62 // inside of staging/, but works when typechecked as part of vendor/. 63 "staging", 64 // OS-specific vendor code tends to be imported by OS-specific 65 // packages. We recursively typecheck imported vendored packages for 66 // each OS, but don't typecheck everything for every OS. 67 "vendor", 68 "_output", 69 // This is a weird one. /testdata/ is *mostly* ignored by Go, 70 // and this translates to kubernetes/vendor not working. 71 // edit/record.go doesn't compile without gopkg.in/yaml.v2 72 // in $GOSRC/$GOROOT (both typecheck and the shell script). 73 "pkg/kubectl/cmd/testdata/edit", 74 // Tools we use for maintaining the code base but not necessarily 75 // ship as part of the release 76 "hack/tools", 77 } 78 ) 79 80 func newConfig(platform string) *packages.Config { 81 platSplit := strings.Split(platform, "/") 82 goos, goarch := platSplit[0], platSplit[1] 83 mode := packages.NeedName | packages.NeedFiles | packages.NeedTypes | packages.NeedSyntax | packages.NeedDeps | packages.NeedImports | packages.NeedModule 84 if *defuses { 85 mode = mode | packages.NeedTypesInfo 86 } 87 env := append(os.Environ(), 88 "CGO_ENABLED=1", 89 fmt.Sprintf("GOOS=%s", goos), 90 fmt.Sprintf("GOARCH=%s", goarch)) 91 tagstr := "selinux" 92 if *tags != "" { 93 tagstr = tagstr + "," + *tags 94 } 95 flags := []string{"-tags", tagstr} 96 97 return &packages.Config{ 98 Mode: mode, 99 Env: env, 100 BuildFlags: flags, 101 Tests: !(*skipTest), 102 } 103 } 104 105 type collector struct { 106 dirs []string 107 ignoreDirs []string 108 } 109 110 func newCollector(ignoreDirs string) collector { 111 c := collector{ 112 ignoreDirs: append([]string(nil), standardIgnoreDirs...), 113 } 114 if ignoreDirs != "" { 115 c.ignoreDirs = append(c.ignoreDirs, strings.Split(ignoreDirs, ",")...) 116 } 117 return c 118 } 119 120 func (c *collector) walk(roots []string) error { 121 for _, root := range roots { 122 err := filepath.Walk(root, c.handlePath) 123 if err != nil { 124 return err 125 } 126 } 127 sort.Strings(c.dirs) 128 return nil 129 } 130 131 // handlePath walks the filesystem recursively, collecting directories, 132 // ignoring some unneeded directories (hidden/vendored) that are handled 133 // specially later. 134 func (c *collector) handlePath(path string, info os.FileInfo, err error) error { 135 if err != nil { 136 return err 137 } 138 if info.IsDir() { 139 name := info.Name() 140 // Ignore hidden directories (.git, .cache, etc) 141 if (len(name) > 1 && (name[0] == '.' || name[0] == '_')) || name == "testdata" { 142 if *verbose { 143 fmt.Printf("DBG: skipping dir %s\n", path) 144 } 145 return filepath.SkipDir 146 } 147 for _, dir := range c.ignoreDirs { 148 if path == dir { 149 if *verbose { 150 fmt.Printf("DBG: ignoring dir %s\n", path) 151 } 152 return filepath.SkipDir 153 } 154 } 155 // Make dirs into relative pkg names. 156 // NOTE: can't use filepath.Join because it elides the leading "./" 157 pkg := path 158 if !strings.HasPrefix(pkg, "./") { 159 pkg = "./" + pkg 160 } 161 c.dirs = append(c.dirs, pkg) 162 if *verbose { 163 fmt.Printf("DBG: added dir %s\n", path) 164 } 165 } 166 return nil 167 } 168 169 func (c *collector) verify(plat string) ([]string, error) { 170 errors := []packages.Error{} 171 start := time.Now() 172 config := newConfig(plat) 173 174 rootPkgs, err := packages.Load(config, c.dirs...) 175 if err != nil { 176 return nil, err 177 } 178 179 // Recursively import all deps and flatten to one list. 180 allMap := map[string]*packages.Package{} 181 for _, pkg := range rootPkgs { 182 if *verbose { 183 serialFprintf(os.Stdout, "pkg %q has %d GoFiles\n", pkg.PkgPath, len(pkg.GoFiles)) 184 } 185 allMap[pkg.PkgPath] = pkg 186 if len(pkg.Imports) > 0 { 187 for _, imp := range pkg.Imports { 188 if *verbose { 189 serialFprintf(os.Stdout, "pkg %q imports %q\n", pkg.PkgPath, imp.PkgPath) 190 } 191 allMap[imp.PkgPath] = imp 192 } 193 } 194 } 195 keys := make([]string, 0, len(allMap)) 196 for k := range allMap { 197 keys = append(keys, k) 198 } 199 sort.Strings(keys) 200 allList := make([]*packages.Package, 0, len(keys)) 201 for _, k := range keys { 202 allList = append(allList, allMap[k]) 203 } 204 205 for _, pkg := range allList { 206 if len(pkg.GoFiles) > 0 { 207 if len(pkg.Errors) > 0 && (pkg.PkgPath == "main" || strings.Contains(pkg.PkgPath, ".")) { 208 errors = append(errors, pkg.Errors...) 209 } 210 } 211 if *defuses { 212 for id, obj := range pkg.TypesInfo.Defs { 213 serialFprintf(os.Stdout, "%s: %q defines %v\n", 214 pkg.Fset.Position(id.Pos()), id.Name, obj) 215 } 216 for id, obj := range pkg.TypesInfo.Uses { 217 serialFprintf(os.Stdout, "%s: %q uses %v\n", 218 pkg.Fset.Position(id.Pos()), id.Name, obj) 219 } 220 } 221 } 222 if *timings { 223 serialFprintf(os.Stdout, "%s took %.1fs\n", plat, time.Since(start).Seconds()) 224 } 225 return dedup(errors), nil 226 } 227 228 func dedup(errors []packages.Error) []string { 229 ret := []string{} 230 231 m := map[string]bool{} 232 for _, e := range errors { 233 es := e.Error() 234 if !m[es] { 235 ret = append(ret, es) 236 m[es] = true 237 } 238 } 239 return ret 240 } 241 242 var outMu sync.Mutex 243 244 func serialFprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { 245 outMu.Lock() 246 defer outMu.Unlock() 247 return fmt.Fprintf(w, format, a...) 248 } 249 250 func main() { 251 flag.Parse() 252 args := flag.Args() 253 254 if *verbose { 255 *serial = true // to avoid confusing interleaved logs 256 } 257 258 if len(args) == 0 { 259 args = append(args, ".") 260 } 261 262 c := newCollector(*ignoreDirs) 263 264 if err := c.walk(args); err != nil { 265 log.Fatalf("Error walking: %v", err) 266 } 267 268 plats := crossPlatforms[:] 269 if *platforms != "" { 270 plats = strings.Split(*platforms, ",") 271 } else if !*cross { 272 plats = plats[:1] 273 } 274 275 var wg sync.WaitGroup 276 var failMu sync.Mutex 277 failed := false 278 279 if *serial { 280 *parallel = 1 281 } else if *parallel == 0 { 282 *parallel = len(plats) 283 } 284 throttle := make(chan int, *parallel) 285 286 for _, plat := range plats { 287 wg.Add(1) 288 go func(plat string) { 289 // block until there's room for this task 290 throttle <- 1 291 defer func() { 292 // indicate this task is done 293 <-throttle 294 }() 295 296 f := false 297 serialFprintf(os.Stdout, "type-checking %s\n", plat) 298 errors, err := c.verify(plat) 299 if err != nil { 300 serialFprintf(os.Stderr, "ERROR(%s): failed to verify: %v\n", plat, err) 301 f = true 302 } else if len(errors) > 0 { 303 for _, e := range errors { 304 // Special case CGo errors which may depend on headers we 305 // don't have. 306 if !strings.HasSuffix(e, "could not import C (no metadata for C)") { 307 f = true 308 serialFprintf(os.Stderr, "ERROR(%s): %s\n", plat, e) 309 } 310 } 311 } 312 failMu.Lock() 313 failed = failed || f 314 failMu.Unlock() 315 wg.Done() 316 }(plat) 317 } 318 wg.Wait() 319 if failed { 320 os.Exit(1) 321 } 322 }