golang.org/x/tools@v0.21.0/go/analysis/unitchecker/separate_test.go (about) 1 // Copyright 2023 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package unitchecker_test 6 7 // This file illustrates separate analysis with an example. 8 9 import ( 10 "bytes" 11 "encoding/json" 12 "fmt" 13 "go/token" 14 "go/types" 15 "io" 16 "os" 17 "path/filepath" 18 "strings" 19 "sync/atomic" 20 "testing" 21 22 "golang.org/x/tools/go/analysis/passes/printf" 23 "golang.org/x/tools/go/analysis/unitchecker" 24 "golang.org/x/tools/go/gcexportdata" 25 "golang.org/x/tools/go/packages" 26 "golang.org/x/tools/internal/testenv" 27 "golang.org/x/tools/internal/testfiles" 28 "golang.org/x/tools/txtar" 29 ) 30 31 // TestExampleSeparateAnalysis demonstrates the principle of separate 32 // analysis, the distribution of units of type-checking and analysis 33 // work across several processes, using serialized summaries to 34 // communicate between them. 35 // 36 // It uses two different kinds of task, "manager" and "worker": 37 // 38 // - The manager computes the graph of package dependencies, and makes 39 // a request to the worker for each package. It does not parse, 40 // type-check, or analyze Go code. It is analogous "go vet". 41 // 42 // - The worker, which contains the Analyzers, reads each request, 43 // loads, parses, and type-checks the files of one package, 44 // applies all necessary analyzers to the package, then writes 45 // its results to a file. It is a unitchecker-based driver, 46 // analogous to the program specified by go vet -vettool= flag. 47 // 48 // In practice these would be separate executables, but for simplicity 49 // of this example they are provided by one executable in two 50 // different modes: the Example function is the manager, and the same 51 // executable invoked with ENTRYPOINT=worker is the worker. 52 // (See TestIntegration for how this happens.) 53 // 54 // Unfortunately this can't be a true Example because of the skip, 55 // which requires a testing.T. 56 func TestExampleSeparateAnalysis(t *testing.T) { 57 testenv.NeedsGoPackages(t) 58 59 // src is an archive containing a module with a printf mistake. 60 const src = ` 61 -- go.mod -- 62 module separate 63 go 1.18 64 65 -- main/main.go -- 66 package main 67 68 import "separate/lib" 69 70 func main() { 71 lib.MyPrintf("%s", 123) 72 } 73 74 -- lib/lib.go -- 75 package lib 76 77 import "fmt" 78 79 func MyPrintf(format string, args ...any) { 80 fmt.Printf(format, args...) 81 } 82 ` 83 84 // Expand archive into tmp tree. 85 tmpdir := t.TempDir() 86 if err := testfiles.ExtractTxtar(tmpdir, txtar.Parse([]byte(src))); err != nil { 87 t.Fatal(err) 88 } 89 90 // Load metadata for the main package and all its dependencies. 91 cfg := &packages.Config{ 92 Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedModule, 93 Dir: tmpdir, 94 Env: append(os.Environ(), 95 "GOPROXY=off", // disable network 96 "GOWORK=off", // an ambient GOWORK value would break package loading 97 ), 98 Logf: t.Logf, 99 } 100 pkgs, err := packages.Load(cfg, "separate/main") 101 if err != nil { 102 t.Fatal(err) 103 } 104 // Stop if any package had a metadata error. 105 if packages.PrintErrors(pkgs) > 0 { 106 t.Fatal("there were errors among loaded packages") 107 } 108 109 // Now we have loaded the import graph, 110 // let's begin the proper work of the manager. 111 112 // Gather root packages. They will get all analyzers, 113 // whereas dependencies get only the subset that 114 // produce facts or are required by them. 115 roots := make(map[*packages.Package]bool) 116 for _, pkg := range pkgs { 117 roots[pkg] = true 118 } 119 120 // nextID generates sequence numbers for each unit of work. 121 // We use it to create names of temporary files. 122 var nextID atomic.Int32 123 124 var allDiagnostics []string 125 126 // Visit all packages in postorder: dependencies first. 127 // TODO(adonovan): opt: use parallel postorder. 128 packages.Visit(pkgs, nil, func(pkg *packages.Package) { 129 if pkg.PkgPath == "unsafe" { 130 return 131 } 132 133 // Choose a unique prefix for temporary files 134 // (.cfg .types .facts) produced by this package. 135 // We stow it in an otherwise unused field of 136 // Package so it can be accessed by our importers. 137 prefix := fmt.Sprintf("%s/%d", tmpdir, nextID.Add(1)) 138 pkg.ExportFile = prefix 139 140 // Construct the request to the worker. 141 var ( 142 importMap = make(map[string]string) 143 packageFile = make(map[string]string) 144 packageVetx = make(map[string]string) 145 ) 146 for importPath, dep := range pkg.Imports { 147 importMap[importPath] = dep.PkgPath 148 if depPrefix := dep.ExportFile; depPrefix != "" { // skip "unsafe" 149 packageFile[dep.PkgPath] = depPrefix + ".types" 150 packageVetx[dep.PkgPath] = depPrefix + ".facts" 151 } 152 } 153 cfg := unitchecker.Config{ 154 ID: pkg.ID, 155 ImportPath: pkg.PkgPath, 156 GoFiles: pkg.CompiledGoFiles, 157 NonGoFiles: pkg.OtherFiles, 158 IgnoredFiles: pkg.IgnoredFiles, 159 ImportMap: importMap, 160 PackageFile: packageFile, 161 PackageVetx: packageVetx, 162 VetxOnly: !roots[pkg], 163 VetxOutput: prefix + ".facts", 164 } 165 if pkg.Module != nil { 166 if v := pkg.Module.GoVersion; v != "" { 167 cfg.GoVersion = "go" + v 168 } 169 } 170 171 // Write the JSON configuration message to a file. 172 cfgData, err := json.Marshal(cfg) 173 if err != nil { 174 t.Fatalf("internal error in json.Marshal: %v", err) 175 } 176 cfgFile := prefix + ".cfg" 177 if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil { 178 t.Fatal(err) 179 } 180 181 // Send the request to the worker. 182 cmd := testenv.Command(t, os.Args[0], "-json", cfgFile) 183 cmd.Stderr = os.Stderr 184 cmd.Stdout = new(bytes.Buffer) 185 cmd.Env = append(os.Environ(), "ENTRYPOINT=worker") 186 if err := cmd.Run(); err != nil { 187 t.Fatal(err) 188 } 189 190 // Parse JSON output and gather in allDiagnostics. 191 dec := json.NewDecoder(cmd.Stdout.(io.Reader)) 192 for { 193 type jsonDiagnostic struct { 194 Posn string `json:"posn"` 195 Message string `json:"message"` 196 } 197 // 'results' maps Package.Path -> Analyzer.Name -> diagnostics 198 var results map[string]map[string][]jsonDiagnostic 199 if err := dec.Decode(&results); err != nil { 200 if err == io.EOF { 201 break 202 } 203 t.Fatalf("internal error decoding JSON: %v", err) 204 } 205 for _, result := range results { 206 for analyzer, diags := range result { 207 for _, diag := range diags { 208 rel := strings.ReplaceAll(diag.Posn, tmpdir, "") 209 rel = filepath.ToSlash(rel) 210 msg := fmt.Sprintf("%s: [%s] %s", rel, analyzer, diag.Message) 211 allDiagnostics = append(allDiagnostics, msg) 212 } 213 } 214 } 215 } 216 }) 217 218 // Observe that the example produces a fact-based diagnostic 219 // from separate analysis of "main", "lib", and "fmt": 220 221 const want = `/main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int` 222 if got := strings.Join(allDiagnostics, "\n"); got != want { 223 t.Errorf("Got: %s\nWant: %s", got, want) 224 } 225 } 226 227 // -- worker process -- 228 229 // worker is the main entry point for a unitchecker-based driver 230 // with only a single analyzer, for illustration. 231 func worker() { 232 // Currently the unitchecker API doesn't allow clients to 233 // control exactly how and where fact and type information 234 // is produced and consumed. 235 // 236 // So, for example, it assumes that type information has 237 // already been produced by the compiler, which is true when 238 // running under "go vet", but isn't necessary. It may be more 239 // convenient and efficient for a distributed analysis system 240 // if the worker generates both of them, which is the approach 241 // taken in this example; they could even be saved as two 242 // sections of a single file. 243 // 244 // Consequently, this test currently needs special access to 245 // private hooks in unitchecker to control how and where facts 246 // and types are produced and consumed. In due course this 247 // will become a respectable public API. In the meantime, it 248 // should at least serve as a demonstration of how one could 249 // fork unitchecker to achieve separate analysis without go vet. 250 unitchecker.SetTypeImportExport(makeTypesImporter, exportTypes) 251 252 unitchecker.Main(printf.Analyzer) 253 } 254 255 func makeTypesImporter(cfg *unitchecker.Config, fset *token.FileSet) types.Importer { 256 imports := make(map[string]*types.Package) 257 return importerFunc(func(importPath string) (*types.Package, error) { 258 // Resolve import path to package path (vendoring, etc) 259 path, ok := cfg.ImportMap[importPath] 260 if !ok { 261 return nil, fmt.Errorf("can't resolve import %q", path) 262 } 263 if path == "unsafe" { 264 return types.Unsafe, nil 265 } 266 267 // Find, read, and decode file containing type information. 268 file, ok := cfg.PackageFile[path] 269 if !ok { 270 return nil, fmt.Errorf("no package file for %q", path) 271 } 272 f, err := os.Open(file) 273 if err != nil { 274 return nil, err 275 } 276 defer f.Close() // ignore error 277 return gcexportdata.Read(f, fset, imports, path) 278 }) 279 } 280 281 func exportTypes(cfg *unitchecker.Config, fset *token.FileSet, pkg *types.Package) error { 282 var out bytes.Buffer 283 if err := gcexportdata.Write(&out, fset, pkg); err != nil { 284 return err 285 } 286 typesFile := strings.TrimSuffix(cfg.VetxOutput, ".facts") + ".types" 287 return os.WriteFile(typesFile, out.Bytes(), 0666) 288 } 289 290 // -- helpers -- 291 292 type importerFunc func(path string) (*types.Package, error) 293 294 func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }