golang.org/x/tools@v0.21.0/go/packages/packagestest/export.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 /* 6 Package packagestest creates temporary projects on disk for testing go tools on. 7 8 By changing the exporter used, you can create projects for multiple build 9 systems from the same description, and run the same tests on them in many 10 cases. 11 12 # Example 13 14 As an example of packagestest use, consider the following test that runs 15 the 'go list' command on the specified modules: 16 17 // TestGoList exercises the 'go list' command in module mode and in GOPATH mode. 18 func TestGoList(t *testing.T) { packagestest.TestAll(t, testGoList) } 19 func testGoList(t *testing.T, x packagestest.Exporter) { 20 e := packagestest.Export(t, x, []packagestest.Module{ 21 { 22 Name: "gopher.example/repoa", 23 Files: map[string]interface{}{ 24 "a/a.go": "package a", 25 }, 26 }, 27 { 28 Name: "gopher.example/repob", 29 Files: map[string]interface{}{ 30 "b/b.go": "package b", 31 }, 32 }, 33 }) 34 defer e.Cleanup() 35 36 cmd := exec.Command("go", "list", "gopher.example/...") 37 cmd.Dir = e.Config.Dir 38 cmd.Env = e.Config.Env 39 out, err := cmd.Output() 40 if err != nil { 41 t.Fatal(err) 42 } 43 t.Logf("'go list gopher.example/...' with %s mode layout:\n%s", x.Name(), out) 44 } 45 46 TestGoList uses TestAll to exercise the 'go list' command with all 47 exporters known to packagestest. Currently, packagestest includes 48 exporters that produce module mode layouts and GOPATH mode layouts. 49 Running the test with verbose output will print: 50 51 === RUN TestGoList 52 === RUN TestGoList/GOPATH 53 === RUN TestGoList/Modules 54 --- PASS: TestGoList (0.21s) 55 --- PASS: TestGoList/GOPATH (0.03s) 56 main_test.go:36: 'go list gopher.example/...' with GOPATH mode layout: 57 gopher.example/repoa/a 58 gopher.example/repob/b 59 --- PASS: TestGoList/Modules (0.18s) 60 main_test.go:36: 'go list gopher.example/...' with Modules mode layout: 61 gopher.example/repoa/a 62 gopher.example/repob/b 63 */ 64 package packagestest 65 66 import ( 67 "errors" 68 "flag" 69 "fmt" 70 "go/token" 71 "io" 72 "log" 73 "os" 74 "path/filepath" 75 "runtime" 76 "strings" 77 "testing" 78 79 "golang.org/x/tools/go/expect" 80 "golang.org/x/tools/go/packages" 81 "golang.org/x/tools/internal/testenv" 82 ) 83 84 var ( 85 skipCleanup = flag.Bool("skip-cleanup", false, "Do not delete the temporary export folders") // for debugging 86 ) 87 88 // ErrUnsupported indicates an error due to an operation not supported on the 89 // current platform. 90 var ErrUnsupported = errors.New("operation is not supported") 91 92 // Module is a representation of a go module. 93 type Module struct { 94 // Name is the base name of the module as it would be in the go.mod file. 95 Name string 96 // Files is the set of source files for all packages that make up the module. 97 // The keys are the file fragment that follows the module name, the value can 98 // be a string or byte slice, in which case it is the contents of the 99 // file, otherwise it must be a Writer function. 100 Files map[string]interface{} 101 102 // Overlay is the set of source file overlays for the module. 103 // The keys are the file fragment as in the Files configuration. 104 // The values are the in memory overlay content for the file. 105 Overlay map[string][]byte 106 } 107 108 // A Writer is a function that writes out a test file. 109 // It is provided the name of the file to write, and may return an error if it 110 // cannot write the file. 111 // These are used as the content of the Files map in a Module. 112 type Writer func(filename string) error 113 114 // Exported is returned by the Export function to report the structure that was produced on disk. 115 type Exported struct { 116 // Config is a correctly configured packages.Config ready to be passed to packages.Load. 117 // Exactly what it will contain varies depending on the Exporter being used. 118 Config *packages.Config 119 120 // Modules is the module description that was used to produce this exported data set. 121 Modules []Module 122 123 ExpectFileSet *token.FileSet // The file set used when parsing expectations 124 125 Exporter Exporter // the exporter used 126 temp string // the temporary directory that was exported to 127 primary string // the first non GOROOT module that was exported 128 written map[string]map[string]string // the full set of exported files 129 notes []*expect.Note // The list of expectations extracted from go source files 130 markers map[string]Range // The set of markers extracted from go source files 131 } 132 133 // Exporter implementations are responsible for converting from the generic description of some 134 // test data to a driver specific file layout. 135 type Exporter interface { 136 // Name reports the name of the exporter, used in logging and sub-test generation. 137 Name() string 138 // Filename reports the system filename for test data source file. 139 // It is given the base directory, the module the file is part of and the filename fragment to 140 // work from. 141 Filename(exported *Exported, module, fragment string) string 142 // Finalize is called once all files have been written to write any extra data needed and modify 143 // the Config to match. It is handed the full list of modules that were encountered while writing 144 // files. 145 Finalize(exported *Exported) error 146 } 147 148 // All is the list of known exporters. 149 // This is used by TestAll to run tests with all the exporters. 150 var All = []Exporter{GOPATH, Modules} 151 152 // TestAll invokes the testing function once for each exporter registered in 153 // the All global. 154 // Each exporter will be run as a sub-test named after the exporter being used. 155 func TestAll(t *testing.T, f func(*testing.T, Exporter)) { 156 t.Helper() 157 for _, e := range All { 158 e := e // in case f calls t.Parallel 159 t.Run(e.Name(), func(t *testing.T) { 160 t.Helper() 161 f(t, e) 162 }) 163 } 164 } 165 166 // BenchmarkAll invokes the testing function once for each exporter registered in 167 // the All global. 168 // Each exporter will be run as a sub-test named after the exporter being used. 169 func BenchmarkAll(b *testing.B, f func(*testing.B, Exporter)) { 170 b.Helper() 171 for _, e := range All { 172 e := e // in case f calls t.Parallel 173 b.Run(e.Name(), func(b *testing.B) { 174 b.Helper() 175 f(b, e) 176 }) 177 } 178 } 179 180 // Export is called to write out a test directory from within a test function. 181 // It takes the exporter and the build system agnostic module descriptions, and 182 // uses them to build a temporary directory. 183 // It returns an Exported with the results of the export. 184 // The Exported.Config is prepared for loading from the exported data. 185 // You must invoke Exported.Cleanup on the returned value to clean up. 186 // The file deletion in the cleanup can be skipped by setting the skip-cleanup 187 // flag when invoking the test, allowing the temporary directory to be left for 188 // debugging tests. 189 // 190 // If the Writer for any file within any module returns an error equivalent to 191 // ErrUnspported, Export skips the test. 192 func Export(t testing.TB, exporter Exporter, modules []Module) *Exported { 193 t.Helper() 194 if exporter == Modules { 195 testenv.NeedsTool(t, "go") 196 } 197 198 dirname := strings.Replace(t.Name(), "/", "_", -1) 199 dirname = strings.Replace(dirname, "#", "_", -1) // duplicate subtests get a #NNN suffix. 200 temp, err := os.MkdirTemp("", dirname) 201 if err != nil { 202 t.Fatal(err) 203 } 204 exported := &Exported{ 205 Config: &packages.Config{ 206 Dir: temp, 207 Env: append(os.Environ(), "GOPACKAGESDRIVER=off", "GOROOT="), // Clear GOROOT to work around #32849. 208 Overlay: make(map[string][]byte), 209 Tests: true, 210 Mode: packages.LoadImports, 211 }, 212 Modules: modules, 213 Exporter: exporter, 214 temp: temp, 215 primary: modules[0].Name, 216 written: map[string]map[string]string{}, 217 ExpectFileSet: token.NewFileSet(), 218 } 219 if testing.Verbose() { 220 exported.Config.Logf = t.Logf 221 } 222 defer func() { 223 if t.Failed() || t.Skipped() { 224 exported.Cleanup() 225 } 226 }() 227 for _, module := range modules { 228 // Create all parent directories before individual files. If any file is a 229 // symlink to a directory, that directory must exist before the symlink is 230 // created or else it may be created with the wrong type on Windows. 231 // (See https://golang.org/issue/39183.) 232 for fragment := range module.Files { 233 fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) 234 if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil { 235 t.Fatal(err) 236 } 237 } 238 239 for fragment, value := range module.Files { 240 fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) 241 written, ok := exported.written[module.Name] 242 if !ok { 243 written = map[string]string{} 244 exported.written[module.Name] = written 245 } 246 written[fragment] = fullpath 247 switch value := value.(type) { 248 case Writer: 249 if err := value(fullpath); err != nil { 250 if errors.Is(err, ErrUnsupported) { 251 t.Skip(err) 252 } 253 t.Fatal(err) 254 } 255 case string: 256 if err := os.WriteFile(fullpath, []byte(value), 0644); err != nil { 257 t.Fatal(err) 258 } 259 default: 260 t.Fatalf("Invalid type %T in files, must be string or Writer", value) 261 } 262 } 263 for fragment, value := range module.Overlay { 264 fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment)) 265 exported.Config.Overlay[fullpath] = value 266 } 267 } 268 if err := exporter.Finalize(exported); err != nil { 269 t.Fatal(err) 270 } 271 testenv.NeedsGoPackagesEnv(t, exported.Config.Env) 272 return exported 273 } 274 275 // Script returns a Writer that writes out contents to the file and sets the 276 // executable bit on the created file. 277 // It is intended for source files that are shell scripts. 278 func Script(contents string) Writer { 279 return func(filename string) error { 280 return os.WriteFile(filename, []byte(contents), 0755) 281 } 282 } 283 284 // Link returns a Writer that creates a hard link from the specified source to 285 // the required file. 286 // This is used to link testdata files into the generated testing tree. 287 // 288 // If hard links to source are not supported on the destination filesystem, the 289 // returned Writer returns an error for which errors.Is(_, ErrUnsupported) 290 // returns true. 291 func Link(source string) Writer { 292 return func(filename string) error { 293 linkErr := os.Link(source, filename) 294 295 if linkErr != nil && !builderMustSupportLinks() { 296 // Probe to figure out whether Link failed because the Link operation 297 // isn't supported. 298 if stat, err := openAndStat(source); err == nil { 299 if err := createEmpty(filename, stat.Mode()); err == nil { 300 // Successfully opened the source and created the destination, 301 // but the result is empty and not a hard-link. 302 return &os.PathError{Op: "Link", Path: filename, Err: ErrUnsupported} 303 } 304 } 305 } 306 307 return linkErr 308 } 309 } 310 311 // Symlink returns a Writer that creates a symlink from the specified source to the 312 // required file. 313 // This is used to link testdata files into the generated testing tree. 314 // 315 // If symlinks to source are not supported on the destination filesystem, the 316 // returned Writer returns an error for which errors.Is(_, ErrUnsupported) 317 // returns true. 318 func Symlink(source string) Writer { 319 if !strings.HasPrefix(source, ".") { 320 if absSource, err := filepath.Abs(source); err == nil { 321 if _, err := os.Stat(source); !os.IsNotExist(err) { 322 source = absSource 323 } 324 } 325 } 326 return func(filename string) error { 327 symlinkErr := os.Symlink(source, filename) 328 329 if symlinkErr != nil && !builderMustSupportLinks() { 330 // Probe to figure out whether Symlink failed because the Symlink 331 // operation isn't supported. 332 fullSource := source 333 if !filepath.IsAbs(source) { 334 // Compute the target path relative to the parent of filename, not the 335 // current working directory. 336 fullSource = filepath.Join(filename, "..", source) 337 } 338 stat, err := openAndStat(fullSource) 339 mode := os.ModePerm 340 if err == nil { 341 mode = stat.Mode() 342 } else if !errors.Is(err, os.ErrNotExist) { 343 // We couldn't open the source, but it might exist. We don't expect to be 344 // able to portably create a symlink to a file we can't see. 345 return symlinkErr 346 } 347 348 if err := createEmpty(filename, mode|0644); err == nil { 349 // Successfully opened the source (or verified that it does not exist) and 350 // created the destination, but we couldn't create it as a symlink. 351 // Probably the OS just doesn't support symlinks in this context. 352 return &os.PathError{Op: "Symlink", Path: filename, Err: ErrUnsupported} 353 } 354 } 355 356 return symlinkErr 357 } 358 } 359 360 // builderMustSupportLinks reports whether we are running on a Go builder 361 // that is known to support hard and symbolic links. 362 func builderMustSupportLinks() bool { 363 if os.Getenv("GO_BUILDER_NAME") == "" { 364 // Any OS can be configured to mount an exotic filesystem. 365 // Don't make assumptions about what users are running. 366 return false 367 } 368 369 switch runtime.GOOS { 370 case "windows", "plan9": 371 // Some versions of Windows and all versions of plan9 do not support 372 // symlinks by default. 373 return false 374 375 default: 376 // All other platforms should support symlinks by default, and our builders 377 // should not do anything unusual that would violate that. 378 return true 379 } 380 } 381 382 // openAndStat attempts to open source for reading. 383 func openAndStat(source string) (os.FileInfo, error) { 384 src, err := os.Open(source) 385 if err != nil { 386 return nil, err 387 } 388 stat, err := src.Stat() 389 src.Close() 390 if err != nil { 391 return nil, err 392 } 393 return stat, nil 394 } 395 396 // createEmpty creates an empty file or directory (depending on mode) 397 // at dst, with the same permissions as mode. 398 func createEmpty(dst string, mode os.FileMode) error { 399 if mode.IsDir() { 400 return os.Mkdir(dst, mode.Perm()) 401 } 402 403 f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode.Perm()) 404 if err != nil { 405 return err 406 } 407 if err := f.Close(); err != nil { 408 os.Remove(dst) // best-effort 409 return err 410 } 411 412 return nil 413 } 414 415 // Copy returns a Writer that copies a file from the specified source to the 416 // required file. 417 // This is used to copy testdata files into the generated testing tree. 418 func Copy(source string) Writer { 419 return func(filename string) error { 420 stat, err := os.Stat(source) 421 if err != nil { 422 return err 423 } 424 if !stat.Mode().IsRegular() { 425 // cannot copy non-regular files (e.g., directories, 426 // symlinks, devices, etc.) 427 return fmt.Errorf("cannot copy non regular file %s", source) 428 } 429 return copyFile(filename, source, stat.Mode().Perm()) 430 } 431 } 432 433 func copyFile(dest, source string, perm os.FileMode) error { 434 src, err := os.Open(source) 435 if err != nil { 436 return err 437 } 438 defer src.Close() 439 440 dst, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) 441 if err != nil { 442 return err 443 } 444 445 _, err = io.Copy(dst, src) 446 if closeErr := dst.Close(); err == nil { 447 err = closeErr 448 } 449 return err 450 } 451 452 // GroupFilesByModules attempts to map directories to the modules within each directory. 453 // This function assumes that the folder is structured in the following way: 454 // 455 // dir/ 456 // primarymod/ 457 // *.go files 458 // packages 459 // go.mod (optional) 460 // modules/ 461 // repoa/ 462 // mod1/ 463 // *.go files 464 // packages 465 // go.mod (optional) 466 // 467 // It scans the directory tree anchored at root and adds a Copy writer to the 468 // map for every file found. 469 // This is to enable the common case in tests where you have a full copy of the 470 // package in your testdata. 471 func GroupFilesByModules(root string) ([]Module, error) { 472 root = filepath.FromSlash(root) 473 primarymodPath := filepath.Join(root, "primarymod") 474 475 _, err := os.Stat(primarymodPath) 476 if os.IsNotExist(err) { 477 return nil, fmt.Errorf("could not find primarymod folder within %s", root) 478 } 479 480 primarymod := &Module{ 481 Name: root, 482 Files: make(map[string]interface{}), 483 Overlay: make(map[string][]byte), 484 } 485 mods := map[string]*Module{ 486 root: primarymod, 487 } 488 modules := []Module{*primarymod} 489 490 if err := filepath.Walk(primarymodPath, func(path string, info os.FileInfo, err error) error { 491 if err != nil { 492 return err 493 } 494 if info.IsDir() { 495 return nil 496 } 497 fragment, err := filepath.Rel(primarymodPath, path) 498 if err != nil { 499 return err 500 } 501 primarymod.Files[filepath.ToSlash(fragment)] = Copy(path) 502 return nil 503 }); err != nil { 504 return nil, err 505 } 506 507 modulesPath := filepath.Join(root, "modules") 508 if _, err := os.Stat(modulesPath); os.IsNotExist(err) { 509 return modules, nil 510 } 511 512 var currentRepo, currentModule string 513 updateCurrentModule := func(dir string) { 514 if dir == currentModule { 515 return 516 } 517 // Handle the case where we step into a nested directory that is a module 518 // and then step out into the parent which is also a module. 519 // Example: 520 // - repoa 521 // - moda 522 // - go.mod 523 // - v2 524 // - go.mod 525 // - what.go 526 // - modb 527 for dir != root { 528 if mods[dir] != nil { 529 currentModule = dir 530 return 531 } 532 dir = filepath.Dir(dir) 533 } 534 } 535 536 if err := filepath.Walk(modulesPath, func(path string, info os.FileInfo, err error) error { 537 if err != nil { 538 return err 539 } 540 enclosingDir := filepath.Dir(path) 541 // If the path is not a directory, then we want to add the path to 542 // the files map of the currentModule. 543 if !info.IsDir() { 544 updateCurrentModule(enclosingDir) 545 fragment, err := filepath.Rel(currentModule, path) 546 if err != nil { 547 return err 548 } 549 mods[currentModule].Files[filepath.ToSlash(fragment)] = Copy(path) 550 return nil 551 } 552 // If the path is a directory and it's enclosing folder is equal to 553 // the modules folder, then the path is a new repo. 554 if enclosingDir == modulesPath { 555 currentRepo = path 556 return nil 557 } 558 // If the path is a directory and it's enclosing folder is not the same 559 // as the current repo and it is not of the form `v1`,`v2`,... 560 // then the path is a folder/package of the current module. 561 if enclosingDir != currentRepo && !versionSuffixRE.MatchString(filepath.Base(path)) { 562 return nil 563 } 564 // If the path is a directory and it's enclosing folder is the current repo 565 // then the path is a new module. 566 module, err := filepath.Rel(modulesPath, path) 567 if err != nil { 568 return err 569 } 570 mods[path] = &Module{ 571 Name: filepath.ToSlash(module), 572 Files: make(map[string]interface{}), 573 Overlay: make(map[string][]byte), 574 } 575 currentModule = path 576 modules = append(modules, *mods[path]) 577 return nil 578 }); err != nil { 579 return nil, err 580 } 581 return modules, nil 582 } 583 584 // MustCopyFileTree returns a file set for a module based on a real directory tree. 585 // It scans the directory tree anchored at root and adds a Copy writer to the 586 // map for every file found. It skips copying files in nested modules. 587 // This is to enable the common case in tests where you have a full copy of the 588 // package in your testdata. 589 // This will panic if there is any kind of error trying to walk the file tree. 590 func MustCopyFileTree(root string) map[string]interface{} { 591 result := map[string]interface{}{} 592 if err := filepath.Walk(filepath.FromSlash(root), func(path string, info os.FileInfo, err error) error { 593 if err != nil { 594 return err 595 } 596 if info.IsDir() { 597 // skip nested modules. 598 if path != root { 599 if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() { 600 return filepath.SkipDir 601 } 602 } 603 return nil 604 } 605 fragment, err := filepath.Rel(root, path) 606 if err != nil { 607 return err 608 } 609 result[filepath.ToSlash(fragment)] = Copy(path) 610 return nil 611 }); err != nil { 612 log.Panic(fmt.Sprintf("MustCopyFileTree failed: %v", err)) 613 } 614 return result 615 } 616 617 // Cleanup removes the temporary directory (unless the --skip-cleanup flag was set) 618 // It is safe to call cleanup multiple times. 619 func (e *Exported) Cleanup() { 620 if e.temp == "" { 621 return 622 } 623 if *skipCleanup { 624 log.Printf("Skipping cleanup of temp dir: %s", e.temp) 625 return 626 } 627 // Make everything read-write so that the Module exporter's module cache can be deleted. 628 filepath.Walk(e.temp, func(path string, info os.FileInfo, err error) error { 629 if err != nil { 630 return nil 631 } 632 if info.IsDir() { 633 os.Chmod(path, 0777) 634 } 635 return nil 636 }) 637 os.RemoveAll(e.temp) // ignore errors 638 e.temp = "" 639 } 640 641 // Temp returns the temporary directory that was generated. 642 func (e *Exported) Temp() string { 643 return e.temp 644 } 645 646 // File returns the full path for the given module and file fragment. 647 func (e *Exported) File(module, fragment string) string { 648 if m := e.written[module]; m != nil { 649 return m[fragment] 650 } 651 return "" 652 } 653 654 // FileContents returns the contents of the specified file. 655 // It will use the overlay if the file is present, otherwise it will read it 656 // from disk. 657 func (e *Exported) FileContents(filename string) ([]byte, error) { 658 if content, found := e.Config.Overlay[filename]; found { 659 return content, nil 660 } 661 content, err := os.ReadFile(filename) 662 if err != nil { 663 return nil, err 664 } 665 return content, nil 666 }