github.com/penguinn/godep@v0.0.0-20170205210856-a9cd0561f946/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 OnlyInGOPATH: true, 63 } 64 65 var ( 66 saveR, saveT bool 67 ) 68 69 func init() { 70 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 71 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 72 73 } 74 75 func runSave(cmd *Command, args []string) { 76 if VendorExperiment && saveR { 77 log.Println("flag -r is incompatible with the vendoring experiment") 78 cmd.UsageExit() 79 } 80 err := save(args) 81 if err != nil { 82 log.Fatalln(err) 83 } 84 } 85 86 func dotPackage() (*build.Package, error) { 87 dir, err := filepath.Abs(".") 88 if err != nil { 89 return nil, err 90 } 91 return build.ImportDir(dir, build.FindOnly) 92 } 93 94 func projectPackages(dDir string, a []*Package) []*Package { 95 var projPkgs []*Package 96 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 97 for _, p := range a { 98 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 99 if strings.HasPrefix(pkgDir, dotDir) { 100 projPkgs = append(projPkgs, p) 101 } 102 } 103 return projPkgs 104 } 105 106 func save(pkgs []string) error { 107 var err error 108 dp, err := dotPackage() 109 if err != nil { 110 return err 111 } 112 debugln("dotPackageImportPath:", dp.ImportPath) 113 debugln("dotPackageDir:", dp.Dir) 114 115 cv, err := goVersion() 116 if err != nil { 117 return err 118 } 119 verboseln("Go Version:", cv) 120 121 gold, err := loadDefaultGodepsFile() 122 if err != nil { 123 if !os.IsNotExist(err) { 124 return err 125 } 126 verboseln("No old Godeps.json found.") 127 gold.GoVersion = cv 128 } 129 130 printVersionWarnings(gold.GoVersion) 131 if len(gold.GoVersion) == 0 { 132 gold.GoVersion = majorGoVersion 133 } else { 134 majorGoVersion, err = trimGoVersion(gold.GoVersion) 135 if err != nil { 136 log.Fatalf("Unable to determine go major version from value specified in %s: %s\n", gold.file(), gold.GoVersion) 137 } 138 } 139 140 gnew := &Godeps{ 141 ImportPath: dp.ImportPath, 142 GoVersion: gold.GoVersion, 143 } 144 145 switch len(pkgs) { 146 case 0: 147 pkgs = []string{"."} 148 default: 149 gnew.Packages = pkgs 150 } 151 152 verboseln("Finding dependencies for", pkgs) 153 a, err := LoadPackages(pkgs...) 154 if err != nil { 155 return err 156 } 157 158 for _, p := range a { 159 verboseln("Found package:", p.ImportPath) 160 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 161 } 162 ppln(a) 163 164 projA := projectPackages(dp.Dir, a) 165 debugln("Filtered projectPackages") 166 ppln(projA) 167 168 verboseln("Computing new Godeps.json file") 169 err = gnew.fill(a, dp.ImportPath) 170 if err != nil { 171 return err 172 } 173 debugln("New Godeps Filled") 174 ppln(gnew) 175 176 if gnew.Deps == nil { 177 gnew.Deps = make([]Dependency, 0) // produce json [], not null 178 } 179 gdisk := gnew.copy() 180 err = carryVersions(&gold, gnew) 181 if err != nil { 182 return err 183 } 184 185 if gold.isOldFile { 186 // If we are migrating from an old format file, 187 // we require that the listed version of every 188 // dependency must be installed in GOPATH, so it's 189 // available to copy. 190 if !eqDeps(gnew.Deps, gdisk.Deps) { 191 return errors.New(strings.TrimSpace(needRestore)) 192 } 193 gold = Godeps{} 194 } 195 os.Remove("Godeps") // remove regular file if present; ignore error 196 readme := filepath.Join("Godeps", "Readme") 197 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 198 if err != nil { 199 log.Println(err) 200 } 201 _, err = gnew.save() 202 if err != nil { 203 return err 204 } 205 206 verboseln("Computing diff between old and new deps") 207 // We use a name starting with "_" so the go tool 208 // ignores this directory when traversing packages 209 // starting at the project's root. For example, 210 // godep go list ./... 211 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 212 rem := subDeps(gold.Deps, gnew.Deps) 213 ppln(rem) 214 add := subDeps(gnew.Deps, gold.Deps) 215 ppln(add) 216 if len(rem) > 0 { 217 verboseln("Deps to remove:") 218 for _, r := range rem { 219 verboseln("\t", r.ImportPath) 220 } 221 verboseln("Removing unused dependencies") 222 err = removeSrc(srcdir, rem) 223 if err != nil { 224 return err 225 } 226 } 227 if len(add) > 0 { 228 verboseln("Deps to add:") 229 for _, a := range add { 230 verboseln("\t", a.ImportPath) 231 } 232 verboseln("Adding new dependencies") 233 err = copySrc(srcdir, add) 234 if err != nil { 235 return err 236 } 237 } 238 if !VendorExperiment { 239 f, _ := filepath.Split(srcdir) 240 writeVCSIgnore(f) 241 } 242 var rewritePaths []string 243 if saveR { 244 for _, dep := range gnew.Deps { 245 rewritePaths = append(rewritePaths, dep.ImportPath) 246 } 247 } 248 verboseln("Rewriting paths (if necessary)") 249 ppln(rewritePaths) 250 return rewrite(projA, dp.ImportPath, rewritePaths) 251 } 252 253 func printVersionWarnings(ov string) { 254 var warning bool 255 cv, err := goVersion() 256 if err != nil { 257 return 258 } 259 // Trim the old version because we may have saved it w/o trimming it 260 // cv is already trimmed by goVersion() 261 tov, err := trimGoVersion(ov) 262 if err != nil { 263 return 264 } 265 266 if tov != ov { 267 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 268 warning = true 269 } 270 if cv != tov { 271 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, cv) 272 warning = true 273 } 274 if warning { 275 log.Println("To record current major go version run `godep update -goversion`.") 276 } 277 } 278 279 type revError struct { 280 ImportPath string 281 WantRev string 282 HavePath string 283 HaveRev string 284 } 285 286 func (v *revError) Error() string { 287 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 288 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 289 } 290 291 // carryVersions copies Rev and Comment from a to b for 292 // each dependency with an identical ImportPath. For any 293 // dependency in b that appears to be from the same repo 294 // as one in a (for example, a parent or child directory), 295 // the Rev must already match - otherwise it is an error. 296 func carryVersions(a, b *Godeps) error { 297 for i := range b.Deps { 298 err := carryVersion(a, &b.Deps[i]) 299 if err != nil { 300 return err 301 } 302 } 303 return nil 304 } 305 306 func carryVersion(a *Godeps, db *Dependency) error { 307 // First see if this exact package is already in the list. 308 for _, da := range a.Deps { 309 if db.ImportPath == da.ImportPath { 310 db.Rev = da.Rev 311 db.Comment = da.Comment 312 return nil 313 } 314 } 315 // No exact match, check for child or sibling package. 316 // We can't handle mismatched versions for packages in 317 // the same repo, so report that as an error. 318 for _, da := range a.Deps { 319 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 320 strings.HasPrefix(da.ImportPath, db.root+"/") { 321 if da.Rev != db.Rev { 322 return &revError{ 323 ImportPath: db.ImportPath, 324 WantRev: db.Rev, 325 HavePath: da.ImportPath, 326 HaveRev: da.Rev, 327 } 328 } 329 } 330 } 331 // No related package in the list, must be a new repo. 332 return nil 333 } 334 335 // subDeps returns a - b, using ImportPath for equality. 336 func subDeps(a, b []Dependency) (diff []Dependency) { 337 Diff: 338 for _, da := range a { 339 for _, db := range b { 340 if da.ImportPath == db.ImportPath { 341 continue Diff 342 } 343 } 344 diff = append(diff, da) 345 } 346 return diff 347 } 348 349 func removeSrc(srcdir string, deps []Dependency) error { 350 for _, dep := range deps { 351 path := filepath.FromSlash(dep.ImportPath) 352 err := os.RemoveAll(filepath.Join(srcdir, path)) 353 if err != nil { 354 return err 355 } 356 } 357 return nil 358 } 359 360 func copySrc(dir string, deps []Dependency) error { 361 // mapping to see if we visited a parent directory already 362 visited := make(map[string]bool) 363 ok := true 364 for _, dep := range deps { 365 debugln("copySrc for", dep.ImportPath) 366 srcdir := filepath.Join(dep.ws, "src") 367 rel, err := filepath.Rel(srcdir, dep.dir) 368 debugln("srcdir", srcdir) 369 debugln("rel", rel) 370 debugln("err", err) 371 if err != nil { // this should never happen 372 return err 373 } 374 dstpkgroot := filepath.Join(dir, rel) 375 err = os.RemoveAll(dstpkgroot) 376 if err != nil { 377 log.Println(err) 378 ok = false 379 } 380 381 // copy actual dependency 382 vf := dep.vcs.listFiles(dep.dir) 383 debugln("vf", vf) 384 w := fs.Walk(dep.dir) 385 for w.Step() { 386 err = copyPkgFile(vf, dir, srcdir, w) 387 if err != nil { 388 log.Println(err) 389 ok = false 390 } 391 } 392 393 // Look for legal files in root 394 // some packages are imports as a sub-package but license info 395 // is at root: exampleorg/common has license file in exampleorg 396 // 397 if dep.ImportPath == dep.root { 398 // we are already at root 399 continue 400 } 401 402 // prevent copying twice This could happen if we have 403 // two subpackages listed someorg/common and 404 // someorg/anotherpack which has their license in 405 // the parent dir of someorg 406 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 407 if visited[rootdir] { 408 continue 409 } 410 visited[rootdir] = true 411 vf = dep.vcs.listFiles(rootdir) 412 w = fs.Walk(rootdir) 413 for w.Step() { 414 fname := filepath.Base(w.Path()) 415 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 416 err = copyPkgFile(vf, dir, srcdir, w) 417 if err != nil { 418 log.Println(err) 419 ok = false 420 } 421 } 422 } 423 } 424 425 if !ok { 426 return errorCopyingSourceCode 427 } 428 429 return nil 430 } 431 432 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 433 if w.Err() != nil { 434 return w.Err() 435 } 436 name := w.Stat().Name() 437 if w.Stat().IsDir() { 438 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 439 // Skip directories starting with '.' or '_' or 440 // 'testdata' (last is only skipped if saveT is false) 441 w.SkipDir() 442 } 443 return nil 444 } 445 rel, err := filepath.Rel(srcroot, w.Path()) 446 if err != nil { // this should never happen 447 return err 448 } 449 if !saveT && strings.HasSuffix(name, "_test.go") { 450 if verbose { 451 log.Printf("save: skipping test file: %s", w.Path()) 452 } 453 return nil 454 } 455 if !vf.Contains(w.Path()) { 456 if verbose { 457 log.Printf("save: skipping untracked file: %s", w.Path()) 458 } 459 return nil 460 } 461 return copyFile(filepath.Join(dstroot, rel), w.Path()) 462 } 463 464 // copyFile copies a regular file from src to dst. 465 // dst is opened with os.Create. 466 // If the file name ends with .go, 467 // copyFile strips canonical import path annotations. 468 // These are comments of the form: 469 // package foo // import "bar/foo" 470 // package foo /* import "bar/foo" */ 471 func copyFile(dst, src string) error { 472 err := os.MkdirAll(filepath.Dir(dst), 0777) 473 if err != nil { 474 return err 475 } 476 477 linkDst, err := os.Readlink(src) 478 if err == nil { 479 return os.Symlink(linkDst, dst) 480 } 481 482 si, err := stat(src) 483 if err != nil { 484 return err 485 } 486 487 r, err := os.Open(src) 488 if err != nil { 489 return err 490 } 491 defer r.Close() 492 493 w, err := os.Create(dst) 494 if err != nil { 495 return err 496 } 497 if err := os.Chmod(dst, si.Mode()); err != nil { 498 return err 499 } 500 501 if strings.HasSuffix(dst, ".go") { 502 debugln("Copy Without Import Comment", w, r) 503 err = copyWithoutImportComment(w, r) 504 } else { 505 debugln("Copy (plain)", w, r) 506 _, err = io.Copy(w, r) 507 } 508 err1 := w.Close() 509 if err == nil { 510 err = err1 511 } 512 513 return err 514 } 515 516 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 517 b := bufio.NewReader(r) 518 for { 519 l, err := b.ReadBytes('\n') 520 eof := err == io.EOF 521 if err != nil && err != io.EOF { 522 return err 523 } 524 525 // If we have data then write it out... 526 if len(l) > 0 { 527 // Strip off \n if it exists because stripImportComment 528 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 529 if err != nil { 530 return err 531 } 532 } 533 534 if eof { 535 return nil 536 } 537 } 538 } 539 540 const ( 541 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 542 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 543 ) 544 545 var ( 546 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 547 pkgPrefix = []byte("package ") 548 ) 549 550 // stripImportComment returns line with its import comment removed. 551 // If s is not a package statement containing an import comment, 552 // it is returned unaltered. 553 // FIXME: expects lines w/o a \n at the end 554 // See also http://golang.org/s/go14customimport. 555 func stripImportComment(line []byte) []byte { 556 if !bytes.HasPrefix(line, pkgPrefix) { 557 // Fast path; this will skip all but one line in the file. 558 // This assumes there is no whitespace before the keyword. 559 return line 560 } 561 if m := importCommentRE.FindSubmatch(line); m != nil { 562 return append(m[1], m[2]...) 563 } 564 return line 565 } 566 567 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 568 // so that dir/pkg and dir/bin don't accidentally get committed. 569 // It logs any errors it encounters. 570 func writeVCSIgnore(dir string) { 571 // Currently git is the only VCS for which we know how to do this. 572 // Mercurial and Bazaar have similar mechanisms, but they apparently 573 // require writing files outside of dir. 574 const ignore = "/pkg\n/bin\n" 575 name := filepath.Join(dir, ".gitignore") 576 err := writeFile(name, ignore) 577 if err != nil { 578 log.Println(err) 579 } 580 } 581 582 // writeFile is like ioutil.WriteFile but it creates 583 // intermediate directories with os.MkdirAll. 584 func writeFile(name, body string) error { 585 err := os.MkdirAll(filepath.Dir(name), 0777) 586 if err != nil { 587 return err 588 } 589 return ioutil.WriteFile(name, []byte(body), 0666) 590 } 591 592 const ( 593 // Readme contains the README text. 594 Readme = ` 595 This directory tree is generated automatically by godep. 596 597 Please do not edit. 598 599 See https://github.com/tools/godep for more information. 600 ` 601 needRestore = ` 602 mismatched versions while migrating 603 604 It looks like you are switching from the old Godeps format 605 (from flag -copy=false). The old format is just a file; it 606 doesn't contain source code. For this migration, godep needs 607 the appropriate version of each dependency to be installed in 608 GOPATH, so that the source code is available to copy. 609 610 To fix this, run 'godep restore'. 611 ` 612 )