gopkg.in/tools/godep.v61@v61.0.0-20160406162537-35ee059b4e6c/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 OnlyInGOPATH: true, 63 } 64 65 var ( 66 saveR, saveT bool 67 ) 68 69 func init() { 70 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 71 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 72 73 } 74 75 func runSave(cmd *Command, args []string) { 76 if VendorExperiment && saveR { 77 log.Println("flag -r is incompatible with the vendoring experiment") 78 cmd.UsageExit() 79 } 80 err := save(args) 81 if err != nil { 82 log.Fatalln(err) 83 } 84 } 85 86 func dotPackage() (*build.Package, error) { 87 dir, err := filepath.Abs(".") 88 if err != nil { 89 return nil, err 90 } 91 return build.ImportDir(dir, build.FindOnly) 92 } 93 94 func projectPackages(dDir string, a []*Package) []*Package { 95 var projPkgs []*Package 96 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 97 for _, p := range a { 98 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 99 if strings.HasPrefix(pkgDir, dotDir) { 100 projPkgs = append(projPkgs, p) 101 } 102 } 103 return projPkgs 104 } 105 106 func save(pkgs []string) error { 107 var err error 108 dp, err := dotPackage() 109 if err != nil { 110 return err 111 } 112 debugln("dotPackageImportPath:", dp.ImportPath) 113 debugln("dotPackageDir:", dp.Dir) 114 115 cv, err := goVersion() 116 if err != nil { 117 return err 118 } 119 verboseln("Go Version:", cv) 120 121 gold, err := loadDefaultGodepsFile() 122 if err != nil { 123 if !os.IsNotExist(err) { 124 return err 125 } 126 verboseln("No old Godeps.json found.") 127 gold.GoVersion = cv 128 } 129 130 printVersionWarnings(gold.GoVersion) 131 if len(gold.GoVersion) == 0 { 132 gold.GoVersion = majorGoVersion 133 } else { 134 majorGoVersion, err = trimGoVersion(gold.GoVersion) 135 if err != nil { 136 log.Fatalf("Unable to determine go major version from value specified in %s: %s\n", gold.file(), gold.GoVersion) 137 } 138 } 139 140 gnew := &Godeps{ 141 ImportPath: dp.ImportPath, 142 GoVersion: gold.GoVersion, 143 } 144 145 switch len(pkgs) { 146 case 0: 147 pkgs = []string{"."} 148 default: 149 gnew.Packages = pkgs 150 } 151 152 verboseln("Finding dependencies for", pkgs) 153 a, err := LoadPackages(pkgs...) 154 if err != nil { 155 return err 156 } 157 158 for _, p := range a { 159 verboseln("Found package:", p.ImportPath) 160 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 161 } 162 ppln(a) 163 164 projA := projectPackages(dp.Dir, a) 165 debugln("Filtered projectPackages") 166 ppln(projA) 167 168 verboseln("Computing new Godeps.json file") 169 err = gnew.fill(a, dp.ImportPath) 170 if err != nil { 171 return err 172 } 173 debugln("New Godeps Filled") 174 ppln(gnew) 175 176 if gnew.Deps == nil { 177 gnew.Deps = make([]Dependency, 0) // produce json [], not null 178 } 179 gdisk := gnew.copy() 180 err = carryVersions(&gold, gnew) 181 if err != nil { 182 return err 183 } 184 185 if gold.isOldFile { 186 // If we are migrating from an old format file, 187 // we require that the listed version of every 188 // dependency must be installed in GOPATH, so it's 189 // available to copy. 190 if !eqDeps(gnew.Deps, gdisk.Deps) { 191 return errors.New(strings.TrimSpace(needRestore)) 192 } 193 gold = Godeps{} 194 } 195 os.Remove("Godeps") // remove regular file if present; ignore error 196 readme := filepath.Join("Godeps", "Readme") 197 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 198 if err != nil { 199 log.Println(err) 200 } 201 _, err = gnew.save() 202 if err != nil { 203 return err 204 } 205 206 verboseln("Computing diff between old and new deps") 207 // We use a name starting with "_" so the go tool 208 // ignores this directory when traversing packages 209 // starting at the project's root. For example, 210 // godep go list ./... 211 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 212 rem := subDeps(gold.Deps, gnew.Deps) 213 ppln(rem) 214 add := subDeps(gnew.Deps, gold.Deps) 215 ppln(add) 216 if len(rem) > 0 { 217 verboseln("Deps to remove:") 218 for _, r := range rem { 219 verboseln("\t", r.ImportPath) 220 } 221 verboseln("Removing unused dependencies") 222 err = removeSrc(srcdir, rem) 223 if err != nil { 224 return err 225 } 226 } 227 if len(add) > 0 { 228 verboseln("Deps to add:") 229 for _, a := range add { 230 verboseln("\t", a.ImportPath) 231 } 232 verboseln("Adding new dependencies") 233 err = copySrc(srcdir, add) 234 if err != nil { 235 return err 236 } 237 } 238 if !VendorExperiment { 239 f, _ := filepath.Split(srcdir) 240 writeVCSIgnore(f) 241 } 242 var rewritePaths []string 243 if saveR { 244 for _, dep := range gnew.Deps { 245 rewritePaths = append(rewritePaths, dep.ImportPath) 246 } 247 } 248 verboseln("Rewriting paths (if necessary)") 249 ppln(rewritePaths) 250 return rewrite(projA, dp.ImportPath, rewritePaths) 251 } 252 253 func printVersionWarnings(ov string) { 254 var warning bool 255 cv, err := goVersion() 256 if err != nil { 257 return 258 } 259 tov, err := trimGoVersion(ov) 260 if err != nil { 261 return 262 } 263 tcv, err := trimGoVersion(cv) 264 if err != nil { 265 return 266 } 267 268 if tov != ov { 269 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 270 warning = true 271 } 272 if tcv != tov { 273 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 274 warning = true 275 } 276 if warning { 277 log.Println("To record current major go version run `godep update -goversion`.") 278 } 279 } 280 281 type revError struct { 282 ImportPath string 283 WantRev string 284 HavePath string 285 HaveRev string 286 } 287 288 func (v *revError) Error() string { 289 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 290 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 291 } 292 293 // carryVersions copies Rev and Comment from a to b for 294 // each dependency with an identical ImportPath. For any 295 // dependency in b that appears to be from the same repo 296 // as one in a (for example, a parent or child directory), 297 // the Rev must already match - otherwise it is an error. 298 func carryVersions(a, b *Godeps) error { 299 for i := range b.Deps { 300 err := carryVersion(a, &b.Deps[i]) 301 if err != nil { 302 return err 303 } 304 } 305 return nil 306 } 307 308 func carryVersion(a *Godeps, db *Dependency) error { 309 // First see if this exact package is already in the list. 310 for _, da := range a.Deps { 311 if db.ImportPath == da.ImportPath { 312 db.Rev = da.Rev 313 db.Comment = da.Comment 314 return nil 315 } 316 } 317 // No exact match, check for child or sibling package. 318 // We can't handle mismatched versions for packages in 319 // the same repo, so report that as an error. 320 for _, da := range a.Deps { 321 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 322 strings.HasPrefix(da.ImportPath, db.root+"/") { 323 if da.Rev != db.Rev { 324 return &revError{ 325 ImportPath: db.ImportPath, 326 WantRev: db.Rev, 327 HavePath: da.ImportPath, 328 HaveRev: da.Rev, 329 } 330 } 331 } 332 } 333 // No related package in the list, must be a new repo. 334 return nil 335 } 336 337 // subDeps returns a - b, using ImportPath for equality. 338 func subDeps(a, b []Dependency) (diff []Dependency) { 339 Diff: 340 for _, da := range a { 341 for _, db := range b { 342 if da.ImportPath == db.ImportPath { 343 continue Diff 344 } 345 } 346 diff = append(diff, da) 347 } 348 return diff 349 } 350 351 func removeSrc(srcdir string, deps []Dependency) error { 352 for _, dep := range deps { 353 path := filepath.FromSlash(dep.ImportPath) 354 err := os.RemoveAll(filepath.Join(srcdir, path)) 355 if err != nil { 356 return err 357 } 358 } 359 return nil 360 } 361 362 func copySrc(dir string, deps []Dependency) error { 363 // mapping to see if we visited a parent directory already 364 visited := make(map[string]bool) 365 ok := true 366 for _, dep := range deps { 367 debugln("copySrc for", dep.ImportPath) 368 srcdir := filepath.Join(dep.ws, "src") 369 rel, err := filepath.Rel(srcdir, dep.dir) 370 if err != nil { // this should never happen 371 return err 372 } 373 dstpkgroot := filepath.Join(dir, rel) 374 err = os.RemoveAll(dstpkgroot) 375 if err != nil { 376 log.Println(err) 377 ok = false 378 } 379 380 // copy actual dependency 381 vf := dep.vcs.listFiles(dep.dir) 382 debugln("vf", vf) 383 w := fs.Walk(dep.dir) 384 for w.Step() { 385 err = copyPkgFile(vf, dir, srcdir, w) 386 if err != nil { 387 log.Println(err) 388 ok = false 389 } 390 } 391 392 // Look for legal files in root 393 // some packages are imports as a sub-package but license info 394 // is at root: exampleorg/common has license file in exampleorg 395 // 396 if dep.ImportPath == dep.root { 397 // we are already at root 398 continue 399 } 400 401 // prevent copying twice This could happen if we have 402 // two subpackages listed someorg/common and 403 // someorg/anotherpack which has their license in 404 // the parent dir of someorg 405 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 406 if visited[rootdir] { 407 continue 408 } 409 visited[rootdir] = true 410 vf = dep.vcs.listFiles(rootdir) 411 w = fs.Walk(rootdir) 412 for w.Step() { 413 fname := filepath.Base(w.Path()) 414 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 415 err = copyPkgFile(vf, dir, srcdir, w) 416 if err != nil { 417 log.Println(err) 418 ok = false 419 } 420 } 421 } 422 } 423 424 if !ok { 425 return errorCopyingSourceCode 426 } 427 428 return nil 429 } 430 431 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 432 if w.Err() != nil { 433 return w.Err() 434 } 435 name := w.Stat().Name() 436 if w.Stat().IsDir() { 437 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 438 // Skip directories starting with '.' or '_' or 439 // 'testdata' (last is only skipped if saveT is false) 440 w.SkipDir() 441 } 442 return nil 443 } 444 rel, err := filepath.Rel(srcroot, w.Path()) 445 if err != nil { // this should never happen 446 return err 447 } 448 if !saveT && strings.HasSuffix(name, "_test.go") { 449 if verbose { 450 log.Printf("save: skipping test file: %s", w.Path()) 451 } 452 return nil 453 } 454 if !vf.Contains(w.Path()) { 455 if verbose { 456 log.Printf("save: skipping untracked file: %s", w.Path()) 457 } 458 return nil 459 } 460 return copyFile(filepath.Join(dstroot, rel), w.Path()) 461 } 462 463 // copyFile copies a regular file from src to dst. 464 // dst is opened with os.Create. 465 // If the file name ends with .go, 466 // copyFile strips canonical import path annotations. 467 // These are comments of the form: 468 // package foo // import "bar/foo" 469 // package foo /* import "bar/foo" */ 470 func copyFile(dst, src string) error { 471 err := os.MkdirAll(filepath.Dir(dst), 0777) 472 if err != nil { 473 return err 474 } 475 476 linkDst, err := os.Readlink(src) 477 if err == nil { 478 return os.Symlink(linkDst, dst) 479 } 480 481 r, err := os.Open(src) 482 if err != nil { 483 return err 484 } 485 defer r.Close() 486 487 w, err := os.Create(dst) 488 if err != nil { 489 return err 490 } 491 492 if strings.HasSuffix(dst, ".go") { 493 debugln("Copy Without Import Comment", w, r) 494 err = copyWithoutImportComment(w, r) 495 } else { 496 debugln("Copy (plain)", w, r) 497 _, err = io.Copy(w, r) 498 } 499 err1 := w.Close() 500 if err == nil { 501 err = err1 502 } 503 504 return err 505 } 506 507 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 508 b := bufio.NewReader(r) 509 for { 510 l, err := b.ReadBytes('\n') 511 eof := err == io.EOF 512 if err != nil && err != io.EOF { 513 return err 514 } 515 516 // If we have data then write it out... 517 if len(l) > 0 { 518 // Strip off \n if it exists because stripImportComment 519 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 520 if err != nil { 521 return err 522 } 523 } 524 525 if eof { 526 return nil 527 } 528 } 529 } 530 531 const ( 532 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 533 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 534 ) 535 536 var ( 537 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 538 pkgPrefix = []byte("package ") 539 ) 540 541 // stripImportComment returns line with its import comment removed. 542 // If s is not a package statement containing an import comment, 543 // it is returned unaltered. 544 // FIXME: expects lines w/o a \n at the end 545 // See also http://golang.org/s/go14customimport. 546 func stripImportComment(line []byte) []byte { 547 if !bytes.HasPrefix(line, pkgPrefix) { 548 // Fast path; this will skip all but one line in the file. 549 // This assumes there is no whitespace before the keyword. 550 return line 551 } 552 if m := importCommentRE.FindSubmatch(line); m != nil { 553 return append(m[1], m[2]...) 554 } 555 return line 556 } 557 558 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 559 // so that dir/pkg and dir/bin don't accidentally get committed. 560 // It logs any errors it encounters. 561 func writeVCSIgnore(dir string) { 562 // Currently git is the only VCS for which we know how to do this. 563 // Mercurial and Bazaar have similar mechanisms, but they apparently 564 // require writing files outside of dir. 565 const ignore = "/pkg\n/bin\n" 566 name := filepath.Join(dir, ".gitignore") 567 err := writeFile(name, ignore) 568 if err != nil { 569 log.Println(err) 570 } 571 } 572 573 // writeFile is like ioutil.WriteFile but it creates 574 // intermediate directories with os.MkdirAll. 575 func writeFile(name, body string) error { 576 err := os.MkdirAll(filepath.Dir(name), 0777) 577 if err != nil { 578 return err 579 } 580 return ioutil.WriteFile(name, []byte(body), 0666) 581 } 582 583 const ( 584 // Readme contains the README text. 585 Readme = ` 586 This directory tree is generated automatically by godep. 587 588 Please do not edit. 589 590 See https://github.com/tools/godep for more information. 591 ` 592 needRestore = ` 593 mismatched versions while migrating 594 595 It looks like you are switching from the old Godeps format 596 (from flag -copy=false). The old format is just a file; it 597 doesn't contain source code. For this migration, godep needs 598 the appropriate version of each dependency to be installed in 599 GOPATH, so that the source code is available to copy. 600 601 To fix this, run 'godep restore'. 602 ` 603 )