gopkg.in/tools/godep.v65@v65.0.0-20160509212847-4d9a4c3d91e3/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 OnlyInGOPATH: true, 63 } 64 65 var ( 66 saveR, saveT bool 67 ) 68 69 func init() { 70 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 71 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 72 73 } 74 75 func runSave(cmd *Command, args []string) { 76 if VendorExperiment && saveR { 77 log.Println("flag -r is incompatible with the vendoring experiment") 78 cmd.UsageExit() 79 } 80 err := save(args) 81 if err != nil { 82 log.Fatalln(err) 83 } 84 } 85 86 func dotPackage() (*build.Package, error) { 87 dir, err := filepath.Abs(".") 88 if err != nil { 89 return nil, err 90 } 91 return build.ImportDir(dir, build.FindOnly) 92 } 93 94 func projectPackages(dDir string, a []*Package) []*Package { 95 var projPkgs []*Package 96 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 97 for _, p := range a { 98 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 99 if strings.HasPrefix(pkgDir, dotDir) { 100 projPkgs = append(projPkgs, p) 101 } 102 } 103 return projPkgs 104 } 105 106 func save(pkgs []string) error { 107 var err error 108 dp, err := dotPackage() 109 if err != nil { 110 return err 111 } 112 debugln("dotPackageImportPath:", dp.ImportPath) 113 debugln("dotPackageDir:", dp.Dir) 114 115 cv, err := goVersion() 116 if err != nil { 117 return err 118 } 119 verboseln("Go Version:", cv) 120 121 gold, err := loadDefaultGodepsFile() 122 if err != nil { 123 if !os.IsNotExist(err) { 124 return err 125 } 126 verboseln("No old Godeps.json found.") 127 gold.GoVersion = cv 128 } 129 130 printVersionWarnings(gold.GoVersion) 131 if len(gold.GoVersion) == 0 { 132 gold.GoVersion = majorGoVersion 133 } else { 134 majorGoVersion, err = trimGoVersion(gold.GoVersion) 135 if err != nil { 136 log.Fatalf("Unable to determine go major version from value specified in %s: %s\n", gold.file(), gold.GoVersion) 137 } 138 } 139 140 gnew := &Godeps{ 141 ImportPath: dp.ImportPath, 142 GoVersion: gold.GoVersion, 143 } 144 145 switch len(pkgs) { 146 case 0: 147 pkgs = []string{"."} 148 default: 149 gnew.Packages = pkgs 150 } 151 152 verboseln("Finding dependencies for", pkgs) 153 a, err := LoadPackages(pkgs...) 154 if err != nil { 155 return err 156 } 157 158 for _, p := range a { 159 verboseln("Found package:", p.ImportPath) 160 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 161 } 162 ppln(a) 163 164 projA := projectPackages(dp.Dir, a) 165 debugln("Filtered projectPackages") 166 ppln(projA) 167 168 verboseln("Computing new Godeps.json file") 169 err = gnew.fill(a, dp.ImportPath) 170 if err != nil { 171 return err 172 } 173 debugln("New Godeps Filled") 174 ppln(gnew) 175 176 if gnew.Deps == nil { 177 gnew.Deps = make([]Dependency, 0) // produce json [], not null 178 } 179 gdisk := gnew.copy() 180 err = carryVersions(&gold, gnew) 181 if err != nil { 182 return err 183 } 184 185 if gold.isOldFile { 186 // If we are migrating from an old format file, 187 // we require that the listed version of every 188 // dependency must be installed in GOPATH, so it's 189 // available to copy. 190 if !eqDeps(gnew.Deps, gdisk.Deps) { 191 return errors.New(strings.TrimSpace(needRestore)) 192 } 193 gold = Godeps{} 194 } 195 os.Remove("Godeps") // remove regular file if present; ignore error 196 readme := filepath.Join("Godeps", "Readme") 197 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 198 if err != nil { 199 log.Println(err) 200 } 201 _, err = gnew.save() 202 if err != nil { 203 return err 204 } 205 206 verboseln("Computing diff between old and new deps") 207 // We use a name starting with "_" so the go tool 208 // ignores this directory when traversing packages 209 // starting at the project's root. For example, 210 // godep go list ./... 211 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 212 rem := subDeps(gold.Deps, gnew.Deps) 213 ppln(rem) 214 add := subDeps(gnew.Deps, gold.Deps) 215 ppln(add) 216 if len(rem) > 0 { 217 verboseln("Deps to remove:") 218 for _, r := range rem { 219 verboseln("\t", r.ImportPath) 220 } 221 verboseln("Removing unused dependencies") 222 err = removeSrc(srcdir, rem) 223 if err != nil { 224 return err 225 } 226 } 227 if len(add) > 0 { 228 verboseln("Deps to add:") 229 for _, a := range add { 230 verboseln("\t", a.ImportPath) 231 } 232 verboseln("Adding new dependencies") 233 err = copySrc(srcdir, add) 234 if err != nil { 235 return err 236 } 237 } 238 if !VendorExperiment { 239 f, _ := filepath.Split(srcdir) 240 writeVCSIgnore(f) 241 } 242 var rewritePaths []string 243 if saveR { 244 for _, dep := range gnew.Deps { 245 rewritePaths = append(rewritePaths, dep.ImportPath) 246 } 247 } 248 verboseln("Rewriting paths (if necessary)") 249 ppln(rewritePaths) 250 return rewrite(projA, dp.ImportPath, rewritePaths) 251 } 252 253 func printVersionWarnings(ov string) { 254 var warning bool 255 cv, err := goVersion() 256 if err != nil { 257 return 258 } 259 // Trim the old version because we may have saved it w/o trimming it 260 // cv is already trimmed by goVersion() 261 tov, err := trimGoVersion(ov) 262 if err != nil { 263 return 264 } 265 266 if tov != ov { 267 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 268 warning = true 269 } 270 if cv != tov { 271 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, cv) 272 warning = true 273 } 274 if warning { 275 log.Println("To record current major go version run `godep update -goversion`.") 276 } 277 } 278 279 type revError struct { 280 ImportPath string 281 WantRev string 282 HavePath string 283 HaveRev string 284 } 285 286 func (v *revError) Error() string { 287 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 288 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 289 } 290 291 // carryVersions copies Rev and Comment from a to b for 292 // each dependency with an identical ImportPath. For any 293 // dependency in b that appears to be from the same repo 294 // as one in a (for example, a parent or child directory), 295 // the Rev must already match - otherwise it is an error. 296 func carryVersions(a, b *Godeps) error { 297 for i := range b.Deps { 298 err := carryVersion(a, &b.Deps[i]) 299 if err != nil { 300 return err 301 } 302 } 303 return nil 304 } 305 306 func carryVersion(a *Godeps, db *Dependency) error { 307 // First see if this exact package is already in the list. 308 for _, da := range a.Deps { 309 if db.ImportPath == da.ImportPath { 310 db.Rev = da.Rev 311 db.Comment = da.Comment 312 return nil 313 } 314 } 315 // No exact match, check for child or sibling package. 316 // We can't handle mismatched versions for packages in 317 // the same repo, so report that as an error. 318 for _, da := range a.Deps { 319 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 320 strings.HasPrefix(da.ImportPath, db.root+"/") { 321 if da.Rev != db.Rev { 322 return &revError{ 323 ImportPath: db.ImportPath, 324 WantRev: db.Rev, 325 HavePath: da.ImportPath, 326 HaveRev: da.Rev, 327 } 328 } 329 } 330 } 331 // No related package in the list, must be a new repo. 332 return nil 333 } 334 335 // subDeps returns a - b, using ImportPath for equality. 336 func subDeps(a, b []Dependency) (diff []Dependency) { 337 Diff: 338 for _, da := range a { 339 for _, db := range b { 340 if da.ImportPath == db.ImportPath { 341 continue Diff 342 } 343 } 344 diff = append(diff, da) 345 } 346 return diff 347 } 348 349 func removeSrc(srcdir string, deps []Dependency) error { 350 for _, dep := range deps { 351 path := filepath.FromSlash(dep.ImportPath) 352 err := os.RemoveAll(filepath.Join(srcdir, path)) 353 if err != nil { 354 return err 355 } 356 } 357 return nil 358 } 359 360 func copySrc(dir string, deps []Dependency) error { 361 // mapping to see if we visited a parent directory already 362 visited := make(map[string]bool) 363 ok := true 364 for _, dep := range deps { 365 debugln("copySrc for", dep.ImportPath) 366 srcdir := filepath.Join(dep.ws, "src") 367 rel, err := filepath.Rel(srcdir, dep.dir) 368 debugln("srcdir", srcdir) 369 debugln("rel", rel) 370 debugln("err", err) 371 if err != nil { // this should never happen 372 return err 373 } 374 dstpkgroot := filepath.Join(dir, rel) 375 err = os.RemoveAll(dstpkgroot) 376 if err != nil { 377 log.Println(err) 378 ok = false 379 } 380 381 // copy actual dependency 382 vf := dep.vcs.listFiles(dep.dir) 383 debugln("vf", vf) 384 w := fs.Walk(dep.dir) 385 for w.Step() { 386 err = copyPkgFile(vf, dir, srcdir, w) 387 if err != nil { 388 log.Println(err) 389 ok = false 390 } 391 } 392 393 // Look for legal files in root 394 // some packages are imports as a sub-package but license info 395 // is at root: exampleorg/common has license file in exampleorg 396 // 397 if dep.ImportPath == dep.root { 398 // we are already at root 399 continue 400 } 401 402 // prevent copying twice This could happen if we have 403 // two subpackages listed someorg/common and 404 // someorg/anotherpack which has their license in 405 // the parent dir of someorg 406 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 407 if visited[rootdir] { 408 continue 409 } 410 visited[rootdir] = true 411 vf = dep.vcs.listFiles(rootdir) 412 w = fs.Walk(rootdir) 413 for w.Step() { 414 fname := filepath.Base(w.Path()) 415 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 416 err = copyPkgFile(vf, dir, srcdir, w) 417 if err != nil { 418 log.Println(err) 419 ok = false 420 } 421 } 422 } 423 } 424 425 if !ok { 426 return errorCopyingSourceCode 427 } 428 429 return nil 430 } 431 432 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 433 if w.Err() != nil { 434 return w.Err() 435 } 436 name := w.Stat().Name() 437 if w.Stat().IsDir() { 438 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 439 // Skip directories starting with '.' or '_' or 440 // 'testdata' (last is only skipped if saveT is false) 441 w.SkipDir() 442 } 443 return nil 444 } 445 rel, err := filepath.Rel(srcroot, w.Path()) 446 if err != nil { // this should never happen 447 return err 448 } 449 if !saveT && strings.HasSuffix(name, "_test.go") { 450 if verbose { 451 log.Printf("save: skipping test file: %s", w.Path()) 452 } 453 return nil 454 } 455 if !vf.Contains(w.Path()) { 456 if verbose { 457 log.Printf("save: skipping untracked file: %s", w.Path()) 458 } 459 return nil 460 } 461 return copyFile(filepath.Join(dstroot, rel), w.Path()) 462 } 463 464 // copyFile copies a regular file from src to dst. 465 // dst is opened with os.Create. 466 // If the file name ends with .go, 467 // copyFile strips canonical import path annotations. 468 // These are comments of the form: 469 // package foo // import "bar/foo" 470 // package foo /* import "bar/foo" */ 471 func copyFile(dst, src string) error { 472 err := os.MkdirAll(filepath.Dir(dst), 0777) 473 if err != nil { 474 return err 475 } 476 477 linkDst, err := os.Readlink(src) 478 if err == nil { 479 return os.Symlink(linkDst, dst) 480 } 481 482 r, err := os.Open(src) 483 if err != nil { 484 return err 485 } 486 defer r.Close() 487 488 w, err := os.Create(dst) 489 if err != nil { 490 return err 491 } 492 493 if strings.HasSuffix(dst, ".go") { 494 debugln("Copy Without Import Comment", w, r) 495 err = copyWithoutImportComment(w, r) 496 } else { 497 debugln("Copy (plain)", w, r) 498 _, err = io.Copy(w, r) 499 } 500 err1 := w.Close() 501 if err == nil { 502 err = err1 503 } 504 505 return err 506 } 507 508 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 509 b := bufio.NewReader(r) 510 for { 511 l, err := b.ReadBytes('\n') 512 eof := err == io.EOF 513 if err != nil && err != io.EOF { 514 return err 515 } 516 517 // If we have data then write it out... 518 if len(l) > 0 { 519 // Strip off \n if it exists because stripImportComment 520 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 521 if err != nil { 522 return err 523 } 524 } 525 526 if eof { 527 return nil 528 } 529 } 530 } 531 532 const ( 533 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 534 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 535 ) 536 537 var ( 538 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 539 pkgPrefix = []byte("package ") 540 ) 541 542 // stripImportComment returns line with its import comment removed. 543 // If s is not a package statement containing an import comment, 544 // it is returned unaltered. 545 // FIXME: expects lines w/o a \n at the end 546 // See also http://golang.org/s/go14customimport. 547 func stripImportComment(line []byte) []byte { 548 if !bytes.HasPrefix(line, pkgPrefix) { 549 // Fast path; this will skip all but one line in the file. 550 // This assumes there is no whitespace before the keyword. 551 return line 552 } 553 if m := importCommentRE.FindSubmatch(line); m != nil { 554 return append(m[1], m[2]...) 555 } 556 return line 557 } 558 559 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 560 // so that dir/pkg and dir/bin don't accidentally get committed. 561 // It logs any errors it encounters. 562 func writeVCSIgnore(dir string) { 563 // Currently git is the only VCS for which we know how to do this. 564 // Mercurial and Bazaar have similar mechanisms, but they apparently 565 // require writing files outside of dir. 566 const ignore = "/pkg\n/bin\n" 567 name := filepath.Join(dir, ".gitignore") 568 err := writeFile(name, ignore) 569 if err != nil { 570 log.Println(err) 571 } 572 } 573 574 // writeFile is like ioutil.WriteFile but it creates 575 // intermediate directories with os.MkdirAll. 576 func writeFile(name, body string) error { 577 err := os.MkdirAll(filepath.Dir(name), 0777) 578 if err != nil { 579 return err 580 } 581 return ioutil.WriteFile(name, []byte(body), 0666) 582 } 583 584 const ( 585 // Readme contains the README text. 586 Readme = ` 587 This directory tree is generated automatically by godep. 588 589 Please do not edit. 590 591 See https://github.com/tools/godep for more information. 592 ` 593 needRestore = ` 594 mismatched versions while migrating 595 596 It looks like you are switching from the old Godeps format 597 (from flag -copy=false). The old format is just a file; it 598 doesn't contain source code. For this migration, godep needs 599 the appropriate version of each dependency to be installed in 600 GOPATH, so that the source code is available to copy. 601 602 To fix this, run 'godep restore'. 603 ` 604 )