gopkg.in/tools/godep.v60@v60.0.0-20160318205742-530caa84a45c/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 OnlyInGOPATH: true, 63 } 64 65 var ( 66 saveR, saveT bool 67 ) 68 69 func init() { 70 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 71 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 72 73 } 74 75 func runSave(cmd *Command, args []string) { 76 if VendorExperiment && saveR { 77 log.Println("flag -r is incompatible with the vendoring experiment") 78 cmd.UsageExit() 79 } 80 err := save(args) 81 if err != nil { 82 log.Fatalln(err) 83 } 84 } 85 86 func dotPackage() (*build.Package, error) { 87 dir, err := filepath.Abs(".") 88 if err != nil { 89 return nil, err 90 } 91 return build.ImportDir(dir, build.FindOnly) 92 } 93 94 func projectPackages(dDir string, a []*Package) []*Package { 95 var projPkgs []*Package 96 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 97 for _, p := range a { 98 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 99 if strings.HasPrefix(pkgDir, dotDir) { 100 projPkgs = append(projPkgs, p) 101 } 102 } 103 return projPkgs 104 } 105 106 func save(pkgs []string) error { 107 dp, err := dotPackage() 108 if err != nil { 109 return err 110 } 111 debugln("dotPackageImportPath:", dp.ImportPath) 112 debugln("dotPackageDir:", dp.Dir) 113 114 cv, err := goVersion() 115 if err != nil { 116 return err 117 } 118 verboseln("Go Version:", cv) 119 120 gold, err := loadDefaultGodepsFile() 121 if err != nil { 122 if !os.IsNotExist(err) { 123 return err 124 } 125 verboseln("No old Godeps.json found.") 126 gold.GoVersion = cv 127 } 128 129 printVersionWarnings(gold.GoVersion) 130 131 gnew := &Godeps{ 132 ImportPath: dp.ImportPath, 133 GoVersion: gold.GoVersion, 134 } 135 136 switch len(pkgs) { 137 case 0: 138 pkgs = []string{"."} 139 default: 140 gnew.Packages = pkgs 141 } 142 143 verboseln("Finding dependencies for", pkgs) 144 a, err := LoadPackages(pkgs...) 145 if err != nil { 146 return err 147 } 148 149 for _, p := range a { 150 verboseln("Found package:", p.ImportPath) 151 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 152 } 153 ppln(a) 154 155 projA := projectPackages(dp.Dir, a) 156 debugln("Filtered projectPackages") 157 ppln(projA) 158 159 verboseln("Computing new Godeps.json file") 160 err = gnew.fill(a, dp.ImportPath) 161 if err != nil { 162 return err 163 } 164 debugln("New Godeps Filled") 165 ppln(gnew) 166 167 if gnew.Deps == nil { 168 gnew.Deps = make([]Dependency, 0) // produce json [], not null 169 } 170 gdisk := gnew.copy() 171 err = carryVersions(&gold, gnew) 172 if err != nil { 173 return err 174 } 175 176 if gold.isOldFile { 177 // If we are migrating from an old format file, 178 // we require that the listed version of every 179 // dependency must be installed in GOPATH, so it's 180 // available to copy. 181 if !eqDeps(gnew.Deps, gdisk.Deps) { 182 return errors.New(strings.TrimSpace(needRestore)) 183 } 184 gold = Godeps{} 185 } 186 os.Remove("Godeps") // remove regular file if present; ignore error 187 readme := filepath.Join("Godeps", "Readme") 188 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 189 if err != nil { 190 log.Println(err) 191 } 192 _, err = gnew.save() 193 if err != nil { 194 return err 195 } 196 197 verboseln("Computing diff between old and new deps") 198 // We use a name starting with "_" so the go tool 199 // ignores this directory when traversing packages 200 // starting at the project's root. For example, 201 // godep go list ./... 202 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 203 rem := subDeps(gold.Deps, gnew.Deps) 204 ppln(rem) 205 add := subDeps(gnew.Deps, gold.Deps) 206 ppln(add) 207 if len(rem) > 0 { 208 verboseln("Deps to remove:") 209 for _, r := range rem { 210 verboseln("\t", r.ImportPath) 211 } 212 verboseln("Removing unused dependencies") 213 err = removeSrc(srcdir, rem) 214 if err != nil { 215 return err 216 } 217 } 218 if len(add) > 0 { 219 verboseln("Deps to add:") 220 for _, a := range add { 221 verboseln("\t", a.ImportPath) 222 } 223 verboseln("Adding new dependencies") 224 err = copySrc(srcdir, add) 225 if err != nil { 226 return err 227 } 228 } 229 if !VendorExperiment { 230 f, _ := filepath.Split(srcdir) 231 writeVCSIgnore(f) 232 } 233 var rewritePaths []string 234 if saveR { 235 for _, dep := range gnew.Deps { 236 rewritePaths = append(rewritePaths, dep.ImportPath) 237 } 238 } 239 verboseln("Rewriting paths (if necessary)") 240 ppln(rewritePaths) 241 return rewrite(projA, dp.ImportPath, rewritePaths) 242 } 243 244 func printVersionWarnings(ov string) { 245 var warning bool 246 cv, err := goVersion() 247 if err != nil { 248 return 249 } 250 tov, err := trimGoVersion(ov) 251 if err != nil { 252 return 253 } 254 tcv, err := trimGoVersion(cv) 255 if err != nil { 256 return 257 } 258 259 if tov != ov { 260 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 261 warning = true 262 } 263 if tcv != tov { 264 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 265 warning = true 266 } 267 if warning { 268 log.Println("To record current major go version run `godep update -goversion`.") 269 } 270 } 271 272 type revError struct { 273 ImportPath string 274 WantRev string 275 HavePath string 276 HaveRev string 277 } 278 279 func (v *revError) Error() string { 280 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 281 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 282 } 283 284 // carryVersions copies Rev and Comment from a to b for 285 // each dependency with an identical ImportPath. For any 286 // dependency in b that appears to be from the same repo 287 // as one in a (for example, a parent or child directory), 288 // the Rev must already match - otherwise it is an error. 289 func carryVersions(a, b *Godeps) error { 290 for i := range b.Deps { 291 err := carryVersion(a, &b.Deps[i]) 292 if err != nil { 293 return err 294 } 295 } 296 return nil 297 } 298 299 func carryVersion(a *Godeps, db *Dependency) error { 300 // First see if this exact package is already in the list. 301 for _, da := range a.Deps { 302 if db.ImportPath == da.ImportPath { 303 db.Rev = da.Rev 304 db.Comment = da.Comment 305 return nil 306 } 307 } 308 // No exact match, check for child or sibling package. 309 // We can't handle mismatched versions for packages in 310 // the same repo, so report that as an error. 311 for _, da := range a.Deps { 312 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 313 strings.HasPrefix(da.ImportPath, db.root+"/") { 314 if da.Rev != db.Rev { 315 return &revError{ 316 ImportPath: db.ImportPath, 317 WantRev: db.Rev, 318 HavePath: da.ImportPath, 319 HaveRev: da.Rev, 320 } 321 } 322 } 323 } 324 // No related package in the list, must be a new repo. 325 return nil 326 } 327 328 // subDeps returns a - b, using ImportPath for equality. 329 func subDeps(a, b []Dependency) (diff []Dependency) { 330 Diff: 331 for _, da := range a { 332 for _, db := range b { 333 if da.ImportPath == db.ImportPath { 334 continue Diff 335 } 336 } 337 diff = append(diff, da) 338 } 339 return diff 340 } 341 342 func removeSrc(srcdir string, deps []Dependency) error { 343 for _, dep := range deps { 344 path := filepath.FromSlash(dep.ImportPath) 345 err := os.RemoveAll(filepath.Join(srcdir, path)) 346 if err != nil { 347 return err 348 } 349 } 350 return nil 351 } 352 353 func copySrc(dir string, deps []Dependency) error { 354 // mapping to see if we visited a parent directory already 355 visited := make(map[string]bool) 356 ok := true 357 for _, dep := range deps { 358 debugln("copySrc for", dep.ImportPath) 359 srcdir := filepath.Join(dep.ws, "src") 360 rel, err := filepath.Rel(srcdir, dep.dir) 361 if err != nil { // this should never happen 362 return err 363 } 364 dstpkgroot := filepath.Join(dir, rel) 365 err = os.RemoveAll(dstpkgroot) 366 if err != nil { 367 log.Println(err) 368 ok = false 369 } 370 371 // copy actual dependency 372 vf := dep.vcs.listFiles(dep.dir) 373 debugln("vf", vf) 374 w := fs.Walk(dep.dir) 375 for w.Step() { 376 err = copyPkgFile(vf, dir, srcdir, w) 377 if err != nil { 378 log.Println(err) 379 ok = false 380 } 381 } 382 383 // Look for legal files in root 384 // some packages are imports as a sub-package but license info 385 // is at root: exampleorg/common has license file in exampleorg 386 // 387 if dep.ImportPath == dep.root { 388 // we are already at root 389 continue 390 } 391 392 // prevent copying twice This could happen if we have 393 // two subpackages listed someorg/common and 394 // someorg/anotherpack which has their license in 395 // the parent dir of someorg 396 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 397 if visited[rootdir] { 398 continue 399 } 400 visited[rootdir] = true 401 vf = dep.vcs.listFiles(rootdir) 402 w = fs.Walk(rootdir) 403 for w.Step() { 404 fname := filepath.Base(w.Path()) 405 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 406 err = copyPkgFile(vf, dir, srcdir, w) 407 if err != nil { 408 log.Println(err) 409 ok = false 410 } 411 } 412 } 413 } 414 415 if !ok { 416 return errorCopyingSourceCode 417 } 418 419 return nil 420 } 421 422 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 423 if w.Err() != nil { 424 return w.Err() 425 } 426 name := w.Stat().Name() 427 if w.Stat().IsDir() { 428 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 429 // Skip directories starting with '.' or '_' or 430 // 'testdata' (last is only skipped if saveT is false) 431 w.SkipDir() 432 } 433 return nil 434 } 435 rel, err := filepath.Rel(srcroot, w.Path()) 436 if err != nil { // this should never happen 437 return err 438 } 439 if !saveT && strings.HasSuffix(name, "_test.go") { 440 if verbose { 441 log.Printf("save: skipping test file: %s", w.Path()) 442 } 443 return nil 444 } 445 if !vf.Contains(w.Path()) { 446 if verbose { 447 log.Printf("save: skipping untracked file: %s", w.Path()) 448 } 449 return nil 450 } 451 return copyFile(filepath.Join(dstroot, rel), w.Path()) 452 } 453 454 // copyFile copies a regular file from src to dst. 455 // dst is opened with os.Create. 456 // If the file name ends with .go, 457 // copyFile strips canonical import path annotations. 458 // These are comments of the form: 459 // package foo // import "bar/foo" 460 // package foo /* import "bar/foo" */ 461 func copyFile(dst, src string) error { 462 err := os.MkdirAll(filepath.Dir(dst), 0777) 463 if err != nil { 464 return err 465 } 466 467 linkDst, err := os.Readlink(src) 468 if err == nil { 469 return os.Symlink(linkDst, dst) 470 } 471 472 r, err := os.Open(src) 473 if err != nil { 474 return err 475 } 476 defer r.Close() 477 478 w, err := os.Create(dst) 479 if err != nil { 480 return err 481 } 482 483 if strings.HasSuffix(dst, ".go") { 484 debugln("Copy Without Import Comment", w, r) 485 err = copyWithoutImportComment(w, r) 486 } else { 487 debugln("Copy (plain)", w, r) 488 _, err = io.Copy(w, r) 489 } 490 err1 := w.Close() 491 if err == nil { 492 err = err1 493 } 494 495 return err 496 } 497 498 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 499 b := bufio.NewReader(r) 500 for { 501 l, err := b.ReadBytes('\n') 502 eof := err == io.EOF 503 if err != nil && err != io.EOF { 504 return err 505 } 506 507 // If we have data then write it out... 508 if len(l) > 0 { 509 // Strip off \n if it exists because stripImportComment 510 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 511 if err != nil { 512 return err 513 } 514 } 515 516 if eof { 517 return nil 518 } 519 } 520 } 521 522 const ( 523 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 524 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 525 ) 526 527 var ( 528 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 529 pkgPrefix = []byte("package ") 530 ) 531 532 // stripImportComment returns line with its import comment removed. 533 // If s is not a package statement containing an import comment, 534 // it is returned unaltered. 535 // FIXME: expects lines w/o a \n at the end 536 // See also http://golang.org/s/go14customimport. 537 func stripImportComment(line []byte) []byte { 538 if !bytes.HasPrefix(line, pkgPrefix) { 539 // Fast path; this will skip all but one line in the file. 540 // This assumes there is no whitespace before the keyword. 541 return line 542 } 543 if m := importCommentRE.FindSubmatch(line); m != nil { 544 return append(m[1], m[2]...) 545 } 546 return line 547 } 548 549 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 550 // so that dir/pkg and dir/bin don't accidentally get committed. 551 // It logs any errors it encounters. 552 func writeVCSIgnore(dir string) { 553 // Currently git is the only VCS for which we know how to do this. 554 // Mercurial and Bazaar have similar mechanisms, but they apparently 555 // require writing files outside of dir. 556 const ignore = "/pkg\n/bin\n" 557 name := filepath.Join(dir, ".gitignore") 558 err := writeFile(name, ignore) 559 if err != nil { 560 log.Println(err) 561 } 562 } 563 564 // writeFile is like ioutil.WriteFile but it creates 565 // intermediate directories with os.MkdirAll. 566 func writeFile(name, body string) error { 567 err := os.MkdirAll(filepath.Dir(name), 0777) 568 if err != nil { 569 return err 570 } 571 return ioutil.WriteFile(name, []byte(body), 0666) 572 } 573 574 const ( 575 // Readme contains the README text. 576 Readme = ` 577 This directory tree is generated automatically by godep. 578 579 Please do not edit. 580 581 See https://github.com/tools/godep for more information. 582 ` 583 needRestore = ` 584 mismatched versions while migrating 585 586 It looks like you are switching from the old Godeps format 587 (from flag -copy=false). The old format is just a file; it 588 doesn't contain source code. For this migration, godep needs 589 the appropriate version of each dependency to be installed in 590 GOPATH, so that the source code is available to copy. 591 592 To fix this, run 'godep restore'. 593 ` 594 )