gopkg.in/tools/godep.v63@v63.0.0-20160503185544-51f9ea00dbee/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 OnlyInGOPATH: true, 63 } 64 65 var ( 66 saveR, saveT bool 67 ) 68 69 func init() { 70 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 71 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 72 73 } 74 75 func runSave(cmd *Command, args []string) { 76 if VendorExperiment && saveR { 77 log.Println("flag -r is incompatible with the vendoring experiment") 78 cmd.UsageExit() 79 } 80 err := save(args) 81 if err != nil { 82 log.Fatalln(err) 83 } 84 } 85 86 func dotPackage() (*build.Package, error) { 87 dir, err := filepath.Abs(".") 88 if err != nil { 89 return nil, err 90 } 91 return build.ImportDir(dir, build.FindOnly) 92 } 93 94 func projectPackages(dDir string, a []*Package) []*Package { 95 var projPkgs []*Package 96 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 97 for _, p := range a { 98 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 99 if strings.HasPrefix(pkgDir, dotDir) { 100 projPkgs = append(projPkgs, p) 101 } 102 } 103 return projPkgs 104 } 105 106 func save(pkgs []string) error { 107 var err error 108 dp, err := dotPackage() 109 if err != nil { 110 return err 111 } 112 debugln("dotPackageImportPath:", dp.ImportPath) 113 debugln("dotPackageDir:", dp.Dir) 114 115 cv, err := goVersion() 116 if err != nil { 117 return err 118 } 119 verboseln("Go Version:", cv) 120 121 gold, err := loadDefaultGodepsFile() 122 if err != nil { 123 if !os.IsNotExist(err) { 124 return err 125 } 126 verboseln("No old Godeps.json found.") 127 gold.GoVersion = cv 128 } 129 130 printVersionWarnings(gold.GoVersion) 131 if len(gold.GoVersion) == 0 { 132 gold.GoVersion = majorGoVersion 133 } else { 134 majorGoVersion, err = trimGoVersion(gold.GoVersion) 135 if err != nil { 136 log.Fatalf("Unable to determine go major version from value specified in %s: %s\n", gold.file(), gold.GoVersion) 137 } 138 } 139 140 gnew := &Godeps{ 141 ImportPath: dp.ImportPath, 142 GoVersion: gold.GoVersion, 143 } 144 145 switch len(pkgs) { 146 case 0: 147 pkgs = []string{"."} 148 default: 149 gnew.Packages = pkgs 150 } 151 152 verboseln("Finding dependencies for", pkgs) 153 a, err := LoadPackages(pkgs...) 154 if err != nil { 155 return err 156 } 157 158 for _, p := range a { 159 verboseln("Found package:", p.ImportPath) 160 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 161 } 162 ppln(a) 163 164 projA := projectPackages(dp.Dir, a) 165 debugln("Filtered projectPackages") 166 ppln(projA) 167 168 verboseln("Computing new Godeps.json file") 169 err = gnew.fill(a, dp.ImportPath) 170 if err != nil { 171 return err 172 } 173 debugln("New Godeps Filled") 174 ppln(gnew) 175 176 if gnew.Deps == nil { 177 gnew.Deps = make([]Dependency, 0) // produce json [], not null 178 } 179 gdisk := gnew.copy() 180 err = carryVersions(&gold, gnew) 181 if err != nil { 182 return err 183 } 184 185 if gold.isOldFile { 186 // If we are migrating from an old format file, 187 // we require that the listed version of every 188 // dependency must be installed in GOPATH, so it's 189 // available to copy. 190 if !eqDeps(gnew.Deps, gdisk.Deps) { 191 return errors.New(strings.TrimSpace(needRestore)) 192 } 193 gold = Godeps{} 194 } 195 os.Remove("Godeps") // remove regular file if present; ignore error 196 readme := filepath.Join("Godeps", "Readme") 197 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 198 if err != nil { 199 log.Println(err) 200 } 201 _, err = gnew.save() 202 if err != nil { 203 return err 204 } 205 206 verboseln("Computing diff between old and new deps") 207 // We use a name starting with "_" so the go tool 208 // ignores this directory when traversing packages 209 // starting at the project's root. For example, 210 // godep go list ./... 211 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 212 rem := subDeps(gold.Deps, gnew.Deps) 213 ppln(rem) 214 add := subDeps(gnew.Deps, gold.Deps) 215 ppln(add) 216 if len(rem) > 0 { 217 verboseln("Deps to remove:") 218 for _, r := range rem { 219 verboseln("\t", r.ImportPath) 220 } 221 verboseln("Removing unused dependencies") 222 err = removeSrc(srcdir, rem) 223 if err != nil { 224 return err 225 } 226 } 227 if len(add) > 0 { 228 verboseln("Deps to add:") 229 for _, a := range add { 230 verboseln("\t", a.ImportPath) 231 } 232 verboseln("Adding new dependencies") 233 err = copySrc(srcdir, add) 234 if err != nil { 235 return err 236 } 237 } 238 if !VendorExperiment { 239 f, _ := filepath.Split(srcdir) 240 writeVCSIgnore(f) 241 } 242 var rewritePaths []string 243 if saveR { 244 for _, dep := range gnew.Deps { 245 rewritePaths = append(rewritePaths, dep.ImportPath) 246 } 247 } 248 verboseln("Rewriting paths (if necessary)") 249 ppln(rewritePaths) 250 return rewrite(projA, dp.ImportPath, rewritePaths) 251 } 252 253 func printVersionWarnings(ov string) { 254 var warning bool 255 cv, err := goVersion() 256 if err != nil { 257 return 258 } 259 // Trim the old version because we may have saved it w/o trimming it 260 // cv is already trimmed by goVersion() 261 tov, err := trimGoVersion(ov) 262 if err != nil { 263 return 264 } 265 266 if tov != ov { 267 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 268 warning = true 269 } 270 if cv != tov { 271 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, cv) 272 warning = true 273 } 274 if warning { 275 log.Println("To record current major go version run `godep update -goversion`.") 276 } 277 } 278 279 type revError struct { 280 ImportPath string 281 WantRev string 282 HavePath string 283 HaveRev string 284 } 285 286 func (v *revError) Error() string { 287 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 288 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 289 } 290 291 // carryVersions copies Rev and Comment from a to b for 292 // each dependency with an identical ImportPath. For any 293 // dependency in b that appears to be from the same repo 294 // as one in a (for example, a parent or child directory), 295 // the Rev must already match - otherwise it is an error. 296 func carryVersions(a, b *Godeps) error { 297 for i := range b.Deps { 298 err := carryVersion(a, &b.Deps[i]) 299 if err != nil { 300 return err 301 } 302 } 303 return nil 304 } 305 306 func carryVersion(a *Godeps, db *Dependency) error { 307 // First see if this exact package is already in the list. 308 for _, da := range a.Deps { 309 if db.ImportPath == da.ImportPath { 310 db.Rev = da.Rev 311 db.Comment = da.Comment 312 return nil 313 } 314 } 315 // No exact match, check for child or sibling package. 316 // We can't handle mismatched versions for packages in 317 // the same repo, so report that as an error. 318 for _, da := range a.Deps { 319 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 320 strings.HasPrefix(da.ImportPath, db.root+"/") { 321 if da.Rev != db.Rev { 322 return &revError{ 323 ImportPath: db.ImportPath, 324 WantRev: db.Rev, 325 HavePath: da.ImportPath, 326 HaveRev: da.Rev, 327 } 328 } 329 } 330 } 331 // No related package in the list, must be a new repo. 332 return nil 333 } 334 335 // subDeps returns a - b, using ImportPath for equality. 336 func subDeps(a, b []Dependency) (diff []Dependency) { 337 Diff: 338 for _, da := range a { 339 for _, db := range b { 340 if da.ImportPath == db.ImportPath { 341 continue Diff 342 } 343 } 344 diff = append(diff, da) 345 } 346 return diff 347 } 348 349 func removeSrc(srcdir string, deps []Dependency) error { 350 for _, dep := range deps { 351 path := filepath.FromSlash(dep.ImportPath) 352 err := os.RemoveAll(filepath.Join(srcdir, path)) 353 if err != nil { 354 return err 355 } 356 } 357 return nil 358 } 359 360 func copySrc(dir string, deps []Dependency) error { 361 // mapping to see if we visited a parent directory already 362 visited := make(map[string]bool) 363 ok := true 364 for _, dep := range deps { 365 debugln("copySrc for", dep.ImportPath) 366 srcdir := filepath.Join(dep.ws, "src") 367 rel, err := filepath.Rel(srcdir, dep.dir) 368 if err != nil { // this should never happen 369 return err 370 } 371 dstpkgroot := filepath.Join(dir, rel) 372 err = os.RemoveAll(dstpkgroot) 373 if err != nil { 374 log.Println(err) 375 ok = false 376 } 377 378 // copy actual dependency 379 vf := dep.vcs.listFiles(dep.dir) 380 debugln("vf", vf) 381 w := fs.Walk(dep.dir) 382 for w.Step() { 383 err = copyPkgFile(vf, dir, srcdir, w) 384 if err != nil { 385 log.Println(err) 386 ok = false 387 } 388 } 389 390 // Look for legal files in root 391 // some packages are imports as a sub-package but license info 392 // is at root: exampleorg/common has license file in exampleorg 393 // 394 if dep.ImportPath == dep.root { 395 // we are already at root 396 continue 397 } 398 399 // prevent copying twice This could happen if we have 400 // two subpackages listed someorg/common and 401 // someorg/anotherpack which has their license in 402 // the parent dir of someorg 403 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 404 if visited[rootdir] { 405 continue 406 } 407 visited[rootdir] = true 408 vf = dep.vcs.listFiles(rootdir) 409 w = fs.Walk(rootdir) 410 for w.Step() { 411 fname := filepath.Base(w.Path()) 412 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 413 err = copyPkgFile(vf, dir, srcdir, w) 414 if err != nil { 415 log.Println(err) 416 ok = false 417 } 418 } 419 } 420 } 421 422 if !ok { 423 return errorCopyingSourceCode 424 } 425 426 return nil 427 } 428 429 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 430 if w.Err() != nil { 431 return w.Err() 432 } 433 name := w.Stat().Name() 434 if w.Stat().IsDir() { 435 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 436 // Skip directories starting with '.' or '_' or 437 // 'testdata' (last is only skipped if saveT is false) 438 w.SkipDir() 439 } 440 return nil 441 } 442 rel, err := filepath.Rel(srcroot, w.Path()) 443 if err != nil { // this should never happen 444 return err 445 } 446 if !saveT && strings.HasSuffix(name, "_test.go") { 447 if verbose { 448 log.Printf("save: skipping test file: %s", w.Path()) 449 } 450 return nil 451 } 452 if !vf.Contains(w.Path()) { 453 if verbose { 454 log.Printf("save: skipping untracked file: %s", w.Path()) 455 } 456 return nil 457 } 458 return copyFile(filepath.Join(dstroot, rel), w.Path()) 459 } 460 461 // copyFile copies a regular file from src to dst. 462 // dst is opened with os.Create. 463 // If the file name ends with .go, 464 // copyFile strips canonical import path annotations. 465 // These are comments of the form: 466 // package foo // import "bar/foo" 467 // package foo /* import "bar/foo" */ 468 func copyFile(dst, src string) error { 469 err := os.MkdirAll(filepath.Dir(dst), 0777) 470 if err != nil { 471 return err 472 } 473 474 linkDst, err := os.Readlink(src) 475 if err == nil { 476 return os.Symlink(linkDst, dst) 477 } 478 479 r, err := os.Open(src) 480 if err != nil { 481 return err 482 } 483 defer r.Close() 484 485 w, err := os.Create(dst) 486 if err != nil { 487 return err 488 } 489 490 if strings.HasSuffix(dst, ".go") { 491 debugln("Copy Without Import Comment", w, r) 492 err = copyWithoutImportComment(w, r) 493 } else { 494 debugln("Copy (plain)", w, r) 495 _, err = io.Copy(w, r) 496 } 497 err1 := w.Close() 498 if err == nil { 499 err = err1 500 } 501 502 return err 503 } 504 505 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 506 b := bufio.NewReader(r) 507 for { 508 l, err := b.ReadBytes('\n') 509 eof := err == io.EOF 510 if err != nil && err != io.EOF { 511 return err 512 } 513 514 // If we have data then write it out... 515 if len(l) > 0 { 516 // Strip off \n if it exists because stripImportComment 517 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 518 if err != nil { 519 return err 520 } 521 } 522 523 if eof { 524 return nil 525 } 526 } 527 } 528 529 const ( 530 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 531 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 532 ) 533 534 var ( 535 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 536 pkgPrefix = []byte("package ") 537 ) 538 539 // stripImportComment returns line with its import comment removed. 540 // If s is not a package statement containing an import comment, 541 // it is returned unaltered. 542 // FIXME: expects lines w/o a \n at the end 543 // See also http://golang.org/s/go14customimport. 544 func stripImportComment(line []byte) []byte { 545 if !bytes.HasPrefix(line, pkgPrefix) { 546 // Fast path; this will skip all but one line in the file. 547 // This assumes there is no whitespace before the keyword. 548 return line 549 } 550 if m := importCommentRE.FindSubmatch(line); m != nil { 551 return append(m[1], m[2]...) 552 } 553 return line 554 } 555 556 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 557 // so that dir/pkg and dir/bin don't accidentally get committed. 558 // It logs any errors it encounters. 559 func writeVCSIgnore(dir string) { 560 // Currently git is the only VCS for which we know how to do this. 561 // Mercurial and Bazaar have similar mechanisms, but they apparently 562 // require writing files outside of dir. 563 const ignore = "/pkg\n/bin\n" 564 name := filepath.Join(dir, ".gitignore") 565 err := writeFile(name, ignore) 566 if err != nil { 567 log.Println(err) 568 } 569 } 570 571 // writeFile is like ioutil.WriteFile but it creates 572 // intermediate directories with os.MkdirAll. 573 func writeFile(name, body string) error { 574 err := os.MkdirAll(filepath.Dir(name), 0777) 575 if err != nil { 576 return err 577 } 578 return ioutil.WriteFile(name, []byte(body), 0666) 579 } 580 581 const ( 582 // Readme contains the README text. 583 Readme = ` 584 This directory tree is generated automatically by godep. 585 586 Please do not edit. 587 588 See https://github.com/tools/godep for more information. 589 ` 590 needRestore = ` 591 mismatched versions while migrating 592 593 It looks like you are switching from the old Godeps format 594 (from flag -copy=false). The old format is just a file; it 595 doesn't contain source code. For this migration, godep needs 596 the appropriate version of each dependency to be installed in 597 GOPATH, so that the source code is available to copy. 598 599 To fix this, run 'godep restore'. 600 ` 601 )