gopkg.in/tools/godep.v54@v54.0.0-20160222173036-53bf4cf4341d/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 } 63 64 var ( 65 saveR, saveT bool 66 ) 67 68 func init() { 69 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 70 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 71 72 } 73 74 func runSave(cmd *Command, args []string) { 75 if VendorExperiment && saveR { 76 log.Println("flag -r is incompatible with the vendoring experiment") 77 cmd.UsageExit() 78 } 79 err := save(args) 80 if err != nil { 81 log.Fatalln(err) 82 } 83 } 84 85 func dotPackage() (*build.Package, error) { 86 dir, err := filepath.Abs(".") 87 if err != nil { 88 return nil, err 89 } 90 return build.ImportDir(dir, build.FindOnly) 91 } 92 93 func projectPackages(dDir string, a []*Package) []*Package { 94 var projPkgs []*Package 95 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 96 for _, p := range a { 97 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 98 if strings.HasPrefix(pkgDir, dotDir) { 99 projPkgs = append(projPkgs, p) 100 } 101 } 102 return projPkgs 103 } 104 105 func save(pkgs []string) error { 106 dp, err := dotPackage() 107 if err != nil { 108 return err 109 } 110 debugln("dotPackageImportPath:", dp.ImportPath) 111 debugln("dotPackageDir:", dp.Dir) 112 113 cv, err := goVersion() 114 if err != nil { 115 return err 116 } 117 verboseln("Go Version:", cv) 118 119 gold, err := loadDefaultGodepsFile() 120 if err != nil { 121 if !os.IsNotExist(err) { 122 return err 123 } 124 verboseln("No old Godeps.json found.") 125 gold.GoVersion = cv 126 } 127 128 printVersionWarnings(gold.GoVersion) 129 130 gnew := &Godeps{ 131 ImportPath: dp.ImportPath, 132 GoVersion: gold.GoVersion, 133 } 134 135 switch len(pkgs) { 136 case 0: 137 pkgs = []string{"."} 138 default: 139 gnew.Packages = pkgs 140 } 141 142 verboseln("Finding dependencies for", pkgs) 143 a, err := LoadPackages(pkgs...) 144 if err != nil { 145 return err 146 } 147 148 for _, p := range a { 149 verboseln("Found package:", p.ImportPath) 150 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 151 } 152 ppln(a) 153 154 projA := projectPackages(dp.Dir, a) 155 debugln("Filtered projectPackages") 156 ppln(projA) 157 158 verboseln("Computing new Godeps.json file") 159 err = gnew.fill(a, dp.ImportPath) 160 if err != nil { 161 return err 162 } 163 debugln("New Godeps Filled") 164 ppln(gnew) 165 166 if gnew.Deps == nil { 167 gnew.Deps = make([]Dependency, 0) // produce json [], not null 168 } 169 gdisk := gnew.copy() 170 err = carryVersions(&gold, gnew) 171 if err != nil { 172 return err 173 } 174 175 if gold.isOldFile { 176 // If we are migrating from an old format file, 177 // we require that the listed version of every 178 // dependency must be installed in GOPATH, so it's 179 // available to copy. 180 if !eqDeps(gnew.Deps, gdisk.Deps) { 181 return errors.New(strings.TrimSpace(needRestore)) 182 } 183 gold = Godeps{} 184 } 185 os.Remove("Godeps") // remove regular file if present; ignore error 186 readme := filepath.Join("Godeps", "Readme") 187 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 188 if err != nil { 189 log.Println(err) 190 } 191 _, err = gnew.save() 192 if err != nil { 193 return err 194 } 195 196 verboseln("Computing diff between old and new deps") 197 // We use a name starting with "_" so the go tool 198 // ignores this directory when traversing packages 199 // starting at the project's root. For example, 200 // godep go list ./... 201 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 202 rem := subDeps(gold.Deps, gnew.Deps) 203 ppln(rem) 204 add := subDeps(gnew.Deps, gold.Deps) 205 ppln(add) 206 if len(rem) > 0 { 207 verboseln("Deps to remove:") 208 for _, r := range rem { 209 verboseln("\t", r.ImportPath) 210 } 211 verboseln("Removing unused dependencies") 212 err = removeSrc(srcdir, rem) 213 if err != nil { 214 return err 215 } 216 } 217 if len(add) > 0 { 218 verboseln("Deps to add:") 219 for _, a := range add { 220 verboseln("\t", a.ImportPath) 221 } 222 verboseln("Adding new dependencies") 223 err = copySrc(srcdir, add) 224 if err != nil { 225 return err 226 } 227 } 228 if !VendorExperiment { 229 f, _ := filepath.Split(srcdir) 230 writeVCSIgnore(f) 231 } 232 var rewritePaths []string 233 if saveR { 234 for _, dep := range gnew.Deps { 235 rewritePaths = append(rewritePaths, dep.ImportPath) 236 } 237 } 238 verboseln("Rewriting paths (if necessary)") 239 ppln(rewritePaths) 240 return rewrite(projA, dp.ImportPath, rewritePaths) 241 } 242 243 func printVersionWarnings(ov string) { 244 var warning bool 245 cv, err := goVersion() 246 if err != nil { 247 return 248 } 249 tov, err := trimGoVersion(ov) 250 if err != nil { 251 return 252 } 253 tcv, err := trimGoVersion(cv) 254 if err != nil { 255 return 256 } 257 258 if tov != ov { 259 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 260 warning = true 261 } 262 if tcv != tov { 263 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 264 warning = true 265 } 266 if warning { 267 log.Println("To record current major go version run `godep update -goversion`.") 268 } 269 } 270 271 type revError struct { 272 ImportPath string 273 WantRev string 274 HavePath string 275 HaveRev string 276 } 277 278 func (v *revError) Error() string { 279 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 280 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 281 } 282 283 // carryVersions copies Rev and Comment from a to b for 284 // each dependency with an identical ImportPath. For any 285 // dependency in b that appears to be from the same repo 286 // as one in a (for example, a parent or child directory), 287 // the Rev must already match - otherwise it is an error. 288 func carryVersions(a, b *Godeps) error { 289 for i := range b.Deps { 290 err := carryVersion(a, &b.Deps[i]) 291 if err != nil { 292 return err 293 } 294 } 295 return nil 296 } 297 298 func carryVersion(a *Godeps, db *Dependency) error { 299 // First see if this exact package is already in the list. 300 for _, da := range a.Deps { 301 if db.ImportPath == da.ImportPath { 302 db.Rev = da.Rev 303 db.Comment = da.Comment 304 return nil 305 } 306 } 307 // No exact match, check for child or sibling package. 308 // We can't handle mismatched versions for packages in 309 // the same repo, so report that as an error. 310 for _, da := range a.Deps { 311 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 312 strings.HasPrefix(da.ImportPath, db.root+"/") { 313 if da.Rev != db.Rev { 314 return &revError{ 315 ImportPath: db.ImportPath, 316 WantRev: db.Rev, 317 HavePath: da.ImportPath, 318 HaveRev: da.Rev, 319 } 320 } 321 } 322 } 323 // No related package in the list, must be a new repo. 324 return nil 325 } 326 327 // subDeps returns a - b, using ImportPath for equality. 328 func subDeps(a, b []Dependency) (diff []Dependency) { 329 Diff: 330 for _, da := range a { 331 for _, db := range b { 332 if da.ImportPath == db.ImportPath { 333 continue Diff 334 } 335 } 336 diff = append(diff, da) 337 } 338 return diff 339 } 340 341 func removeSrc(srcdir string, deps []Dependency) error { 342 for _, dep := range deps { 343 path := filepath.FromSlash(dep.ImportPath) 344 err := os.RemoveAll(filepath.Join(srcdir, path)) 345 if err != nil { 346 return err 347 } 348 } 349 return nil 350 } 351 352 func copySrc(dir string, deps []Dependency) error { 353 // mapping to see if we visited a parent directory already 354 visited := make(map[string]bool) 355 ok := true 356 for _, dep := range deps { 357 debugln("copySrc for", dep.ImportPath) 358 srcdir := filepath.Join(dep.ws, "src") 359 rel, err := filepath.Rel(srcdir, dep.dir) 360 if err != nil { // this should never happen 361 return err 362 } 363 dstpkgroot := filepath.Join(dir, rel) 364 err = os.RemoveAll(dstpkgroot) 365 if err != nil { 366 log.Println(err) 367 ok = false 368 } 369 370 // copy actual dependency 371 vf := dep.vcs.listFiles(dep.dir) 372 debugln("vf", vf) 373 w := fs.Walk(dep.dir) 374 for w.Step() { 375 err = copyPkgFile(vf, dir, srcdir, w) 376 if err != nil { 377 log.Println(err) 378 ok = false 379 } 380 } 381 382 // Look for legal files in root 383 // some packages are imports as a sub-package but license info 384 // is at root: exampleorg/common has license file in exampleorg 385 // 386 if dep.ImportPath == dep.root { 387 // we are already at root 388 continue 389 } 390 391 // prevent copying twice This could happen if we have 392 // two subpackages listed someorg/common and 393 // someorg/anotherpack which has their license in 394 // the parent dir of someorg 395 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 396 if visited[rootdir] { 397 continue 398 } 399 visited[rootdir] = true 400 vf = dep.vcs.listFiles(rootdir) 401 w = fs.Walk(rootdir) 402 for w.Step() { 403 fname := filepath.Base(w.Path()) 404 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 405 err = copyPkgFile(vf, dir, srcdir, w) 406 if err != nil { 407 log.Println(err) 408 ok = false 409 } 410 } 411 } 412 } 413 414 if !ok { 415 return errorCopyingSourceCode 416 } 417 418 return nil 419 } 420 421 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 422 if w.Err() != nil { 423 return w.Err() 424 } 425 name := w.Stat().Name() 426 if w.Stat().IsDir() { 427 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 428 // Skip directories starting with '.' or '_' or 429 // 'testdata' (last is only skipped if saveT is false) 430 w.SkipDir() 431 } 432 return nil 433 } 434 rel, err := filepath.Rel(srcroot, w.Path()) 435 if err != nil { // this should never happen 436 return err 437 } 438 if !saveT && strings.HasSuffix(name, "_test.go") { 439 if verbose { 440 log.Printf("save: skipping test file: %s", w.Path()) 441 } 442 return nil 443 } 444 if !vf.Contains(w.Path()) { 445 if verbose { 446 log.Printf("save: skipping untracked file: %s", w.Path()) 447 } 448 return nil 449 } 450 return copyFile(filepath.Join(dstroot, rel), w.Path()) 451 } 452 453 // copyFile copies a regular file from src to dst. 454 // dst is opened with os.Create. 455 // If the file name ends with .go, 456 // copyFile strips canonical import path annotations. 457 // These are comments of the form: 458 // package foo // import "bar/foo" 459 // package foo /* import "bar/foo" */ 460 func copyFile(dst, src string) error { 461 err := os.MkdirAll(filepath.Dir(dst), 0777) 462 if err != nil { 463 return err 464 } 465 466 linkDst, err := os.Readlink(src) 467 if err == nil { 468 return os.Symlink(linkDst, dst) 469 } 470 471 r, err := os.Open(src) 472 if err != nil { 473 return err 474 } 475 defer r.Close() 476 477 w, err := os.Create(dst) 478 if err != nil { 479 return err 480 } 481 482 if strings.HasSuffix(dst, ".go") { 483 debugln("Copy Without Import Comment", w, r) 484 err = copyWithoutImportComment(w, r) 485 } else { 486 debugln("Copy (plain)", w, r) 487 _, err = io.Copy(w, r) 488 } 489 err1 := w.Close() 490 if err == nil { 491 err = err1 492 } 493 494 return err 495 } 496 497 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 498 b := bufio.NewReader(r) 499 for { 500 l, err := b.ReadBytes('\n') 501 eof := err == io.EOF 502 if err != nil && err != io.EOF { 503 return err 504 } 505 506 // If we have data then write it out... 507 if len(l) > 0 { 508 // Strip off \n if it exists because stripImportComment 509 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 510 if err != nil { 511 return err 512 } 513 } 514 515 if eof { 516 return nil 517 } 518 } 519 } 520 521 const ( 522 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 523 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 524 ) 525 526 var ( 527 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 528 pkgPrefix = []byte("package ") 529 ) 530 531 // stripImportComment returns line with its import comment removed. 532 // If s is not a package statement containing an import comment, 533 // it is returned unaltered. 534 // FIXME: expects lines w/o a \n at the end 535 // See also http://golang.org/s/go14customimport. 536 func stripImportComment(line []byte) []byte { 537 if !bytes.HasPrefix(line, pkgPrefix) { 538 // Fast path; this will skip all but one line in the file. 539 // This assumes there is no whitespace before the keyword. 540 return line 541 } 542 if m := importCommentRE.FindSubmatch(line); m != nil { 543 return append(m[1], m[2]...) 544 } 545 return line 546 } 547 548 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 549 // so that dir/pkg and dir/bin don't accidentally get committed. 550 // It logs any errors it encounters. 551 func writeVCSIgnore(dir string) { 552 // Currently git is the only VCS for which we know how to do this. 553 // Mercurial and Bazaar have similar mechanisms, but they apparently 554 // require writing files outside of dir. 555 const ignore = "/pkg\n/bin\n" 556 name := filepath.Join(dir, ".gitignore") 557 err := writeFile(name, ignore) 558 if err != nil { 559 log.Println(err) 560 } 561 } 562 563 // writeFile is like ioutil.WriteFile but it creates 564 // intermediate directories with os.MkdirAll. 565 func writeFile(name, body string) error { 566 err := os.MkdirAll(filepath.Dir(name), 0777) 567 if err != nil { 568 return err 569 } 570 return ioutil.WriteFile(name, []byte(body), 0666) 571 } 572 573 const ( 574 // Readme contains the README text. 575 Readme = ` 576 This directory tree is generated automatically by godep. 577 578 Please do not edit. 579 580 See https://github.com/tools/godep for more information. 581 ` 582 needRestore = ` 583 mismatched versions while migrating 584 585 It looks like you are switching from the old Godeps format 586 (from flag -copy=false). The old format is just a file; it 587 doesn't contain source code. For this migration, godep needs 588 the appropriate version of each dependency to be installed in 589 GOPATH, so that the source code is available to copy. 590 591 To fix this, run 'godep restore'. 592 ` 593 )