gopkg.in/tools/godep.v51@v51.0.0-20160121191931-64044a295f54/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 } 63 64 var ( 65 saveR, saveT bool 66 ) 67 68 func init() { 69 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 70 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 71 72 } 73 74 func runSave(cmd *Command, args []string) { 75 if VendorExperiment && saveR { 76 log.Println("flag -r is incompatible with the vendoring experiment") 77 cmd.UsageExit() 78 } 79 err := save(args) 80 if err != nil { 81 log.Fatalln(err) 82 } 83 } 84 85 func dotPackage() (*build.Package, error) { 86 dir, err := filepath.Abs(".") 87 if err != nil { 88 return nil, err 89 } 90 return build.ImportDir(dir, build.FindOnly) 91 } 92 93 func projectPackages(dDir string, a []*Package) []*Package { 94 var projPkgs []*Package 95 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 96 for _, p := range a { 97 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 98 if strings.HasPrefix(pkgDir, dotDir) { 99 projPkgs = append(projPkgs, p) 100 } 101 } 102 return projPkgs 103 } 104 105 func save(pkgs []string) error { 106 dp, err := dotPackage() 107 if err != nil { 108 return err 109 } 110 debugln("dotPackageImportPath:", dp.ImportPath) 111 debugln("dotPackageDir:", dp.Dir) 112 113 cv, err := goVersion() 114 if err != nil { 115 return err 116 } 117 verboseln("Go Version:", cv) 118 119 gold, err := loadDefaultGodepsFile() 120 if err != nil { 121 if !os.IsNotExist(err) { 122 return err 123 } 124 verboseln("No old Godeps.json found.") 125 gold.GoVersion = cv 126 } 127 128 printVersionWarnings(gold.GoVersion) 129 130 gnew := &Godeps{ 131 ImportPath: dp.ImportPath, 132 GoVersion: gold.GoVersion, 133 } 134 135 switch len(pkgs) { 136 case 0: 137 pkgs = []string{"."} 138 default: 139 gnew.Packages = pkgs 140 } 141 142 verboseln("Finding dependencies for", pkgs) 143 a, err := LoadPackages(pkgs...) 144 if err != nil { 145 return err 146 } 147 148 for _, p := range a { 149 verboseln("Found package:", p.ImportPath) 150 verboseln("\tDeps:", strings.Join(p.Deps, " ")) 151 } 152 ppln(a) 153 154 projA := projectPackages(dp.Dir, a) 155 debugln("Filtered projectPackages") 156 ppln(projA) 157 158 verboseln("Computing new Godeps.json file") 159 err = gnew.fill(a, dp.ImportPath) 160 if err != nil { 161 return err 162 } 163 debugln("New Godeps Filled") 164 ppln(gnew) 165 166 if gnew.Deps == nil { 167 gnew.Deps = make([]Dependency, 0) // produce json [], not null 168 } 169 gdisk := gnew.copy() 170 err = carryVersions(&gold, gnew) 171 if err != nil { 172 return err 173 } 174 175 if gold.isOldFile { 176 // If we are migrating from an old format file, 177 // we require that the listed version of every 178 // dependency must be installed in GOPATH, so it's 179 // available to copy. 180 if !eqDeps(gnew.Deps, gdisk.Deps) { 181 return errors.New(strings.TrimSpace(needRestore)) 182 } 183 gold = Godeps{} 184 } 185 os.Remove("Godeps") // remove regular file if present; ignore error 186 readme := filepath.Join("Godeps", "Readme") 187 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 188 if err != nil { 189 log.Println(err) 190 } 191 _, err = gnew.save() 192 if err != nil { 193 return err 194 } 195 196 verboseln("Computing diff between old and new deps") 197 // We use a name starting with "_" so the go tool 198 // ignores this directory when traversing packages 199 // starting at the project's root. For example, 200 // godep go list ./... 201 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 202 rem := subDeps(gold.Deps, gnew.Deps) 203 ppln(rem) 204 add := subDeps(gnew.Deps, gold.Deps) 205 ppln(add) 206 if len(rem) > 0 { 207 verboseln("Deps to remove:") 208 for _, r := range rem { 209 verboseln("\t", r.ImportPath) 210 } 211 verboseln("Removing unused dependencies") 212 err = removeSrc(srcdir, rem) 213 if err != nil { 214 return err 215 } 216 } 217 if len(add) > 0 { 218 verboseln("Deps to add:") 219 for _, a := range add { 220 verboseln("\t", a.ImportPath) 221 } 222 verboseln("Adding new dependencies") 223 err = copySrc(srcdir, add) 224 if err != nil { 225 return err 226 } 227 } 228 if !VendorExperiment { 229 f, _ := filepath.Split(srcdir) 230 writeVCSIgnore(f) 231 } 232 var rewritePaths []string 233 if saveR { 234 for _, dep := range gnew.Deps { 235 rewritePaths = append(rewritePaths, dep.ImportPath) 236 } 237 } 238 verboseln("Rewriting paths (if necessary)") 239 ppln(rewritePaths) 240 return rewrite(projA, dp.ImportPath, rewritePaths) 241 } 242 243 func printVersionWarnings(ov string) { 244 var warning bool 245 cv, err := goVersion() 246 if err != nil { 247 return 248 } 249 tov, err := trimGoVersion(ov) 250 if err != nil { 251 return 252 } 253 tcv, err := trimGoVersion(cv) 254 if err != nil { 255 return 256 } 257 258 if tov != ov { 259 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 260 warning = true 261 } 262 if tcv != tov { 263 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 264 warning = true 265 } 266 if warning { 267 log.Println("To record current major go version run `godep update -goversion`.") 268 } 269 } 270 271 type revError struct { 272 ImportPath string 273 WantRev string 274 HavePath string 275 HaveRev string 276 } 277 278 func (v *revError) Error() string { 279 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 280 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 281 } 282 283 // carryVersions copies Rev and Comment from a to b for 284 // each dependency with an identical ImportPath. For any 285 // dependency in b that appears to be from the same repo 286 // as one in a (for example, a parent or child directory), 287 // the Rev must already match - otherwise it is an error. 288 func carryVersions(a, b *Godeps) error { 289 for i := range b.Deps { 290 err := carryVersion(a, &b.Deps[i]) 291 if err != nil { 292 return err 293 } 294 } 295 return nil 296 } 297 298 func carryVersion(a *Godeps, db *Dependency) error { 299 // First see if this exact package is already in the list. 300 for _, da := range a.Deps { 301 if db.ImportPath == da.ImportPath { 302 db.Rev = da.Rev 303 db.Comment = da.Comment 304 return nil 305 } 306 } 307 // No exact match, check for child or sibling package. 308 // We can't handle mismatched versions for packages in 309 // the same repo, so report that as an error. 310 for _, da := range a.Deps { 311 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 312 strings.HasPrefix(da.ImportPath, db.root+"/") { 313 if da.Rev != db.Rev { 314 return &revError{ 315 ImportPath: db.ImportPath, 316 WantRev: db.Rev, 317 HavePath: da.ImportPath, 318 HaveRev: da.Rev, 319 } 320 } 321 } 322 } 323 // No related package in the list, must be a new repo. 324 return nil 325 } 326 327 // subDeps returns a - b, using ImportPath for equality. 328 func subDeps(a, b []Dependency) (diff []Dependency) { 329 Diff: 330 for _, da := range a { 331 for _, db := range b { 332 if da.ImportPath == db.ImportPath { 333 continue Diff 334 } 335 } 336 diff = append(diff, da) 337 } 338 return diff 339 } 340 341 func removeSrc(srcdir string, deps []Dependency) error { 342 for _, dep := range deps { 343 path := filepath.FromSlash(dep.ImportPath) 344 err := os.RemoveAll(filepath.Join(srcdir, path)) 345 if err != nil { 346 return err 347 } 348 } 349 return nil 350 } 351 352 func copySrc(dir string, deps []Dependency) error { 353 // mapping to see if we visited a parent directory already 354 visited := make(map[string]bool) 355 ok := true 356 for _, dep := range deps { 357 srcdir := filepath.Join(dep.ws, "src") 358 rel, err := filepath.Rel(srcdir, dep.dir) 359 if err != nil { // this should never happen 360 return err 361 } 362 dstpkgroot := filepath.Join(dir, rel) 363 err = os.RemoveAll(dstpkgroot) 364 if err != nil { 365 log.Println(err) 366 ok = false 367 } 368 369 // copy actual dependency 370 vf := dep.vcs.listFiles(dep.dir) 371 w := fs.Walk(dep.dir) 372 for w.Step() { 373 err = copyPkgFile(vf, dir, srcdir, w) 374 if err != nil { 375 log.Println(err) 376 ok = false 377 } 378 } 379 380 // Look for legal files in root 381 // some packages are imports as a sub-package but license info 382 // is at root: exampleorg/common has license file in exampleorg 383 // 384 if dep.ImportPath == dep.root { 385 // we are already at root 386 continue 387 } 388 389 // prevent copying twice This could happen if we have 390 // two subpackages listed someorg/common and 391 // someorg/anotherpack which has their license in 392 // the parent dir of someorg 393 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 394 if visited[rootdir] { 395 continue 396 } 397 visited[rootdir] = true 398 vf = dep.vcs.listFiles(rootdir) 399 w = fs.Walk(rootdir) 400 for w.Step() { 401 fname := filepath.Base(w.Path()) 402 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 403 err = copyPkgFile(vf, dir, srcdir, w) 404 if err != nil { 405 log.Println(err) 406 ok = false 407 } 408 } 409 } 410 } 411 412 if !ok { 413 return errorCopyingSourceCode 414 } 415 416 return nil 417 } 418 419 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 420 if w.Err() != nil { 421 return w.Err() 422 } 423 name := w.Stat().Name() 424 if w.Stat().IsDir() { 425 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 426 // Skip directories starting with '.' or '_' or 427 // 'testdata' (last is only skipped if saveT is false) 428 w.SkipDir() 429 } 430 return nil 431 } 432 rel, err := filepath.Rel(srcroot, w.Path()) 433 if err != nil { // this should never happen 434 return err 435 } 436 if !saveT && strings.HasSuffix(name, "_test.go") { 437 if verbose { 438 log.Printf("save: skipping test file: %s", w.Path()) 439 } 440 return nil 441 } 442 if !vf.Contains(w.Path()) { 443 if verbose { 444 log.Printf("save: skipping untracked file: %s", w.Path()) 445 } 446 return nil 447 } 448 return copyFile(filepath.Join(dstroot, rel), w.Path()) 449 } 450 451 // copyFile copies a regular file from src to dst. 452 // dst is opened with os.Create. 453 // If the file name ends with .go, 454 // copyFile strips canonical import path annotations. 455 // These are comments of the form: 456 // package foo // import "bar/foo" 457 // package foo /* import "bar/foo" */ 458 func copyFile(dst, src string) error { 459 err := os.MkdirAll(filepath.Dir(dst), 0777) 460 if err != nil { 461 return err 462 } 463 464 linkDst, err := os.Readlink(src) 465 if err == nil { 466 return os.Symlink(linkDst, dst) 467 } 468 469 r, err := os.Open(src) 470 if err != nil { 471 return err 472 } 473 defer r.Close() 474 475 w, err := os.Create(dst) 476 if err != nil { 477 return err 478 } 479 480 if strings.HasSuffix(dst, ".go") { 481 err = copyWithoutImportComment(w, r) 482 } else { 483 _, err = io.Copy(w, r) 484 } 485 err1 := w.Close() 486 if err == nil { 487 err = err1 488 } 489 490 return err 491 } 492 493 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 494 b := bufio.NewReader(r) 495 for { 496 l, err := b.ReadBytes('\n') 497 eof := err == io.EOF 498 if err != nil && err != io.EOF { 499 return err 500 } 501 502 // If we have data then write it out... 503 if len(l) > 0 { 504 // Strip off \n if it exists because stripImportComment 505 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 506 if err != nil { 507 return err 508 } 509 } 510 511 if eof { 512 return nil 513 } 514 } 515 } 516 517 const ( 518 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 519 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 520 ) 521 522 var ( 523 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 524 pkgPrefix = []byte("package ") 525 ) 526 527 // stripImportComment returns line with its import comment removed. 528 // If s is not a package statement containing an import comment, 529 // it is returned unaltered. 530 // FIXME: expects lines w/o a \n at the end 531 // See also http://golang.org/s/go14customimport. 532 func stripImportComment(line []byte) []byte { 533 if !bytes.HasPrefix(line, pkgPrefix) { 534 // Fast path; this will skip all but one line in the file. 535 // This assumes there is no whitespace before the keyword. 536 return line 537 } 538 if m := importCommentRE.FindSubmatch(line); m != nil { 539 return append(m[1], m[2]...) 540 } 541 return line 542 } 543 544 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 545 // so that dir/pkg and dir/bin don't accidentally get committed. 546 // It logs any errors it encounters. 547 func writeVCSIgnore(dir string) { 548 // Currently git is the only VCS for which we know how to do this. 549 // Mercurial and Bazaar have similar mechanisms, but they apparently 550 // require writing files outside of dir. 551 const ignore = "/pkg\n/bin\n" 552 name := filepath.Join(dir, ".gitignore") 553 err := writeFile(name, ignore) 554 if err != nil { 555 log.Println(err) 556 } 557 } 558 559 // writeFile is like ioutil.WriteFile but it creates 560 // intermediate directories with os.MkdirAll. 561 func writeFile(name, body string) error { 562 err := os.MkdirAll(filepath.Dir(name), 0777) 563 if err != nil { 564 return err 565 } 566 return ioutil.WriteFile(name, []byte(body), 0666) 567 } 568 569 const ( 570 // Readme contains the README text. 571 Readme = ` 572 This directory tree is generated automatically by godep. 573 574 Please do not edit. 575 576 See https://github.com/tools/godep for more information. 577 ` 578 needRestore = ` 579 mismatched versions while migrating 580 581 It looks like you are switching from the old Godeps format 582 (from flag -copy=false). The old format is just a file; it 583 doesn't contain source code. For this migration, godep needs 584 the appropriate version of each dependency to be installed in 585 GOPATH, so that the source code is available to copy. 586 587 To fix this, run 'godep restore'. 588 ` 589 )