gopkg.in/tools/godep.v42@v42.0.0-20151223001600-f221061cd941/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 } 63 64 var ( 65 saveR, saveT bool 66 ) 67 68 func init() { 69 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 70 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 71 72 } 73 74 func runSave(cmd *Command, args []string) { 75 if VendorExperiment && saveR { 76 log.Println("flag -r is incompatible with the vendoring experiment") 77 cmd.UsageExit() 78 } 79 err := save(args) 80 if err != nil { 81 log.Fatalln(err) 82 } 83 } 84 85 func dotPackage() (*build.Package, error) { 86 dir, err := filepath.Abs(".") 87 if err != nil { 88 return nil, err 89 } 90 return build.ImportDir(dir, build.FindOnly) 91 } 92 93 func projectPackages(dDir string, a []*Package) []*Package { 94 var projPkgs []*Package 95 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 96 for _, p := range a { 97 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 98 if strings.HasPrefix(pkgDir, dotDir) { 99 projPkgs = append(projPkgs, p) 100 } 101 } 102 return projPkgs 103 } 104 105 func save(pkgs []string) error { 106 dp, err := dotPackage() 107 if err != nil { 108 return err 109 } 110 debugln("dotPackageImportPath:", dp.ImportPath) 111 debugln("dotPackageDir:", dp.Dir) 112 113 cv, err := goVersion() 114 if err != nil { 115 return err 116 } 117 debugln("goVersion:", cv) 118 119 gold, err := loadDefaultGodepsFile() 120 if err != nil { 121 if !os.IsNotExist(err) { 122 return err 123 } 124 gold.GoVersion = cv 125 } 126 127 printVersionWarnings(gold.GoVersion) 128 129 gnew := &Godeps{ 130 ImportPath: dp.ImportPath, 131 GoVersion: gold.GoVersion, 132 } 133 134 switch len(pkgs) { 135 case 0: 136 pkgs = []string{"."} 137 default: 138 gnew.Packages = pkgs 139 } 140 141 a, err := LoadPackages(pkgs...) 142 if err != nil { 143 return err 144 } 145 debugln("Project Packages", pkgs) 146 ppln(a) 147 148 projA := projectPackages(dp.Dir, a) 149 debugln("Filtered projectPackages") 150 ppln(projA) 151 152 err = gnew.fill(a, dp.ImportPath) 153 if err != nil { 154 return err 155 } 156 debugln("New Godeps Filled") 157 ppln(gnew) 158 159 if gnew.Deps == nil { 160 gnew.Deps = make([]Dependency, 0) // produce json [], not null 161 } 162 gdisk := gnew.copy() 163 err = carryVersions(&gold, gnew) 164 if err != nil { 165 return err 166 } 167 168 if gold.isOldFile { 169 // If we are migrating from an old format file, 170 // we require that the listed version of every 171 // dependency must be installed in GOPATH, so it's 172 // available to copy. 173 if !eqDeps(gnew.Deps, gdisk.Deps) { 174 return errors.New(strings.TrimSpace(needRestore)) 175 } 176 gold = Godeps{} 177 } 178 os.Remove("Godeps") // remove regular file if present; ignore error 179 readme := filepath.Join("Godeps", "Readme") 180 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 181 if err != nil { 182 log.Println(err) 183 } 184 _, err = gnew.save() 185 if err != nil { 186 return err 187 } 188 // We use a name starting with "_" so the go tool 189 // ignores this directory when traversing packages 190 // starting at the project's root. For example, 191 // godep go list ./... 192 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 193 rem := subDeps(gold.Deps, gnew.Deps) 194 debugln("Deps to remove") 195 ppln(rem) 196 add := subDeps(gnew.Deps, gold.Deps) 197 debugln("Deps to add") 198 ppln(add) 199 err = removeSrc(srcdir, rem) 200 if err != nil { 201 return err 202 } 203 err = copySrc(srcdir, add) 204 if err != nil { 205 return err 206 } 207 if !VendorExperiment { 208 f, _ := filepath.Split(srcdir) 209 writeVCSIgnore(f) 210 } 211 var rewritePaths []string 212 if saveR { 213 for _, dep := range gnew.Deps { 214 rewritePaths = append(rewritePaths, dep.ImportPath) 215 } 216 } 217 debugln("rewritePaths") 218 ppln(rewritePaths) 219 return rewrite(projA, dp.ImportPath, rewritePaths) 220 } 221 222 func printVersionWarnings(ov string) { 223 var warning bool 224 cv, err := goVersion() 225 if err != nil { 226 return 227 } 228 tov, err := trimGoVersion(ov) 229 if err != nil { 230 return 231 } 232 tcv, err := trimGoVersion(cv) 233 if err != nil { 234 return 235 } 236 237 if tov != ov { 238 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 239 warning = true 240 } 241 if tcv != tov { 242 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 243 warning = true 244 } 245 if warning { 246 log.Println("To record current major go version run `godep update -goversion`.") 247 } 248 } 249 250 type revError struct { 251 ImportPath string 252 WantRev string 253 HavePath string 254 HaveRev string 255 } 256 257 func (v *revError) Error() string { 258 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 259 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 260 } 261 262 // carryVersions copies Rev and Comment from a to b for 263 // each dependency with an identical ImportPath. For any 264 // dependency in b that appears to be from the same repo 265 // as one in a (for example, a parent or child directory), 266 // the Rev must already match - otherwise it is an error. 267 func carryVersions(a, b *Godeps) error { 268 for i := range b.Deps { 269 err := carryVersion(a, &b.Deps[i]) 270 if err != nil { 271 return err 272 } 273 } 274 return nil 275 } 276 277 func carryVersion(a *Godeps, db *Dependency) error { 278 // First see if this exact package is already in the list. 279 for _, da := range a.Deps { 280 if db.ImportPath == da.ImportPath { 281 db.Rev = da.Rev 282 db.Comment = da.Comment 283 return nil 284 } 285 } 286 // No exact match, check for child or sibling package. 287 // We can't handle mismatched versions for packages in 288 // the same repo, so report that as an error. 289 for _, da := range a.Deps { 290 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 291 strings.HasPrefix(da.ImportPath, db.root+"/") { 292 if da.Rev != db.Rev { 293 return &revError{ 294 ImportPath: db.ImportPath, 295 WantRev: db.Rev, 296 HavePath: da.ImportPath, 297 HaveRev: da.Rev, 298 } 299 } 300 } 301 } 302 // No related package in the list, must be a new repo. 303 return nil 304 } 305 306 // subDeps returns a - b, using ImportPath for equality. 307 func subDeps(a, b []Dependency) (diff []Dependency) { 308 Diff: 309 for _, da := range a { 310 for _, db := range b { 311 if da.ImportPath == db.ImportPath { 312 continue Diff 313 } 314 } 315 diff = append(diff, da) 316 } 317 return diff 318 } 319 320 func removeSrc(srcdir string, deps []Dependency) error { 321 for _, dep := range deps { 322 path := filepath.FromSlash(dep.ImportPath) 323 err := os.RemoveAll(filepath.Join(srcdir, path)) 324 if err != nil { 325 return err 326 } 327 } 328 return nil 329 } 330 331 func copySrc(dir string, deps []Dependency) error { 332 // mapping to see if we visited a parent directory already 333 visited := make(map[string]bool) 334 ok := true 335 for _, dep := range deps { 336 srcdir := filepath.Join(dep.ws, "src") 337 rel, err := filepath.Rel(srcdir, dep.dir) 338 if err != nil { // this should never happen 339 return err 340 } 341 dstpkgroot := filepath.Join(dir, rel) 342 err = os.RemoveAll(dstpkgroot) 343 if err != nil { 344 log.Println(err) 345 ok = false 346 } 347 348 // copy actual dependency 349 vf := dep.vcs.listFiles(dep.dir) 350 w := fs.Walk(dep.dir) 351 for w.Step() { 352 err = copyPkgFile(vf, dir, srcdir, w) 353 if err != nil { 354 log.Println(err) 355 ok = false 356 } 357 } 358 359 // Look for legal files in root 360 // some packages are imports as a sub-package but license info 361 // is at root: exampleorg/common has license file in exampleorg 362 // 363 if dep.ImportPath == dep.root { 364 // we are already at root 365 continue 366 } 367 368 // prevent copying twice This could happen if we have 369 // two subpackages listed someorg/common and 370 // someorg/anotherpack which has their license in 371 // the parent dir of someorg 372 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 373 if visited[rootdir] { 374 continue 375 } 376 visited[rootdir] = true 377 vf = dep.vcs.listFiles(rootdir) 378 w = fs.Walk(rootdir) 379 for w.Step() { 380 fname := filepath.Base(w.Path()) 381 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 382 err = copyPkgFile(vf, dir, srcdir, w) 383 if err != nil { 384 log.Println(err) 385 ok = false 386 } 387 } 388 } 389 } 390 391 if !ok { 392 return errorCopyingSourceCode 393 } 394 395 return nil 396 } 397 398 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 399 if w.Err() != nil { 400 return w.Err() 401 } 402 name := w.Stat().Name() 403 if w.Stat().IsDir() { 404 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 405 // Skip directories starting with '.' or '_' or 406 // 'testdata' (last is only skipped if saveT is false) 407 w.SkipDir() 408 } 409 return nil 410 } 411 rel, err := filepath.Rel(srcroot, w.Path()) 412 if err != nil { // this should never happen 413 return err 414 } 415 if !saveT && strings.HasSuffix(name, "_test.go") { 416 if verbose { 417 log.Printf("save: skipping test file: %s", w.Path()) 418 } 419 return nil 420 } 421 if !vf.Contains(w.Path()) { 422 if verbose { 423 log.Printf("save: skipping untracked file: %s", w.Path()) 424 } 425 return nil 426 } 427 return copyFile(filepath.Join(dstroot, rel), w.Path()) 428 } 429 430 // copyFile copies a regular file from src to dst. 431 // dst is opened with os.Create. 432 // If the file name ends with .go, 433 // copyFile strips canonical import path annotations. 434 // These are comments of the form: 435 // package foo // import "bar/foo" 436 // package foo /* import "bar/foo" */ 437 func copyFile(dst, src string) error { 438 err := os.MkdirAll(filepath.Dir(dst), 0777) 439 if err != nil { 440 return err 441 } 442 443 linkDst, err := os.Readlink(src) 444 if err == nil { 445 return os.Symlink(linkDst, dst) 446 } 447 448 r, err := os.Open(src) 449 if err != nil { 450 return err 451 } 452 defer r.Close() 453 454 w, err := os.Create(dst) 455 if err != nil { 456 return err 457 } 458 459 if strings.HasSuffix(dst, ".go") { 460 err = copyWithoutImportComment(w, r) 461 } else { 462 _, err = io.Copy(w, r) 463 } 464 err1 := w.Close() 465 if err == nil { 466 err = err1 467 } 468 469 return err 470 } 471 472 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 473 b := bufio.NewReader(r) 474 for { 475 l, err := b.ReadBytes('\n') 476 eof := err == io.EOF 477 if err != nil && err != io.EOF { 478 return err 479 } 480 481 // If we have data then write it out... 482 if len(l) > 0 { 483 // Strip off \n if it exists because stripImportComment 484 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 485 if err != nil { 486 return err 487 } 488 } 489 490 if eof { 491 return nil 492 } 493 } 494 } 495 496 const ( 497 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 498 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 499 ) 500 501 var ( 502 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 503 pkgPrefix = []byte("package ") 504 ) 505 506 // stripImportComment returns line with its import comment removed. 507 // If s is not a package statement containing an import comment, 508 // it is returned unaltered. 509 // FIXME: expects lines w/o a \n at the end 510 // See also http://golang.org/s/go14customimport. 511 func stripImportComment(line []byte) []byte { 512 if !bytes.HasPrefix(line, pkgPrefix) { 513 // Fast path; this will skip all but one line in the file. 514 // This assumes there is no whitespace before the keyword. 515 return line 516 } 517 if m := importCommentRE.FindSubmatch(line); m != nil { 518 return append(m[1], m[2]...) 519 } 520 return line 521 } 522 523 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 524 // so that dir/pkg and dir/bin don't accidentally get committed. 525 // It logs any errors it encounters. 526 func writeVCSIgnore(dir string) { 527 // Currently git is the only VCS for which we know how to do this. 528 // Mercurial and Bazaar have similar mechanisms, but they apparently 529 // require writing files outside of dir. 530 const ignore = "/pkg\n/bin\n" 531 name := filepath.Join(dir, ".gitignore") 532 err := writeFile(name, ignore) 533 if err != nil { 534 log.Println(err) 535 } 536 } 537 538 // writeFile is like ioutil.WriteFile but it creates 539 // intermediate directories with os.MkdirAll. 540 func writeFile(name, body string) error { 541 err := os.MkdirAll(filepath.Dir(name), 0777) 542 if err != nil { 543 return err 544 } 545 return ioutil.WriteFile(name, []byte(body), 0666) 546 } 547 548 const ( 549 // Readme contains the README text. 550 Readme = ` 551 This directory tree is generated automatically by godep. 552 553 Please do not edit. 554 555 See https://github.com/tools/godep for more information. 556 ` 557 needRestore = ` 558 mismatched versions while migrating 559 560 It looks like you are switching from the old Godeps format 561 (from flag -copy=false). The old format is just a file; it 562 doesn't contain source code. For this migration, godep needs 563 the appropriate version of each dependency to be installed in 564 GOPATH, so that the source code is available to copy. 565 566 To fix this, run 'godep restore'. 567 ` 568 )