gopkg.in/tools/godep.v41@v41.0.0-20151217180337-fe5ce707f879/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer directly 52 to the copied source code. This is not compatible with the vendor 53 experiment. Note that this will not rewrite the statements in the 54 files outside the project. 55 56 If -t is given, test files (*_test.go files + testdata directories) are 57 also saved. 58 59 For more about specifying packages, see 'go help packages'. 60 `, 61 Run: runSave, 62 } 63 64 var ( 65 saveR, saveT bool 66 ) 67 68 func init() { 69 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 70 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 71 72 } 73 74 func runSave(cmd *Command, args []string) { 75 if VendorExperiment && saveR { 76 log.Println("flag -r is incompatible with the vendoring experiment") 77 cmd.UsageExit() 78 } 79 err := save(args) 80 if err != nil { 81 log.Fatalln(err) 82 } 83 } 84 85 func dotPackage() (*build.Package, error) { 86 dir, err := filepath.Abs(".") 87 if err != nil { 88 return nil, err 89 } 90 return build.ImportDir(dir, build.FindOnly) 91 } 92 93 func projectPackages(dDir string, a []*Package) []*Package { 94 var projPkgs []*Package 95 dotDir := fmt.Sprintf("%s%c", dDir, filepath.Separator) 96 for _, p := range a { 97 pkgDir := fmt.Sprintf("%s%c", p.Dir, filepath.Separator) 98 if strings.HasPrefix(pkgDir, dotDir) { 99 projPkgs = append(projPkgs, p) 100 } 101 } 102 return projPkgs 103 } 104 105 func save(pkgs []string) error { 106 dp, err := dotPackage() 107 if err != nil { 108 return err 109 } 110 debugln("dotPackageImportPath:", dp.ImportPath) 111 debugln("dotPackageDir:", dp.Dir) 112 113 cv, err := goVersion() 114 if err != nil { 115 return err 116 } 117 debugln("goVersion:", cv) 118 119 gold, err := loadDefaultGodepsFile() 120 if err != nil { 121 if !os.IsNotExist(err) { 122 return err 123 } 124 gold.GoVersion = cv 125 } 126 127 printVersionWarnings(gold.GoVersion) 128 129 gnew := &Godeps{ 130 ImportPath: dp.ImportPath, 131 GoVersion: gold.GoVersion, 132 } 133 134 switch len(pkgs) { 135 case 0: 136 pkgs = []string{"."} 137 default: 138 gnew.Packages = pkgs 139 } 140 141 a, err := LoadPackages(pkgs...) 142 if err != nil { 143 return err 144 } 145 debugln("LoadPackages", pkgs) 146 ppln(a) 147 148 projA := projectPackages(dp.Dir, a) 149 err = gnew.fill(a, dp.ImportPath) 150 if err != nil { 151 return err 152 } 153 debugln("New Godeps Filled") 154 155 if gnew.Deps == nil { 156 gnew.Deps = make([]Dependency, 0) // produce json [], not null 157 } 158 gdisk := gnew.copy() 159 err = carryVersions(&gold, gnew) 160 if err != nil { 161 return err 162 } 163 if gold.isOldFile { 164 // If we are migrating from an old format file, 165 // we require that the listed version of every 166 // dependency must be installed in GOPATH, so it's 167 // available to copy. 168 if !eqDeps(gnew.Deps, gdisk.Deps) { 169 return errors.New(strings.TrimSpace(needRestore)) 170 } 171 gold = Godeps{} 172 } 173 os.Remove("Godeps") // remove regular file if present; ignore error 174 readme := filepath.Join("Godeps", "Readme") 175 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 176 if err != nil { 177 log.Println(err) 178 } 179 _, err = gnew.save() 180 if err != nil { 181 return err 182 } 183 // We use a name starting with "_" so the go tool 184 // ignores this directory when traversing packages 185 // starting at the project's root. For example, 186 // godep go list ./... 187 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 188 rem := subDeps(gold.Deps, gnew.Deps) 189 add := subDeps(gnew.Deps, gold.Deps) 190 err = removeSrc(srcdir, rem) 191 if err != nil { 192 return err 193 } 194 err = copySrc(srcdir, add) 195 if err != nil { 196 return err 197 } 198 if !VendorExperiment { 199 f, _ := filepath.Split(srcdir) 200 writeVCSIgnore(f) 201 } 202 var rewritePaths []string 203 if saveR { 204 for _, dep := range gnew.Deps { 205 rewritePaths = append(rewritePaths, dep.ImportPath) 206 } 207 } 208 return rewrite(projA, dp.ImportPath, rewritePaths) 209 } 210 211 func printVersionWarnings(ov string) { 212 var warning bool 213 cv, err := goVersion() 214 if err != nil { 215 return 216 } 217 tov, err := trimGoVersion(ov) 218 if err != nil { 219 return 220 } 221 tcv, err := trimGoVersion(cv) 222 if err != nil { 223 return 224 } 225 226 if tov != ov { 227 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 228 warning = true 229 } 230 if tcv != tov { 231 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 232 warning = true 233 } 234 if warning { 235 log.Println("To record current major go version run `godep update -goversion`.") 236 } 237 } 238 239 type revError struct { 240 ImportPath string 241 WantRev string 242 HavePath string 243 HaveRev string 244 } 245 246 func (v *revError) Error() string { 247 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 248 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 249 } 250 251 // carryVersions copies Rev and Comment from a to b for 252 // each dependency with an identical ImportPath. For any 253 // dependency in b that appears to be from the same repo 254 // as one in a (for example, a parent or child directory), 255 // the Rev must already match - otherwise it is an error. 256 func carryVersions(a, b *Godeps) error { 257 for i := range b.Deps { 258 err := carryVersion(a, &b.Deps[i]) 259 if err != nil { 260 return err 261 } 262 } 263 return nil 264 } 265 266 func carryVersion(a *Godeps, db *Dependency) error { 267 // First see if this exact package is already in the list. 268 for _, da := range a.Deps { 269 if db.ImportPath == da.ImportPath { 270 db.Rev = da.Rev 271 db.Comment = da.Comment 272 return nil 273 } 274 } 275 // No exact match, check for child or sibling package. 276 // We can't handle mismatched versions for packages in 277 // the same repo, so report that as an error. 278 for _, da := range a.Deps { 279 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 280 strings.HasPrefix(da.ImportPath, db.root+"/") { 281 if da.Rev != db.Rev { 282 return &revError{ 283 ImportPath: db.ImportPath, 284 WantRev: db.Rev, 285 HavePath: da.ImportPath, 286 HaveRev: da.Rev, 287 } 288 } 289 } 290 } 291 // No related package in the list, must be a new repo. 292 return nil 293 } 294 295 // subDeps returns a - b, using ImportPath for equality. 296 func subDeps(a, b []Dependency) (diff []Dependency) { 297 Diff: 298 for _, da := range a { 299 for _, db := range b { 300 if da.ImportPath == db.ImportPath { 301 continue Diff 302 } 303 } 304 diff = append(diff, da) 305 } 306 return diff 307 } 308 309 func removeSrc(srcdir string, deps []Dependency) error { 310 for _, dep := range deps { 311 path := filepath.FromSlash(dep.ImportPath) 312 err := os.RemoveAll(filepath.Join(srcdir, path)) 313 if err != nil { 314 return err 315 } 316 } 317 return nil 318 } 319 320 func copySrc(dir string, deps []Dependency) error { 321 // mapping to see if we visited a parent directory already 322 visited := make(map[string]bool) 323 ok := true 324 for _, dep := range deps { 325 srcdir := filepath.Join(dep.ws, "src") 326 rel, err := filepath.Rel(srcdir, dep.dir) 327 if err != nil { // this should never happen 328 return err 329 } 330 dstpkgroot := filepath.Join(dir, rel) 331 err = os.RemoveAll(dstpkgroot) 332 if err != nil { 333 log.Println(err) 334 ok = false 335 } 336 337 // copy actual dependency 338 vf := dep.vcs.listFiles(dep.dir) 339 w := fs.Walk(dep.dir) 340 for w.Step() { 341 err = copyPkgFile(vf, dir, srcdir, w) 342 if err != nil { 343 log.Println(err) 344 ok = false 345 } 346 } 347 348 // Look for legal files in root 349 // some packages are imports as a sub-package but license info 350 // is at root: exampleorg/common has license file in exampleorg 351 // 352 if dep.ImportPath == dep.root { 353 // we are already at root 354 continue 355 } 356 357 // prevent copying twice This could happen if we have 358 // two subpackages listed someorg/common and 359 // someorg/anotherpack which has their license in 360 // the parent dir of someorg 361 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 362 if visited[rootdir] { 363 continue 364 } 365 visited[rootdir] = true 366 vf = dep.vcs.listFiles(rootdir) 367 w = fs.Walk(rootdir) 368 for w.Step() { 369 fname := filepath.Base(w.Path()) 370 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 371 err = copyPkgFile(vf, dir, srcdir, w) 372 if err != nil { 373 log.Println(err) 374 ok = false 375 } 376 } 377 } 378 } 379 380 if !ok { 381 return errorCopyingSourceCode 382 } 383 384 return nil 385 } 386 387 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 388 if w.Err() != nil { 389 return w.Err() 390 } 391 name := w.Stat().Name() 392 if w.Stat().IsDir() { 393 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 394 // Skip directories starting with '.' or '_' or 395 // 'testdata' (last is only skipped if saveT is false) 396 w.SkipDir() 397 } 398 return nil 399 } 400 rel, err := filepath.Rel(srcroot, w.Path()) 401 if err != nil { // this should never happen 402 return err 403 } 404 if !saveT && strings.HasSuffix(name, "_test.go") { 405 if verbose { 406 log.Printf("save: skipping test file: %s", w.Path()) 407 } 408 return nil 409 } 410 if !vf.Contains(w.Path()) { 411 if verbose { 412 log.Printf("save: skipping untracked file: %s", w.Path()) 413 } 414 return nil 415 } 416 return copyFile(filepath.Join(dstroot, rel), w.Path()) 417 } 418 419 // copyFile copies a regular file from src to dst. 420 // dst is opened with os.Create. 421 // If the file name ends with .go, 422 // copyFile strips canonical import path annotations. 423 // These are comments of the form: 424 // package foo // import "bar/foo" 425 // package foo /* import "bar/foo" */ 426 func copyFile(dst, src string) error { 427 err := os.MkdirAll(filepath.Dir(dst), 0777) 428 if err != nil { 429 return err 430 } 431 432 linkDst, err := os.Readlink(src) 433 if err == nil { 434 return os.Symlink(linkDst, dst) 435 } 436 437 r, err := os.Open(src) 438 if err != nil { 439 return err 440 } 441 defer r.Close() 442 443 w, err := os.Create(dst) 444 if err != nil { 445 return err 446 } 447 448 if strings.HasSuffix(dst, ".go") { 449 err = copyWithoutImportComment(w, r) 450 } else { 451 _, err = io.Copy(w, r) 452 } 453 err1 := w.Close() 454 if err == nil { 455 err = err1 456 } 457 458 return err 459 } 460 461 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 462 b := bufio.NewReader(r) 463 for { 464 l, err := b.ReadBytes('\n') 465 eof := err == io.EOF 466 if err != nil && err != io.EOF { 467 return err 468 } 469 470 // If we have data then write it out... 471 if len(l) > 0 { 472 // Strip off \n if it exists because stripImportComment 473 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 474 if err != nil { 475 return err 476 } 477 } 478 479 if eof { 480 return nil 481 } 482 } 483 } 484 485 const ( 486 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 487 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 488 ) 489 490 var ( 491 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 492 pkgPrefix = []byte("package ") 493 ) 494 495 // stripImportComment returns line with its import comment removed. 496 // If s is not a package statement containing an import comment, 497 // it is returned unaltered. 498 // FIXME: expects lines w/o a \n at the end 499 // See also http://golang.org/s/go14customimport. 500 func stripImportComment(line []byte) []byte { 501 if !bytes.HasPrefix(line, pkgPrefix) { 502 // Fast path; this will skip all but one line in the file. 503 // This assumes there is no whitespace before the keyword. 504 return line 505 } 506 if m := importCommentRE.FindSubmatch(line); m != nil { 507 return append(m[1], m[2]...) 508 } 509 return line 510 } 511 512 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 513 // so that dir/pkg and dir/bin don't accidentally get committed. 514 // It logs any errors it encounters. 515 func writeVCSIgnore(dir string) { 516 // Currently git is the only VCS for which we know how to do this. 517 // Mercurial and Bazaar have similar mechanisms, but they apparently 518 // require writing files outside of dir. 519 const ignore = "/pkg\n/bin\n" 520 name := filepath.Join(dir, ".gitignore") 521 err := writeFile(name, ignore) 522 if err != nil { 523 log.Println(err) 524 } 525 } 526 527 // writeFile is like ioutil.WriteFile but it creates 528 // intermediate directories with os.MkdirAll. 529 func writeFile(name, body string) error { 530 err := os.MkdirAll(filepath.Dir(name), 0777) 531 if err != nil { 532 return err 533 } 534 return ioutil.WriteFile(name, []byte(body), 0666) 535 } 536 537 const ( 538 // Readme contains the README text. 539 Readme = ` 540 This directory tree is generated automatically by godep. 541 542 Please do not edit. 543 544 See https://github.com/tools/godep for more information. 545 ` 546 needRestore = ` 547 mismatched versions while migrating 548 549 It looks like you are switching from the old Godeps format 550 (from flag -copy=false). The old format is just a file; it 551 doesn't contain source code. For this migration, godep needs 552 the appropriate version of each dependency to be installed in 553 GOPATH, so that the source code is available to copy. 554 555 To fix this, run 'godep restore'. 556 ` 557 )