gopkg.in/tools/godep.v39@v39.0.0-20151216234527-2721f686f09e/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer 52 directly to the copied source code. This is not compatible with the 53 vendor experiment. 54 55 If -t is given, test files (*_test.go files + testdata directories) are 56 also saved. 57 58 For more about specifying packages, see 'go help packages'. 59 `, 60 Run: runSave, 61 } 62 63 var ( 64 saveR, saveT bool 65 ) 66 67 func init() { 68 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 69 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 70 71 } 72 73 func runSave(cmd *Command, args []string) { 74 if VendorExperiment && saveR { 75 log.Println("flag -r is incompatible with the vendoring experiment") 76 cmd.UsageExit() 77 } 78 err := save(args) 79 if err != nil { 80 log.Fatalln(err) 81 } 82 } 83 84 func dotPackageImportPath() (string, error) { 85 dir, err := filepath.Abs(".") 86 if err != nil { 87 return "", err 88 } 89 p, err := build.ImportDir(dir, build.FindOnly) 90 return p.ImportPath, err 91 } 92 93 func save(pkgs []string) error { 94 dip, err := dotPackageImportPath() 95 if err != nil { 96 return err 97 } 98 debugln("dotPackageImportPath:", dip) 99 100 cv, err := goVersion() 101 if err != nil { 102 return err 103 } 104 debugln("goVersion:", cv) 105 106 gold, err := loadDefaultGodepsFile() 107 if err != nil { 108 if !os.IsNotExist(err) { 109 return err 110 } 111 gold.GoVersion = cv 112 } 113 114 printVersionWarnings(gold.GoVersion) 115 116 gnew := &Godeps{ 117 ImportPath: dip, 118 GoVersion: gold.GoVersion, 119 } 120 121 switch len(pkgs) { 122 case 0: 123 pkgs = []string{"."} 124 default: 125 gnew.Packages = pkgs 126 } 127 128 a, err := LoadPackages(pkgs...) 129 if err != nil { 130 return err 131 } 132 debugln("LoadPackages", pkgs) 133 ppln(a) 134 135 err = gnew.fill(a, dip) 136 if err != nil { 137 return err 138 } 139 debugln("New Godeps Filled") 140 141 if gnew.Deps == nil { 142 gnew.Deps = make([]Dependency, 0) // produce json [], not null 143 } 144 gdisk := gnew.copy() 145 err = carryVersions(&gold, gnew) 146 if err != nil { 147 return err 148 } 149 if gold.isOldFile { 150 // If we are migrating from an old format file, 151 // we require that the listed version of every 152 // dependency must be installed in GOPATH, so it's 153 // available to copy. 154 if !eqDeps(gnew.Deps, gdisk.Deps) { 155 return errors.New(strings.TrimSpace(needRestore)) 156 } 157 gold = Godeps{} 158 } 159 os.Remove("Godeps") // remove regular file if present; ignore error 160 readme := filepath.Join("Godeps", "Readme") 161 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 162 if err != nil { 163 log.Println(err) 164 } 165 _, err = gnew.save() 166 if err != nil { 167 return err 168 } 169 // We use a name starting with "_" so the go tool 170 // ignores this directory when traversing packages 171 // starting at the project's root. For example, 172 // godep go list ./... 173 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 174 rem := subDeps(gold.Deps, gnew.Deps) 175 add := subDeps(gnew.Deps, gold.Deps) 176 err = removeSrc(srcdir, rem) 177 if err != nil { 178 return err 179 } 180 err = copySrc(srcdir, add) 181 if err != nil { 182 return err 183 } 184 if !VendorExperiment { 185 f, _ := filepath.Split(srcdir) 186 writeVCSIgnore(f) 187 } 188 var rewritePaths []string 189 if saveR { 190 for _, dep := range gnew.Deps { 191 rewritePaths = append(rewritePaths, dep.ImportPath) 192 } 193 } 194 return rewrite(a, dip, rewritePaths) 195 } 196 197 func printVersionWarnings(ov string) { 198 var warning bool 199 cv, err := goVersion() 200 if err != nil { 201 return 202 } 203 tov, err := trimGoVersion(ov) 204 if err != nil { 205 return 206 } 207 tcv, err := trimGoVersion(cv) 208 if err != nil { 209 return 210 } 211 212 if tov != ov { 213 log.Printf("WARNING: Recorded go version (%s) with minor version string found.\n", ov) 214 warning = true 215 } 216 if tcv != tov { 217 log.Printf("WARNING: Recorded major go version (%s) and in-use major go version (%s) differ.\n", tov, tcv) 218 warning = true 219 } 220 if warning { 221 log.Println("To record current major go version run `godep update -goversion`.") 222 } 223 } 224 225 type revError struct { 226 ImportPath string 227 WantRev string 228 HavePath string 229 HaveRev string 230 } 231 232 func (v *revError) Error() string { 233 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 234 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 235 } 236 237 // carryVersions copies Rev and Comment from a to b for 238 // each dependency with an identical ImportPath. For any 239 // dependency in b that appears to be from the same repo 240 // as one in a (for example, a parent or child directory), 241 // the Rev must already match - otherwise it is an error. 242 func carryVersions(a, b *Godeps) error { 243 for i := range b.Deps { 244 err := carryVersion(a, &b.Deps[i]) 245 if err != nil { 246 return err 247 } 248 } 249 return nil 250 } 251 252 func carryVersion(a *Godeps, db *Dependency) error { 253 // First see if this exact package is already in the list. 254 for _, da := range a.Deps { 255 if db.ImportPath == da.ImportPath { 256 db.Rev = da.Rev 257 db.Comment = da.Comment 258 return nil 259 } 260 } 261 // No exact match, check for child or sibling package. 262 // We can't handle mismatched versions for packages in 263 // the same repo, so report that as an error. 264 for _, da := range a.Deps { 265 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 266 strings.HasPrefix(da.ImportPath, db.root+"/") { 267 if da.Rev != db.Rev { 268 return &revError{ 269 ImportPath: db.ImportPath, 270 WantRev: db.Rev, 271 HavePath: da.ImportPath, 272 HaveRev: da.Rev, 273 } 274 } 275 } 276 } 277 // No related package in the list, must be a new repo. 278 return nil 279 } 280 281 // subDeps returns a - b, using ImportPath for equality. 282 func subDeps(a, b []Dependency) (diff []Dependency) { 283 Diff: 284 for _, da := range a { 285 for _, db := range b { 286 if da.ImportPath == db.ImportPath { 287 continue Diff 288 } 289 } 290 diff = append(diff, da) 291 } 292 return diff 293 } 294 295 func removeSrc(srcdir string, deps []Dependency) error { 296 for _, dep := range deps { 297 path := filepath.FromSlash(dep.ImportPath) 298 err := os.RemoveAll(filepath.Join(srcdir, path)) 299 if err != nil { 300 return err 301 } 302 } 303 return nil 304 } 305 306 func copySrc(dir string, deps []Dependency) error { 307 // mapping to see if we visited a parent directory already 308 visited := make(map[string]bool) 309 ok := true 310 for _, dep := range deps { 311 srcdir := filepath.Join(dep.ws, "src") 312 rel, err := filepath.Rel(srcdir, dep.dir) 313 if err != nil { // this should never happen 314 return err 315 } 316 dstpkgroot := filepath.Join(dir, rel) 317 err = os.RemoveAll(dstpkgroot) 318 if err != nil { 319 log.Println(err) 320 ok = false 321 } 322 323 // copy actual dependency 324 vf := dep.vcs.listFiles(dep.dir) 325 w := fs.Walk(dep.dir) 326 for w.Step() { 327 err = copyPkgFile(vf, dir, srcdir, w) 328 if err != nil { 329 log.Println(err) 330 ok = false 331 } 332 } 333 334 // Look for legal files in root 335 // some packages are imports as a sub-package but license info 336 // is at root: exampleorg/common has license file in exampleorg 337 // 338 if dep.ImportPath == dep.root { 339 // we are already at root 340 continue 341 } 342 343 // prevent copying twice This could happen if we have 344 // two subpackages listed someorg/common and 345 // someorg/anotherpack which has their license in 346 // the parent dir of someorg 347 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 348 if visited[rootdir] { 349 continue 350 } 351 visited[rootdir] = true 352 vf = dep.vcs.listFiles(rootdir) 353 w = fs.Walk(rootdir) 354 for w.Step() { 355 fname := filepath.Base(w.Path()) 356 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 357 err = copyPkgFile(vf, dir, srcdir, w) 358 if err != nil { 359 log.Println(err) 360 ok = false 361 } 362 } 363 } 364 } 365 366 if !ok { 367 return errorCopyingSourceCode 368 } 369 370 return nil 371 } 372 373 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 374 if w.Err() != nil { 375 return w.Err() 376 } 377 name := w.Stat().Name() 378 if w.Stat().IsDir() { 379 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 380 // Skip directories starting with '.' or '_' or 381 // 'testdata' (last is only skipped if saveT is false) 382 w.SkipDir() 383 } 384 return nil 385 } 386 rel, err := filepath.Rel(srcroot, w.Path()) 387 if err != nil { // this should never happen 388 return err 389 } 390 if !saveT && strings.HasSuffix(name, "_test.go") { 391 if verbose { 392 log.Printf("save: skipping test file: %s", w.Path()) 393 } 394 return nil 395 } 396 if !vf.Contains(w.Path()) { 397 if verbose { 398 log.Printf("save: skipping untracked file: %s", w.Path()) 399 } 400 return nil 401 } 402 return copyFile(filepath.Join(dstroot, rel), w.Path()) 403 } 404 405 // copyFile copies a regular file from src to dst. 406 // dst is opened with os.Create. 407 // If the file name ends with .go, 408 // copyFile strips canonical import path annotations. 409 // These are comments of the form: 410 // package foo // import "bar/foo" 411 // package foo /* import "bar/foo" */ 412 func copyFile(dst, src string) error { 413 err := os.MkdirAll(filepath.Dir(dst), 0777) 414 if err != nil { 415 return err 416 } 417 418 linkDst, err := os.Readlink(src) 419 if err == nil { 420 return os.Symlink(linkDst, dst) 421 } 422 423 r, err := os.Open(src) 424 if err != nil { 425 return err 426 } 427 defer r.Close() 428 429 w, err := os.Create(dst) 430 if err != nil { 431 return err 432 } 433 434 if strings.HasSuffix(dst, ".go") { 435 err = copyWithoutImportComment(w, r) 436 } else { 437 _, err = io.Copy(w, r) 438 } 439 err1 := w.Close() 440 if err == nil { 441 err = err1 442 } 443 444 return err 445 } 446 447 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 448 b := bufio.NewReader(r) 449 for { 450 l, err := b.ReadBytes('\n') 451 eof := err == io.EOF 452 if err != nil && err != io.EOF { 453 return err 454 } 455 456 // If we have data then write it out... 457 if len(l) > 0 { 458 // Strip off \n if it exists because stripImportComment 459 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 460 if err != nil { 461 return err 462 } 463 } 464 465 if eof { 466 return nil 467 } 468 } 469 } 470 471 const ( 472 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 473 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 474 ) 475 476 var ( 477 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 478 pkgPrefix = []byte("package ") 479 ) 480 481 // stripImportComment returns line with its import comment removed. 482 // If s is not a package statement containing an import comment, 483 // it is returned unaltered. 484 // FIXME: expects lines w/o a \n at the end 485 // See also http://golang.org/s/go14customimport. 486 func stripImportComment(line []byte) []byte { 487 if !bytes.HasPrefix(line, pkgPrefix) { 488 // Fast path; this will skip all but one line in the file. 489 // This assumes there is no whitespace before the keyword. 490 return line 491 } 492 if m := importCommentRE.FindSubmatch(line); m != nil { 493 return append(m[1], m[2]...) 494 } 495 return line 496 } 497 498 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 499 // so that dir/pkg and dir/bin don't accidentally get committed. 500 // It logs any errors it encounters. 501 func writeVCSIgnore(dir string) { 502 // Currently git is the only VCS for which we know how to do this. 503 // Mercurial and Bazaar have similar mechanisms, but they apparently 504 // require writing files outside of dir. 505 const ignore = "/pkg\n/bin\n" 506 name := filepath.Join(dir, ".gitignore") 507 err := writeFile(name, ignore) 508 if err != nil { 509 log.Println(err) 510 } 511 } 512 513 // writeFile is like ioutil.WriteFile but it creates 514 // intermediate directories with os.MkdirAll. 515 func writeFile(name, body string) error { 516 err := os.MkdirAll(filepath.Dir(name), 0777) 517 if err != nil { 518 return err 519 } 520 return ioutil.WriteFile(name, []byte(body), 0666) 521 } 522 523 const ( 524 // Readme contains the README text. 525 Readme = ` 526 This directory tree is generated automatically by godep. 527 528 Please do not edit. 529 530 See https://github.com/tools/godep for more information. 531 ` 532 needRestore = ` 533 mismatched versions while migrating 534 535 It looks like you are switching from the old Godeps format 536 (from flag -copy=false). The old format is just a file; it 537 doesn't contain source code. For this migration, godep needs 538 the appropriate version of each dependency to be installed in 539 GOPATH, so that the source code is available to copy. 540 541 To fix this, run 'godep restore'. 542 ` 543 )