gopkg.in/tools/godep.v38@v38.0.0-20151216225452-4154dbb67855/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "go/build" 9 "io" 10 "io/ioutil" 11 "log" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 17 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 18 ) 19 20 var cmdSave = &Command{ 21 Name: "save", 22 Args: "[-r] [-t] [packages]", 23 Short: "list and copy dependencies into Godeps", 24 Long: ` 25 26 Save writes a list of the named packages and their dependencies along 27 with the exact source control revision of each package, and copies 28 their source code into a subdirectory. Packages inside "." are excluded 29 from the list to be copied. 30 31 The list is written to Godeps/Godeps.json, and source code for all 32 dependencies is copied into either Godeps/_workspace or, if the vendor 33 experiment is turned on, vendor/. 34 35 The dependency list is a JSON document with the following structure: 36 37 type Godeps struct { 38 ImportPath string 39 GoVersion string // Abridged output of 'go version'. 40 Packages []string // Arguments to godep save, if any. 41 Deps []struct { 42 ImportPath string 43 Comment string // Tag or description of commit. 44 Rev string // VCS-specific commit ID. 45 } 46 } 47 48 Any packages already present in the list will be left unchanged. 49 To update a dependency to a newer revision, use 'godep update'. 50 51 If -r is given, import statements will be rewritten to refer 52 directly to the copied source code. This is not compatible with the 53 vendor experiment. 54 55 If -t is given, test files (*_test.go files + testdata directories) are 56 also saved. 57 58 For more about specifying packages, see 'go help packages'. 59 `, 60 Run: runSave, 61 } 62 63 var ( 64 saveR, saveT bool 65 ) 66 67 func init() { 68 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 69 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 70 71 } 72 73 func runSave(cmd *Command, args []string) { 74 if VendorExperiment && saveR { 75 log.Println("flag -r is incompatible with the vendoring experiment") 76 cmd.UsageExit() 77 } 78 err := save(args) 79 if err != nil { 80 log.Fatalln(err) 81 } 82 } 83 84 func dotPackageImportPath() (string, error) { 85 dir, err := filepath.Abs(".") 86 if err != nil { 87 return "", err 88 } 89 p, err := build.ImportDir(dir, build.FindOnly) 90 return p.ImportPath, err 91 } 92 93 func save(pkgs []string) error { 94 dip, err := dotPackageImportPath() 95 if err != nil { 96 return err 97 } 98 debugln("dotPackageImportPath:", dip) 99 100 ver, err := goVersion() 101 if err != nil { 102 return err 103 } 104 debugln("goVersion:", ver) 105 106 gold, err := loadDefaultGodepsFile() 107 if err != nil { 108 if !os.IsNotExist(err) { 109 return err 110 } 111 } 112 113 gnew := &Godeps{ 114 ImportPath: dip, 115 GoVersion: ver, 116 } 117 118 switch len(pkgs) { 119 case 0: 120 pkgs = []string{"."} 121 default: 122 gnew.Packages = pkgs 123 } 124 125 a, err := LoadPackages(pkgs...) 126 if err != nil { 127 return err 128 } 129 debugln("LoadPackages", pkgs) 130 ppln(a) 131 132 err = gnew.fill(a, dip) 133 if err != nil { 134 return err 135 } 136 debugln("New Godeps Filled") 137 138 if gnew.Deps == nil { 139 gnew.Deps = make([]Dependency, 0) // produce json [], not null 140 } 141 gdisk := gnew.copy() 142 err = carryVersions(&gold, gnew) 143 if err != nil { 144 return err 145 } 146 if gold.isOldFile { 147 // If we are migrating from an old format file, 148 // we require that the listed version of every 149 // dependency must be installed in GOPATH, so it's 150 // available to copy. 151 if !eqDeps(gnew.Deps, gdisk.Deps) { 152 return errors.New(strings.TrimSpace(needRestore)) 153 } 154 gold = Godeps{} 155 } 156 os.Remove("Godeps") // remove regular file if present; ignore error 157 readme := filepath.Join("Godeps", "Readme") 158 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 159 if err != nil { 160 log.Println(err) 161 } 162 _, err = gnew.save() 163 if err != nil { 164 return err 165 } 166 // We use a name starting with "_" so the go tool 167 // ignores this directory when traversing packages 168 // starting at the project's root. For example, 169 // godep go list ./... 170 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 171 rem := subDeps(gold.Deps, gnew.Deps) 172 add := subDeps(gnew.Deps, gold.Deps) 173 err = removeSrc(srcdir, rem) 174 if err != nil { 175 return err 176 } 177 err = copySrc(srcdir, add) 178 if err != nil { 179 return err 180 } 181 if !VendorExperiment { 182 f, _ := filepath.Split(srcdir) 183 writeVCSIgnore(f) 184 } 185 var rewritePaths []string 186 if saveR { 187 for _, dep := range gnew.Deps { 188 rewritePaths = append(rewritePaths, dep.ImportPath) 189 } 190 } 191 return rewrite(a, dip, rewritePaths) 192 } 193 194 type revError struct { 195 ImportPath string 196 WantRev string 197 HavePath string 198 HaveRev string 199 } 200 201 func (v *revError) Error() string { 202 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 203 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 204 } 205 206 // carryVersions copies Rev and Comment from a to b for 207 // each dependency with an identical ImportPath. For any 208 // dependency in b that appears to be from the same repo 209 // as one in a (for example, a parent or child directory), 210 // the Rev must already match - otherwise it is an error. 211 func carryVersions(a, b *Godeps) error { 212 for i := range b.Deps { 213 err := carryVersion(a, &b.Deps[i]) 214 if err != nil { 215 return err 216 } 217 } 218 return nil 219 } 220 221 func carryVersion(a *Godeps, db *Dependency) error { 222 // First see if this exact package is already in the list. 223 for _, da := range a.Deps { 224 if db.ImportPath == da.ImportPath { 225 db.Rev = da.Rev 226 db.Comment = da.Comment 227 return nil 228 } 229 } 230 // No exact match, check for child or sibling package. 231 // We can't handle mismatched versions for packages in 232 // the same repo, so report that as an error. 233 for _, da := range a.Deps { 234 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 235 strings.HasPrefix(da.ImportPath, db.root+"/") { 236 if da.Rev != db.Rev { 237 return &revError{ 238 ImportPath: db.ImportPath, 239 WantRev: db.Rev, 240 HavePath: da.ImportPath, 241 HaveRev: da.Rev, 242 } 243 } 244 } 245 } 246 // No related package in the list, must be a new repo. 247 return nil 248 } 249 250 // subDeps returns a - b, using ImportPath for equality. 251 func subDeps(a, b []Dependency) (diff []Dependency) { 252 Diff: 253 for _, da := range a { 254 for _, db := range b { 255 if da.ImportPath == db.ImportPath { 256 continue Diff 257 } 258 } 259 diff = append(diff, da) 260 } 261 return diff 262 } 263 264 func removeSrc(srcdir string, deps []Dependency) error { 265 for _, dep := range deps { 266 path := filepath.FromSlash(dep.ImportPath) 267 err := os.RemoveAll(filepath.Join(srcdir, path)) 268 if err != nil { 269 return err 270 } 271 } 272 return nil 273 } 274 275 func copySrc(dir string, deps []Dependency) error { 276 // mapping to see if we visited a parent directory already 277 visited := make(map[string]bool) 278 ok := true 279 for _, dep := range deps { 280 srcdir := filepath.Join(dep.ws, "src") 281 rel, err := filepath.Rel(srcdir, dep.dir) 282 if err != nil { // this should never happen 283 return err 284 } 285 dstpkgroot := filepath.Join(dir, rel) 286 err = os.RemoveAll(dstpkgroot) 287 if err != nil { 288 log.Println(err) 289 ok = false 290 } 291 292 // copy actual dependency 293 vf := dep.vcs.listFiles(dep.dir) 294 w := fs.Walk(dep.dir) 295 for w.Step() { 296 err = copyPkgFile(vf, dir, srcdir, w) 297 if err != nil { 298 log.Println(err) 299 ok = false 300 } 301 } 302 303 // Look for legal files in root 304 // some packages are imports as a sub-package but license info 305 // is at root: exampleorg/common has license file in exampleorg 306 // 307 if dep.ImportPath == dep.root { 308 // we are already at root 309 continue 310 } 311 312 // prevent copying twice This could happen if we have 313 // two subpackages listed someorg/common and 314 // someorg/anotherpack which has their license in 315 // the parent dir of someorg 316 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 317 if visited[rootdir] { 318 continue 319 } 320 visited[rootdir] = true 321 vf = dep.vcs.listFiles(rootdir) 322 w = fs.Walk(rootdir) 323 for w.Step() { 324 fname := filepath.Base(w.Path()) 325 if IsLegalFile(fname) && !strings.Contains(w.Path(), sep) { 326 err = copyPkgFile(vf, dir, srcdir, w) 327 if err != nil { 328 log.Println(err) 329 ok = false 330 } 331 } 332 } 333 } 334 335 if !ok { 336 return errorCopyingSourceCode 337 } 338 339 return nil 340 } 341 342 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 343 if w.Err() != nil { 344 return w.Err() 345 } 346 name := w.Stat().Name() 347 if w.Stat().IsDir() { 348 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 349 // Skip directories starting with '.' or '_' or 350 // 'testdata' (last is only skipped if saveT is false) 351 w.SkipDir() 352 } 353 return nil 354 } 355 rel, err := filepath.Rel(srcroot, w.Path()) 356 if err != nil { // this should never happen 357 return err 358 } 359 if !saveT && strings.HasSuffix(name, "_test.go") { 360 if verbose { 361 log.Printf("save: skipping test file: %s", w.Path()) 362 } 363 return nil 364 } 365 if !vf.Contains(w.Path()) { 366 if verbose { 367 log.Printf("save: skipping untracked file: %s", w.Path()) 368 } 369 return nil 370 } 371 return copyFile(filepath.Join(dstroot, rel), w.Path()) 372 } 373 374 // copyFile copies a regular file from src to dst. 375 // dst is opened with os.Create. 376 // If the file name ends with .go, 377 // copyFile strips canonical import path annotations. 378 // These are comments of the form: 379 // package foo // import "bar/foo" 380 // package foo /* import "bar/foo" */ 381 func copyFile(dst, src string) error { 382 err := os.MkdirAll(filepath.Dir(dst), 0777) 383 if err != nil { 384 return err 385 } 386 387 linkDst, err := os.Readlink(src) 388 if err == nil { 389 return os.Symlink(linkDst, dst) 390 } 391 392 r, err := os.Open(src) 393 if err != nil { 394 return err 395 } 396 defer r.Close() 397 398 w, err := os.Create(dst) 399 if err != nil { 400 return err 401 } 402 403 if strings.HasSuffix(dst, ".go") { 404 err = copyWithoutImportComment(w, r) 405 } else { 406 _, err = io.Copy(w, r) 407 } 408 err1 := w.Close() 409 if err == nil { 410 err = err1 411 } 412 413 return err 414 } 415 416 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 417 b := bufio.NewReader(r) 418 for { 419 l, err := b.ReadBytes('\n') 420 eof := err == io.EOF 421 if err != nil && err != io.EOF { 422 return err 423 } 424 425 // If we have data then write it out... 426 if len(l) > 0 { 427 // Strip off \n if it exists because stripImportComment 428 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 429 if err != nil { 430 return err 431 } 432 } 433 434 if eof { 435 return nil 436 } 437 } 438 } 439 440 const ( 441 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 442 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 443 ) 444 445 var ( 446 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 447 pkgPrefix = []byte("package ") 448 ) 449 450 // stripImportComment returns line with its import comment removed. 451 // If s is not a package statement containing an import comment, 452 // it is returned unaltered. 453 // FIXME: expects lines w/o a \n at the end 454 // See also http://golang.org/s/go14customimport. 455 func stripImportComment(line []byte) []byte { 456 if !bytes.HasPrefix(line, pkgPrefix) { 457 // Fast path; this will skip all but one line in the file. 458 // This assumes there is no whitespace before the keyword. 459 return line 460 } 461 if m := importCommentRE.FindSubmatch(line); m != nil { 462 return append(m[1], m[2]...) 463 } 464 return line 465 } 466 467 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 468 // so that dir/pkg and dir/bin don't accidentally get committed. 469 // It logs any errors it encounters. 470 func writeVCSIgnore(dir string) { 471 // Currently git is the only VCS for which we know how to do this. 472 // Mercurial and Bazaar have similar mechanisms, but they apparently 473 // require writing files outside of dir. 474 const ignore = "/pkg\n/bin\n" 475 name := filepath.Join(dir, ".gitignore") 476 err := writeFile(name, ignore) 477 if err != nil { 478 log.Println(err) 479 } 480 } 481 482 // writeFile is like ioutil.WriteFile but it creates 483 // intermediate directories with os.MkdirAll. 484 func writeFile(name, body string) error { 485 err := os.MkdirAll(filepath.Dir(name), 0777) 486 if err != nil { 487 return err 488 } 489 return ioutil.WriteFile(name, []byte(body), 0666) 490 } 491 492 const ( 493 // Readme contains the README text. 494 Readme = ` 495 This directory tree is generated automatically by godep. 496 497 Please do not edit. 498 499 See https://github.com/tools/godep for more information. 500 ` 501 needRestore = ` 502 mismatched versions while migrating 503 504 It looks like you are switching from the old Godeps format 505 (from flag -copy=false). The old format is just a file; it 506 doesn't contain source code. For this migration, godep needs 507 the appropriate version of each dependency to be installed in 508 GOPATH, so that the source code is available to copy. 509 510 To fix this, run 'godep restore'. 511 ` 512 )