gopkg.in/tools/godep.v21@v21.0.0-20151104013723-2cf1d6e3f557/save.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "bytes" 6 "errors" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "log" 11 "os" 12 "path/filepath" 13 "regexp" 14 "strings" 15 16 "github.com/tools/godep/Godeps/_workspace/src/github.com/kr/fs" 17 ) 18 19 var cmdSave = &Command{ 20 Usage: "save [-r] [-v] [-t] [packages]", 21 Short: "list and copy dependencies into Godeps", 22 Long: ` 23 24 Save writes a list of the named packages and their dependencies along 25 with the exact source control revision of each package, and copies 26 their source code into a subdirectory. Packages inside "." are excluded 27 from the list to be copied. 28 29 The list is written to Godeps/Godeps.json, and source code for all 30 dependencies is copied into either Godeps/_workspace or, if the vendor 31 experiment is turned on, vendor/. 32 33 The dependency list is a JSON document with the following structure: 34 35 type Godeps struct { 36 ImportPath string 37 GoVersion string // Abridged output of 'go version'. 38 Packages []string // Arguments to godep save, if any. 39 Deps []struct { 40 ImportPath string 41 Comment string // Tag or description of commit. 42 Rev string // VCS-specific commit ID. 43 } 44 } 45 46 Any packages already present in the list will be left unchanged. 47 To update a dependency to a newer revision, use 'godep update'. 48 49 If -r is given, import statements will be rewritten to refer 50 directly to the copied source code. This is not compatible with the 51 vendor experiment. 52 53 If -v is given, verbose output is enabled. 54 55 If -t is given, test files (*_test.go files + testdata directories) are 56 also saved. 57 58 For more about specifying packages, see 'go help packages'. 59 `, 60 Run: runSave, 61 } 62 63 var ( 64 saveR, saveT bool 65 ) 66 67 func init() { 68 cmdSave.Flag.BoolVar(&verbose, "v", false, "enable verbose output") 69 cmdSave.Flag.BoolVar(&saveR, "r", false, "rewrite import paths") 70 cmdSave.Flag.BoolVar(&saveT, "t", false, "save test files") 71 } 72 73 func runSave(cmd *Command, args []string) { 74 if VendorExperiment && saveR { 75 log.Println("flag -r is incompatible with the vendoring experiment") 76 cmd.UsageExit() 77 } 78 err := save(args) 79 if err != nil { 80 log.Fatalln(err) 81 } 82 } 83 84 func dotPackage() (*Package, error) { 85 p, err := LoadPackages(".") 86 if err != nil { 87 return nil, err 88 } 89 if len(p) > 1 { 90 panic("Impossible number of packages") 91 } 92 return p[0], nil 93 } 94 95 func save(pkgs []string) error { 96 dot, err := dotPackage() 97 if err != nil { 98 return err 99 } 100 ver, err := goVersion() 101 if err != nil { 102 return err 103 } 104 105 gold, err := loadDefaultGodepsFile() 106 if err != nil { 107 if !os.IsNotExist(err) { 108 return err 109 } 110 } 111 112 gnew := &Godeps{ 113 ImportPath: dot.ImportPath, 114 GoVersion: ver, 115 } 116 117 switch len(pkgs) { 118 case 0: 119 pkgs = []string{"."} 120 default: 121 gnew.Packages = pkgs 122 } 123 124 a, err := LoadPackages(pkgs...) 125 if err != nil { 126 return err 127 } 128 err = gnew.fill(a, dot.ImportPath) 129 if err != nil { 130 return err 131 } 132 if gnew.Deps == nil { 133 gnew.Deps = make([]Dependency, 0) // produce json [], not null 134 } 135 gdisk := gnew.copy() 136 err = carryVersions(&gold, gnew) 137 if err != nil { 138 return err 139 } 140 if gold.isOldFile { 141 // If we are migrating from an old format file, 142 // we require that the listed version of every 143 // dependency must be installed in GOPATH, so it's 144 // available to copy. 145 if !eqDeps(gnew.Deps, gdisk.Deps) { 146 return errors.New(strings.TrimSpace(needRestore)) 147 } 148 gold = Godeps{} 149 } 150 os.Remove("Godeps") // remove regular file if present; ignore error 151 readme := filepath.Join("Godeps", "Readme") 152 err = writeFile(readme, strings.TrimSpace(Readme)+"\n") 153 if err != nil { 154 log.Println(err) 155 } 156 _, err = gnew.save() 157 if err != nil { 158 return err 159 } 160 // We use a name starting with "_" so the go tool 161 // ignores this directory when traversing packages 162 // starting at the project's root. For example, 163 // godep go list ./... 164 srcdir := filepath.FromSlash(strings.Trim(sep, "/")) 165 rem := subDeps(gold.Deps, gnew.Deps) 166 add := subDeps(gnew.Deps, gold.Deps) 167 err = removeSrc(srcdir, rem) 168 if err != nil { 169 return err 170 } 171 err = copySrc(srcdir, add) 172 if err != nil { 173 return err 174 } 175 if !VendorExperiment { 176 f, _ := filepath.Split(srcdir) 177 writeVCSIgnore(f) 178 } 179 var rewritePaths []string 180 if saveR { 181 for _, dep := range gnew.Deps { 182 rewritePaths = append(rewritePaths, dep.ImportPath) 183 } 184 } 185 return rewrite(a, dot.ImportPath, rewritePaths) 186 } 187 188 type revError struct { 189 ImportPath string 190 WantRev string 191 HavePath string 192 HaveRev string 193 } 194 195 func (v *revError) Error() string { 196 return fmt.Sprintf("cannot save %s at revision %s: already have %s at revision %s.\n"+ 197 "Run `godep update %s' first.", v.ImportPath, v.WantRev, v.HavePath, v.HaveRev, v.HavePath) 198 } 199 200 // carryVersions copies Rev and Comment from a to b for 201 // each dependency with an identical ImportPath. For any 202 // dependency in b that appears to be from the same repo 203 // as one in a (for example, a parent or child directory), 204 // the Rev must already match - otherwise it is an error. 205 func carryVersions(a, b *Godeps) error { 206 for i := range b.Deps { 207 err := carryVersion(a, &b.Deps[i]) 208 if err != nil { 209 return err 210 } 211 } 212 return nil 213 } 214 215 func carryVersion(a *Godeps, db *Dependency) error { 216 // First see if this exact package is already in the list. 217 for _, da := range a.Deps { 218 if db.ImportPath == da.ImportPath { 219 db.Rev = da.Rev 220 db.Comment = da.Comment 221 return nil 222 } 223 } 224 // No exact match, check for child or sibling package. 225 // We can't handle mismatched versions for packages in 226 // the same repo, so report that as an error. 227 for _, da := range a.Deps { 228 if strings.HasPrefix(db.ImportPath, da.ImportPath+"/") || 229 strings.HasPrefix(da.ImportPath, db.root+"/") { 230 if da.Rev != db.Rev { 231 return &revError{ 232 ImportPath: db.ImportPath, 233 WantRev: db.Rev, 234 HavePath: da.ImportPath, 235 HaveRev: da.Rev, 236 } 237 } 238 } 239 } 240 // No related package in the list, must be a new repo. 241 return nil 242 } 243 244 // subDeps returns a - b, using ImportPath for equality. 245 func subDeps(a, b []Dependency) (diff []Dependency) { 246 Diff: 247 for _, da := range a { 248 for _, db := range b { 249 if da.ImportPath == db.ImportPath { 250 continue Diff 251 } 252 } 253 diff = append(diff, da) 254 } 255 return diff 256 } 257 258 func removeSrc(srcdir string, deps []Dependency) error { 259 for _, dep := range deps { 260 path := filepath.FromSlash(dep.ImportPath) 261 err := os.RemoveAll(filepath.Join(srcdir, path)) 262 if err != nil { 263 return err 264 } 265 } 266 return nil 267 } 268 269 func copySrc(dir string, deps []Dependency) error { 270 // mapping to see if we visited a parent directory already 271 visited := make(map[string]bool) 272 ok := true 273 for _, dep := range deps { 274 srcdir := filepath.Join(dep.ws, "src") 275 rel, err := filepath.Rel(srcdir, dep.dir) 276 if err != nil { // this should never happen 277 return err 278 } 279 dstpkgroot := filepath.Join(dir, rel) 280 err = os.RemoveAll(dstpkgroot) 281 if err != nil { 282 log.Println(err) 283 ok = false 284 } 285 286 // copy actual dependency 287 vf := dep.vcs.listFiles(dep.dir) 288 w := fs.Walk(dep.dir) 289 for w.Step() { 290 err = copyPkgFile(vf, dir, srcdir, w) 291 if err != nil { 292 log.Println(err) 293 ok = false 294 } 295 } 296 297 // Look for legal files in root 298 // some packages are imports as a sub-package but license info 299 // is at root: exampleorg/common has license file in exampleorg 300 // 301 if dep.ImportPath == dep.root { 302 // we are already at root 303 continue 304 } 305 306 // prevent copying twice This could happen if we have 307 // two subpackages listed someorg/common and 308 // someorg/anotherpack which has their license in 309 // the parent dir of someorg 310 rootdir := filepath.Join(srcdir, filepath.FromSlash(dep.root)) 311 if visited[rootdir] { 312 continue 313 } 314 visited[rootdir] = true 315 vf = dep.vcs.listFiles(rootdir) 316 w = fs.Walk(rootdir) 317 for w.Step() { 318 fname := filepath.Base(w.Path()) 319 if IsLegalFile(fname) { 320 err = copyPkgFile(vf, dir, srcdir, w) 321 if err != nil { 322 log.Println(err) 323 ok = false 324 } 325 } 326 } 327 } 328 329 if !ok { 330 return errorCopyingSourceCode 331 } 332 333 return nil 334 } 335 336 func copyPkgFile(vf vcsFiles, dstroot, srcroot string, w *fs.Walker) error { 337 if w.Err() != nil { 338 return w.Err() 339 } 340 name := w.Stat().Name() 341 if w.Stat().IsDir() { 342 if name[0] == '.' || name[0] == '_' || (!saveT && name == "testdata") { 343 // Skip directories starting with '.' or '_' or 344 // 'testdata' (last is only skipped if saveT is false) 345 w.SkipDir() 346 } 347 return nil 348 } 349 rel, err := filepath.Rel(srcroot, w.Path()) 350 if err != nil { // this should never happen 351 return err 352 } 353 if !saveT && strings.HasSuffix(name, "_test.go") { 354 if verbose { 355 log.Printf("save: skipping test file: %s", w.Path()) 356 } 357 return nil 358 } 359 if !vf.Contains(w.Path()) { 360 if verbose { 361 log.Printf("save: skipping untracked file: %s", w.Path()) 362 } 363 return nil 364 } 365 return copyFile(filepath.Join(dstroot, rel), w.Path()) 366 } 367 368 // copyFile copies a regular file from src to dst. 369 // dst is opened with os.Create. 370 // If the file name ends with .go, 371 // copyFile strips canonical import path annotations. 372 // These are comments of the form: 373 // package foo // import "bar/foo" 374 // package foo /* import "bar/foo" */ 375 func copyFile(dst, src string) error { 376 err := os.MkdirAll(filepath.Dir(dst), 0777) 377 if err != nil { 378 return err 379 } 380 381 linkDst, err := os.Readlink(src) 382 if err == nil { 383 return os.Symlink(linkDst, dst) 384 } 385 386 r, err := os.Open(src) 387 if err != nil { 388 return err 389 } 390 defer r.Close() 391 392 w, err := os.Create(dst) 393 if err != nil { 394 return err 395 } 396 397 if strings.HasSuffix(dst, ".go") { 398 err = copyWithoutImportComment(w, r) 399 } else { 400 _, err = io.Copy(w, r) 401 } 402 err1 := w.Close() 403 if err == nil { 404 err = err1 405 } 406 407 return err 408 } 409 410 func copyWithoutImportComment(w io.Writer, r io.Reader) error { 411 b := bufio.NewReader(r) 412 for { 413 l, err := b.ReadBytes('\n') 414 eof := err == io.EOF 415 if err != nil && err != io.EOF { 416 return err 417 } 418 419 // If we have data then write it out... 420 if len(l) > 0 { 421 // Strip off \n if it exists because stripImportComment 422 _, err := w.Write(append(stripImportComment(bytes.TrimRight(l, "\n")), '\n')) 423 if err != nil { 424 return err 425 } 426 } 427 428 if eof { 429 return nil 430 } 431 } 432 } 433 434 const ( 435 importAnnotation = `import\s+(?:"[^"]*"|` + "`[^`]*`" + `)` 436 importComment = `(?://\s*` + importAnnotation + `\s*$|/\*\s*` + importAnnotation + `\s*\*/)` 437 ) 438 439 var ( 440 importCommentRE = regexp.MustCompile(`^\s*(package\s+\w+)\s+` + importComment + `(.*)`) 441 pkgPrefix = []byte("package ") 442 ) 443 444 // stripImportComment returns line with its import comment removed. 445 // If s is not a package statement containing an import comment, 446 // it is returned unaltered. 447 // FIXME: expects lines w/o a \n at the end 448 // See also http://golang.org/s/go14customimport. 449 func stripImportComment(line []byte) []byte { 450 if !bytes.HasPrefix(line, pkgPrefix) { 451 // Fast path; this will skip all but one line in the file. 452 // This assumes there is no whitespace before the keyword. 453 return line 454 } 455 if m := importCommentRE.FindSubmatch(line); m != nil { 456 return append(m[1], m[2]...) 457 } 458 return line 459 } 460 461 // Func writeVCSIgnore writes "ignore" files inside dir for known VCSs, 462 // so that dir/pkg and dir/bin don't accidentally get committed. 463 // It logs any errors it encounters. 464 func writeVCSIgnore(dir string) { 465 // Currently git is the only VCS for which we know how to do this. 466 // Mercurial and Bazaar have similar mechanisms, but they apparently 467 // require writing files outside of dir. 468 const ignore = "/pkg\n/bin\n" 469 name := filepath.Join(dir, ".gitignore") 470 err := writeFile(name, ignore) 471 if err != nil { 472 log.Println(err) 473 } 474 } 475 476 // writeFile is like ioutil.WriteFile but it creates 477 // intermediate directories with os.MkdirAll. 478 func writeFile(name, body string) error { 479 err := os.MkdirAll(filepath.Dir(name), 0777) 480 if err != nil { 481 return err 482 } 483 return ioutil.WriteFile(name, []byte(body), 0666) 484 } 485 486 const ( 487 // Readme contains the README text. 488 Readme = ` 489 This directory tree is generated automatically by godep. 490 491 Please do not edit. 492 493 See https://github.com/tools/godep for more information. 494 ` 495 needRestore = ` 496 mismatched versions while migrating 497 498 It looks like you are switching from the old Godeps format 499 (from flag -copy=false). The old format is just a file; it 500 doesn't contain source code. For this migration, godep needs 501 the appropriate version of each dependency to be installed in 502 GOPATH, so that the source code is available to copy. 503 504 To fix this, run 'godep restore'. 505 ` 506 )