github.com/letsencrypt/go@v0.0.0-20160714163537-4054769a31f6/src/cmd/go/get.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "fmt" 9 "go/build" 10 "os" 11 "path/filepath" 12 "regexp" 13 "runtime" 14 "strconv" 15 "strings" 16 ) 17 18 var cmdGet = &Command{ 19 UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [build flags] [packages]", 20 Short: "download and install packages and dependencies", 21 Long: ` 22 Get downloads the packages named by the import paths, along with their 23 dependencies. It then installs the named packages, like 'go install'. 24 25 The -d flag instructs get to stop after downloading the packages; that is, 26 it instructs get not to install the packages. 27 28 The -f flag, valid only when -u is set, forces get -u not to verify that 29 each package has been checked out from the source control repository 30 implied by its import path. This can be useful if the source is a local fork 31 of the original. 32 33 The -fix flag instructs get to run the fix tool on the downloaded packages 34 before resolving dependencies or building the code. 35 36 The -insecure flag permits fetching from repositories and resolving 37 custom domains using insecure schemes such as HTTP. Use with caution. 38 39 The -t flag instructs get to also download the packages required to build 40 the tests for the specified packages. 41 42 The -u flag instructs get to use the network to update the named packages 43 and their dependencies. By default, get uses the network to check out 44 missing packages but does not use it to look for updates to existing packages. 45 46 Get also accepts build flags to control the installation. See 'go help build'. 47 48 When checking out a new package, get creates the target directory 49 GOPATH/src/<import-path>. If the GOPATH contains multiple entries, 50 get uses the first one. See 'go help gopath'. 51 52 When checking out or updating a package, get looks for a branch or tag 53 that matches the locally installed version of Go. The most important 54 rule is that if the local installation is running version "go1", get 55 searches for a branch or tag named "go1". If no such version exists it 56 retrieves the most recent version of the package. 57 58 When go get checks out or updates a Git repository, 59 it also updates any git submodules referenced by the repository. 60 61 Get never checks out or updates code stored in vendor directories. 62 63 For more about specifying packages, see 'go help packages'. 64 65 For more about how 'go get' finds source code to 66 download, see 'go help importpath'. 67 68 See also: go build, go install, go clean. 69 `, 70 } 71 72 var getD = cmdGet.Flag.Bool("d", false, "") 73 var getF = cmdGet.Flag.Bool("f", false, "") 74 var getT = cmdGet.Flag.Bool("t", false, "") 75 var getU = cmdGet.Flag.Bool("u", false, "") 76 var getFix = cmdGet.Flag.Bool("fix", false, "") 77 var getInsecure = cmdGet.Flag.Bool("insecure", false, "") 78 79 func init() { 80 addBuildFlags(cmdGet) 81 cmdGet.Run = runGet // break init loop 82 } 83 84 func runGet(cmd *Command, args []string) { 85 if *getF && !*getU { 86 fatalf("go get: cannot use -f flag without -u") 87 } 88 89 // Disable any prompting for passwords by Git. 90 // Only has an effect for 2.3.0 or later, but avoiding 91 // the prompt in earlier versions is just too hard. 92 // If user has explicitly set GIT_TERMINAL_PROMPT=1, keep 93 // prompting. 94 // See golang.org/issue/9341 and golang.org/issue/12706. 95 if os.Getenv("GIT_TERMINAL_PROMPT") == "" { 96 os.Setenv("GIT_TERMINAL_PROMPT", "0") 97 } 98 99 // Phase 1. Download/update. 100 var stk importStack 101 mode := 0 102 if *getT { 103 mode |= getTestDeps 104 } 105 for _, arg := range downloadPaths(args) { 106 download(arg, nil, &stk, mode) 107 } 108 exitIfErrors() 109 110 // Phase 2. Rescan packages and re-evaluate args list. 111 112 // Code we downloaded and all code that depends on it 113 // needs to be evicted from the package cache so that 114 // the information will be recomputed. Instead of keeping 115 // track of the reverse dependency information, evict 116 // everything. 117 for name := range packageCache { 118 delete(packageCache, name) 119 } 120 121 // In order to rebuild packages information completely, 122 // we need to clear commands cache. Command packages are 123 // referring to evicted packages from the package cache. 124 // This leads to duplicated loads of the standard packages. 125 for name := range cmdCache { 126 delete(cmdCache, name) 127 } 128 129 args = importPaths(args) 130 packagesForBuild(args) 131 132 // Phase 3. Install. 133 if *getD { 134 // Download only. 135 // Check delayed until now so that importPaths 136 // and packagesForBuild have a chance to print errors. 137 return 138 } 139 140 runInstall(cmd, args) 141 } 142 143 // downloadPaths prepares the list of paths to pass to download. 144 // It expands ... patterns that can be expanded. If there is no match 145 // for a particular pattern, downloadPaths leaves it in the result list, 146 // in the hope that we can figure out the repository from the 147 // initial ...-free prefix. 148 func downloadPaths(args []string) []string { 149 args = importPathsNoDotExpansion(args) 150 var out []string 151 for _, a := range args { 152 if strings.Contains(a, "...") { 153 var expand []string 154 // Use matchPackagesInFS to avoid printing 155 // warnings. They will be printed by the 156 // eventual call to importPaths instead. 157 if build.IsLocalImport(a) { 158 expand = matchPackagesInFS(a) 159 } else { 160 expand = matchPackages(a) 161 } 162 if len(expand) > 0 { 163 out = append(out, expand...) 164 continue 165 } 166 } 167 out = append(out, a) 168 } 169 return out 170 } 171 172 // downloadCache records the import paths we have already 173 // considered during the download, to avoid duplicate work when 174 // there is more than one dependency sequence leading to 175 // a particular package. 176 var downloadCache = map[string]bool{} 177 178 // downloadRootCache records the version control repository 179 // root directories we have already considered during the download. 180 // For example, all the packages in the code.google.com/p/codesearch repo 181 // share the same root (the directory for that path), and we only need 182 // to run the hg commands to consider each repository once. 183 var downloadRootCache = map[string]bool{} 184 185 // download runs the download half of the get command 186 // for the package named by the argument. 187 func download(arg string, parent *Package, stk *importStack, mode int) { 188 load := func(path string, mode int) *Package { 189 if parent == nil { 190 return loadPackage(path, stk) 191 } 192 return loadImport(path, parent.Dir, parent, stk, nil, mode) 193 } 194 195 p := load(arg, mode) 196 if p.Error != nil && p.Error.hard { 197 errorf("%s", p.Error) 198 return 199 } 200 201 // loadPackage inferred the canonical ImportPath from arg. 202 // Use that in the following to prevent hysteresis effects 203 // in e.g. downloadCache and packageCache. 204 // This allows invocations such as: 205 // mkdir -p $GOPATH/src/github.com/user 206 // cd $GOPATH/src/github.com/user 207 // go get ./foo 208 // see: golang.org/issue/9767 209 arg = p.ImportPath 210 211 // There's nothing to do if this is a package in the standard library. 212 if p.Standard { 213 return 214 } 215 216 // Only process each package once. 217 // (Unless we're fetching test dependencies for this package, 218 // in which case we want to process it again.) 219 if downloadCache[arg] && mode&getTestDeps == 0 { 220 return 221 } 222 downloadCache[arg] = true 223 224 pkgs := []*Package{p} 225 wildcardOkay := len(*stk) == 0 226 isWildcard := false 227 228 // Download if the package is missing, or update if we're using -u. 229 if p.Dir == "" || *getU { 230 // The actual download. 231 stk.push(arg) 232 err := downloadPackage(p) 233 if err != nil { 234 errorf("%s", &PackageError{ImportStack: stk.copy(), Err: err.Error()}) 235 stk.pop() 236 return 237 } 238 stk.pop() 239 240 args := []string{arg} 241 // If the argument has a wildcard in it, re-evaluate the wildcard. 242 // We delay this until after reloadPackage so that the old entry 243 // for p has been replaced in the package cache. 244 if wildcardOkay && strings.Contains(arg, "...") { 245 if build.IsLocalImport(arg) { 246 args = matchPackagesInFS(arg) 247 } else { 248 args = matchPackages(arg) 249 } 250 isWildcard = true 251 } 252 253 // Clear all relevant package cache entries before 254 // doing any new loads. 255 for _, arg := range args { 256 p := packageCache[arg] 257 if p != nil { 258 delete(packageCache, p.Dir) 259 delete(packageCache, p.ImportPath) 260 } 261 } 262 263 pkgs = pkgs[:0] 264 for _, arg := range args { 265 // Note: load calls loadPackage or loadImport, 266 // which push arg onto stk already. 267 // Do not push here too, or else stk will say arg imports arg. 268 p := load(arg, mode) 269 if p.Error != nil { 270 errorf("%s", p.Error) 271 continue 272 } 273 pkgs = append(pkgs, p) 274 } 275 } 276 277 // Process package, which might now be multiple packages 278 // due to wildcard expansion. 279 for _, p := range pkgs { 280 if *getFix { 281 run(buildToolExec, stringList(tool("fix"), relPaths(p.allgofiles))) 282 283 // The imports might have changed, so reload again. 284 p = reloadPackage(arg, stk) 285 if p.Error != nil { 286 errorf("%s", p.Error) 287 return 288 } 289 } 290 291 if isWildcard { 292 // Report both the real package and the 293 // wildcard in any error message. 294 stk.push(p.ImportPath) 295 } 296 297 // Process dependencies, now that we know what they are. 298 for _, path := range p.Imports { 299 if path == "C" { 300 continue 301 } 302 // Don't get test dependencies recursively. 303 // Imports is already vendor-expanded. 304 download(path, p, stk, 0) 305 } 306 if mode&getTestDeps != 0 { 307 // Process test dependencies when -t is specified. 308 // (Don't get test dependencies for test dependencies.) 309 // We pass useVendor here because p.load does not 310 // vendor-expand TestImports and XTestImports. 311 // The call to loadImport inside download needs to do that. 312 for _, path := range p.TestImports { 313 if path == "C" { 314 continue 315 } 316 download(path, p, stk, useVendor) 317 } 318 for _, path := range p.XTestImports { 319 if path == "C" { 320 continue 321 } 322 download(path, p, stk, useVendor) 323 } 324 } 325 326 if isWildcard { 327 stk.pop() 328 } 329 } 330 } 331 332 // downloadPackage runs the create or download command 333 // to make the first copy of or update a copy of the given package. 334 func downloadPackage(p *Package) error { 335 var ( 336 vcs *vcsCmd 337 repo, rootPath string 338 err error 339 ) 340 341 security := secure 342 if *getInsecure { 343 security = insecure 344 } 345 346 if p.build.SrcRoot != "" { 347 // Directory exists. Look for checkout along path to src. 348 vcs, rootPath, err = vcsFromDir(p.Dir, p.build.SrcRoot) 349 if err != nil { 350 return err 351 } 352 repo = "<local>" // should be unused; make distinctive 353 354 // Double-check where it came from. 355 if *getU && vcs.remoteRepo != nil { 356 dir := filepath.Join(p.build.SrcRoot, filepath.FromSlash(rootPath)) 357 remote, err := vcs.remoteRepo(vcs, dir) 358 if err != nil { 359 return err 360 } 361 repo = remote 362 if !*getF { 363 if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil { 364 repo := rr.repo 365 if rr.vcs.resolveRepo != nil { 366 resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo) 367 if err == nil { 368 repo = resolved 369 } 370 } 371 if remote != repo && p.ImportComment != "" { 372 return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote) 373 } 374 } 375 } 376 } 377 } else { 378 // Analyze the import path to determine the version control system, 379 // repository, and the import path for the root of the repository. 380 rr, err := repoRootForImportPath(p.ImportPath, security) 381 if err != nil { 382 return err 383 } 384 vcs, repo, rootPath = rr.vcs, rr.repo, rr.root 385 } 386 if !vcs.isSecure(repo) && !*getInsecure { 387 return fmt.Errorf("cannot download, %v uses insecure protocol", repo) 388 } 389 390 if p.build.SrcRoot == "" { 391 // Package not found. Put in first directory of $GOPATH. 392 list := filepath.SplitList(buildContext.GOPATH) 393 if len(list) == 0 { 394 return fmt.Errorf("cannot download, $GOPATH not set. For more details see: go help gopath") 395 } 396 // Guard against people setting GOPATH=$GOROOT. 397 if list[0] == goroot { 398 return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: go help gopath") 399 } 400 p.build.SrcRoot = filepath.Join(list[0], "src") 401 p.build.PkgRoot = filepath.Join(list[0], "pkg") 402 } 403 root := filepath.Join(p.build.SrcRoot, filepath.FromSlash(rootPath)) 404 // If we've considered this repository already, don't do it again. 405 if downloadRootCache[root] { 406 return nil 407 } 408 downloadRootCache[root] = true 409 410 if buildV { 411 fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath) 412 } 413 414 // Check that this is an appropriate place for the repo to be checked out. 415 // The target directory must either not exist or have a repo checked out already. 416 meta := filepath.Join(root, "."+vcs.cmd) 417 st, err := os.Stat(meta) 418 if err == nil && !st.IsDir() { 419 return fmt.Errorf("%s exists but is not a directory", meta) 420 } 421 if err != nil { 422 // Metadata directory does not exist. Prepare to checkout new copy. 423 // Some version control tools require the target directory not to exist. 424 // We require that too, just to avoid stepping on existing work. 425 if _, err := os.Stat(root); err == nil { 426 return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta) 427 } 428 // Some version control tools require the parent of the target to exist. 429 parent, _ := filepath.Split(root) 430 if err = os.MkdirAll(parent, 0777); err != nil { 431 return err 432 } 433 if err = vcs.create(root, repo); err != nil { 434 return err 435 } 436 } else { 437 // Metadata directory does exist; download incremental updates. 438 if err = vcs.download(root); err != nil { 439 return err 440 } 441 } 442 443 if buildN { 444 // Do not show tag sync in -n; it's noise more than anything, 445 // and since we're not running commands, no tag will be found. 446 // But avoid printing nothing. 447 fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd) 448 return nil 449 } 450 451 // Select and sync to appropriate version of the repository. 452 tags, err := vcs.tags(root) 453 if err != nil { 454 return err 455 } 456 vers := runtime.Version() 457 if i := strings.Index(vers, " "); i >= 0 { 458 vers = vers[:i] 459 } 460 if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil { 461 return err 462 } 463 464 return nil 465 } 466 467 // goTag matches go release tags such as go1 and go1.2.3. 468 // The numbers involved must be small (at most 4 digits), 469 // have no unnecessary leading zeros, and the version cannot 470 // end in .0 - it is go1, not go1.0 or go1.0.0. 471 var goTag = regexp.MustCompile( 472 `^go((0|[1-9][0-9]{0,3})\.)*([1-9][0-9]{0,3})$`, 473 ) 474 475 // selectTag returns the closest matching tag for a given version. 476 // Closest means the latest one that is not after the current release. 477 // Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form. 478 // Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number). 479 // Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD". 480 // 481 // NOTE(rsc): Eventually we will need to decide on some logic here. 482 // For now, there is only "go1". This matches the docs in go help get. 483 func selectTag(goVersion string, tags []string) (match string) { 484 for _, t := range tags { 485 if t == "go1" { 486 return "go1" 487 } 488 } 489 return "" 490 491 /* 492 if goTag.MatchString(goVersion) { 493 v := goVersion 494 for _, t := range tags { 495 if !goTag.MatchString(t) { 496 continue 497 } 498 if cmpGoVersion(match, t) < 0 && cmpGoVersion(t, v) <= 0 { 499 match = t 500 } 501 } 502 } 503 504 return match 505 */ 506 } 507 508 // cmpGoVersion returns -1, 0, +1 reporting whether 509 // x < y, x == y, or x > y. 510 func cmpGoVersion(x, y string) int { 511 // Malformed strings compare less than well-formed strings. 512 if !goTag.MatchString(x) { 513 return -1 514 } 515 if !goTag.MatchString(y) { 516 return +1 517 } 518 519 // Compare numbers in sequence. 520 xx := strings.Split(x[len("go"):], ".") 521 yy := strings.Split(y[len("go"):], ".") 522 523 for i := 0; i < len(xx) && i < len(yy); i++ { 524 // The Atoi are guaranteed to succeed 525 // because the versions match goTag. 526 xi, _ := strconv.Atoi(xx[i]) 527 yi, _ := strconv.Atoi(yy[i]) 528 if xi < yi { 529 return -1 530 } else if xi > yi { 531 return +1 532 } 533 } 534 535 if len(xx) < len(yy) { 536 return -1 537 } 538 if len(xx) > len(yy) { 539 return +1 540 } 541 return 0 542 }