github.com/zxy12/go_duplicate_112_new@v0.0.0-20200807091221-747231827200/src/cmd/go/internal/modfetch/fetch.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package modfetch 6 7 import ( 8 "archive/zip" 9 "bytes" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "sort" 16 "strings" 17 "sync" 18 19 "cmd/go/internal/base" 20 "cmd/go/internal/cfg" 21 "cmd/go/internal/dirhash" 22 "cmd/go/internal/module" 23 "cmd/go/internal/par" 24 "cmd/go/internal/renameio" 25 ) 26 27 var downloadCache par.Cache 28 29 // Download downloads the specific module version to the 30 // local download cache and returns the name of the directory 31 // corresponding to the root of the module's file tree. 32 func Download(mod module.Version) (dir string, err error) { 33 if PkgMod == "" { 34 // Do not download to current directory. 35 return "", fmt.Errorf("missing modfetch.PkgMod") 36 } 37 38 // The par.Cache here avoids duplicate work. 39 type cached struct { 40 dir string 41 err error 42 } 43 c := downloadCache.Do(mod, func() interface{} { 44 dir, err := DownloadDir(mod) 45 if err != nil { 46 return cached{"", err} 47 } 48 if err := download(mod, dir); err != nil { 49 return cached{"", err} 50 } 51 checkSum(mod) 52 return cached{dir, nil} 53 }).(cached) 54 return c.dir, c.err 55 } 56 57 func download(mod module.Version, dir string) (err error) { 58 // If the directory exists, the module has already been extracted. 59 fi, err := os.Stat(dir) 60 if err == nil && fi.IsDir() { 61 return nil 62 } 63 64 // To avoid cluttering the cache with extraneous files, 65 // DownloadZip uses the same lockfile as Download. 66 // Invoke DownloadZip before locking the file. 67 zipfile, err := DownloadZip(mod) 68 if err != nil { 69 return err 70 } 71 72 if cfg.CmdName != "mod download" { 73 fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version) 74 } 75 76 unlock, err := lockVersion(mod) 77 if err != nil { 78 return err 79 } 80 defer unlock() 81 82 // Check whether the directory was populated while we were waiting on the lock. 83 fi, err = os.Stat(dir) 84 if err == nil && fi.IsDir() { 85 return nil 86 } 87 88 // Clean up any remaining temporary directories from previous runs. 89 // This is only safe to do because the lock file ensures that their writers 90 // are no longer active. 91 parentDir := filepath.Dir(dir) 92 tmpPrefix := filepath.Base(dir) + ".tmp-" 93 if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil { 94 for _, path := range old { 95 RemoveAll(path) // best effort 96 } 97 } 98 99 // Extract the zip file to a temporary directory, then rename it to the 100 // final path. That way, we can use the existence of the source directory to 101 // signal that it has been extracted successfully, and if someone deletes 102 // the entire directory (e.g. as an attempt to prune out file corruption) 103 // the module cache will still be left in a recoverable state. 104 if err := os.MkdirAll(parentDir, 0777); err != nil { 105 return err 106 } 107 tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix) 108 if err != nil { 109 return err 110 } 111 defer func() { 112 if err != nil { 113 RemoveAll(tmpDir) 114 } 115 }() 116 117 modpath := mod.Path + "@" + mod.Version 118 if err := Unzip(tmpDir, zipfile, modpath, 0); err != nil { 119 fmt.Fprintf(os.Stderr, "-> %s\n", err) 120 return err 121 } 122 123 if err := os.Rename(tmpDir, dir); err != nil { 124 return err 125 } 126 127 // Make dir read-only only *after* renaming it. 128 // os.Rename was observed to fail for read-only directories on macOS. 129 makeDirsReadOnly(dir) 130 return nil 131 } 132 133 var downloadZipCache par.Cache 134 135 // DownloadZip downloads the specific module version to the 136 // local zip cache and returns the name of the zip file. 137 func DownloadZip(mod module.Version) (zipfile string, err error) { 138 // The par.Cache here avoids duplicate work. 139 type cached struct { 140 zipfile string 141 err error 142 } 143 c := downloadZipCache.Do(mod, func() interface{} { 144 zipfile, err := CachePath(mod, "zip") 145 if err != nil { 146 return cached{"", err} 147 } 148 149 // Skip locking if the zipfile already exists. 150 if _, err := os.Stat(zipfile); err == nil { 151 return cached{zipfile, nil} 152 } 153 154 // The zip file does not exist. Acquire the lock and create it. 155 if cfg.CmdName != "mod download" { 156 fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version) 157 } 158 unlock, err := lockVersion(mod) 159 if err != nil { 160 return cached{"", err} 161 } 162 defer unlock() 163 164 // Double-check that the zipfile was not created while we were waiting for 165 // the lock. 166 if _, err := os.Stat(zipfile); err == nil { 167 return cached{zipfile, nil} 168 } 169 if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { 170 return cached{"", err} 171 } 172 if err := downloadZip(mod, zipfile); err != nil { 173 return cached{"", err} 174 } 175 return cached{zipfile, nil} 176 }).(cached) 177 return c.zipfile, c.err 178 } 179 180 func downloadZip(mod module.Version, zipfile string) (err error) { 181 // Clean up any remaining tempfiles from previous runs. 182 // This is only safe to do because the lock file ensures that their 183 // writers are no longer active. 184 for _, base := range []string{zipfile, zipfile + "hash"} { 185 if old, err := filepath.Glob(renameio.Pattern(base)); err == nil { 186 for _, path := range old { 187 os.Remove(path) // best effort 188 } 189 } 190 } 191 192 // From here to the os.Rename call below is functionally almost equivalent to 193 // renameio.WriteToFile, with one key difference: we want to validate the 194 // contents of the file (by hashing it) before we commit it. Because the file 195 // is zip-compressed, we need an actual file — or at least an io.ReaderAt — to 196 // validate it: we can't just tee the stream as we write it. 197 f, err := ioutil.TempFile(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile))) 198 if err != nil { 199 return err 200 } 201 defer func() { 202 if err != nil { 203 f.Close() 204 os.Remove(f.Name()) 205 } 206 }() 207 208 repo, err := Lookup(mod.Path) 209 if err != nil { 210 return err 211 } 212 if err := repo.Zip(f, mod.Version); err != nil { 213 return err 214 } 215 216 // Double-check that the paths within the zip file are well-formed. 217 // 218 // TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one? 219 fi, err := f.Stat() 220 if err != nil { 221 return err 222 } 223 z, err := zip.NewReader(f, fi.Size()) 224 if err != nil { 225 return err 226 } 227 prefix := mod.Path + "@" + mod.Version + "/" 228 for _, f := range z.File { 229 if !strings.HasPrefix(f.Name, prefix) { 230 return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name) 231 } 232 } 233 234 // Sync the file before renaming it: otherwise, after a crash the reader may 235 // observe a 0-length file instead of the actual contents. 236 // See https://golang.org/issue/22397#issuecomment-380831736. 237 if err := f.Sync(); err != nil { 238 return err 239 } 240 if err := f.Close(); err != nil { 241 return err 242 } 243 244 // Hash the zip file and check the sum before renaming to the final location. 245 hash, err := dirhash.HashZip(f.Name(), dirhash.DefaultHash) 246 if err != nil { 247 return err 248 } 249 checkOneSum(mod, hash) 250 251 if err := renameio.WriteFile(zipfile+"hash", []byte(hash)); err != nil { 252 return err 253 } 254 if err := os.Rename(f.Name(), zipfile); err != nil { 255 return err 256 } 257 258 // TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering? 259 260 return nil 261 } 262 263 var GoSumFile string // path to go.sum; set by package modload 264 265 type modSum struct { 266 mod module.Version 267 sum string 268 } 269 270 var goSum struct { 271 mu sync.Mutex 272 m map[module.Version][]string // content of go.sum file (+ go.modverify if present) 273 checked map[modSum]bool // sums actually checked during execution 274 dirty bool // whether we added any new sums to m 275 overwrite bool // if true, overwrite go.sum without incorporating its contents 276 enabled bool // whether to use go.sum at all 277 modverify string // path to go.modverify, to be deleted 278 } 279 280 // initGoSum initializes the go.sum data. 281 // It reports whether use of go.sum is now enabled. 282 // The goSum lock must be held. 283 func initGoSum() bool { 284 if GoSumFile == "" { 285 return false 286 } 287 if goSum.m != nil { 288 return true 289 } 290 291 goSum.m = make(map[module.Version][]string) 292 goSum.checked = make(map[modSum]bool) 293 data, err := ioutil.ReadFile(GoSumFile) 294 if err != nil && !os.IsNotExist(err) { 295 base.Fatalf("go: %v", err) 296 } 297 goSum.enabled = true 298 readGoSum(goSum.m, GoSumFile, data) 299 300 // Add old go.modverify file. 301 // We'll delete go.modverify in WriteGoSum. 302 alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify" 303 if data, err := ioutil.ReadFile(alt); err == nil { 304 migrate := make(map[module.Version][]string) 305 readGoSum(migrate, alt, data) 306 for mod, sums := range migrate { 307 for _, sum := range sums { 308 checkOneSumLocked(mod, sum) 309 } 310 } 311 goSum.modverify = alt 312 } 313 return true 314 } 315 316 // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod. 317 // A bug caused us to write these into go.sum files for non-modules. 318 // We detect and remove them. 319 const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY=" 320 321 // readGoSum parses data, which is the content of file, 322 // and adds it to goSum.m. The goSum lock must be held. 323 func readGoSum(dst map[module.Version][]string, file string, data []byte) { 324 lineno := 0 325 for len(data) > 0 { 326 var line []byte 327 lineno++ 328 i := bytes.IndexByte(data, '\n') 329 if i < 0 { 330 line, data = data, nil 331 } else { 332 line, data = data[:i], data[i+1:] 333 } 334 f := strings.Fields(string(line)) 335 if len(f) == 0 { 336 // blank line; skip it 337 continue 338 } 339 if len(f) != 3 { 340 base.Fatalf("go: malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f)) 341 } 342 if f[2] == emptyGoModHash { 343 // Old bug; drop it. 344 continue 345 } 346 mod := module.Version{Path: f[0], Version: f[1]} 347 dst[mod] = append(dst[mod], f[2]) 348 } 349 } 350 351 // checkSum checks the given module's checksum. 352 func checkSum(mod module.Version) { 353 if PkgMod == "" { 354 // Do not use current directory. 355 return 356 } 357 358 // Do the file I/O before acquiring the go.sum lock. 359 ziphash, err := CachePath(mod, "ziphash") 360 if err != nil { 361 base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err) 362 } 363 data, err := ioutil.ReadFile(ziphash) 364 if err != nil { 365 if os.IsNotExist(err) { 366 // This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes. 367 return 368 } 369 base.Fatalf("verifying %s@%s: %v", mod.Path, mod.Version, err) 370 } 371 h := strings.TrimSpace(string(data)) 372 if !strings.HasPrefix(h, "h1:") { 373 base.Fatalf("verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h) 374 } 375 376 checkOneSum(mod, h) 377 } 378 379 // goModSum returns the checksum for the go.mod contents. 380 func goModSum(data []byte) (string, error) { 381 return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { 382 return ioutil.NopCloser(bytes.NewReader(data)), nil 383 }) 384 } 385 386 // checkGoMod checks the given module's go.mod checksum; 387 // data is the go.mod content. 388 func checkGoMod(path, version string, data []byte) { 389 h, err := goModSum(data) 390 if err != nil { 391 base.Fatalf("verifying %s %s go.mod: %v", path, version, err) 392 } 393 394 checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h) 395 } 396 397 // checkOneSum checks that the recorded hash for mod is h. 398 func checkOneSum(mod module.Version, h string) { 399 goSum.mu.Lock() 400 defer goSum.mu.Unlock() 401 if initGoSum() { 402 checkOneSumLocked(mod, h) 403 } 404 } 405 406 func checkOneSumLocked(mod module.Version, h string) { 407 goSum.checked[modSum{mod, h}] = true 408 409 for _, vh := range goSum.m[mod] { 410 if h == vh { 411 return 412 } 413 if strings.HasPrefix(vh, "h1:") { 414 base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum: %v", mod.Path, mod.Version, h, vh) 415 } 416 } 417 if len(goSum.m[mod]) > 0 { 418 fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) 419 } 420 goSum.m[mod] = append(goSum.m[mod], h) 421 goSum.dirty = true 422 } 423 424 // Sum returns the checksum for the downloaded copy of the given module, 425 // if present in the download cache. 426 func Sum(mod module.Version) string { 427 if PkgMod == "" { 428 // Do not use current directory. 429 return "" 430 } 431 432 ziphash, err := CachePath(mod, "ziphash") 433 if err != nil { 434 return "" 435 } 436 data, err := ioutil.ReadFile(ziphash) 437 if err != nil { 438 return "" 439 } 440 return strings.TrimSpace(string(data)) 441 } 442 443 // WriteGoSum writes the go.sum file if it needs to be updated. 444 func WriteGoSum() { 445 goSum.mu.Lock() 446 defer goSum.mu.Unlock() 447 448 if !goSum.enabled { 449 // If we haven't read the go.sum file yet, don't bother writing it: at best, 450 // we could rename the go.modverify file if it isn't empty, but we haven't 451 // needed to touch it so far — how important could it be? 452 return 453 } 454 if !goSum.dirty { 455 // Don't bother opening the go.sum file if we don't have anything to add. 456 return 457 } 458 459 // We want to avoid races between creating the lockfile and deleting it, but 460 // we also don't want to leave a permanent lockfile in the user's repository. 461 // 462 // On top of that, if we crash while writing go.sum, we don't want to lose the 463 // sums that were already present in the file, so it's important that we write 464 // the file by renaming rather than truncating — which means that we can't 465 // lock the go.sum file itself. 466 // 467 // Instead, we'll lock a distinguished file in the cache directory: that will 468 // only race if the user runs `go clean -modcache` concurrently with a command 469 // that updates go.sum, and that's already racy to begin with. 470 // 471 // We'll end up slightly over-synchronizing go.sum writes if the user runs a 472 // bunch of go commands that update sums in separate modules simultaneously, 473 // but that's unlikely to matter in practice. 474 475 unlock := SideLock() 476 defer unlock() 477 478 if !goSum.overwrite { 479 // Re-read the go.sum file to incorporate any sums added by other processes 480 // in the meantime. 481 data, err := ioutil.ReadFile(GoSumFile) 482 if err != nil && !os.IsNotExist(err) { 483 base.Fatalf("go: re-reading go.sum: %v", err) 484 } 485 486 // Add only the sums that we actually checked: the user may have edited or 487 // truncated the file to remove erroneous hashes, and we shouldn't restore 488 // them without good reason. 489 goSum.m = make(map[module.Version][]string, len(goSum.m)) 490 readGoSum(goSum.m, GoSumFile, data) 491 for ms := range goSum.checked { 492 checkOneSumLocked(ms.mod, ms.sum) 493 } 494 } 495 496 var mods []module.Version 497 for m := range goSum.m { 498 mods = append(mods, m) 499 } 500 module.Sort(mods) 501 var buf bytes.Buffer 502 for _, m := range mods { 503 list := goSum.m[m] 504 sort.Strings(list) 505 for _, h := range list { 506 fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) 507 } 508 } 509 510 if err := renameio.WriteFile(GoSumFile, buf.Bytes()); err != nil { 511 base.Fatalf("go: writing go.sum: %v", err) 512 } 513 514 goSum.checked = make(map[modSum]bool) 515 goSum.dirty = false 516 goSum.overwrite = false 517 518 if goSum.modverify != "" { 519 os.Remove(goSum.modverify) // best effort 520 } 521 } 522 523 // TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. 524 func TrimGoSum(keep map[module.Version]bool) { 525 goSum.mu.Lock() 526 defer goSum.mu.Unlock() 527 if !initGoSum() { 528 return 529 } 530 531 for m := range goSum.m { 532 // If we're keeping x@v we also keep x@v/go.mod. 533 // Map x@v/go.mod back to x@v for the keep lookup. 534 noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")} 535 if !keep[m] && !keep[noGoMod] { 536 delete(goSum.m, m) 537 goSum.dirty = true 538 goSum.overwrite = true 539 } 540 } 541 }