github.com/zxy12/go_duplicate_112_new@v0.0.0-20200807091221-747231827200/src/cmd/go/internal/modfetch/cache.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package modfetch 6 7 import ( 8 "bytes" 9 "encoding/json" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "strings" 16 17 "cmd/go/internal/base" 18 "cmd/go/internal/lockedfile" 19 "cmd/go/internal/modfetch/codehost" 20 "cmd/go/internal/module" 21 "cmd/go/internal/par" 22 "cmd/go/internal/renameio" 23 "cmd/go/internal/semver" 24 ) 25 26 var QuietLookup bool // do not print about lookups 27 28 var PkgMod string // $GOPATH/pkg/mod; set by package modload 29 30 func cacheDir(path string) (string, error) { 31 if PkgMod == "" { 32 return "", fmt.Errorf("internal error: modfetch.PkgMod not set") 33 } 34 enc, err := module.EncodePath(path) 35 if err != nil { 36 return "", err 37 } 38 return filepath.Join(PkgMod, "cache/download", enc, "/@v"), nil 39 } 40 41 func CachePath(m module.Version, suffix string) (string, error) { 42 dir, err := cacheDir(m.Path) 43 if err != nil { 44 return "", err 45 } 46 if !semver.IsValid(m.Version) { 47 return "", fmt.Errorf("non-semver module version %q", m.Version) 48 } 49 if module.CanonicalVersion(m.Version) != m.Version { 50 return "", fmt.Errorf("non-canonical module version %q", m.Version) 51 } 52 encVer, err := module.EncodeVersion(m.Version) 53 if err != nil { 54 return "", err 55 } 56 return filepath.Join(dir, encVer+"."+suffix), nil 57 } 58 59 // DownloadDir returns the directory to which m should be downloaded. 60 // Note that the directory may not yet exist. 61 func DownloadDir(m module.Version) (string, error) { 62 if PkgMod == "" { 63 return "", fmt.Errorf("internal error: modfetch.PkgMod not set") 64 } 65 enc, err := module.EncodePath(m.Path) 66 if err != nil { 67 return "", err 68 } 69 if !semver.IsValid(m.Version) { 70 return "", fmt.Errorf("non-semver module version %q", m.Version) 71 } 72 if module.CanonicalVersion(m.Version) != m.Version { 73 return "", fmt.Errorf("non-canonical module version %q", m.Version) 74 } 75 encVer, err := module.EncodeVersion(m.Version) 76 if err != nil { 77 return "", err 78 } 79 return filepath.Join(PkgMod, enc+"@"+encVer), nil 80 } 81 82 // lockVersion locks a file within the module cache that guards the downloading 83 // and extraction of the zipfile for the given module version. 84 func lockVersion(mod module.Version) (unlock func(), err error) { 85 path, err := CachePath(mod, "lock") 86 if err != nil { 87 return nil, err 88 } 89 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { 90 return nil, err 91 } 92 return lockedfile.MutexAt(path).Lock() 93 } 94 95 // SideLock locks a file within the module cache that that guards edits to files 96 // outside the cache, such as go.sum and go.mod files in the user's working 97 // directory. It returns a function that must be called to unlock the file. 98 func SideLock() (unlock func()) { 99 if PkgMod == "" { 100 base.Fatalf("go: internal error: modfetch.PkgMod not set") 101 } 102 path := filepath.Join(PkgMod, "cache", "lock") 103 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { 104 base.Fatalf("go: failed to create cache directory %s: %v", filepath.Dir(path), err) 105 } 106 unlock, err := lockedfile.MutexAt(path).Lock() 107 if err != nil { 108 base.Fatalf("go: failed to lock file at %v", path) 109 } 110 return unlock 111 } 112 113 // A cachingRepo is a cache around an underlying Repo, 114 // avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip). 115 // It is also safe for simultaneous use by multiple goroutines 116 // (so that it can be returned from Lookup multiple times). 117 // It serializes calls to the underlying Repo. 118 type cachingRepo struct { 119 path string 120 cache par.Cache // cache for all operations 121 r Repo 122 } 123 124 func newCachingRepo(r Repo) *cachingRepo { 125 return &cachingRepo{ 126 r: r, 127 path: r.ModulePath(), 128 } 129 } 130 131 func (r *cachingRepo) ModulePath() string { 132 return r.path 133 } 134 135 func (r *cachingRepo) Versions(prefix string) ([]string, error) { 136 type cached struct { 137 list []string 138 err error 139 } 140 c := r.cache.Do("versions:"+prefix, func() interface{} { 141 list, err := r.r.Versions(prefix) 142 return cached{list, err} 143 }).(cached) 144 145 if c.err != nil { 146 return nil, c.err 147 } 148 return append([]string(nil), c.list...), nil 149 } 150 151 type cachedInfo struct { 152 info *RevInfo 153 err error 154 } 155 156 func (r *cachingRepo) Stat(rev string) (*RevInfo, error) { 157 c := r.cache.Do("stat:"+rev, func() interface{} { 158 file, info, err := readDiskStat(r.path, rev) 159 if err == nil { 160 return cachedInfo{info, nil} 161 } 162 163 if !QuietLookup { 164 fmt.Fprintf(os.Stderr, "go: finding %s %s\n", r.path, rev) 165 } 166 info, err = r.r.Stat(rev) 167 if err == nil { 168 // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, 169 // then save the information under the proper version, for future use. 170 if info.Version != rev { 171 file, _ = CachePath(module.Version{Path: r.path, Version: info.Version}, "info") 172 r.cache.Do("stat:"+info.Version, func() interface{} { 173 return cachedInfo{info, err} 174 }) 175 } 176 177 if err := writeDiskStat(file, info); err != nil { 178 fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err) 179 } 180 } 181 return cachedInfo{info, err} 182 }).(cachedInfo) 183 184 if c.err != nil { 185 return nil, c.err 186 } 187 info := *c.info 188 return &info, nil 189 } 190 191 func (r *cachingRepo) Latest() (*RevInfo, error) { 192 c := r.cache.Do("latest:", func() interface{} { 193 if !QuietLookup { 194 fmt.Fprintf(os.Stderr, "go: finding %s latest\n", r.path) 195 } 196 info, err := r.r.Latest() 197 198 // Save info for likely future Stat call. 199 if err == nil { 200 r.cache.Do("stat:"+info.Version, func() interface{} { 201 return cachedInfo{info, err} 202 }) 203 if file, _, err := readDiskStat(r.path, info.Version); err != nil { 204 writeDiskStat(file, info) 205 } 206 } 207 208 return cachedInfo{info, err} 209 }).(cachedInfo) 210 211 if c.err != nil { 212 return nil, c.err 213 } 214 info := *c.info 215 return &info, nil 216 } 217 218 func (r *cachingRepo) GoMod(rev string) ([]byte, error) { 219 type cached struct { 220 text []byte 221 err error 222 } 223 c := r.cache.Do("gomod:"+rev, func() interface{} { 224 file, text, err := readDiskGoMod(r.path, rev) 225 if err == nil { 226 // Note: readDiskGoMod already called checkGoMod. 227 return cached{text, nil} 228 } 229 230 // Convert rev to canonical version 231 // so that we use the right identifier in the go.sum check. 232 info, err := r.Stat(rev) 233 if err != nil { 234 return cached{nil, err} 235 } 236 rev = info.Version 237 238 text, err = r.r.GoMod(rev) 239 if err == nil { 240 checkGoMod(r.path, rev, text) 241 if err := writeDiskGoMod(file, text); err != nil { 242 fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err) 243 } 244 } 245 return cached{text, err} 246 }).(cached) 247 248 if c.err != nil { 249 return nil, c.err 250 } 251 return append([]byte(nil), c.text...), nil 252 } 253 254 func (r *cachingRepo) Zip(dst io.Writer, version string) error { 255 return r.r.Zip(dst, version) 256 } 257 258 // Stat is like Lookup(path).Stat(rev) but avoids the 259 // repository path resolution in Lookup if the result is 260 // already cached on local disk. 261 func Stat(path, rev string) (*RevInfo, error) { 262 _, info, err := readDiskStat(path, rev) 263 if err == nil { 264 return info, nil 265 } 266 repo, err := Lookup(path) 267 if err != nil { 268 return nil, err 269 } 270 return repo.Stat(rev) 271 } 272 273 // InfoFile is like Stat but returns the name of the file containing 274 // the cached information. 275 func InfoFile(path, version string) (string, error) { 276 if !semver.IsValid(version) { 277 return "", fmt.Errorf("invalid version %q", version) 278 } 279 if _, err := Stat(path, version); err != nil { 280 return "", err 281 } 282 // Stat should have populated the disk cache for us. 283 file, _, err := readDiskStat(path, version) 284 if err != nil { 285 return "", err 286 } 287 return file, nil 288 } 289 290 // GoMod is like Lookup(path).GoMod(rev) but avoids the 291 // repository path resolution in Lookup if the result is 292 // already cached on local disk. 293 func GoMod(path, rev string) ([]byte, error) { 294 // Convert commit hash to pseudo-version 295 // to increase cache hit rate. 296 if !semver.IsValid(rev) { 297 info, err := Stat(path, rev) 298 if err != nil { 299 return nil, err 300 } 301 rev = info.Version 302 } 303 _, data, err := readDiskGoMod(path, rev) 304 if err == nil { 305 return data, nil 306 } 307 repo, err := Lookup(path) 308 if err != nil { 309 return nil, err 310 } 311 return repo.GoMod(rev) 312 } 313 314 // GoModFile is like GoMod but returns the name of the file containing 315 // the cached information. 316 func GoModFile(path, version string) (string, error) { 317 if !semver.IsValid(version) { 318 return "", fmt.Errorf("invalid version %q", version) 319 } 320 if _, err := GoMod(path, version); err != nil { 321 return "", err 322 } 323 // GoMod should have populated the disk cache for us. 324 file, _, err := readDiskGoMod(path, version) 325 if err != nil { 326 return "", err 327 } 328 return file, nil 329 } 330 331 // GoModSum returns the go.sum entry for the module version's go.mod file. 332 // (That is, it returns the entry listed in go.sum as "path version/go.mod".) 333 func GoModSum(path, version string) (string, error) { 334 if !semver.IsValid(version) { 335 return "", fmt.Errorf("invalid version %q", version) 336 } 337 data, err := GoMod(path, version) 338 if err != nil { 339 return "", err 340 } 341 sum, err := goModSum(data) 342 if err != nil { 343 return "", err 344 } 345 return sum, nil 346 } 347 348 var errNotCached = fmt.Errorf("not in cache") 349 350 // readDiskStat reads a cached stat result from disk, 351 // returning the name of the cache file and the result. 352 // If the read fails, the caller can use 353 // writeDiskStat(file, info) to write a new cache entry. 354 func readDiskStat(path, rev string) (file string, info *RevInfo, err error) { 355 file, data, err := readDiskCache(path, rev, "info") 356 if err != nil { 357 if file, info, err := readDiskStatByHash(path, rev); err == nil { 358 return file, info, nil 359 } 360 return file, nil, err 361 } 362 info = new(RevInfo) 363 if err := json.Unmarshal(data, info); err != nil { 364 return file, nil, errNotCached 365 } 366 // The disk might have stale .info files that have Name and Short fields set. 367 // We want to canonicalize to .info files with those fields omitted. 368 // Remarshal and update the cache file if needed. 369 data2, err := json.Marshal(info) 370 if err == nil && !bytes.Equal(data2, data) { 371 writeDiskCache(file, data) 372 } 373 return file, info, nil 374 } 375 376 // readDiskStatByHash is a fallback for readDiskStat for the case 377 // where rev is a commit hash instead of a proper semantic version. 378 // In that case, we look for a cached pseudo-version that matches 379 // the commit hash. If we find one, we use it. 380 // This matters most for converting legacy package management 381 // configs, when we are often looking up commits by full hash. 382 // Without this check we'd be doing network I/O to the remote repo 383 // just to find out about a commit we already know about 384 // (and have cached under its pseudo-version). 385 func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error) { 386 if PkgMod == "" { 387 // Do not download to current directory. 388 return "", nil, errNotCached 389 } 390 391 if !codehost.AllHex(rev) || len(rev) < 12 { 392 return "", nil, errNotCached 393 } 394 rev = rev[:12] 395 cdir, err := cacheDir(path) 396 if err != nil { 397 return "", nil, errNotCached 398 } 399 dir, err := os.Open(cdir) 400 if err != nil { 401 return "", nil, errNotCached 402 } 403 names, err := dir.Readdirnames(-1) 404 dir.Close() 405 if err != nil { 406 return "", nil, errNotCached 407 } 408 suffix := "-" + rev + ".info" 409 for _, name := range names { 410 if strings.HasSuffix(name, suffix) && IsPseudoVersion(strings.TrimSuffix(name, ".info")) { 411 return readDiskStat(path, strings.TrimSuffix(name, ".info")) 412 } 413 } 414 return "", nil, errNotCached 415 } 416 417 // oldVgoPrefix is the prefix in the old auto-generated cached go.mod files. 418 // We stopped trying to auto-generate the go.mod files. Now we use a trivial 419 // go.mod with only a module line, and we've dropped the version prefix 420 // entirely. If we see a version prefix, that means we're looking at an old copy 421 // and should ignore it. 422 var oldVgoPrefix = []byte("//vgo 0.0.") 423 424 // readDiskGoMod reads a cached go.mod file from disk, 425 // returning the name of the cache file and the result. 426 // If the read fails, the caller can use 427 // writeDiskGoMod(file, data) to write a new cache entry. 428 func readDiskGoMod(path, rev string) (file string, data []byte, err error) { 429 file, data, err = readDiskCache(path, rev, "mod") 430 431 // If the file has an old auto-conversion prefix, pretend it's not there. 432 if bytes.HasPrefix(data, oldVgoPrefix) { 433 err = errNotCached 434 data = nil 435 } 436 437 if err == nil { 438 checkGoMod(path, rev, data) 439 } 440 441 return file, data, err 442 } 443 444 // readDiskCache is the generic "read from a cache file" implementation. 445 // It takes the revision and an identifying suffix for the kind of data being cached. 446 // It returns the name of the cache file and the content of the file. 447 // If the read fails, the caller can use 448 // writeDiskCache(file, data) to write a new cache entry. 449 func readDiskCache(path, rev, suffix string) (file string, data []byte, err error) { 450 file, err = CachePath(module.Version{Path: path, Version: rev}, suffix) 451 if err != nil { 452 return "", nil, errNotCached 453 } 454 data, err = ioutil.ReadFile(file) 455 if err != nil { 456 return file, nil, errNotCached 457 } 458 return file, data, nil 459 } 460 461 // writeDiskStat writes a stat result cache entry. 462 // The file name must have been returned by a previous call to readDiskStat. 463 func writeDiskStat(file string, info *RevInfo) error { 464 if file == "" { 465 return nil 466 } 467 js, err := json.Marshal(info) 468 if err != nil { 469 return err 470 } 471 return writeDiskCache(file, js) 472 } 473 474 // writeDiskGoMod writes a go.mod cache entry. 475 // The file name must have been returned by a previous call to readDiskGoMod. 476 func writeDiskGoMod(file string, text []byte) error { 477 return writeDiskCache(file, text) 478 } 479 480 // writeDiskCache is the generic "write to a cache file" implementation. 481 // The file must have been returned by a previous call to readDiskCache. 482 func writeDiskCache(file string, data []byte) error { 483 if file == "" { 484 return nil 485 } 486 // Make sure directory for file exists. 487 if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil { 488 return err 489 } 490 491 if err := renameio.WriteFile(file, data); err != nil { 492 return err 493 } 494 495 if strings.HasSuffix(file, ".mod") { 496 rewriteVersionList(filepath.Dir(file)) 497 } 498 return nil 499 } 500 501 // rewriteVersionList rewrites the version list in dir 502 // after a new *.mod file has been written. 503 func rewriteVersionList(dir string) { 504 if filepath.Base(dir) != "@v" { 505 base.Fatalf("go: internal error: misuse of rewriteVersionList") 506 } 507 508 listFile := filepath.Join(dir, "list") 509 510 // We use a separate lockfile here instead of locking listFile itself because 511 // we want to use Rename to write the file atomically. The list may be read by 512 // a GOPROXY HTTP server, and if we crash midway through a rewrite (or if the 513 // HTTP server ignores our locking and serves the file midway through a 514 // rewrite) it's better to serve a stale list than a truncated one. 515 unlock, err := lockedfile.MutexAt(listFile + ".lock").Lock() 516 if err != nil { 517 base.Fatalf("go: can't lock version list lockfile: %v", err) 518 } 519 defer unlock() 520 521 infos, err := ioutil.ReadDir(dir) 522 if err != nil { 523 return 524 } 525 var list []string 526 for _, info := range infos { 527 // We look for *.mod files on the theory that if we can't supply 528 // the .mod file then there's no point in listing that version, 529 // since it's unusable. (We can have *.info without *.mod.) 530 // We don't require *.zip files on the theory that for code only 531 // involved in module graph construction, many *.zip files 532 // will never be requested. 533 name := info.Name() 534 if strings.HasSuffix(name, ".mod") { 535 v := strings.TrimSuffix(name, ".mod") 536 if v != "" && module.CanonicalVersion(v) == v { 537 list = append(list, v) 538 } 539 } 540 } 541 SortVersions(list) 542 543 var buf bytes.Buffer 544 for _, v := range list { 545 buf.WriteString(v) 546 buf.WriteString("\n") 547 } 548 old, _ := ioutil.ReadFile(listFile) 549 if bytes.Equal(buf.Bytes(), old) { 550 return 551 } 552 553 if err := renameio.WriteFile(listFile, buf.Bytes()); err != nil { 554 base.Fatalf("go: failed to write version list: %v", err) 555 } 556 }