github.com/bir3/gocompiler@v0.3.205/src/cmd/gocmd/internal/modfetch/codehost/codehost.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package codehost defines the interface implemented by a code hosting source, 6 // along with support code for use by implementations. 7 package codehost 8 9 import ( 10 "bytes" 11 "crypto/sha256" 12 "fmt" 13 "io" 14 "io/fs" 15 "os" 16 "os/exec" 17 "path/filepath" 18 "strings" 19 "sync" 20 "time" 21 22 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/cfg" 23 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/lockedfile" 24 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/str" 25 26 "github.com/bir3/gocompiler/src/xvendor/golang.org/x/mod/module" 27 "github.com/bir3/gocompiler/src/xvendor/golang.org/x/mod/semver" 28 ) 29 30 // Downloaded size limits. 31 const ( 32 MaxGoMod = 16 << 20 // maximum size of go.mod file 33 MaxLICENSE = 16 << 20 // maximum size of LICENSE file 34 MaxZipFile = 500 << 20 // maximum size of downloaded zip file 35 ) 36 37 // A Repo represents a code hosting source. 38 // Typical implementations include local version control repositories, 39 // remote version control servers, and code hosting sites. 40 // 41 // A Repo must be safe for simultaneous use by multiple goroutines, 42 // and callers must not modify returned values, which may be cached and shared. 43 type Repo interface { 44 // CheckReuse checks whether the old origin information 45 // remains up to date. If so, whatever cached object it was 46 // taken from can be reused. 47 // The subdir gives subdirectory name where the module root is expected to be found, 48 // "" for the root or "sub/dir" for a subdirectory (no trailing slash). 49 CheckReuse(old *Origin, subdir string) error 50 51 // List lists all tags with the given prefix. 52 Tags(prefix string) (*Tags, error) 53 54 // Stat returns information about the revision rev. 55 // A revision can be any identifier known to the underlying service: 56 // commit hash, branch, tag, and so on. 57 Stat(rev string) (*RevInfo, error) 58 59 // Latest returns the latest revision on the default branch, 60 // whatever that means in the underlying implementation. 61 Latest() (*RevInfo, error) 62 63 // ReadFile reads the given file in the file tree corresponding to revision rev. 64 // It should refuse to read more than maxSize bytes. 65 // 66 // If the requested file does not exist it should return an error for which 67 // os.IsNotExist(err) returns true. 68 ReadFile(rev, file string, maxSize int64) (data []byte, err error) 69 70 // ReadZip downloads a zip file for the subdir subdirectory 71 // of the given revision to a new file in a given temporary directory. 72 // It should refuse to read more than maxSize bytes. 73 // It returns a ReadCloser for a streamed copy of the zip file. 74 // All files in the zip file are expected to be 75 // nested in a single top-level directory, whose name is not specified. 76 ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) 77 78 // RecentTag returns the most recent tag on rev or one of its predecessors 79 // with the given prefix. allowed may be used to filter out unwanted versions. 80 RecentTag(rev, prefix string, allowed func(tag string) bool) (tag string, err error) 81 82 // DescendsFrom reports whether rev or any of its ancestors has the given tag. 83 // 84 // DescendsFrom must return true for any tag returned by RecentTag for the 85 // same revision. 86 DescendsFrom(rev, tag string) (bool, error) 87 } 88 89 // An Origin describes the provenance of a given repo method result. 90 // It can be passed to CheckReuse (usually in a different go command invocation) 91 // to see whether the result remains up-to-date. 92 type Origin struct { 93 VCS string `json:",omitempty"` // "git" etc 94 URL string `json:",omitempty"` // URL of repository 95 Subdir string `json:",omitempty"` // subdirectory in repo 96 97 // If TagSum is non-empty, then the resolution of this module version 98 // depends on the set of tags present in the repo, specifically the tags 99 // of the form TagPrefix + a valid semver version. 100 // If the matching repo tags and their commit hashes still hash to TagSum, 101 // the Origin is still valid (at least as far as the tags are concerned). 102 // The exact checksum is up to the Repo implementation; see (*gitRepo).Tags. 103 TagPrefix string `json:",omitempty"` 104 TagSum string `json:",omitempty"` 105 106 // If Ref is non-empty, then the resolution of this module version 107 // depends on Ref resolving to the revision identified by Hash. 108 // If Ref still resolves to Hash, the Origin is still valid (at least as far as Ref is concerned). 109 // For Git, the Ref is a full ref like "refs/heads/main" or "refs/tags/v1.2.3", 110 // and the Hash is the Git object hash the ref maps to. 111 // Other VCS might choose differently, but the idea is that Ref is the name 112 // with a mutable meaning while Hash is a name with an immutable meaning. 113 Ref string `json:",omitempty"` 114 Hash string `json:",omitempty"` 115 116 // If RepoSum is non-empty, then the resolution of this module version 117 // failed due to the repo being available but the version not being present. 118 // This depends on the entire state of the repo, which RepoSum summarizes. 119 // For Git, this is a hash of all the refs and their hashes. 120 RepoSum string `json:",omitempty"` 121 } 122 123 // Checkable reports whether the Origin contains anything that can be checked. 124 // If not, the Origin is purely informational and should fail a CheckReuse call. 125 func (o *Origin) Checkable() bool { 126 return o.TagSum != "" || o.Ref != "" || o.Hash != "" || o.RepoSum != "" 127 } 128 129 // ClearCheckable clears the Origin enough to make Checkable return false. 130 func (o *Origin) ClearCheckable() { 131 o.TagSum = "" 132 o.TagPrefix = "" 133 o.Ref = "" 134 o.Hash = "" 135 o.RepoSum = "" 136 } 137 138 // A Tags describes the available tags in a code repository. 139 type Tags struct { 140 Origin *Origin 141 List []Tag 142 } 143 144 // A Tag describes a single tag in a code repository. 145 type Tag struct { 146 Name string 147 Hash string // content hash identifying tag's content, if available 148 } 149 150 // isOriginTag reports whether tag should be preserved 151 // in the Tags method's Origin calculation. 152 // We can safely ignore tags that are not look like pseudo-versions, 153 // because ../coderepo.go's (*codeRepo).Versions ignores them too. 154 // We can also ignore non-semver tags, but we have to include semver 155 // tags with extra suffixes, because the pseudo-version base finder uses them. 156 func isOriginTag(tag string) bool { 157 // modfetch.(*codeRepo).Versions uses Canonical == tag, 158 // but pseudo-version calculation has a weaker condition that 159 // the canonical is a prefix of the tag. 160 // Include those too, so that if any new one appears, we'll invalidate the cache entry. 161 // This will lead to spurious invalidation of version list results, 162 // but tags of this form being created should be fairly rare 163 // (and invalidate pseudo-version results anyway). 164 c := semver.Canonical(tag) 165 return c != "" && strings.HasPrefix(tag, c) && !module.IsPseudoVersion(tag) 166 } 167 168 // A RevInfo describes a single revision in a source code repository. 169 type RevInfo struct { 170 Origin *Origin 171 Name string // complete ID in underlying repository 172 Short string // shortened ID, for use in pseudo-version 173 Version string // version used in lookup 174 Time time.Time // commit time 175 Tags []string // known tags for commit 176 } 177 178 // UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a 179 // revision rather than a file. 180 type UnknownRevisionError struct { 181 Rev string 182 } 183 184 func (e *UnknownRevisionError) Error() string { 185 return "unknown revision " + e.Rev 186 } 187 func (UnknownRevisionError) Is(err error) bool { 188 return err == fs.ErrNotExist 189 } 190 191 // ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given 192 // repository or module contains no commits. 193 var ErrNoCommits error = noCommitsError{} 194 195 type noCommitsError struct{} 196 197 func (noCommitsError) Error() string { 198 return "no commits" 199 } 200 func (noCommitsError) Is(err error) bool { 201 return err == fs.ErrNotExist 202 } 203 204 // ErrUnsupported indicates that a requested operation cannot be performed, 205 // because it is unsupported. This error indicates that there is no alternative 206 // way to perform the operation. 207 // 208 // TODO(#41198): Remove this declaration and use errors.ErrUnsupported instead. 209 var ErrUnsupported = unsupportedOperationError{} 210 211 type unsupportedOperationError struct{} 212 213 func (unsupportedOperationError) Error() string { 214 return "unsupported operation" 215 } 216 217 // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. 218 func AllHex(rev string) bool { 219 for i := 0; i < len(rev); i++ { 220 c := rev[i] 221 if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' { 222 continue 223 } 224 return false 225 } 226 return true 227 } 228 229 // ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length 230 // used in pseudo-versions (12 hex digits). 231 func ShortenSHA1(rev string) string { 232 if AllHex(rev) && len(rev) == 40 { 233 return rev[:12] 234 } 235 return rev 236 } 237 238 // WorkDir returns the name of the cached work directory to use for the 239 // given repository type and name. 240 func WorkDir(typ, name string) (dir, lockfile string, err error) { 241 if cfg.GOMODCACHE == "" { 242 return "", "", fmt.Errorf("neither GOPATH nor GOMODCACHE are set") 243 } 244 245 // We name the work directory for the SHA256 hash of the type and name. 246 // We intentionally avoid the actual name both because of possible 247 // conflicts with valid file system paths and because we want to ensure 248 // that one checkout is never nested inside another. That nesting has 249 // led to security problems in the past. 250 if strings.Contains(typ, ":") { 251 return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon") 252 } 253 key := typ + ":" + name 254 dir = filepath.Join(cfg.GOMODCACHE, "cache/vcs", fmt.Sprintf("%x", sha256.Sum256([]byte(key)))) 255 256 if cfg.BuildX { 257 fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name) 258 } 259 if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil { 260 return "", "", err 261 } 262 263 lockfile = dir + ".lock" 264 if cfg.BuildX { 265 fmt.Fprintf(os.Stderr, "# lock %s\n", lockfile) 266 } 267 268 unlock, err := lockedfile.MutexAt(lockfile).Lock() 269 if err != nil { 270 return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err) 271 } 272 defer unlock() 273 274 data, err := os.ReadFile(dir + ".info") 275 info, err2 := os.Stat(dir) 276 if err == nil && err2 == nil && info.IsDir() { 277 // Info file and directory both already exist: reuse. 278 have := strings.TrimSuffix(string(data), "\n") 279 if have != key { 280 return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key) 281 } 282 if cfg.BuildX { 283 fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name) 284 } 285 return dir, lockfile, nil 286 } 287 288 // Info file or directory missing. Start from scratch. 289 if cfg.BuildX { 290 fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name) 291 } 292 os.RemoveAll(dir) 293 if err := os.MkdirAll(dir, 0777); err != nil { 294 return "", "", err 295 } 296 if err := os.WriteFile(dir+".info", []byte(key), 0666); err != nil { 297 os.RemoveAll(dir) 298 return "", "", err 299 } 300 return dir, lockfile, nil 301 } 302 303 type RunError struct { 304 Cmd string 305 Err error 306 Stderr []byte 307 HelpText string 308 } 309 310 func (e *RunError) Error() string { 311 text := e.Cmd + ": " + e.Err.Error() 312 stderr := bytes.TrimRight(e.Stderr, "\n") 313 if len(stderr) > 0 { 314 text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t") 315 } 316 if len(e.HelpText) > 0 { 317 text += "\n" + e.HelpText 318 } 319 return text 320 } 321 322 var dirLock sync.Map 323 324 // Run runs the command line in the given directory 325 // (an empty dir means the current directory). 326 // It returns the standard output and, for a non-zero exit, 327 // a *RunError indicating the command, exit status, and standard error. 328 // Standard error is unavailable for commands that exit successfully. 329 func Run(dir string, cmdline ...any) ([]byte, error) { 330 return RunWithStdin(dir, nil, cmdline...) 331 } 332 333 // bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell. 334 // See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html. 335 var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`) 336 337 func RunWithStdin(dir string, stdin io.Reader, cmdline ...any) ([]byte, error) { 338 if dir != "" { 339 muIface, ok := dirLock.Load(dir) 340 if !ok { 341 muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex)) 342 } 343 mu := muIface.(*sync.Mutex) 344 mu.Lock() 345 defer mu.Unlock() 346 } 347 348 cmd := str.StringList(cmdline...) 349 if os.Getenv("TESTGOVCS") == "panic" { 350 panic(fmt.Sprintf("use of vcs: %v", cmd)) 351 } 352 if cfg.BuildX { 353 text := new(strings.Builder) 354 if dir != "" { 355 text.WriteString("cd ") 356 text.WriteString(dir) 357 text.WriteString("; ") 358 } 359 for i, arg := range cmd { 360 if i > 0 { 361 text.WriteByte(' ') 362 } 363 switch { 364 case strings.ContainsAny(arg, "'"): 365 // Quote args that could be mistaken for quoted args. 366 text.WriteByte('"') 367 text.WriteString(bashQuoter.Replace(arg)) 368 text.WriteByte('"') 369 case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"): 370 // Quote args that contain special characters, glob patterns, or spaces. 371 text.WriteByte('\'') 372 text.WriteString(arg) 373 text.WriteByte('\'') 374 default: 375 text.WriteString(arg) 376 } 377 } 378 fmt.Fprintf(os.Stderr, "%s\n", text) 379 start := time.Now() 380 defer func() { 381 fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text) 382 }() 383 } 384 // TODO: Impose limits on command output size. 385 // TODO: Set environment to get English error messages. 386 var stderr bytes.Buffer 387 var stdout bytes.Buffer 388 c := exec.Command(cmd[0], cmd[1:]...) 389 c.Dir = dir 390 c.Stdin = stdin 391 c.Stderr = &stderr 392 c.Stdout = &stdout 393 err := c.Run() 394 if err != nil { 395 err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err} 396 } 397 return stdout.Bytes(), err 398 }