github.com/thrawn01/glide@v0.12.3/repo/vcs.go (about) 1 package repo 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "net/http" 8 "net/url" 9 "os" 10 "path/filepath" 11 "runtime" 12 "sort" 13 "strings" 14 15 cp "github.com/Masterminds/glide/cache" 16 "github.com/Masterminds/glide/cfg" 17 "github.com/Masterminds/glide/msg" 18 gpath "github.com/Masterminds/glide/path" 19 "github.com/Masterminds/semver" 20 v "github.com/Masterminds/vcs" 21 ) 22 23 // VcsUpdate updates to a particular checkout based on the VCS setting. 24 func VcsUpdate(dep *cfg.Dependency, force bool, updated *UpdateTracker) error { 25 26 // If the dependency has already been pinned we can skip it. This is a 27 // faster path so we don't need to resolve it again. 28 if dep.Pin != "" { 29 msg.Debug("Dependency %s has already been pinned. Fetching updates skipped.", dep.Name) 30 return nil 31 } 32 33 if updated.Check(dep.Name) { 34 msg.Debug("%s was already updated, skipping.", dep.Name) 35 return nil 36 } 37 updated.Add(dep.Name) 38 39 if filterArchOs(dep) { 40 msg.Info("%s is not used for %s/%s.\n", dep.Name, runtime.GOOS, runtime.GOARCH) 41 return nil 42 } 43 44 key, err := cp.Key(dep.Remote()) 45 if err != nil { 46 msg.Die("Cache key generation error: %s", err) 47 } 48 location := cp.Location() 49 dest := filepath.Join(location, "src", key) 50 51 // If destination doesn't exist we need to perform an initial checkout. 52 if _, err := os.Stat(dest); os.IsNotExist(err) { 53 msg.Info("--> Fetching %s.", dep.Name) 54 if err = VcsGet(dep); err != nil { 55 msg.Warn("Unable to checkout %s\n", dep.Name) 56 return err 57 } 58 } else { 59 // At this point we have a directory for the package. 60 msg.Info("--> Fetching updates for %s.", dep.Name) 61 62 // When the directory is not empty and has no VCS directory it's 63 // a vendored files situation. 64 empty, err := gpath.IsDirectoryEmpty(dest) 65 if err != nil { 66 return err 67 } 68 _, err = v.DetectVcsFromFS(dest) 69 if empty == true && err == v.ErrCannotDetectVCS { 70 msg.Warn("Cached version of %s is an empty directory. Fetching a new copy of the dependency.", dep.Name) 71 msg.Debug("Removing empty directory %s", dest) 72 err := os.RemoveAll(dest) 73 if err != nil { 74 return err 75 } 76 if err = VcsGet(dep); err != nil { 77 msg.Warn("Unable to checkout %s\n", dep.Name) 78 return err 79 } 80 } else { 81 repo, err := dep.GetRepo(dest) 82 83 // Tried to checkout a repo to a path that does not work. Either the 84 // type or endpoint has changed. Force is being passed in so the old 85 // location can be removed and replaced with the new one. 86 // Warning, any changes in the old location will be deleted. 87 // TODO: Put dirty checking in on the existing local checkout. 88 if (err == v.ErrWrongVCS || err == v.ErrWrongRemote) && force == true { 89 newRemote := dep.Remote() 90 91 msg.Warn("Replacing %s with contents from %s\n", dep.Name, newRemote) 92 rerr := os.RemoveAll(dest) 93 if rerr != nil { 94 return rerr 95 } 96 if err = VcsGet(dep); err != nil { 97 msg.Warn("Unable to checkout %s\n", dep.Name) 98 return err 99 } 100 101 repo, err = dep.GetRepo(dest) 102 if err != nil { 103 return err 104 } 105 } else if err != nil { 106 return err 107 } else if repo.IsDirty() { 108 return fmt.Errorf("%s contains uncommitted changes. Skipping update", dep.Name) 109 } 110 111 ver := dep.Reference 112 if ver == "" { 113 ver = defaultBranch(repo) 114 } 115 // Check if the current version is a tag or commit id. If it is 116 // and that version is already checked out we can skip updating 117 // which is faster than going out to the Internet to perform 118 // an update. 119 if ver != "" { 120 version, err := repo.Version() 121 if err != nil { 122 return err 123 } 124 ib, err := isBranch(ver, repo) 125 if err != nil { 126 return err 127 } 128 129 // If the current version equals the ref and it's not a 130 // branch it's a tag or commit id so we can skip 131 // performing an update. 132 if version == ver && !ib { 133 msg.Debug("%s is already set to version %s. Skipping update.", dep.Name, dep.Reference) 134 return nil 135 } 136 } 137 138 if err := repo.Update(); err != nil { 139 msg.Warn("Download failed.\n") 140 return err 141 } 142 } 143 } 144 145 return nil 146 } 147 148 // VcsVersion set the VCS version for a checkout. 149 func VcsVersion(dep *cfg.Dependency) error { 150 151 // If the dependency has already been pinned we can skip it. This is a 152 // faster path so we don't need to resolve it again. 153 if dep.Pin != "" { 154 msg.Debug("Dependency %s has already been pinned. Setting version skipped.", dep.Name) 155 return nil 156 } 157 158 key, err := cp.Key(dep.Remote()) 159 if err != nil { 160 msg.Die("Cache key generation error: %s", err) 161 } 162 location := cp.Location() 163 cwd := filepath.Join(location, "src", key) 164 165 // If there is no reference configured there is nothing to set. 166 if dep.Reference == "" { 167 // Before exiting update the pinned version 168 repo, err := dep.GetRepo(cwd) 169 if err != nil { 170 return err 171 } 172 dep.Pin, err = repo.Version() 173 if err != nil { 174 return err 175 } 176 return nil 177 } 178 179 // When the directory is not empty and has no VCS directory it's 180 // a vendored files situation. 181 empty, err := gpath.IsDirectoryEmpty(cwd) 182 if err != nil { 183 return err 184 } 185 _, err = v.DetectVcsFromFS(cwd) 186 if empty == false && err == v.ErrCannotDetectVCS { 187 return fmt.Errorf("Cache directory missing VCS information for %s", dep.Name) 188 } 189 190 repo, err := dep.GetRepo(cwd) 191 if err != nil { 192 return err 193 } 194 195 ver := dep.Reference 196 // References in Git can begin with a ^ which is similar to semver. 197 // If there is a ^ prefix we assume it's a semver constraint rather than 198 // part of the git/VCS commit id. 199 if repo.IsReference(ver) && !strings.HasPrefix(ver, "^") { 200 msg.Info("--> Setting version for %s to %s.\n", dep.Name, ver) 201 } else { 202 203 // Create the constraint first to make sure it's valid before 204 // working on the repo. 205 constraint, err := semver.NewConstraint(ver) 206 207 // Make sure the constriant is valid. At this point it's not a valid 208 // reference so if it's not a valid constrint we can exit early. 209 if err != nil { 210 msg.Warn("The reference '%s' is not valid\n", ver) 211 return err 212 } 213 214 // Get the tags and branches (in that order) 215 refs, err := getAllVcsRefs(repo) 216 if err != nil { 217 return err 218 } 219 220 // Convert and filter the list to semver.Version instances 221 semvers := getSemVers(refs) 222 223 // Sort semver list 224 sort.Sort(sort.Reverse(semver.Collection(semvers))) 225 found := false 226 for _, v := range semvers { 227 if constraint.Check(v) { 228 found = true 229 // If the constrint passes get the original reference 230 ver = v.Original() 231 break 232 } 233 } 234 if found { 235 msg.Info("--> Detected semantic version. Setting version for %s to %s.", dep.Name, ver) 236 } else { 237 msg.Warn("--> Unable to find semantic version for constraint %s %s", dep.Name, ver) 238 } 239 } 240 if err := repo.UpdateVersion(ver); err != nil { 241 return err 242 } 243 dep.Pin, err = repo.Version() 244 if err != nil { 245 return err 246 } 247 248 return nil 249 } 250 251 // VcsGet figures out how to fetch a dependency, and then gets it. 252 // 253 // VcsGet installs into the cache. 254 func VcsGet(dep *cfg.Dependency) error { 255 256 key, err := cp.Key(dep.Remote()) 257 if err != nil { 258 msg.Die("Cache key generation error: %s", err) 259 } 260 location := cp.Location() 261 d := filepath.Join(location, "src", key) 262 263 repo, err := dep.GetRepo(d) 264 if err != nil { 265 return err 266 } 267 // If the directory does not exist this is a first cache. 268 if _, err = os.Stat(d); os.IsNotExist(err) { 269 msg.Debug("Adding %s to the cache for the first time", dep.Name) 270 err = repo.Get() 271 if err != nil { 272 return err 273 } 274 branch := findCurrentBranch(repo) 275 if branch != "" { 276 msg.Debug("Saving default branch for %s", repo.Remote()) 277 c := cp.RepoInfo{DefaultBranch: branch} 278 err = cp.SaveRepoData(key, c) 279 if err == cp.ErrCacheDisabled { 280 msg.Debug("Unable to cache default branch because caching is disabled") 281 } else if err != nil { 282 msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) 283 } 284 } 285 } else { 286 msg.Debug("Updating %s in the cache", dep.Name) 287 err = repo.Update() 288 if err != nil { 289 return err 290 } 291 } 292 293 return nil 294 } 295 296 // filterArchOs indicates a dependency should be filtered out because it is 297 // the wrong GOOS or GOARCH. 298 // 299 // FIXME: Should this be moved to the dependency package? 300 func filterArchOs(dep *cfg.Dependency) bool { 301 found := false 302 if len(dep.Arch) > 0 { 303 for _, a := range dep.Arch { 304 if a == runtime.GOARCH { 305 found = true 306 } 307 } 308 // If it's not found, it should be filtered out. 309 if !found { 310 return true 311 } 312 } 313 314 found = false 315 if len(dep.Os) > 0 { 316 for _, o := range dep.Os { 317 if o == runtime.GOOS { 318 found = true 319 } 320 } 321 if !found { 322 return true 323 } 324 325 } 326 327 return false 328 } 329 330 // isBranch returns true if the given string is a branch in VCS. 331 func isBranch(branch string, repo v.Repo) (bool, error) { 332 branches, err := repo.Branches() 333 if err != nil { 334 return false, err 335 } 336 for _, b := range branches { 337 if b == branch { 338 return true, nil 339 } 340 } 341 return false, nil 342 } 343 344 // defaultBranch tries to ascertain the default branch for the given repo. 345 // Some repos will have multiple branches in them (e.g. Git) while others 346 // (e.g. Svn) will not. 347 func defaultBranch(repo v.Repo) string { 348 349 // Svn and Bzr use different locations (paths or entire locations) 350 // for branches so we won't have a default branch. 351 if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr { 352 return "" 353 } 354 355 // Check the cache for a value. 356 key, kerr := cp.Key(repo.Remote()) 357 var d cp.RepoInfo 358 if kerr == nil { 359 d, err := cp.RepoData(key) 360 if err == nil { 361 if d.DefaultBranch != "" { 362 return d.DefaultBranch 363 } 364 } 365 } 366 367 // If we don't have it in the store try some APIs 368 r := repo.Remote() 369 u, err := url.Parse(r) 370 if err != nil { 371 return "" 372 } 373 if u.Scheme == "" { 374 // Where there is no scheme we try urls like git@github.com:foo/bar 375 r = strings.Replace(r, ":", "/", -1) 376 r = "ssh://" + r 377 u, err = url.Parse(r) 378 if err != nil { 379 return "" 380 } 381 u.Scheme = "" 382 } 383 if u.Host == "github.com" { 384 parts := strings.Split(u.Path, "/") 385 if len(parts) != 2 { 386 return "" 387 } 388 api := fmt.Sprintf("https://api.github.com/repos/%s/%s", parts[0], parts[1]) 389 resp, err := http.Get(api) 390 if err != nil { 391 return "" 392 } 393 defer resp.Body.Close() 394 if resp.StatusCode >= 300 || resp.StatusCode < 200 { 395 return "" 396 } 397 body, err := ioutil.ReadAll(resp.Body) 398 var data interface{} 399 err = json.Unmarshal(body, &data) 400 if err != nil { 401 return "" 402 } 403 gh := data.(map[string]interface{}) 404 db := gh["default_branch"].(string) 405 if kerr == nil { 406 d.DefaultBranch = db 407 err := cp.SaveRepoData(key, d) 408 if err == cp.ErrCacheDisabled { 409 msg.Debug("Unable to cache default branch because caching is disabled") 410 } else if err != nil { 411 msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) 412 } 413 } 414 return db 415 } 416 417 if u.Host == "bitbucket.org" { 418 parts := strings.Split(u.Path, "/") 419 if len(parts) != 2 { 420 return "" 421 } 422 api := fmt.Sprintf("https://bitbucket.org/api/1.0/repositories/%s/%s/main-branch/", parts[0], parts[1]) 423 resp, err := http.Get(api) 424 if err != nil { 425 return "" 426 } 427 defer resp.Body.Close() 428 if resp.StatusCode >= 300 || resp.StatusCode < 200 { 429 return "" 430 } 431 body, err := ioutil.ReadAll(resp.Body) 432 var data interface{} 433 err = json.Unmarshal(body, &data) 434 if err != nil { 435 return "" 436 } 437 bb := data.(map[string]interface{}) 438 db := bb["name"].(string) 439 if kerr == nil { 440 d.DefaultBranch = db 441 err := cp.SaveRepoData(key, d) 442 if err == cp.ErrCacheDisabled { 443 msg.Debug("Unable to cache default branch because caching is disabled") 444 } else if err != nil { 445 msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) 446 } 447 } 448 return db 449 } 450 451 return "" 452 } 453 454 // From a local repo find out the current branch name if there is one. 455 // Note, this should only be used right after a fresh clone to get accurate 456 // information. 457 func findCurrentBranch(repo v.Repo) string { 458 msg.Debug("Attempting to find current branch for %s", repo.Remote()) 459 // Svn and Bzr don't have default branches. 460 if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr { 461 return "" 462 } 463 464 if repo.Vcs() == v.Git || repo.Vcs() == v.Hg { 465 ver, err := repo.Current() 466 if err != nil { 467 msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err) 468 return "" 469 } 470 return ver 471 } 472 473 return "" 474 } 475 476 func envForDir(dir string) []string { 477 env := os.Environ() 478 return mergeEnvLists([]string{"PWD=" + dir}, env) 479 } 480 481 func mergeEnvLists(in, out []string) []string { 482 NextVar: 483 for _, inkv := range in { 484 k := strings.SplitAfterN(inkv, "=", 2)[0] 485 for i, outkv := range out { 486 if strings.HasPrefix(outkv, k) { 487 out[i] = inkv 488 continue NextVar 489 } 490 } 491 out = append(out, inkv) 492 } 493 return out 494 }