github.com/umeshredd/helm@v3.0.0-alpha.1+incompatible/pkg/downloader/manager.go (about) 1 /* 2 Copyright The Helm Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package downloader 17 18 import ( 19 "fmt" 20 "io" 21 "io/ioutil" 22 "net/url" 23 "os" 24 "path" 25 "path/filepath" 26 "strings" 27 "sync" 28 29 "github.com/Masterminds/semver" 30 "github.com/ghodss/yaml" 31 "github.com/pkg/errors" 32 33 "helm.sh/helm/pkg/chart" 34 "helm.sh/helm/pkg/chart/loader" 35 "helm.sh/helm/pkg/chartutil" 36 "helm.sh/helm/pkg/getter" 37 "helm.sh/helm/pkg/helmpath" 38 "helm.sh/helm/pkg/repo" 39 "helm.sh/helm/pkg/resolver" 40 "helm.sh/helm/pkg/urlutil" 41 ) 42 43 // Manager handles the lifecycle of fetching, resolving, and storing dependencies. 44 type Manager struct { 45 // Out is used to print warnings and notifications. 46 Out io.Writer 47 // ChartPath is the path to the unpacked base chart upon which this operates. 48 ChartPath string 49 // HelmHome is the $HELM_HOME directory 50 HelmHome helmpath.Home 51 // Verification indicates whether the chart should be verified. 52 Verify VerificationStrategy 53 // Debug is the global "--debug" flag 54 Debug bool 55 // Keyring is the key ring file. 56 Keyring string 57 // SkipUpdate indicates that the repository should not be updated first. 58 SkipUpdate bool 59 // Getter collection for the operation 60 Getters []getter.Provider 61 } 62 63 // Build rebuilds a local charts directory from a lockfile. 64 // 65 // If the lockfile is not present, this will run a Manager.Update() 66 // 67 // If SkipUpdate is set, this will not update the repository. 68 func (m *Manager) Build() error { 69 c, err := m.loadChartDir() 70 if err != nil { 71 return err 72 } 73 74 // If a lock file is found, run a build from that. Otherwise, just do 75 // an update. 76 lock := c.Lock 77 if lock == nil { 78 return m.Update() 79 } 80 81 req := c.Metadata.Dependencies 82 if sum, err := resolver.HashReq(req); err != nil || sum != lock.Digest { 83 return errors.New("Chart.lock is out of sync with Chart.yaml") 84 } 85 86 // Check that all of the repos we're dependent on actually exist. 87 if err := m.hasAllRepos(lock.Dependencies); err != nil { 88 return err 89 } 90 91 if !m.SkipUpdate { 92 // For each repo in the file, update the cached copy of that repo 93 if err := m.UpdateRepositories(); err != nil { 94 return err 95 } 96 } 97 98 // Now we need to fetch every package here into charts/ 99 if err := m.downloadAll(lock.Dependencies); err != nil { 100 return err 101 } 102 103 return nil 104 } 105 106 // Update updates a local charts directory. 107 // 108 // It first reads the Chart.yaml file, and then attempts to 109 // negotiate versions based on that. It will download the versions 110 // from remote chart repositories unless SkipUpdate is true. 111 func (m *Manager) Update() error { 112 c, err := m.loadChartDir() 113 if err != nil { 114 return err 115 } 116 117 // If no dependencies are found, we consider this a successful 118 // completion. 119 req := c.Metadata.Dependencies 120 if req == nil { 121 return nil 122 } 123 124 // Hash dependencies 125 // FIXME should this hash all of Chart.yaml 126 hash, err := resolver.HashReq(req) 127 if err != nil { 128 return err 129 } 130 131 // Check that all of the repos we're dependent on actually exist and 132 // the repo index names. 133 repoNames, err := m.getRepoNames(req) 134 if err != nil { 135 return err 136 } 137 138 // For each repo in the file, update the cached copy of that repo 139 if !m.SkipUpdate { 140 if err := m.UpdateRepositories(); err != nil { 141 return err 142 } 143 } 144 145 // Now we need to find out which version of a chart best satisfies the 146 // dependencies in the Chart.yaml 147 lock, err := m.resolve(req, repoNames, hash) 148 if err != nil { 149 return err 150 } 151 152 // Now we need to fetch every package here into charts/ 153 if err := m.downloadAll(lock.Dependencies); err != nil { 154 return err 155 } 156 157 // If the lock file hasn't changed, don't write a new one. 158 oldLock := c.Lock 159 if oldLock != nil && oldLock.Digest == lock.Digest { 160 return nil 161 } 162 163 // Finally, we need to write the lockfile. 164 return writeLock(m.ChartPath, lock) 165 } 166 167 func (m *Manager) loadChartDir() (*chart.Chart, error) { 168 if fi, err := os.Stat(m.ChartPath); err != nil { 169 return nil, errors.Wrapf(err, "could not find %s", m.ChartPath) 170 } else if !fi.IsDir() { 171 return nil, errors.New("only unpacked charts can be updated") 172 } 173 return loader.LoadDir(m.ChartPath) 174 } 175 176 // resolve takes a list of dependencies and translates them into an exact version to download. 177 // 178 // This returns a lock file, which has all of the dependencies normalized to a specific version. 179 func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string, hash string) (*chart.Lock, error) { 180 res := resolver.New(m.ChartPath, m.HelmHome) 181 return res.Resolve(req, repoNames, hash) 182 } 183 184 // downloadAll takes a list of dependencies and downloads them into charts/ 185 // 186 // It will delete versions of the chart that exist on disk and might cause 187 // a conflict. 188 func (m *Manager) downloadAll(deps []*chart.Dependency) error { 189 repos, err := m.loadChartRepositories() 190 if err != nil { 191 return err 192 } 193 194 destPath := filepath.Join(m.ChartPath, "charts") 195 tmpPath := filepath.Join(m.ChartPath, "tmpcharts") 196 197 // Create 'charts' directory if it doesn't already exist. 198 if fi, err := os.Stat(destPath); err != nil { 199 if err := os.MkdirAll(destPath, 0755); err != nil { 200 return err 201 } 202 } else if !fi.IsDir() { 203 return errors.Errorf("%q is not a directory", destPath) 204 } 205 206 if err := os.Rename(destPath, tmpPath); err != nil { 207 return errors.Wrap(err, "unable to move current charts to tmp dir") 208 } 209 210 if err := os.MkdirAll(destPath, 0755); err != nil { 211 return err 212 } 213 214 fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) 215 var saveError error 216 for _, dep := range deps { 217 if strings.HasPrefix(dep.Repository, "file://") { 218 if m.Debug { 219 fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository) 220 } 221 ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version) 222 if err != nil { 223 saveError = err 224 break 225 } 226 dep.Version = ver 227 continue 228 } 229 230 fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) 231 232 // Any failure to resolve/download a chart should fail: 233 // https://github.com/helm/helm/issues/1439 234 churl, username, password, err := findChartURL(dep.Name, dep.Version, dep.Repository, repos) 235 if err != nil { 236 saveError = errors.Wrapf(err, "could not find %s", churl) 237 break 238 } 239 240 dl := ChartDownloader{ 241 Out: m.Out, 242 Verify: m.Verify, 243 Keyring: m.Keyring, 244 HelmHome: m.HelmHome, 245 Getters: m.Getters, 246 Username: username, 247 Password: password, 248 } 249 250 if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil { 251 saveError = errors.Wrapf(err, "could not download %s", churl) 252 break 253 } 254 } 255 256 if saveError == nil { 257 fmt.Fprintln(m.Out, "Deleting outdated charts") 258 for _, dep := range deps { 259 if err := m.safeDeleteDep(dep.Name, tmpPath); err != nil { 260 return err 261 } 262 } 263 if err := move(tmpPath, destPath); err != nil { 264 return err 265 } 266 if err := os.RemoveAll(tmpPath); err != nil { 267 return errors.Wrapf(err, "failed to remove %v", tmpPath) 268 } 269 } else { 270 fmt.Fprintln(m.Out, "Save error occurred: ", saveError) 271 fmt.Fprintln(m.Out, "Deleting newly downloaded charts, restoring pre-update state") 272 for _, dep := range deps { 273 if err := m.safeDeleteDep(dep.Name, destPath); err != nil { 274 return err 275 } 276 } 277 if err := os.RemoveAll(destPath); err != nil { 278 return errors.Wrapf(err, "failed to remove %v", destPath) 279 } 280 if err := os.Rename(tmpPath, destPath); err != nil { 281 return errors.Wrap(err, "unable to move current charts to tmp dir") 282 } 283 return saveError 284 } 285 return nil 286 } 287 288 // safeDeleteDep deletes any versions of the given dependency in the given directory. 289 // 290 // It does this by first matching the file name to an expected pattern, then loading 291 // the file to verify that it is a chart with the same name as the given name. 292 // 293 // Because it requires tar file introspection, it is more intensive than a basic delete. 294 // 295 // This will only return errors that should stop processing entirely. Other errors 296 // will emit log messages or be ignored. 297 func (m *Manager) safeDeleteDep(name, dir string) error { 298 files, err := filepath.Glob(filepath.Join(dir, name+"-*.tgz")) 299 if err != nil { 300 // Only for ErrBadPattern 301 return err 302 } 303 for _, fname := range files { 304 ch, err := loader.LoadFile(fname) 305 if err != nil { 306 fmt.Fprintf(m.Out, "Could not verify %s for deletion: %s (Skipping)", fname, err) 307 continue 308 } 309 if ch.Name() != name { 310 // This is not the file you are looking for. 311 continue 312 } 313 if err := os.Remove(fname); err != nil { 314 fmt.Fprintf(m.Out, "Could not delete %s: %s (Skipping)", fname, err) 315 continue 316 } 317 } 318 return nil 319 } 320 321 // hasAllRepos ensures that all of the referenced deps are in the local repo cache. 322 func (m *Manager) hasAllRepos(deps []*chart.Dependency) error { 323 rf, err := repo.LoadFile(m.HelmHome.RepositoryFile()) 324 if err != nil { 325 return err 326 } 327 repos := rf.Repositories 328 329 // Verify that all repositories referenced in the deps are actually known 330 // by Helm. 331 missing := []string{} 332 Loop: 333 for _, dd := range deps { 334 // If repo is from local path, continue 335 if strings.HasPrefix(dd.Repository, "file://") { 336 continue 337 } 338 339 if dd.Repository == "" { 340 continue 341 } 342 for _, repo := range repos { 343 if urlutil.Equal(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { 344 continue Loop 345 } 346 } 347 missing = append(missing, dd.Repository) 348 } 349 if len(missing) > 0 { 350 return errors.Errorf("no repository definition for %s. Please add the missing repos via 'helm repo add'", strings.Join(missing, ", ")) 351 } 352 return nil 353 } 354 355 // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file. 356 func (m *Manager) getRepoNames(deps []*chart.Dependency) (map[string]string, error) { 357 rf, err := repo.LoadFile(m.HelmHome.RepositoryFile()) 358 if err != nil { 359 return nil, err 360 } 361 repos := rf.Repositories 362 363 reposMap := make(map[string]string) 364 365 // Verify that all repositories referenced in the deps are actually known 366 // by Helm. 367 missing := []string{} 368 for _, dd := range deps { 369 // if dep chart is from local path, verify the path is valid 370 if strings.HasPrefix(dd.Repository, "file://") { 371 if _, err := resolver.GetLocalPath(dd.Repository, m.ChartPath); err != nil { 372 return nil, err 373 } 374 375 if m.Debug { 376 fmt.Fprintf(m.Out, "Repository from local path: %s\n", dd.Repository) 377 } 378 reposMap[dd.Name] = dd.Repository 379 continue 380 } 381 382 found := false 383 384 for _, repo := range repos { 385 if (strings.HasPrefix(dd.Repository, "@") && strings.TrimPrefix(dd.Repository, "@") == repo.Name) || 386 (strings.HasPrefix(dd.Repository, "alias:") && strings.TrimPrefix(dd.Repository, "alias:") == repo.Name) { 387 found = true 388 dd.Repository = repo.URL 389 reposMap[dd.Name] = repo.Name 390 break 391 } else if urlutil.Equal(repo.URL, dd.Repository) { 392 found = true 393 reposMap[dd.Name] = repo.Name 394 break 395 } 396 } 397 if !found { 398 missing = append(missing, dd.Repository) 399 } 400 } 401 if len(missing) > 0 { 402 errorMessage := fmt.Sprintf("no repository definition for %s. Please add them via 'helm repo add'", strings.Join(missing, ", ")) 403 // It is common for people to try to enter "stable" as a repository instead of the actual URL. 404 // For this case, let's give them a suggestion. 405 containsNonURL := false 406 for _, repo := range missing { 407 if !strings.Contains(repo, "//") && !strings.HasPrefix(repo, "@") && !strings.HasPrefix(repo, "alias:") { 408 containsNonURL = true 409 } 410 } 411 if containsNonURL { 412 errorMessage += ` 413 Note that repositories must be URLs or aliases. For example, to refer to the stable 414 repository, use "https://kubernetes-charts.storage.googleapis.com/" or "@stable" instead of 415 "stable". Don't forget to add the repo, too ('helm repo add').` 416 } 417 return nil, errors.New(errorMessage) 418 } 419 return reposMap, nil 420 } 421 422 // UpdateRepositories updates all of the local repos to the latest. 423 func (m *Manager) UpdateRepositories() error { 424 rf, err := repo.LoadFile(m.HelmHome.RepositoryFile()) 425 if err != nil { 426 return err 427 } 428 repos := rf.Repositories 429 if len(repos) > 0 { 430 // This prints warnings straight to out. 431 if err := m.parallelRepoUpdate(repos); err != nil { 432 return err 433 } 434 } 435 return nil 436 } 437 438 func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { 439 out := m.Out 440 fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") 441 var wg sync.WaitGroup 442 for _, c := range repos { 443 r, err := repo.NewChartRepository(c, m.Getters) 444 if err != nil { 445 return err 446 } 447 wg.Add(1) 448 go func(r *repo.ChartRepository) { 449 if err := r.DownloadIndexFile(m.HelmHome.Cache()); err != nil { 450 fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", r.Config.Name, r.Config.URL, err) 451 } else { 452 fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", r.Config.Name) 453 } 454 wg.Done() 455 }(r) 456 } 457 wg.Wait() 458 fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈") 459 return nil 460 } 461 462 // findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. 463 // 464 // 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the 465 // newest version will be returned. 466 // 467 // repoURL is the repository to search 468 // 469 // If it finds a URL that is "relative", it will prepend the repoURL. 470 func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) { 471 for _, cr := range repos { 472 if urlutil.Equal(repoURL, cr.Config.URL) { 473 var entry repo.ChartVersions 474 entry, err = findEntryByName(name, cr) 475 if err != nil { 476 return 477 } 478 var ve *repo.ChartVersion 479 ve, err = findVersionedEntry(version, entry) 480 if err != nil { 481 return 482 } 483 url, err = normalizeURL(repoURL, ve.URLs[0]) 484 if err != nil { 485 return 486 } 487 username = cr.Config.Username 488 password = cr.Config.Password 489 return 490 } 491 } 492 err = errors.Errorf("chart %s not found in %s", name, repoURL) 493 return 494 } 495 496 // findEntryByName finds an entry in the chart repository whose name matches the given name. 497 // 498 // It returns the ChartVersions for that entry. 499 func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) { 500 for ename, entry := range cr.IndexFile.Entries { 501 if ename == name { 502 return entry, nil 503 } 504 } 505 return nil, errors.New("entry not found") 506 } 507 508 // findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints. 509 // 510 // If version is empty, the first chart found is returned. 511 func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) { 512 for _, verEntry := range vers { 513 if len(verEntry.URLs) == 0 { 514 // Not a legit entry. 515 continue 516 } 517 518 if version == "" || versionEquals(version, verEntry.Version) { 519 return verEntry, nil 520 } 521 } 522 return nil, errors.New("no matching version") 523 } 524 525 func versionEquals(v1, v2 string) bool { 526 sv1, err := semver.NewVersion(v1) 527 if err != nil { 528 // Fallback to string comparison. 529 return v1 == v2 530 } 531 sv2, err := semver.NewVersion(v2) 532 if err != nil { 533 return false 534 } 535 return sv1.Equal(sv2) 536 } 537 538 func normalizeURL(baseURL, urlOrPath string) (string, error) { 539 u, err := url.Parse(urlOrPath) 540 if err != nil { 541 return urlOrPath, err 542 } 543 if u.IsAbs() { 544 return u.String(), nil 545 } 546 u2, err := url.Parse(baseURL) 547 if err != nil { 548 return urlOrPath, errors.Wrap(err, "base URL failed to parse") 549 } 550 551 u2.Path = path.Join(u2.Path, urlOrPath) 552 return u2.String(), nil 553 } 554 555 // loadChartRepositories reads the repositories.yaml, and then builds a map of 556 // ChartRepositories. 557 // 558 // The key is the local name (which is only present in the repositories.yaml). 559 func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) { 560 indices := map[string]*repo.ChartRepository{} 561 repoyaml := m.HelmHome.RepositoryFile() 562 563 // Load repositories.yaml file 564 rf, err := repo.LoadFile(repoyaml) 565 if err != nil { 566 return indices, errors.Wrapf(err, "failed to load %s", repoyaml) 567 } 568 569 for _, re := range rf.Repositories { 570 lname := re.Name 571 cacheindex := m.HelmHome.CacheIndex(lname) 572 index, err := repo.LoadIndexFile(cacheindex) 573 if err != nil { 574 return indices, err 575 } 576 577 // TODO: use constructor 578 cr := &repo.ChartRepository{ 579 Config: re, 580 IndexFile: index, 581 } 582 indices[lname] = cr 583 } 584 return indices, nil 585 } 586 587 // writeLock writes a lockfile to disk 588 func writeLock(chartpath string, lock *chart.Lock) error { 589 data, err := yaml.Marshal(lock) 590 if err != nil { 591 return err 592 } 593 dest := filepath.Join(chartpath, "Chart.lock") 594 return ioutil.WriteFile(dest, data, 0644) 595 } 596 597 // archive a dep chart from local directory and save it into charts/ 598 func tarFromLocalDir(chartpath, name, repo, version string) (string, error) { 599 destPath := filepath.Join(chartpath, "charts") 600 601 if !strings.HasPrefix(repo, "file://") { 602 return "", errors.Errorf("wrong format: chart %s repository %s", name, repo) 603 } 604 605 origPath, err := resolver.GetLocalPath(repo, chartpath) 606 if err != nil { 607 return "", err 608 } 609 610 ch, err := loader.LoadDir(origPath) 611 if err != nil { 612 return "", err 613 } 614 615 constraint, err := semver.NewConstraint(version) 616 if err != nil { 617 return "", errors.Wrapf(err, "dependency %s has an invalid version/constraint format", name) 618 } 619 620 v, err := semver.NewVersion(ch.Metadata.Version) 621 if err != nil { 622 return "", err 623 } 624 625 if constraint.Check(v) { 626 _, err = chartutil.Save(ch, destPath) 627 return ch.Metadata.Version, err 628 } 629 630 return "", errors.Errorf("can't get a valid version for dependency %s", name) 631 } 632 633 // move files from tmppath to destpath 634 func move(tmpPath, destPath string) error { 635 files, _ := ioutil.ReadDir(tmpPath) 636 for _, file := range files { 637 filename := file.Name() 638 tmpfile := filepath.Join(tmpPath, filename) 639 destfile := filepath.Join(destPath, filename) 640 if err := os.Rename(tmpfile, destfile); err != nil { 641 return errors.Wrap(err, "unable to move local charts to charts dir") 642 } 643 } 644 return nil 645 }