github.com/Beeketing/helm@v2.12.1+incompatible/pkg/downloader/manager.go (about) 1 /* 2 Copyright The Helm Authors. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package downloader 17 18 import ( 19 "errors" 20 "fmt" 21 "io" 22 "io/ioutil" 23 "net/url" 24 "os" 25 "path" 26 "path/filepath" 27 "strings" 28 "sync" 29 30 "github.com/Masterminds/semver" 31 "github.com/ghodss/yaml" 32 33 "k8s.io/helm/pkg/chartutil" 34 "k8s.io/helm/pkg/getter" 35 "k8s.io/helm/pkg/helm/helmpath" 36 "k8s.io/helm/pkg/proto/hapi/chart" 37 "k8s.io/helm/pkg/repo" 38 "k8s.io/helm/pkg/resolver" 39 "k8s.io/helm/pkg/urlutil" 40 ) 41 42 // Manager handles the lifecycle of fetching, resolving, and storing dependencies. 43 type Manager struct { 44 // Out is used to print warnings and notifications. 45 Out io.Writer 46 // ChartPath is the path to the unpacked base chart upon which this operates. 47 ChartPath string 48 // HelmHome is the $HELM_HOME directory 49 HelmHome helmpath.Home 50 // Verification indicates whether the chart should be verified. 51 Verify VerificationStrategy 52 // Debug is the global "--debug" flag 53 Debug bool 54 // Keyring is the key ring file. 55 Keyring string 56 // SkipUpdate indicates that the repository should not be updated first. 57 SkipUpdate bool 58 // Getter collection for the operation 59 Getters []getter.Provider 60 } 61 62 // Build rebuilds a local charts directory from a lockfile. 63 // 64 // If the lockfile is not present, this will run a Manager.Update() 65 // 66 // If SkipUpdate is set, this will not update the repository. 67 func (m *Manager) Build() error { 68 c, err := m.loadChartDir() 69 if err != nil { 70 return err 71 } 72 73 // If a lock file is found, run a build from that. Otherwise, just do 74 // an update. 75 lock, err := chartutil.LoadRequirementsLock(c) 76 if err != nil { 77 return m.Update() 78 } 79 80 // A lock must accompany a requirements.yaml file. 81 req, err := chartutil.LoadRequirements(c) 82 if err != nil { 83 return fmt.Errorf("requirements.yaml cannot be opened: %s", err) 84 } 85 if sum, err := resolver.HashReq(req); err != nil || sum != lock.Digest { 86 return fmt.Errorf("requirements.lock is out of sync with requirements.yaml") 87 } 88 89 // Check that all of the repos we're dependent on actually exist. 90 if err := m.hasAllRepos(lock.Dependencies); err != nil { 91 return err 92 } 93 94 if !m.SkipUpdate { 95 // For each repo in the file, update the cached copy of that repo 96 if err := m.UpdateRepositories(); err != nil { 97 return err 98 } 99 } 100 101 // Now we need to fetch every package here into charts/ 102 return m.downloadAll(lock.Dependencies) 103 } 104 105 // Update updates a local charts directory. 106 // 107 // It first reads the requirements.yaml file, and then attempts to 108 // negotiate versions based on that. It will download the versions 109 // from remote chart repositories unless SkipUpdate is true. 110 func (m *Manager) Update() error { 111 c, err := m.loadChartDir() 112 if err != nil { 113 return err 114 } 115 116 // If no requirements file is found, we consider this a successful 117 // completion. 118 req, err := chartutil.LoadRequirements(c) 119 if err != nil { 120 if err == chartutil.ErrRequirementsNotFound { 121 fmt.Fprintf(m.Out, "No requirements found in %s/charts.\n", m.ChartPath) 122 return nil 123 } 124 return err 125 } 126 127 // Hash requirements.yaml 128 hash, err := resolver.HashReq(req) 129 if err != nil { 130 return err 131 } 132 133 // Check that all of the repos we're dependent on actually exist and 134 // the repo index names. 135 repoNames, err := m.getRepoNames(req.Dependencies) 136 if err != nil { 137 return err 138 } 139 140 // For each repo in the file, update the cached copy of that repo 141 if !m.SkipUpdate { 142 if err := m.UpdateRepositories(); err != nil { 143 return err 144 } 145 } 146 147 // Now we need to find out which version of a chart best satisfies the 148 // requirements the requirements.yaml 149 lock, err := m.resolve(req, repoNames, hash) 150 if err != nil { 151 return err 152 } 153 154 // Now we need to fetch every package here into charts/ 155 if err := m.downloadAll(lock.Dependencies); err != nil { 156 return err 157 } 158 159 // If the lock file hasn't changed, don't write a new one. 160 oldLock, err := chartutil.LoadRequirementsLock(c) 161 if err == nil && oldLock.Digest == lock.Digest { 162 return nil 163 } 164 165 // Finally, we need to write the lockfile. 166 return writeLock(m.ChartPath, lock) 167 } 168 169 func (m *Manager) loadChartDir() (*chart.Chart, error) { 170 if fi, err := os.Stat(m.ChartPath); err != nil { 171 return nil, fmt.Errorf("could not find %s: %s", m.ChartPath, err) 172 } else if !fi.IsDir() { 173 return nil, errors.New("only unpacked charts can be updated") 174 } 175 return chartutil.LoadDir(m.ChartPath) 176 } 177 178 // resolve takes a list of requirements and translates them into an exact version to download. 179 // 180 // This returns a lock file, which has all of the requirements normalized to a specific version. 181 func (m *Manager) resolve(req *chartutil.Requirements, repoNames map[string]string, hash string) (*chartutil.RequirementsLock, error) { 182 res := resolver.New(m.ChartPath, m.HelmHome) 183 return res.Resolve(req, repoNames, hash) 184 } 185 186 // downloadAll takes a list of dependencies and downloads them into charts/ 187 // 188 // It will delete versions of the chart that exist on disk and might cause 189 // a conflict. 190 func (m *Manager) downloadAll(deps []*chartutil.Dependency) error { 191 repos, err := m.loadChartRepositories() 192 if err != nil { 193 return err 194 } 195 196 destPath := filepath.Join(m.ChartPath, "charts") 197 tmpPath := filepath.Join(m.ChartPath, "tmpcharts") 198 199 // Create 'charts' directory if it doesn't already exist. 200 if fi, err := os.Stat(destPath); err != nil { 201 if err := os.MkdirAll(destPath, 0755); err != nil { 202 return err 203 } 204 } else if !fi.IsDir() { 205 return fmt.Errorf("%q is not a directory", destPath) 206 } 207 208 if err := os.Rename(destPath, tmpPath); err != nil { 209 return fmt.Errorf("Unable to move current charts to tmp dir: %v", err) 210 } 211 212 if err := os.MkdirAll(destPath, 0755); err != nil { 213 return err 214 } 215 216 fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) 217 var saveError error 218 for _, dep := range deps { 219 if strings.HasPrefix(dep.Repository, "file://") { 220 if m.Debug { 221 fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository) 222 } 223 ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version) 224 if err != nil { 225 saveError = err 226 break 227 } 228 dep.Version = ver 229 continue 230 } 231 232 fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) 233 234 // Any failure to resolve/download a chart should fail: 235 // https://github.com/kubernetes/helm/issues/1439 236 churl, username, password, err := findChartURL(dep.Name, dep.Version, dep.Repository, repos) 237 if err != nil { 238 saveError = fmt.Errorf("could not find %s: %s", churl, err) 239 break 240 } 241 242 dl := ChartDownloader{ 243 Out: m.Out, 244 Verify: m.Verify, 245 Keyring: m.Keyring, 246 HelmHome: m.HelmHome, 247 Getters: m.Getters, 248 Username: username, 249 Password: password, 250 } 251 252 if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil { 253 saveError = fmt.Errorf("could not download %s: %s", churl, err) 254 break 255 } 256 } 257 258 if saveError == nil { 259 fmt.Fprintln(m.Out, "Deleting outdated charts") 260 for _, dep := range deps { 261 if err := m.safeDeleteDep(dep.Name, tmpPath); err != nil { 262 return err 263 } 264 } 265 if err := move(tmpPath, destPath); err != nil { 266 return err 267 } 268 if err := os.RemoveAll(tmpPath); err != nil { 269 return fmt.Errorf("Failed to remove %v: %v", tmpPath, err) 270 } 271 } else { 272 fmt.Fprintln(m.Out, "Save error occurred: ", saveError) 273 fmt.Fprintln(m.Out, "Deleting newly downloaded charts, restoring pre-update state") 274 for _, dep := range deps { 275 if err := m.safeDeleteDep(dep.Name, destPath); err != nil { 276 return err 277 } 278 } 279 if err := os.RemoveAll(destPath); err != nil { 280 return fmt.Errorf("Failed to remove %v: %v", destPath, err) 281 } 282 if err := os.Rename(tmpPath, destPath); err != nil { 283 return fmt.Errorf("Unable to move current charts to tmp dir: %v", err) 284 } 285 return saveError 286 } 287 return nil 288 } 289 290 // safeDeleteDep deletes any versions of the given dependency in the given directory. 291 // 292 // It does this by first matching the file name to an expected pattern, then loading 293 // the file to verify that it is a chart with the same name as the given name. 294 // 295 // Because it requires tar file introspection, it is more intensive than a basic delete. 296 // 297 // This will only return errors that should stop processing entirely. Other errors 298 // will emit log messages or be ignored. 299 func (m *Manager) safeDeleteDep(name, dir string) error { 300 files, err := filepath.Glob(filepath.Join(dir, name+"-*.tgz")) 301 if err != nil { 302 // Only for ErrBadPattern 303 return err 304 } 305 for _, fname := range files { 306 ch, err := chartutil.LoadFile(fname) 307 if err != nil { 308 fmt.Fprintf(m.Out, "Could not verify %s for deletion: %s (Skipping)", fname, err) 309 continue 310 } 311 if ch.Metadata.Name != name { 312 // This is not the file you are looking for. 313 continue 314 } 315 if err := os.Remove(fname); err != nil { 316 fmt.Fprintf(m.Out, "Could not delete %s: %s (Skipping)", fname, err) 317 continue 318 } 319 } 320 return nil 321 } 322 323 // hasAllRepos ensures that all of the referenced deps are in the local repo cache. 324 func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { 325 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 326 if err != nil { 327 return err 328 } 329 repos := rf.Repositories 330 331 // Verify that all repositories referenced in the deps are actually known 332 // by Helm. 333 missing := []string{} 334 for _, dd := range deps { 335 // If repo is from local path, continue 336 if strings.HasPrefix(dd.Repository, "file://") { 337 continue 338 } 339 340 found := false 341 if dd.Repository == "" { 342 found = true 343 } else { 344 for _, repo := range repos { 345 if urlutil.Equal(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { 346 found = true 347 } 348 } 349 } 350 if !found { 351 missing = append(missing, dd.Repository) 352 } 353 } 354 if len(missing) > 0 { 355 return fmt.Errorf("no repository definition for %s. Please add the missing repos via 'helm repo add'", strings.Join(missing, ", ")) 356 } 357 return nil 358 } 359 360 // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file. 361 func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, error) { 362 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 363 if err != nil { 364 return nil, err 365 } 366 repos := rf.Repositories 367 368 reposMap := make(map[string]string) 369 370 // Verify that all repositories referenced in the deps are actually known 371 // by Helm. 372 missing := []string{} 373 for _, dd := range deps { 374 // if dep chart is from local path, verify the path is valid 375 if strings.HasPrefix(dd.Repository, "file://") { 376 if _, err := resolver.GetLocalPath(dd.Repository, m.ChartPath); err != nil { 377 return nil, err 378 } 379 380 if m.Debug { 381 fmt.Fprintf(m.Out, "Repository from local path: %s\n", dd.Repository) 382 } 383 reposMap[dd.Name] = dd.Repository 384 continue 385 } 386 387 found := false 388 389 for _, repo := range repos { 390 if (strings.HasPrefix(dd.Repository, "@") && strings.TrimPrefix(dd.Repository, "@") == repo.Name) || 391 (strings.HasPrefix(dd.Repository, "alias:") && strings.TrimPrefix(dd.Repository, "alias:") == repo.Name) { 392 found = true 393 dd.Repository = repo.URL 394 reposMap[dd.Name] = repo.Name 395 break 396 } else if urlutil.Equal(repo.URL, dd.Repository) { 397 found = true 398 reposMap[dd.Name] = repo.Name 399 break 400 } 401 } 402 if !found { 403 missing = append(missing, dd.Repository) 404 } 405 } 406 if len(missing) > 0 { 407 if len(missing) > 0 { 408 errorMessage := fmt.Sprintf("no repository definition for %s. Please add them via 'helm repo add'", strings.Join(missing, ", ")) 409 // It is common for people to try to enter "stable" as a repository instead of the actual URL. 410 // For this case, let's give them a suggestion. 411 containsNonURL := false 412 for _, repo := range missing { 413 if !strings.Contains(repo, "//") && !strings.HasPrefix(repo, "@") && !strings.HasPrefix(repo, "alias:") { 414 containsNonURL = true 415 } 416 } 417 if containsNonURL { 418 errorMessage += ` 419 Note that repositories must be URLs or aliases. For example, to refer to the stable 420 repository, use "https://kubernetes-charts.storage.googleapis.com/" or "@stable" instead of 421 "stable". Don't forget to add the repo, too ('helm repo add').` 422 } 423 return nil, errors.New(errorMessage) 424 } 425 } 426 return reposMap, nil 427 } 428 429 // UpdateRepositories updates all of the local repos to the latest. 430 func (m *Manager) UpdateRepositories() error { 431 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 432 if err != nil { 433 return err 434 } 435 repos := rf.Repositories 436 if len(repos) > 0 { 437 // This prints warnings straight to out. 438 if err := m.parallelRepoUpdate(repos); err != nil { 439 return err 440 } 441 } 442 return nil 443 } 444 445 func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { 446 out := m.Out 447 fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") 448 var wg sync.WaitGroup 449 for _, c := range repos { 450 r, err := repo.NewChartRepository(c, m.Getters) 451 if err != nil { 452 return err 453 } 454 wg.Add(1) 455 go func(r *repo.ChartRepository) { 456 if err := r.DownloadIndexFile(m.HelmHome.Cache()); err != nil { 457 fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", r.Config.Name, r.Config.URL, err) 458 } else { 459 fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", r.Config.Name) 460 } 461 wg.Done() 462 }(r) 463 } 464 wg.Wait() 465 fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈") 466 return nil 467 } 468 469 // findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. 470 // 471 // 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the 472 // newest version will be returned. 473 // 474 // repoURL is the repository to search 475 // 476 // If it finds a URL that is "relative", it will prepend the repoURL. 477 func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) { 478 for _, cr := range repos { 479 if urlutil.Equal(repoURL, cr.Config.URL) { 480 var entry repo.ChartVersions 481 entry, err = findEntryByName(name, cr) 482 if err != nil { 483 return 484 } 485 var ve *repo.ChartVersion 486 ve, err = findVersionedEntry(version, entry) 487 if err != nil { 488 return 489 } 490 url, err = normalizeURL(repoURL, ve.URLs[0]) 491 if err != nil { 492 return 493 } 494 username = cr.Config.Username 495 password = cr.Config.Password 496 return 497 } 498 } 499 err = fmt.Errorf("chart %s not found in %s", name, repoURL) 500 return 501 } 502 503 // findEntryByName finds an entry in the chart repository whose name matches the given name. 504 // 505 // It returns the ChartVersions for that entry. 506 func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) { 507 for ename, entry := range cr.IndexFile.Entries { 508 if ename == name { 509 return entry, nil 510 } 511 } 512 return nil, errors.New("entry not found") 513 } 514 515 // findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints. 516 // 517 // If version is empty, the first chart found is returned. 518 func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) { 519 for _, verEntry := range vers { 520 if len(verEntry.URLs) == 0 { 521 // Not a legit entry. 522 continue 523 } 524 525 if version == "" || versionEquals(version, verEntry.Version) { 526 return verEntry, nil 527 } 528 } 529 return nil, errors.New("no matching version") 530 } 531 532 func versionEquals(v1, v2 string) bool { 533 sv1, err := semver.NewVersion(v1) 534 if err != nil { 535 // Fallback to string comparison. 536 return v1 == v2 537 } 538 sv2, err := semver.NewVersion(v2) 539 if err != nil { 540 return false 541 } 542 return sv1.Equal(sv2) 543 } 544 545 func normalizeURL(baseURL, urlOrPath string) (string, error) { 546 u, err := url.Parse(urlOrPath) 547 if err != nil { 548 return urlOrPath, err 549 } 550 if u.IsAbs() { 551 return u.String(), nil 552 } 553 u2, err := url.Parse(baseURL) 554 if err != nil { 555 return urlOrPath, fmt.Errorf("Base URL failed to parse: %s", err) 556 } 557 558 u2.Path = path.Join(u2.Path, urlOrPath) 559 return u2.String(), nil 560 } 561 562 // loadChartRepositories reads the repositories.yaml, and then builds a map of 563 // ChartRepositories. 564 // 565 // The key is the local name (which is only present in the repositories.yaml). 566 func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) { 567 indices := map[string]*repo.ChartRepository{} 568 repoyaml := m.HelmHome.RepositoryFile() 569 570 // Load repositories.yaml file 571 rf, err := repo.LoadRepositoriesFile(repoyaml) 572 if err != nil { 573 return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err) 574 } 575 576 for _, re := range rf.Repositories { 577 lname := re.Name 578 cacheindex := m.HelmHome.CacheIndex(lname) 579 index, err := repo.LoadIndexFile(cacheindex) 580 if err != nil { 581 return indices, err 582 } 583 584 // TODO: use constructor 585 cr := &repo.ChartRepository{ 586 Config: re, 587 IndexFile: index, 588 } 589 indices[lname] = cr 590 } 591 return indices, nil 592 } 593 594 // writeLock writes a lockfile to disk 595 func writeLock(chartpath string, lock *chartutil.RequirementsLock) error { 596 data, err := yaml.Marshal(lock) 597 if err != nil { 598 return err 599 } 600 dest := filepath.Join(chartpath, "requirements.lock") 601 return ioutil.WriteFile(dest, data, 0644) 602 } 603 604 // archive a dep chart from local directory and save it into charts/ 605 func tarFromLocalDir(chartpath string, name string, repo string, version string) (string, error) { 606 destPath := filepath.Join(chartpath, "charts") 607 608 if !strings.HasPrefix(repo, "file://") { 609 return "", fmt.Errorf("wrong format: chart %s repository %s", name, repo) 610 } 611 612 origPath, err := resolver.GetLocalPath(repo, chartpath) 613 if err != nil { 614 return "", err 615 } 616 617 ch, err := chartutil.LoadDir(origPath) 618 if err != nil { 619 return "", err 620 } 621 622 constraint, err := semver.NewConstraint(version) 623 if err != nil { 624 return "", fmt.Errorf("dependency %s has an invalid version/constraint format: %s", name, err) 625 } 626 627 v, err := semver.NewVersion(ch.Metadata.Version) 628 if err != nil { 629 return "", err 630 } 631 632 if constraint.Check(v) { 633 _, err = chartutil.Save(ch, destPath) 634 return ch.Metadata.Version, err 635 } 636 637 return "", fmt.Errorf("can't get a valid version for dependency %s", name) 638 } 639 640 // move files from tmppath to destpath 641 func move(tmpPath, destPath string) error { 642 files, _ := ioutil.ReadDir(tmpPath) 643 for _, file := range files { 644 filename := file.Name() 645 tmpfile := filepath.Join(tmpPath, filename) 646 destfile := filepath.Join(destPath, filename) 647 if err := os.Rename(tmpfile, destfile); err != nil { 648 return fmt.Errorf("Unable to move local charts to charts dir: %v", err) 649 } 650 } 651 return nil 652 }