github.com/zoumo/helm@v2.5.0+incompatible/pkg/downloader/manager.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors All rights reserved. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package downloader 17 18 import ( 19 "errors" 20 "fmt" 21 "io" 22 "io/ioutil" 23 "net/url" 24 "os" 25 "path" 26 "path/filepath" 27 "strings" 28 "sync" 29 30 "github.com/Masterminds/semver" 31 "github.com/ghodss/yaml" 32 33 "k8s.io/helm/pkg/chartutil" 34 "k8s.io/helm/pkg/getter" 35 "k8s.io/helm/pkg/helm/helmpath" 36 "k8s.io/helm/pkg/proto/hapi/chart" 37 "k8s.io/helm/pkg/repo" 38 "k8s.io/helm/pkg/resolver" 39 "k8s.io/helm/pkg/urlutil" 40 ) 41 42 // Manager handles the lifecycle of fetching, resolving, and storing dependencies. 43 type Manager struct { 44 // Out is used to print warnings and notifications. 45 Out io.Writer 46 // ChartPath is the path to the unpacked base chart upon which this operates. 47 ChartPath string 48 // HelmHome is the $HELM_HOME directory 49 HelmHome helmpath.Home 50 // Verification indicates whether the chart should be verified. 51 Verify VerificationStrategy 52 // Debug is the global "--debug" flag 53 Debug bool 54 // Keyring is the key ring file. 55 Keyring string 56 // SkipUpdate indicates that the repository should not be updated first. 57 SkipUpdate bool 58 // Getter collection for the operation 59 Getters []getter.Provider 60 } 61 62 // Build rebuilds a local charts directory from a lockfile. 63 // 64 // If the lockfile is not present, this will run a Manager.Update() 65 // 66 // If SkipUpdate is set, this will not update the repository. 67 func (m *Manager) Build() error { 68 c, err := m.loadChartDir() 69 if err != nil { 70 return err 71 } 72 73 // If a lock file is found, run a build from that. Otherwise, just do 74 // an update. 75 lock, err := chartutil.LoadRequirementsLock(c) 76 if err != nil { 77 return m.Update() 78 } 79 80 // A lock must accompany a requirements.yaml file. 81 req, err := chartutil.LoadRequirements(c) 82 if err != nil { 83 return fmt.Errorf("requirements.yaml cannot be opened: %s", err) 84 } 85 if sum, err := resolver.HashReq(req); err != nil || sum != lock.Digest { 86 return fmt.Errorf("requirements.lock is out of sync with requirements.yaml") 87 } 88 89 // Check that all of the repos we're dependent on actually exist. 90 if err := m.hasAllRepos(lock.Dependencies); err != nil { 91 return err 92 } 93 94 if !m.SkipUpdate { 95 // For each repo in the file, update the cached copy of that repo 96 if err := m.UpdateRepositories(); err != nil { 97 return err 98 } 99 } 100 101 // Now we need to fetch every package here into charts/ 102 if err := m.downloadAll(lock.Dependencies); err != nil { 103 return err 104 } 105 106 return nil 107 } 108 109 // Update updates a local charts directory. 110 // 111 // It first reads the requirements.yaml file, and then attempts to 112 // negotiate versions based on that. It will download the versions 113 // from remote chart repositories unless SkipUpdate is true. 114 func (m *Manager) Update() error { 115 c, err := m.loadChartDir() 116 if err != nil { 117 return err 118 } 119 120 // If no requirements file is found, we consider this a successful 121 // completion. 122 req, err := chartutil.LoadRequirements(c) 123 if err != nil { 124 if err == chartutil.ErrRequirementsNotFound { 125 fmt.Fprintf(m.Out, "No requirements found in %s/charts.\n", m.ChartPath) 126 return nil 127 } 128 return err 129 } 130 131 // Hash requirements.yaml 132 hash, err := resolver.HashReq(req) 133 if err != nil { 134 return err 135 } 136 137 // Check that all of the repos we're dependent on actually exist and 138 // the repo index names. 139 repoNames, err := m.getRepoNames(req.Dependencies) 140 if err != nil { 141 return err 142 } 143 144 // For each repo in the file, update the cached copy of that repo 145 if !m.SkipUpdate { 146 if err := m.UpdateRepositories(); err != nil { 147 return err 148 } 149 } 150 151 // Now we need to find out which version of a chart best satisfies the 152 // requirements the requirements.yaml 153 lock, err := m.resolve(req, repoNames, hash) 154 if err != nil { 155 return err 156 } 157 158 // Now we need to fetch every package here into charts/ 159 if err := m.downloadAll(lock.Dependencies); err != nil { 160 return err 161 } 162 163 // If the lock file hasn't changed, don't write a new one. 164 oldLock, err := chartutil.LoadRequirementsLock(c) 165 if err == nil && oldLock.Digest == lock.Digest { 166 return nil 167 } 168 169 // Finally, we need to write the lockfile. 170 return writeLock(m.ChartPath, lock) 171 } 172 173 func (m *Manager) loadChartDir() (*chart.Chart, error) { 174 if fi, err := os.Stat(m.ChartPath); err != nil { 175 return nil, fmt.Errorf("could not find %s: %s", m.ChartPath, err) 176 } else if !fi.IsDir() { 177 return nil, errors.New("only unpacked charts can be updated") 178 } 179 return chartutil.LoadDir(m.ChartPath) 180 } 181 182 // resolve takes a list of requirements and translates them into an exact version to download. 183 // 184 // This returns a lock file, which has all of the requirements normalized to a specific version. 185 func (m *Manager) resolve(req *chartutil.Requirements, repoNames map[string]string, hash string) (*chartutil.RequirementsLock, error) { 186 res := resolver.New(m.ChartPath, m.HelmHome) 187 return res.Resolve(req, repoNames, hash) 188 } 189 190 // downloadAll takes a list of dependencies and downloads them into charts/ 191 // 192 // It will delete versions of the chart that exist on disk and might cause 193 // a conflict. 194 func (m *Manager) downloadAll(deps []*chartutil.Dependency) error { 195 repos, err := m.loadChartRepositories() 196 if err != nil { 197 return err 198 } 199 200 dl := ChartDownloader{ 201 Out: m.Out, 202 Verify: m.Verify, 203 Keyring: m.Keyring, 204 HelmHome: m.HelmHome, 205 Getters: m.Getters, 206 } 207 208 destPath := filepath.Join(m.ChartPath, "charts") 209 210 // Create 'charts' directory if it doesn't already exist. 211 if fi, err := os.Stat(destPath); err != nil { 212 if err := os.MkdirAll(destPath, 0755); err != nil { 213 return err 214 } 215 } else if !fi.IsDir() { 216 return fmt.Errorf("%q is not a directory", destPath) 217 } 218 219 fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) 220 for _, dep := range deps { 221 if err := m.safeDeleteDep(dep.Name, destPath); err != nil { 222 return err 223 } 224 225 if strings.HasPrefix(dep.Repository, "file://") { 226 if m.Debug { 227 fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository) 228 } 229 ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version) 230 if err != nil { 231 return err 232 } 233 dep.Version = ver 234 continue 235 } 236 237 fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) 238 239 // Any failure to resolve/download a chart should fail: 240 // https://github.com/kubernetes/helm/issues/1439 241 churl, err := findChartURL(dep.Name, dep.Version, dep.Repository, repos) 242 if err != nil { 243 return fmt.Errorf("could not find %s: %s", churl, err) 244 } 245 246 if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil { 247 return fmt.Errorf("could not download %s: %s", churl, err) 248 } 249 } 250 return nil 251 } 252 253 // safeDeleteDep deletes any versions of the given dependency in the given directory. 254 // 255 // It does this by first matching the file name to an expected pattern, then loading 256 // the file to verify that it is a chart with the same name as the given name. 257 // 258 // Because it requires tar file introspection, it is more intensive than a basic delete. 259 // 260 // This will only return errors that should stop processing entirely. Other errors 261 // will emit log messages or be ignored. 262 func (m *Manager) safeDeleteDep(name, dir string) error { 263 files, err := filepath.Glob(filepath.Join(dir, name+"-*.tgz")) 264 if err != nil { 265 // Only for ErrBadPattern 266 return err 267 } 268 for _, fname := range files { 269 ch, err := chartutil.LoadFile(fname) 270 if err != nil { 271 fmt.Fprintf(m.Out, "Could not verify %s for deletion: %s (Skipping)", fname, err) 272 continue 273 } 274 if ch.Metadata.Name != name { 275 // This is not the file you are looking for. 276 continue 277 } 278 if err := os.Remove(fname); err != nil { 279 fmt.Fprintf(m.Out, "Could not delete %s: %s (Skipping)", fname, err) 280 continue 281 } 282 } 283 return nil 284 } 285 286 // hasAllRepos ensures that all of the referenced deps are in the local repo cache. 287 func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { 288 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 289 if err != nil { 290 return err 291 } 292 repos := rf.Repositories 293 294 // Verify that all repositories referenced in the deps are actually known 295 // by Helm. 296 missing := []string{} 297 for _, dd := range deps { 298 // If repo is from local path, continue 299 if strings.HasPrefix(dd.Repository, "file://") { 300 continue 301 } 302 303 found := false 304 if dd.Repository == "" { 305 found = true 306 } else { 307 for _, repo := range repos { 308 if urlutil.Equal(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { 309 found = true 310 } 311 } 312 } 313 if !found { 314 missing = append(missing, dd.Repository) 315 } 316 } 317 if len(missing) > 0 { 318 return fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", ")) 319 } 320 return nil 321 } 322 323 // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file. 324 func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, error) { 325 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 326 if err != nil { 327 return nil, err 328 } 329 repos := rf.Repositories 330 331 reposMap := make(map[string]string) 332 333 // Verify that all repositories referenced in the deps are actually known 334 // by Helm. 335 missing := []string{} 336 for _, dd := range deps { 337 // if dep chart is from local path, verify the path is valid 338 if strings.HasPrefix(dd.Repository, "file://") { 339 if _, err := resolver.GetLocalPath(dd.Repository, m.ChartPath); err != nil { 340 return nil, err 341 } 342 343 if m.Debug { 344 fmt.Fprintf(m.Out, "Repository from local path: %s\n", dd.Repository) 345 } 346 reposMap[dd.Name] = dd.Repository 347 continue 348 } 349 350 found := false 351 352 for _, repo := range repos { 353 if (strings.HasPrefix(dd.Repository, "@") && strings.TrimPrefix(dd.Repository, "@") == repo.Name) || 354 (strings.HasPrefix(dd.Repository, "alias:") && strings.TrimPrefix(dd.Repository, "alias:") == repo.Name) { 355 found = true 356 dd.Repository = repo.URL 357 reposMap[dd.Name] = repo.Name 358 break 359 } else if urlutil.Equal(repo.URL, dd.Repository) { 360 found = true 361 reposMap[dd.Name] = repo.Name 362 break 363 } 364 } 365 if !found { 366 missing = append(missing, dd.Repository) 367 } 368 } 369 if len(missing) > 0 { 370 return nil, fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", ")) 371 } 372 return reposMap, nil 373 } 374 375 // UpdateRepositories updates all of the local repos to the latest. 376 func (m *Manager) UpdateRepositories() error { 377 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 378 if err != nil { 379 return err 380 } 381 repos := rf.Repositories 382 if len(repos) > 0 { 383 // This prints warnings straight to out. 384 if err := m.parallelRepoUpdate(repos); err != nil { 385 return err 386 } 387 } 388 return nil 389 } 390 391 func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { 392 out := m.Out 393 fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") 394 var wg sync.WaitGroup 395 for _, c := range repos { 396 r, err := repo.NewChartRepository(c, m.Getters) 397 if err != nil { 398 return err 399 } 400 wg.Add(1) 401 go func(r *repo.ChartRepository) { 402 if err := r.DownloadIndexFile(m.HelmHome.Cache()); err != nil { 403 fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", r.Config.Name, r.Config.URL, err) 404 } else { 405 fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", r.Config.Name) 406 } 407 wg.Done() 408 }(r) 409 } 410 wg.Wait() 411 fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈") 412 return nil 413 } 414 415 // findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. 416 // 417 // 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the 418 // newest version will be returned. 419 // 420 // repoURL is the repository to search 421 // 422 // If it finds a URL that is "relative", it will prepend the repoURL. 423 func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (string, error) { 424 for _, cr := range repos { 425 if urlutil.Equal(repoURL, cr.Config.URL) { 426 entry, err := findEntryByName(name, cr) 427 if err != nil { 428 return "", err 429 } 430 ve, err := findVersionedEntry(version, entry) 431 if err != nil { 432 return "", err 433 } 434 435 return normalizeURL(repoURL, ve.URLs[0]) 436 } 437 } 438 return "", fmt.Errorf("chart %s not found in %s", name, repoURL) 439 } 440 441 // findEntryByName finds an entry in the chart repository whose name matches the given name. 442 // 443 // It returns the ChartVersions for that entry. 444 func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) { 445 for ename, entry := range cr.IndexFile.Entries { 446 if ename == name { 447 return entry, nil 448 } 449 } 450 return nil, errors.New("entry not found") 451 } 452 453 // findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints. 454 // 455 // If version is empty, the first chart found is returned. 456 func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) { 457 for _, verEntry := range vers { 458 if len(verEntry.URLs) == 0 { 459 // Not a legit entry. 460 continue 461 } 462 463 if version == "" || versionEquals(version, verEntry.Version) { 464 return verEntry, nil 465 } 466 } 467 return nil, errors.New("no matching version") 468 } 469 470 func versionEquals(v1, v2 string) bool { 471 sv1, err := semver.NewVersion(v1) 472 if err != nil { 473 // Fallback to string comparison. 474 return v1 == v2 475 } 476 sv2, err := semver.NewVersion(v2) 477 if err != nil { 478 return false 479 } 480 return sv1.Equal(sv2) 481 } 482 483 func normalizeURL(baseURL, urlOrPath string) (string, error) { 484 u, err := url.Parse(urlOrPath) 485 if err != nil { 486 return urlOrPath, err 487 } 488 if u.IsAbs() { 489 return u.String(), nil 490 } 491 u2, err := url.Parse(baseURL) 492 if err != nil { 493 return urlOrPath, fmt.Errorf("Base URL failed to parse: %s", err) 494 } 495 496 u2.Path = path.Join(u2.Path, urlOrPath) 497 return u2.String(), nil 498 } 499 500 // loadChartRepositories reads the repositories.yaml, and then builds a map of 501 // ChartRepositories. 502 // 503 // The key is the local name (which is only present in the repositories.yaml). 504 func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) { 505 indices := map[string]*repo.ChartRepository{} 506 repoyaml := m.HelmHome.RepositoryFile() 507 508 // Load repositories.yaml file 509 rf, err := repo.LoadRepositoriesFile(repoyaml) 510 if err != nil { 511 return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err) 512 } 513 514 for _, re := range rf.Repositories { 515 lname := re.Name 516 cacheindex := m.HelmHome.CacheIndex(lname) 517 index, err := repo.LoadIndexFile(cacheindex) 518 if err != nil { 519 return indices, err 520 } 521 522 // TODO: use constructor 523 cr := &repo.ChartRepository{ 524 Config: re, 525 IndexFile: index, 526 } 527 indices[lname] = cr 528 } 529 return indices, nil 530 } 531 532 // writeLock writes a lockfile to disk 533 func writeLock(chartpath string, lock *chartutil.RequirementsLock) error { 534 data, err := yaml.Marshal(lock) 535 if err != nil { 536 return err 537 } 538 dest := filepath.Join(chartpath, "requirements.lock") 539 return ioutil.WriteFile(dest, data, 0644) 540 } 541 542 // archive a dep chart from local directory and save it into charts/ 543 func tarFromLocalDir(chartpath string, name string, repo string, version string) (string, error) { 544 destPath := filepath.Join(chartpath, "charts") 545 546 if !strings.HasPrefix(repo, "file://") { 547 return "", fmt.Errorf("wrong format: chart %s repository %s", name, repo) 548 } 549 550 origPath, err := resolver.GetLocalPath(repo, chartpath) 551 if err != nil { 552 return "", err 553 } 554 555 ch, err := chartutil.LoadDir(origPath) 556 if err != nil { 557 return "", err 558 } 559 560 constraint, err := semver.NewConstraint(version) 561 if err != nil { 562 return "", fmt.Errorf("dependency %s has an invalid version/constraint format: %s", name, err) 563 } 564 565 v, err := semver.NewVersion(ch.Metadata.Version) 566 if err != nil { 567 return "", err 568 } 569 570 if constraint.Check(v) { 571 _, err = chartutil.Save(ch, destPath) 572 return ch.Metadata.Version, err 573 } 574 575 return "", fmt.Errorf("can't get a valid version for dependency %s", name) 576 }