github.com/felipejfc/helm@v2.1.2+incompatible/cmd/helm/downloader/manager.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors All rights reserved. 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 */ 15 16 package downloader 17 18 import ( 19 "errors" 20 "fmt" 21 "io" 22 "io/ioutil" 23 "net/url" 24 "os" 25 "path" 26 "path/filepath" 27 "strings" 28 "sync" 29 30 "github.com/Masterminds/semver" 31 "github.com/ghodss/yaml" 32 33 "k8s.io/helm/cmd/helm/helmpath" 34 "k8s.io/helm/cmd/helm/resolver" 35 "k8s.io/helm/pkg/chartutil" 36 "k8s.io/helm/pkg/proto/hapi/chart" 37 "k8s.io/helm/pkg/repo" 38 ) 39 40 // Manager handles the lifecycle of fetching, resolving, and storing dependencies. 41 type Manager struct { 42 // Out is used to print warnings and notifications. 43 Out io.Writer 44 // ChartPath is the path to the unpacked base chart upon which this operates. 45 ChartPath string 46 // HelmHome is the $HELM_HOME directory 47 HelmHome helmpath.Home 48 // Verification indicates whether the chart should be verified. 49 Verify VerificationStrategy 50 // Keyring is the key ring file. 51 Keyring string 52 } 53 54 // Build rebuilds a local charts directory from a lockfile. 55 // 56 // If the lockfile is not present, this will run a Manager.Update() 57 func (m *Manager) Build() error { 58 c, err := m.loadChartDir() 59 if err != nil { 60 return err 61 } 62 63 // If a lock file is found, run a build from that. Otherwise, just do 64 // an update. 65 lock, err := chartutil.LoadRequirementsLock(c) 66 if err != nil { 67 return m.Update() 68 } 69 70 // A lock must accompany a requirements.yaml file. 71 req, err := chartutil.LoadRequirements(c) 72 if err != nil { 73 return fmt.Errorf("requirements.yaml cannot be opened: %s", err) 74 } 75 if sum, err := resolver.HashReq(req); err != nil || sum != lock.Digest { 76 return fmt.Errorf("requirements.lock is out of sync with requirements.yaml") 77 } 78 79 // Check that all of the repos we're dependent on actually exist. 80 if err := m.hasAllRepos(lock.Dependencies); err != nil { 81 return err 82 } 83 84 // For each repo in the file, update the cached copy of that repo 85 if err := m.UpdateRepositories(); err != nil { 86 return err 87 } 88 89 // Now we need to fetch every package here into charts/ 90 if err := m.downloadAll(lock.Dependencies); err != nil { 91 return err 92 } 93 94 return nil 95 } 96 97 // Update updates a local charts directory. 98 // 99 // It first reads the requirements.yaml file, and then attempts to 100 // negotiate versions based on that. It will download the versions 101 // from remote chart repositories. 102 func (m *Manager) Update() error { 103 c, err := m.loadChartDir() 104 if err != nil { 105 return err 106 } 107 108 // If no requirements file is found, we consider this a successful 109 // completion. 110 req, err := chartutil.LoadRequirements(c) 111 if err != nil { 112 if err == chartutil.ErrRequirementsNotFound { 113 fmt.Fprintf(m.Out, "No requirements found in %s/charts.\n", m.ChartPath) 114 return nil 115 } 116 return err 117 } 118 119 // Check that all of the repos we're dependent on actually exist and 120 // the repo index names. 121 repoNames, err := m.getRepoNames(req.Dependencies) 122 if err != nil { 123 return err 124 } 125 126 // For each repo in the file, update the cached copy of that repo 127 if err := m.UpdateRepositories(); err != nil { 128 return err 129 } 130 131 // Now we need to find out which version of a chart best satisfies the 132 // requirements the requirements.yaml 133 lock, err := m.resolve(req, repoNames) 134 if err != nil { 135 return err 136 } 137 138 // Now we need to fetch every package here into charts/ 139 if err := m.downloadAll(lock.Dependencies); err != nil { 140 return err 141 } 142 143 // If the lock file hasn't changed, don't write a new one. 144 oldLock, err := chartutil.LoadRequirementsLock(c) 145 if err == nil && oldLock.Digest == lock.Digest { 146 return nil 147 } 148 149 // Finally, we need to write the lockfile. 150 return writeLock(m.ChartPath, lock) 151 } 152 153 func (m *Manager) loadChartDir() (*chart.Chart, error) { 154 if fi, err := os.Stat(m.ChartPath); err != nil { 155 return nil, fmt.Errorf("could not find %s: %s", m.ChartPath, err) 156 } else if !fi.IsDir() { 157 return nil, errors.New("only unpacked charts can be updated") 158 } 159 return chartutil.LoadDir(m.ChartPath) 160 } 161 162 // resolve takes a list of requirements and translates them into an exact version to download. 163 // 164 // This returns a lock file, which has all of the requirements normalized to a specific version. 165 func (m *Manager) resolve(req *chartutil.Requirements, repoNames map[string]string) (*chartutil.RequirementsLock, error) { 166 res := resolver.New(m.ChartPath, m.HelmHome) 167 return res.Resolve(req, repoNames) 168 } 169 170 // downloadAll takes a list of dependencies and downloads them into charts/ 171 func (m *Manager) downloadAll(deps []*chartutil.Dependency) error { 172 repos, err := m.loadChartRepositories() 173 if err != nil { 174 return err 175 } 176 177 dl := ChartDownloader{ 178 Out: m.Out, 179 Verify: m.Verify, 180 Keyring: m.Keyring, 181 HelmHome: m.HelmHome, 182 } 183 184 destPath := filepath.Join(m.ChartPath, "charts") 185 186 // Create 'charts' directory if it doesn't already exist. 187 if fi, err := os.Stat(destPath); err != nil { 188 if err := os.MkdirAll(destPath, 0755); err != nil { 189 return err 190 } 191 } else if !fi.IsDir() { 192 return fmt.Errorf("%q is not a directory", destPath) 193 } 194 195 fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) 196 for _, dep := range deps { 197 fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) 198 199 // Any failure to resolve/download a chart should fail: 200 // https://github.com/kubernetes/helm/issues/1439 201 churl, err := findChartURL(dep.Name, dep.Version, dep.Repository, repos) 202 if err != nil { 203 return fmt.Errorf("could not find %s: %s", churl, err) 204 } 205 206 if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil { 207 return fmt.Errorf("could not download %s: %s", churl, err) 208 } 209 } 210 return nil 211 } 212 213 // hasAllRepos ensures that all of the referenced deps are in the local repo cache. 214 func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { 215 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 216 if err != nil { 217 return err 218 } 219 repos := rf.Repositories 220 // Verify that all repositories referenced in the deps are actually known 221 // by Helm. 222 missing := []string{} 223 for _, dd := range deps { 224 found := false 225 if dd.Repository == "" { 226 found = true 227 } else { 228 for _, repo := range repos { 229 if urlsAreEqual(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { 230 found = true 231 } 232 } 233 } 234 if !found { 235 missing = append(missing, dd.Repository) 236 } 237 } 238 if len(missing) > 0 { 239 return fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", ")) 240 } 241 return nil 242 } 243 244 // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file. 245 func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, error) { 246 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 247 if err != nil { 248 return nil, err 249 } 250 repos := rf.Repositories 251 252 reposMap := make(map[string]string) 253 254 // Verify that all repositories referenced in the deps are actually known 255 // by Helm. 256 missing := []string{} 257 for _, dd := range deps { 258 found := false 259 260 for _, repo := range repos { 261 if urlsAreEqual(repo.URL, dd.Repository) { 262 found = true 263 reposMap[dd.Name] = repo.Name 264 break 265 } 266 } 267 if !found { 268 missing = append(missing, dd.Repository) 269 } 270 } 271 if len(missing) > 0 { 272 return nil, fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", ")) 273 } 274 return reposMap, nil 275 } 276 277 // UpdateRepositories updates all of the local repos to the latest. 278 func (m *Manager) UpdateRepositories() error { 279 rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) 280 if err != nil { 281 return err 282 } 283 repos := rf.Repositories 284 if len(repos) > 0 { 285 // This prints warnings straight to out. 286 m.parallelRepoUpdate(repos) 287 } 288 return nil 289 } 290 291 func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) { 292 out := m.Out 293 fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") 294 var wg sync.WaitGroup 295 for _, re := range repos { 296 wg.Add(1) 297 go func(n, u string) { 298 if err := repo.DownloadIndexFile(n, u, m.HelmHome.CacheIndex(n)); err != nil { 299 fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", n, u, err) 300 } else { 301 fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", n) 302 } 303 wg.Done() 304 }(re.Name, re.URL) 305 } 306 wg.Wait() 307 fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈") 308 } 309 310 // urlsAreEqual normalizes two URLs and then compares for equality. 311 // 312 // TODO: This and the urlJoin functions should really be moved to a 'urlutil' package. 313 func urlsAreEqual(a, b string) bool { 314 au, err := url.Parse(a) 315 if err != nil { 316 a = filepath.Clean(a) 317 b = filepath.Clean(b) 318 // If urls are paths, return true only if they are an exact match 319 return a == b 320 } 321 bu, err := url.Parse(b) 322 if err != nil { 323 return false 324 } 325 326 for _, u := range []*url.URL{au, bu} { 327 if u.Path == "" { 328 u.Path = "/" 329 } 330 u.Path = filepath.Clean(u.Path) 331 } 332 return au.String() == bu.String() 333 } 334 335 // findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. 336 // 337 // 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the 338 // newest version will be returned. 339 // 340 // repoURL is the repository to search 341 // 342 // If it finds a URL that is "relative", it will prepend the repoURL. 343 func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (string, error) { 344 for _, cr := range repos { 345 if urlsAreEqual(repoURL, cr.URL) { 346 entry, err := findEntryByName(name, cr) 347 if err != nil { 348 return "", err 349 } 350 ve, err := findVersionedEntry(version, entry) 351 if err != nil { 352 return "", err 353 } 354 355 return normalizeURL(repoURL, ve.URLs[0]) 356 } 357 } 358 return "", fmt.Errorf("chart %s not found in %s", name, repoURL) 359 } 360 361 // findEntryByName finds an entry in the chart repository whose name matches the given name. 362 // 363 // It returns the ChartVersions for that entry. 364 func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) { 365 for ename, entry := range cr.IndexFile.Entries { 366 if ename == name { 367 return entry, nil 368 } 369 } 370 return nil, errors.New("entry not found") 371 } 372 373 // findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints. 374 // 375 // If version is empty, the first chart found is returned. 376 func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) { 377 for _, verEntry := range vers { 378 if len(verEntry.URLs) == 0 { 379 // Not a legit entry. 380 continue 381 } 382 383 if version == "" || versionEquals(version, verEntry.Version) { 384 return verEntry, nil 385 } 386 } 387 return nil, errors.New("no matching version") 388 } 389 390 func versionEquals(v1, v2 string) bool { 391 sv1, err := semver.NewVersion(v1) 392 if err != nil { 393 // Fallback to string comparison. 394 return v1 == v2 395 } 396 sv2, err := semver.NewVersion(v2) 397 if err != nil { 398 return false 399 } 400 return sv1.Equal(sv2) 401 } 402 403 func normalizeURL(baseURL, urlOrPath string) (string, error) { 404 u, err := url.Parse(urlOrPath) 405 if err != nil { 406 return urlOrPath, err 407 } 408 if u.IsAbs() { 409 return u.String(), nil 410 } 411 u2, err := url.Parse(baseURL) 412 if err != nil { 413 return urlOrPath, fmt.Errorf("Base URL failed to parse: %s", err) 414 } 415 416 u2.Path = path.Join(u2.Path, urlOrPath) 417 return u2.String(), nil 418 } 419 420 // loadChartRepositories reads the repositories.yaml, and then builds a map of 421 // ChartRepositories. 422 // 423 // The key is the local name (which is only present in the repositories.yaml). 424 func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) { 425 indices := map[string]*repo.ChartRepository{} 426 repoyaml := m.HelmHome.RepositoryFile() 427 428 // Load repositories.yaml file 429 rf, err := repo.LoadRepositoriesFile(repoyaml) 430 if err != nil { 431 return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err) 432 } 433 434 for _, re := range rf.Repositories { 435 lname := re.Name 436 cacheindex := m.HelmHome.CacheIndex(lname) 437 index, err := repo.LoadIndexFile(cacheindex) 438 if err != nil { 439 return indices, err 440 } 441 442 cr := &repo.ChartRepository{ 443 URL: re.URL, 444 IndexFile: index, 445 } 446 indices[lname] = cr 447 } 448 return indices, nil 449 } 450 451 // writeLock writes a lockfile to disk 452 func writeLock(chartpath string, lock *chartutil.RequirementsLock) error { 453 data, err := yaml.Marshal(lock) 454 if err != nil { 455 return err 456 } 457 dest := filepath.Join(chartpath, "requirements.lock") 458 return ioutil.WriteFile(dest, data, 0644) 459 }