github.com/felipejfc/helm@v2.1.2+incompatible/cmd/helm/downloader/manager.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors All rights reserved.
     3  Licensed under the Apache License, Version 2.0 (the "License");
     4  you may not use this file except in compliance with the License.
     5  You may obtain a copy of the License at
     6  
     7  http://www.apache.org/licenses/LICENSE-2.0
     8  
     9  Unless required by applicable law or agreed to in writing, software
    10  distributed under the License is distributed on an "AS IS" BASIS,
    11  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  See the License for the specific language governing permissions and
    13  limitations under the License.
    14  */
    15  
    16  package downloader
    17  
    18  import (
    19  	"errors"
    20  	"fmt"
    21  	"io"
    22  	"io/ioutil"
    23  	"net/url"
    24  	"os"
    25  	"path"
    26  	"path/filepath"
    27  	"strings"
    28  	"sync"
    29  
    30  	"github.com/Masterminds/semver"
    31  	"github.com/ghodss/yaml"
    32  
    33  	"k8s.io/helm/cmd/helm/helmpath"
    34  	"k8s.io/helm/cmd/helm/resolver"
    35  	"k8s.io/helm/pkg/chartutil"
    36  	"k8s.io/helm/pkg/proto/hapi/chart"
    37  	"k8s.io/helm/pkg/repo"
    38  )
    39  
    40  // Manager handles the lifecycle of fetching, resolving, and storing dependencies.
    41  type Manager struct {
    42  	// Out is used to print warnings and notifications.
    43  	Out io.Writer
    44  	// ChartPath is the path to the unpacked base chart upon which this operates.
    45  	ChartPath string
    46  	// HelmHome is the $HELM_HOME directory
    47  	HelmHome helmpath.Home
    48  	// Verification indicates whether the chart should be verified.
    49  	Verify VerificationStrategy
    50  	// Keyring is the key ring file.
    51  	Keyring string
    52  }
    53  
    54  // Build rebuilds a local charts directory from a lockfile.
    55  //
    56  // If the lockfile is not present, this will run a Manager.Update()
    57  func (m *Manager) Build() error {
    58  	c, err := m.loadChartDir()
    59  	if err != nil {
    60  		return err
    61  	}
    62  
    63  	// If a lock file is found, run a build from that. Otherwise, just do
    64  	// an update.
    65  	lock, err := chartutil.LoadRequirementsLock(c)
    66  	if err != nil {
    67  		return m.Update()
    68  	}
    69  
    70  	// A lock must accompany a requirements.yaml file.
    71  	req, err := chartutil.LoadRequirements(c)
    72  	if err != nil {
    73  		return fmt.Errorf("requirements.yaml cannot be opened: %s", err)
    74  	}
    75  	if sum, err := resolver.HashReq(req); err != nil || sum != lock.Digest {
    76  		return fmt.Errorf("requirements.lock is out of sync with requirements.yaml")
    77  	}
    78  
    79  	// Check that all of the repos we're dependent on actually exist.
    80  	if err := m.hasAllRepos(lock.Dependencies); err != nil {
    81  		return err
    82  	}
    83  
    84  	// For each repo in the file, update the cached copy of that repo
    85  	if err := m.UpdateRepositories(); err != nil {
    86  		return err
    87  	}
    88  
    89  	// Now we need to fetch every package here into charts/
    90  	if err := m.downloadAll(lock.Dependencies); err != nil {
    91  		return err
    92  	}
    93  
    94  	return nil
    95  }
    96  
    97  // Update updates a local charts directory.
    98  //
    99  // It first reads the requirements.yaml file, and then attempts to
   100  // negotiate versions based on that. It will download the versions
   101  // from remote chart repositories.
   102  func (m *Manager) Update() error {
   103  	c, err := m.loadChartDir()
   104  	if err != nil {
   105  		return err
   106  	}
   107  
   108  	// If no requirements file is found, we consider this a successful
   109  	// completion.
   110  	req, err := chartutil.LoadRequirements(c)
   111  	if err != nil {
   112  		if err == chartutil.ErrRequirementsNotFound {
   113  			fmt.Fprintf(m.Out, "No requirements found in %s/charts.\n", m.ChartPath)
   114  			return nil
   115  		}
   116  		return err
   117  	}
   118  
   119  	// Check that all of the repos we're dependent on actually exist and
   120  	// the repo index names.
   121  	repoNames, err := m.getRepoNames(req.Dependencies)
   122  	if err != nil {
   123  		return err
   124  	}
   125  
   126  	// For each repo in the file, update the cached copy of that repo
   127  	if err := m.UpdateRepositories(); err != nil {
   128  		return err
   129  	}
   130  
   131  	// Now we need to find out which version of a chart best satisfies the
   132  	// requirements the requirements.yaml
   133  	lock, err := m.resolve(req, repoNames)
   134  	if err != nil {
   135  		return err
   136  	}
   137  
   138  	// Now we need to fetch every package here into charts/
   139  	if err := m.downloadAll(lock.Dependencies); err != nil {
   140  		return err
   141  	}
   142  
   143  	// If the lock file hasn't changed, don't write a new one.
   144  	oldLock, err := chartutil.LoadRequirementsLock(c)
   145  	if err == nil && oldLock.Digest == lock.Digest {
   146  		return nil
   147  	}
   148  
   149  	// Finally, we need to write the lockfile.
   150  	return writeLock(m.ChartPath, lock)
   151  }
   152  
   153  func (m *Manager) loadChartDir() (*chart.Chart, error) {
   154  	if fi, err := os.Stat(m.ChartPath); err != nil {
   155  		return nil, fmt.Errorf("could not find %s: %s", m.ChartPath, err)
   156  	} else if !fi.IsDir() {
   157  		return nil, errors.New("only unpacked charts can be updated")
   158  	}
   159  	return chartutil.LoadDir(m.ChartPath)
   160  }
   161  
   162  // resolve takes a list of requirements and translates them into an exact version to download.
   163  //
   164  // This returns a lock file, which has all of the requirements normalized to a specific version.
   165  func (m *Manager) resolve(req *chartutil.Requirements, repoNames map[string]string) (*chartutil.RequirementsLock, error) {
   166  	res := resolver.New(m.ChartPath, m.HelmHome)
   167  	return res.Resolve(req, repoNames)
   168  }
   169  
   170  // downloadAll takes a list of dependencies and downloads them into charts/
   171  func (m *Manager) downloadAll(deps []*chartutil.Dependency) error {
   172  	repos, err := m.loadChartRepositories()
   173  	if err != nil {
   174  		return err
   175  	}
   176  
   177  	dl := ChartDownloader{
   178  		Out:      m.Out,
   179  		Verify:   m.Verify,
   180  		Keyring:  m.Keyring,
   181  		HelmHome: m.HelmHome,
   182  	}
   183  
   184  	destPath := filepath.Join(m.ChartPath, "charts")
   185  
   186  	// Create 'charts' directory if it doesn't already exist.
   187  	if fi, err := os.Stat(destPath); err != nil {
   188  		if err := os.MkdirAll(destPath, 0755); err != nil {
   189  			return err
   190  		}
   191  	} else if !fi.IsDir() {
   192  		return fmt.Errorf("%q is not a directory", destPath)
   193  	}
   194  
   195  	fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps))
   196  	for _, dep := range deps {
   197  		fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository)
   198  
   199  		// Any failure to resolve/download a chart should fail:
   200  		// https://github.com/kubernetes/helm/issues/1439
   201  		churl, err := findChartURL(dep.Name, dep.Version, dep.Repository, repos)
   202  		if err != nil {
   203  			return fmt.Errorf("could not find %s: %s", churl, err)
   204  		}
   205  
   206  		if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil {
   207  			return fmt.Errorf("could not download %s: %s", churl, err)
   208  		}
   209  	}
   210  	return nil
   211  }
   212  
   213  // hasAllRepos ensures that all of the referenced deps are in the local repo cache.
   214  func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error {
   215  	rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile())
   216  	if err != nil {
   217  		return err
   218  	}
   219  	repos := rf.Repositories
   220  	// Verify that all repositories referenced in the deps are actually known
   221  	// by Helm.
   222  	missing := []string{}
   223  	for _, dd := range deps {
   224  		found := false
   225  		if dd.Repository == "" {
   226  			found = true
   227  		} else {
   228  			for _, repo := range repos {
   229  				if urlsAreEqual(repo.URL, strings.TrimSuffix(dd.Repository, "/")) {
   230  					found = true
   231  				}
   232  			}
   233  		}
   234  		if !found {
   235  			missing = append(missing, dd.Repository)
   236  		}
   237  	}
   238  	if len(missing) > 0 {
   239  		return fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", "))
   240  	}
   241  	return nil
   242  }
   243  
   244  // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file.
   245  func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, error) {
   246  	rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile())
   247  	if err != nil {
   248  		return nil, err
   249  	}
   250  	repos := rf.Repositories
   251  
   252  	reposMap := make(map[string]string)
   253  
   254  	// Verify that all repositories referenced in the deps are actually known
   255  	// by Helm.
   256  	missing := []string{}
   257  	for _, dd := range deps {
   258  		found := false
   259  
   260  		for _, repo := range repos {
   261  			if urlsAreEqual(repo.URL, dd.Repository) {
   262  				found = true
   263  				reposMap[dd.Name] = repo.Name
   264  				break
   265  			}
   266  		}
   267  		if !found {
   268  			missing = append(missing, dd.Repository)
   269  		}
   270  	}
   271  	if len(missing) > 0 {
   272  		return nil, fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", "))
   273  	}
   274  	return reposMap, nil
   275  }
   276  
   277  // UpdateRepositories updates all of the local repos to the latest.
   278  func (m *Manager) UpdateRepositories() error {
   279  	rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile())
   280  	if err != nil {
   281  		return err
   282  	}
   283  	repos := rf.Repositories
   284  	if len(repos) > 0 {
   285  		// This prints warnings straight to out.
   286  		m.parallelRepoUpdate(repos)
   287  	}
   288  	return nil
   289  }
   290  
   291  func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) {
   292  	out := m.Out
   293  	fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...")
   294  	var wg sync.WaitGroup
   295  	for _, re := range repos {
   296  		wg.Add(1)
   297  		go func(n, u string) {
   298  			if err := repo.DownloadIndexFile(n, u, m.HelmHome.CacheIndex(n)); err != nil {
   299  				fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", n, u, err)
   300  			} else {
   301  				fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", n)
   302  			}
   303  			wg.Done()
   304  		}(re.Name, re.URL)
   305  	}
   306  	wg.Wait()
   307  	fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈")
   308  }
   309  
   310  // urlsAreEqual normalizes two URLs and then compares for equality.
   311  //
   312  // TODO: This and the urlJoin functions should really be moved to a 'urlutil' package.
   313  func urlsAreEqual(a, b string) bool {
   314  	au, err := url.Parse(a)
   315  	if err != nil {
   316  		a = filepath.Clean(a)
   317  		b = filepath.Clean(b)
   318  		// If urls are paths, return true only if they are an exact match
   319  		return a == b
   320  	}
   321  	bu, err := url.Parse(b)
   322  	if err != nil {
   323  		return false
   324  	}
   325  
   326  	for _, u := range []*url.URL{au, bu} {
   327  		if u.Path == "" {
   328  			u.Path = "/"
   329  		}
   330  		u.Path = filepath.Clean(u.Path)
   331  	}
   332  	return au.String() == bu.String()
   333  }
   334  
   335  // findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified.
   336  //
   337  // 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the
   338  // newest version will be returned.
   339  //
   340  // repoURL is the repository to search
   341  //
   342  // If it finds a URL that is "relative", it will prepend the repoURL.
   343  func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (string, error) {
   344  	for _, cr := range repos {
   345  		if urlsAreEqual(repoURL, cr.URL) {
   346  			entry, err := findEntryByName(name, cr)
   347  			if err != nil {
   348  				return "", err
   349  			}
   350  			ve, err := findVersionedEntry(version, entry)
   351  			if err != nil {
   352  				return "", err
   353  			}
   354  
   355  			return normalizeURL(repoURL, ve.URLs[0])
   356  		}
   357  	}
   358  	return "", fmt.Errorf("chart %s not found in %s", name, repoURL)
   359  }
   360  
   361  // findEntryByName finds an entry in the chart repository whose name matches the given name.
   362  //
   363  // It returns the ChartVersions for that entry.
   364  func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) {
   365  	for ename, entry := range cr.IndexFile.Entries {
   366  		if ename == name {
   367  			return entry, nil
   368  		}
   369  	}
   370  	return nil, errors.New("entry not found")
   371  }
   372  
   373  // findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints.
   374  //
   375  // If version is empty, the first chart found is returned.
   376  func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) {
   377  	for _, verEntry := range vers {
   378  		if len(verEntry.URLs) == 0 {
   379  			// Not a legit entry.
   380  			continue
   381  		}
   382  
   383  		if version == "" || versionEquals(version, verEntry.Version) {
   384  			return verEntry, nil
   385  		}
   386  	}
   387  	return nil, errors.New("no matching version")
   388  }
   389  
   390  func versionEquals(v1, v2 string) bool {
   391  	sv1, err := semver.NewVersion(v1)
   392  	if err != nil {
   393  		// Fallback to string comparison.
   394  		return v1 == v2
   395  	}
   396  	sv2, err := semver.NewVersion(v2)
   397  	if err != nil {
   398  		return false
   399  	}
   400  	return sv1.Equal(sv2)
   401  }
   402  
   403  func normalizeURL(baseURL, urlOrPath string) (string, error) {
   404  	u, err := url.Parse(urlOrPath)
   405  	if err != nil {
   406  		return urlOrPath, err
   407  	}
   408  	if u.IsAbs() {
   409  		return u.String(), nil
   410  	}
   411  	u2, err := url.Parse(baseURL)
   412  	if err != nil {
   413  		return urlOrPath, fmt.Errorf("Base URL failed to parse: %s", err)
   414  	}
   415  
   416  	u2.Path = path.Join(u2.Path, urlOrPath)
   417  	return u2.String(), nil
   418  }
   419  
   420  // loadChartRepositories reads the repositories.yaml, and then builds a map of
   421  // ChartRepositories.
   422  //
   423  // The key is the local name (which is only present in the repositories.yaml).
   424  func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) {
   425  	indices := map[string]*repo.ChartRepository{}
   426  	repoyaml := m.HelmHome.RepositoryFile()
   427  
   428  	// Load repositories.yaml file
   429  	rf, err := repo.LoadRepositoriesFile(repoyaml)
   430  	if err != nil {
   431  		return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err)
   432  	}
   433  
   434  	for _, re := range rf.Repositories {
   435  		lname := re.Name
   436  		cacheindex := m.HelmHome.CacheIndex(lname)
   437  		index, err := repo.LoadIndexFile(cacheindex)
   438  		if err != nil {
   439  			return indices, err
   440  		}
   441  
   442  		cr := &repo.ChartRepository{
   443  			URL:       re.URL,
   444  			IndexFile: index,
   445  		}
   446  		indices[lname] = cr
   447  	}
   448  	return indices, nil
   449  }
   450  
   451  // writeLock writes a lockfile to disk
   452  func writeLock(chartpath string, lock *chartutil.RequirementsLock) error {
   453  	data, err := yaml.Marshal(lock)
   454  	if err != nil {
   455  		return err
   456  	}
   457  	dest := filepath.Join(chartpath, "requirements.lock")
   458  	return ioutil.WriteFile(dest, data, 0644)
   459  }