github.com/Pankov404/juju@v0.0.0-20150703034450-be266991dceb/environs/sync/sync.go (about)

     1  // Copyright 2012, 2013 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package sync
     5  
     6  import (
     7  	"bytes"
     8  	"fmt"
     9  	"io"
    10  	"io/ioutil"
    11  	"os"
    12  	"path/filepath"
    13  
    14  	"github.com/juju/errors"
    15  	"github.com/juju/loggo"
    16  	"github.com/juju/utils"
    17  
    18  	"github.com/juju/juju/environs/filestorage"
    19  	"github.com/juju/juju/environs/simplestreams"
    20  	"github.com/juju/juju/environs/storage"
    21  	envtools "github.com/juju/juju/environs/tools"
    22  	coretools "github.com/juju/juju/tools"
    23  	"github.com/juju/juju/version"
    24  )
    25  
    26  var logger = loggo.GetLogger("juju.environs.sync")
    27  
    28  // SyncContext describes the context for tool synchronization.
    29  type SyncContext struct {
    30  	// TargetToolsFinder is a ToolsFinder provided to find existing
    31  	// tools in the target destination.
    32  	TargetToolsFinder ToolsFinder
    33  
    34  	// TargetToolsUploader is a ToolsUploader provided to upload
    35  	// tools to the target destination.
    36  	TargetToolsUploader ToolsUploader
    37  
    38  	// AllVersions controls the copy of all versions, not only the latest.
    39  	AllVersions bool
    40  
    41  	// Copy tools with major version, if MajorVersion > 0.
    42  	MajorVersion int
    43  
    44  	// Copy tools with minor version, if MinorVersion > 0.
    45  	MinorVersion int
    46  
    47  	// DryRun controls that nothing is copied. Instead it's logged
    48  	// what would be coppied.
    49  	DryRun bool
    50  
    51  	// Stream specifies the simplestreams stream to use (defaults to "Released").
    52  	Stream string
    53  
    54  	// Source, if non-empty, specifies a directory in the local file system
    55  	// to use as a source.
    56  	Source string
    57  }
    58  
    59  // ToolsFinder provides an interface for finding tools of a specified version.
    60  type ToolsFinder interface {
    61  	// FindTools returns a list of tools with the specified major version in the specified stream.
    62  	FindTools(major int, stream string) (coretools.List, error)
    63  }
    64  
    65  // ToolsUploader provides an interface for uploading tools and associated
    66  // metadata.
    67  type ToolsUploader interface {
    68  	// UploadTools uploads the tools with the specified version and tarball contents.
    69  	UploadTools(toolsDir, stream string, tools *coretools.Tools, data []byte) error
    70  }
    71  
    72  // SyncTools copies the Juju tools tarball from the official bucket
    73  // or a specified source directory into the user's environment.
    74  func SyncTools(syncContext *SyncContext) error {
    75  	sourceDataSource, err := selectSourceDatasource(syncContext)
    76  	if err != nil {
    77  		return err
    78  	}
    79  
    80  	logger.Infof("listing available tools")
    81  	if syncContext.MajorVersion == 0 && syncContext.MinorVersion == 0 {
    82  		syncContext.MajorVersion = version.Current.Major
    83  		syncContext.MinorVersion = -1
    84  		if !syncContext.AllVersions {
    85  			syncContext.MinorVersion = version.Current.Minor
    86  		}
    87  	}
    88  
    89  	toolsDir := syncContext.Stream
    90  	// If no stream has been specified, assume "released" for non-devel versions of Juju.
    91  	if syncContext.Stream == "" {
    92  		// We now store the tools in a directory named after their stream, but the
    93  		// legacy behaviour is to store all tools in a single "releases" directory.
    94  		toolsDir = envtools.LegacyReleaseDirectory
    95  		if !version.Current.IsDev() {
    96  			syncContext.Stream = envtools.ReleasedStream
    97  		} else {
    98  			syncContext.Stream = envtools.TestingStream
    99  		}
   100  	}
   101  	sourceTools, err := envtools.FindToolsForCloud(
   102  		[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
   103  		syncContext.Stream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{})
   104  	// For backwards compatibility with cloud storage, if there are no tools in the specified stream,
   105  	// double check the release stream.
   106  	// TODO - remove this when we no longer need to support cloud storage upgrades.
   107  	if err == envtools.ErrNoTools {
   108  		sourceTools, err = envtools.FindToolsForCloud(
   109  			[]simplestreams.DataSource{sourceDataSource}, simplestreams.CloudSpec{},
   110  			envtools.ReleasedStream, syncContext.MajorVersion, syncContext.MinorVersion, coretools.Filter{})
   111  	}
   112  	if err != nil {
   113  		return err
   114  	}
   115  
   116  	logger.Infof("found %d tools", len(sourceTools))
   117  	if !syncContext.AllVersions {
   118  		var latest version.Number
   119  		latest, sourceTools = sourceTools.Newest()
   120  		logger.Infof("found %d recent tools (version %s)", len(sourceTools), latest)
   121  	}
   122  	for _, tool := range sourceTools {
   123  		logger.Debugf("found source tool: %v", tool)
   124  	}
   125  
   126  	logger.Infof("listing target tools storage")
   127  	targetTools, err := syncContext.TargetToolsFinder.FindTools(syncContext.MajorVersion, syncContext.Stream)
   128  	switch err {
   129  	case nil, coretools.ErrNoMatches, envtools.ErrNoTools:
   130  	default:
   131  		return err
   132  	}
   133  	for _, tool := range targetTools {
   134  		logger.Debugf("found target tool: %v", tool)
   135  	}
   136  
   137  	missing := sourceTools.Exclude(targetTools)
   138  	logger.Infof("found %d tools in target; %d tools to be copied", len(targetTools), len(missing))
   139  	if syncContext.DryRun {
   140  		for _, tools := range missing {
   141  			logger.Infof("copying %s from %s", tools.Version, tools.URL)
   142  		}
   143  		return nil
   144  	}
   145  
   146  	err = copyTools(toolsDir, syncContext.Stream, missing, syncContext.TargetToolsUploader)
   147  	if err != nil {
   148  		return err
   149  	}
   150  	logger.Infof("copied %d tools", len(missing))
   151  	return nil
   152  }
   153  
   154  // selectSourceDatasource returns a storage reader based on the source setting.
   155  func selectSourceDatasource(syncContext *SyncContext) (simplestreams.DataSource, error) {
   156  	source := syncContext.Source
   157  	if source == "" {
   158  		source = envtools.DefaultBaseURL
   159  	}
   160  	sourceURL, err := envtools.ToolsURL(source)
   161  	if err != nil {
   162  		return nil, err
   163  	}
   164  	logger.Infof("using sync tools source: %v", sourceURL)
   165  	return simplestreams.NewURLDataSource("sync tools source", sourceURL, utils.VerifySSLHostnames), nil
   166  }
   167  
   168  // copyTools copies a set of tools from the source to the target.
   169  func copyTools(toolsDir, stream string, tools []*coretools.Tools, u ToolsUploader) error {
   170  	for _, tool := range tools {
   171  		logger.Infof("copying %s from %s", tool.Version, tool.URL)
   172  		if err := copyOneToolsPackage(toolsDir, stream, tool, u); err != nil {
   173  			return err
   174  		}
   175  	}
   176  	return nil
   177  }
   178  
   179  // copyOneToolsPackage copies one tool from the source to the target.
   180  func copyOneToolsPackage(toolsDir, stream string, tools *coretools.Tools, u ToolsUploader) error {
   181  	toolsName := envtools.StorageName(tools.Version, toolsDir)
   182  	logger.Infof("downloading %q %v (%v)", stream, toolsName, tools.URL)
   183  	resp, err := utils.GetValidatingHTTPClient().Get(tools.URL)
   184  	if err != nil {
   185  		return err
   186  	}
   187  	defer resp.Body.Close()
   188  	// Verify SHA-256 hash.
   189  	var buf bytes.Buffer
   190  	sha256, size, err := utils.ReadSHA256(io.TeeReader(resp.Body, &buf))
   191  	if err != nil {
   192  		return err
   193  	}
   194  	if tools.SHA256 == "" {
   195  		logger.Warningf("no SHA-256 hash for %v", tools.SHA256)
   196  	} else if sha256 != tools.SHA256 {
   197  		return errors.Errorf("SHA-256 hash mismatch (%v/%v)", sha256, tools.SHA256)
   198  	}
   199  	sizeInKB := (size + 512) / 1024
   200  	logger.Infof("uploading %v (%dkB) to environment", toolsName, sizeInKB)
   201  	return u.UploadTools(toolsDir, stream, tools, buf.Bytes())
   202  }
   203  
   204  // UploadFunc is the type of Upload, which may be
   205  // reassigned to control the behaviour of tools
   206  // uploading.
   207  type UploadFunc func(stor storage.Storage, stream string, forceVersion *version.Number, series ...string) (*coretools.Tools, error)
   208  
   209  // Exported for testing.
   210  var Upload UploadFunc = upload
   211  
   212  // upload builds whatever version of github.com/juju/juju is in $GOPATH,
   213  // uploads it to the given storage, and returns a Tools instance describing
   214  // them. If forceVersion is not nil, the uploaded tools bundle will report
   215  // the given version number; if any fakeSeries are supplied, additional copies
   216  // of the built tools will be uploaded for use by machines of those series.
   217  // Juju tools built for one series do not necessarily run on another, but this
   218  // func exists only for development use cases.
   219  func upload(stor storage.Storage, stream string, forceVersion *version.Number, fakeSeries ...string) (*coretools.Tools, error) {
   220  	builtTools, err := BuildToolsTarball(forceVersion, stream)
   221  	if err != nil {
   222  		return nil, err
   223  	}
   224  	defer os.RemoveAll(builtTools.Dir)
   225  	logger.Debugf("Uploading tools for %v", fakeSeries)
   226  	return syncBuiltTools(stor, stream, builtTools, fakeSeries...)
   227  }
   228  
   229  // cloneToolsForSeries copies the built tools tarball into a tarball for the specified
   230  // stream and series and generates corresponding metadata.
   231  func cloneToolsForSeries(toolsInfo *BuiltTools, stream string, series ...string) error {
   232  	// Copy the tools to the target storage, recording a Tools struct for each one.
   233  	var targetTools coretools.List
   234  	targetTools = append(targetTools, &coretools.Tools{
   235  		Version: toolsInfo.Version,
   236  		Size:    toolsInfo.Size,
   237  		SHA256:  toolsInfo.Sha256Hash,
   238  	})
   239  	putTools := func(vers version.Binary) (string, error) {
   240  		name := envtools.StorageName(vers, stream)
   241  		src := filepath.Join(toolsInfo.Dir, toolsInfo.StorageName)
   242  		dest := filepath.Join(toolsInfo.Dir, name)
   243  		destDir := filepath.Dir(dest)
   244  		if err := os.MkdirAll(destDir, 0755); err != nil {
   245  			return "", err
   246  		}
   247  		if err := utils.CopyFile(dest, src); err != nil {
   248  			return "", err
   249  		}
   250  		// Append to targetTools the attributes required to write out tools metadata.
   251  		targetTools = append(targetTools, &coretools.Tools{
   252  			Version: vers,
   253  			Size:    toolsInfo.Size,
   254  			SHA256:  toolsInfo.Sha256Hash,
   255  		})
   256  		return name, nil
   257  	}
   258  	logger.Debugf("generating tarballs for %v", series)
   259  	for _, series := range series {
   260  		_, err := version.SeriesVersion(series)
   261  		if err != nil {
   262  			return err
   263  		}
   264  		if series != toolsInfo.Version.Series {
   265  			fakeVersion := toolsInfo.Version
   266  			fakeVersion.Series = series
   267  			if _, err := putTools(fakeVersion); err != nil {
   268  				return err
   269  			}
   270  		}
   271  	}
   272  	// The tools have been copied to a temp location from which they will be uploaded,
   273  	// now write out the matching simplestreams metadata so that SyncTools can find them.
   274  	metadataStore, err := filestorage.NewFileStorageWriter(toolsInfo.Dir)
   275  	if err != nil {
   276  		return err
   277  	}
   278  	logger.Debugf("generating tools metadata")
   279  	return envtools.MergeAndWriteMetadata(metadataStore, stream, stream, targetTools, false)
   280  }
   281  
   282  // BuiltTools contains metadata for a tools tarball resulting from
   283  // a call to BundleTools.
   284  type BuiltTools struct {
   285  	Version     version.Binary
   286  	Dir         string
   287  	StorageName string
   288  	Sha256Hash  string
   289  	Size        int64
   290  }
   291  
   292  // BuildToolsTarballFunc is a function which can build a tools tarball.
   293  type BuildToolsTarballFunc func(forceVersion *version.Number, stream string) (*BuiltTools, error)
   294  
   295  // Override for testing.
   296  var BuildToolsTarball BuildToolsTarballFunc = buildToolsTarball
   297  
   298  // buildToolsTarball bundles a tools tarball and places it in a temp directory in
   299  // the expected tools path.
   300  func buildToolsTarball(forceVersion *version.Number, stream string) (builtTools *BuiltTools, err error) {
   301  	// TODO(rog) find binaries from $PATH when not using a development
   302  	// version of juju within a $GOPATH.
   303  
   304  	logger.Debugf("Building tools")
   305  	// We create the entire archive before asking the environment to
   306  	// start uploading so that we can be sure we have archived
   307  	// correctly.
   308  	f, err := ioutil.TempFile("", "juju-tgz")
   309  	if err != nil {
   310  		return nil, err
   311  	}
   312  	defer f.Close()
   313  	defer os.Remove(f.Name())
   314  	toolsVersion, sha256Hash, err := envtools.BundleTools(f, forceVersion)
   315  	if err != nil {
   316  		return nil, err
   317  	}
   318  	fileInfo, err := f.Stat()
   319  	if err != nil {
   320  		return nil, fmt.Errorf("cannot stat newly made tools archive: %v", err)
   321  	}
   322  	size := fileInfo.Size()
   323  	logger.Infof("built tools %v (%dkB)", toolsVersion, (size+512)/1024)
   324  	baseToolsDir, err := ioutil.TempDir("", "juju-tools")
   325  	if err != nil {
   326  		return nil, err
   327  	}
   328  
   329  	// If we exit with an error, clean up the built tools directory.
   330  	defer func() {
   331  		if err != nil {
   332  			os.RemoveAll(baseToolsDir)
   333  		}
   334  	}()
   335  
   336  	err = os.MkdirAll(filepath.Join(baseToolsDir, storage.BaseToolsPath, stream), 0755)
   337  	if err != nil {
   338  		return nil, err
   339  	}
   340  	storageName := envtools.StorageName(toolsVersion, stream)
   341  	err = utils.CopyFile(filepath.Join(baseToolsDir, storageName), f.Name())
   342  	if err != nil {
   343  		return nil, err
   344  	}
   345  	return &BuiltTools{
   346  		Version:     toolsVersion,
   347  		Dir:         baseToolsDir,
   348  		StorageName: storageName,
   349  		Size:        size,
   350  		Sha256Hash:  sha256Hash,
   351  	}, nil
   352  }
   353  
   354  // syncBuiltTools copies to storage a tools tarball and cloned copies for each series.
   355  func syncBuiltTools(stor storage.Storage, stream string, builtTools *BuiltTools, fakeSeries ...string) (*coretools.Tools, error) {
   356  	if err := cloneToolsForSeries(builtTools, stream, fakeSeries...); err != nil {
   357  		return nil, err
   358  	}
   359  	syncContext := &SyncContext{
   360  		Source:              builtTools.Dir,
   361  		TargetToolsFinder:   StorageToolsFinder{stor},
   362  		TargetToolsUploader: StorageToolsUploader{stor, false, false},
   363  		AllVersions:         true,
   364  		Stream:              stream,
   365  		MajorVersion:        builtTools.Version.Major,
   366  		MinorVersion:        -1,
   367  	}
   368  	logger.Debugf("uploading tools to cloud storage")
   369  	err := SyncTools(syncContext)
   370  	if err != nil {
   371  		return nil, err
   372  	}
   373  	url, err := stor.URL(builtTools.StorageName)
   374  	if err != nil {
   375  		return nil, err
   376  	}
   377  	return &coretools.Tools{
   378  		Version: builtTools.Version,
   379  		URL:     url,
   380  		Size:    builtTools.Size,
   381  		SHA256:  builtTools.Sha256Hash,
   382  	}, nil
   383  }
   384  
   385  // StorageToolsFinder is an implementation of ToolsFinder
   386  // that searches for tools in the specified storage.
   387  type StorageToolsFinder struct {
   388  	Storage storage.StorageReader
   389  }
   390  
   391  func (f StorageToolsFinder) FindTools(major int, stream string) (coretools.List, error) {
   392  	return envtools.ReadList(f.Storage, stream, major, -1)
   393  }
   394  
   395  // StorageToolsUplader is an implementation of ToolsUploader that
   396  // writes tools to the provided storage and then writes merged
   397  // metadata, optionally with mirrors.
   398  type StorageToolsUploader struct {
   399  	Storage       storage.Storage
   400  	WriteMetadata bool
   401  	WriteMirrors  envtools.ShouldWriteMirrors
   402  }
   403  
   404  func (u StorageToolsUploader) UploadTools(toolsDir, stream string, tools *coretools.Tools, data []byte) error {
   405  	toolsName := envtools.StorageName(tools.Version, toolsDir)
   406  	if err := u.Storage.Put(toolsName, bytes.NewReader(data), int64(len(data))); err != nil {
   407  		return err
   408  	}
   409  	if !u.WriteMetadata {
   410  		return nil
   411  	}
   412  	err := envtools.MergeAndWriteMetadata(u.Storage, toolsDir, stream, coretools.List{tools}, u.WriteMirrors)
   413  	if err != nil {
   414  		logger.Errorf("error writing tools metadata: %v", err)
   415  		return err
   416  	}
   417  	return nil
   418  }