github.com/Pankov404/juju@v0.0.0-20150703034450-be266991dceb/environs/tools/testing/testing.go (about)

     1  // Copyright 2013 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package testing
     5  
     6  import (
     7  	"bytes"
     8  	"crypto/sha256"
     9  	"fmt"
    10  	"io"
    11  	"io/ioutil"
    12  	"os"
    13  	"path"
    14  	"path/filepath"
    15  	"sort"
    16  	"strings"
    17  	"time"
    18  
    19  	jc "github.com/juju/testing/checkers"
    20  	"github.com/juju/utils"
    21  	"github.com/juju/utils/set"
    22  	gc "gopkg.in/check.v1"
    23  
    24  	"github.com/juju/juju/environs/filestorage"
    25  	"github.com/juju/juju/environs/simplestreams"
    26  	"github.com/juju/juju/environs/storage"
    27  	"github.com/juju/juju/environs/sync"
    28  	"github.com/juju/juju/environs/tools"
    29  	"github.com/juju/juju/juju/names"
    30  	coretesting "github.com/juju/juju/testing"
    31  	coretools "github.com/juju/juju/tools"
    32  	"github.com/juju/juju/version"
    33  )
    34  
    35  func GetMockBundleTools(c *gc.C) tools.BundleToolsFunc {
    36  	return func(w io.Writer, forceVersion *version.Number) (vers version.Binary, sha256Hash string, err error) {
    37  		vers = version.Current
    38  		if forceVersion != nil {
    39  			vers.Number = *forceVersion
    40  		}
    41  		sha256Hash = fmt.Sprintf("%x", sha256.New().Sum(nil))
    42  		return vers, sha256Hash, err
    43  	}
    44  }
    45  
    46  // GetMockBuildTools returns a sync.BuildToolsTarballFunc implementation which generates
    47  // a fake tools tarball.
    48  func GetMockBuildTools(c *gc.C) sync.BuildToolsTarballFunc {
    49  	return func(forceVersion *version.Number, stream string) (*sync.BuiltTools, error) {
    50  		vers := version.Current
    51  		if forceVersion != nil {
    52  			vers.Number = *forceVersion
    53  		}
    54  
    55  		tgz, checksum := coretesting.TarGz(
    56  			coretesting.NewTarFile(names.Jujud, 0777, "jujud contents "+vers.String()))
    57  
    58  		toolsDir, err := ioutil.TempDir("", "juju-tools-"+stream)
    59  		c.Assert(err, jc.ErrorIsNil)
    60  		name := "name"
    61  		ioutil.WriteFile(filepath.Join(toolsDir, name), tgz, 0777)
    62  
    63  		return &sync.BuiltTools{
    64  			Dir:         toolsDir,
    65  			StorageName: name,
    66  			Version:     vers,
    67  			Size:        int64(len(tgz)),
    68  			Sha256Hash:  checksum,
    69  		}, nil
    70  	}
    71  }
    72  
    73  // MakeTools creates some fake tools with the given version strings.
    74  func MakeTools(c *gc.C, metadataDir, stream string, versionStrings []string) coretools.List {
    75  	return makeTools(c, metadataDir, stream, versionStrings, false)
    76  }
    77  
    78  // MakeToolsWithCheckSum creates some fake tools (including checksums) with the given version strings.
    79  func MakeToolsWithCheckSum(c *gc.C, metadataDir, stream string, versionStrings []string) coretools.List {
    80  	return makeTools(c, metadataDir, stream, versionStrings, true)
    81  }
    82  
    83  func makeTools(c *gc.C, metadataDir, stream string, versionStrings []string, withCheckSum bool) coretools.List {
    84  	toolsDir := filepath.Join(metadataDir, storage.BaseToolsPath, stream)
    85  	c.Assert(os.MkdirAll(toolsDir, 0755), gc.IsNil)
    86  	var toolsList coretools.List
    87  	for _, versionString := range versionStrings {
    88  		binary, err := version.ParseBinary(versionString)
    89  		if err != nil {
    90  			c.Assert(err, jc.Satisfies, version.IsUnknownOSForSeriesError)
    91  		}
    92  		path := filepath.Join(toolsDir, fmt.Sprintf("juju-%s.tgz", binary))
    93  		data := binary.String()
    94  		err = ioutil.WriteFile(path, []byte(data), 0644)
    95  		c.Assert(err, jc.ErrorIsNil)
    96  		tool := &coretools.Tools{
    97  			Version: binary,
    98  			URL:     path,
    99  		}
   100  		if withCheckSum {
   101  			tool.Size, tool.SHA256 = SHA256sum(c, path)
   102  		}
   103  		toolsList = append(toolsList, tool)
   104  	}
   105  	// Write the tools metadata.
   106  	stor, err := filestorage.NewFileStorageWriter(metadataDir)
   107  	c.Assert(err, jc.ErrorIsNil)
   108  	err = tools.MergeAndWriteMetadata(stor, stream, stream, toolsList, false)
   109  	c.Assert(err, jc.ErrorIsNil)
   110  	return toolsList
   111  }
   112  
   113  // SHA256sum creates the sha256 checksum for the specified file.
   114  func SHA256sum(c *gc.C, path string) (int64, string) {
   115  	if strings.HasPrefix(path, "file://") {
   116  		path = path[len("file://"):]
   117  	}
   118  	hash, size, err := utils.ReadFileSHA256(path)
   119  	c.Assert(err, jc.ErrorIsNil)
   120  	return size, hash
   121  }
   122  
   123  // ParseMetadataFromDir loads ToolsMetadata from the specified directory.
   124  func ParseMetadataFromDir(c *gc.C, metadataDir, stream string, expectMirrors bool) []*tools.ToolsMetadata {
   125  	stor, err := filestorage.NewFileStorageReader(metadataDir)
   126  	c.Assert(err, jc.ErrorIsNil)
   127  	return ParseMetadataFromStorage(c, stor, stream, expectMirrors)
   128  }
   129  
   130  // ParseMetadataFromStorage loads ToolsMetadata from the specified storage reader.
   131  func ParseMetadataFromStorage(c *gc.C, stor storage.StorageReader, stream string, expectMirrors bool) []*tools.ToolsMetadata {
   132  	source := storage.NewStorageSimpleStreamsDataSource("test storage reader", stor, "tools")
   133  	params := simplestreams.ValueParams{
   134  		DataType:      tools.ContentDownload,
   135  		ValueTemplate: tools.ToolsMetadata{},
   136  	}
   137  
   138  	const requireSigned = false
   139  	indexPath := simplestreams.UnsignedIndex("v1", 2)
   140  	mirrorsPath := simplestreams.MirrorsPath("v1")
   141  	indexRef, err := simplestreams.GetIndexWithFormat(
   142  		source, indexPath, "index:1.0", mirrorsPath, requireSigned, simplestreams.CloudSpec{}, params)
   143  	c.Assert(err, jc.ErrorIsNil)
   144  
   145  	toolsIndexMetadata := indexRef.Indexes[tools.ToolsContentId(stream)]
   146  	c.Assert(toolsIndexMetadata, gc.NotNil)
   147  
   148  	// Read the products file contents.
   149  	r, err := stor.Get(path.Join("tools", toolsIndexMetadata.ProductsFilePath))
   150  	defer r.Close()
   151  	c.Assert(err, jc.ErrorIsNil)
   152  	data, err := ioutil.ReadAll(r)
   153  	c.Assert(err, jc.ErrorIsNil)
   154  
   155  	url, err := source.URL(toolsIndexMetadata.ProductsFilePath)
   156  	c.Assert(err, jc.ErrorIsNil)
   157  	cloudMetadata, err := simplestreams.ParseCloudMetadata(data, "products:1.0", url, tools.ToolsMetadata{})
   158  	c.Assert(err, jc.ErrorIsNil)
   159  
   160  	toolsMetadataMap := make(map[string]*tools.ToolsMetadata)
   161  	expectedProductIds := make(set.Strings)
   162  	toolsVersions := make(set.Strings)
   163  	for _, mc := range cloudMetadata.Products {
   164  		for _, items := range mc.Items {
   165  			for key, item := range items.Items {
   166  				toolsMetadata := item.(*tools.ToolsMetadata)
   167  				toolsMetadataMap[key] = toolsMetadata
   168  				toolsVersions.Add(key)
   169  				seriesVersion, err := version.SeriesVersion(toolsMetadata.Release)
   170  				if err != nil {
   171  					c.Assert(err, jc.Satisfies, version.IsUnknownSeriesVersionError)
   172  				}
   173  				productId := fmt.Sprintf("com.ubuntu.juju:%s:%s", seriesVersion, toolsMetadata.Arch)
   174  				expectedProductIds.Add(productId)
   175  			}
   176  		}
   177  	}
   178  
   179  	// Make sure index's product IDs are all represented in the products metadata.
   180  	sort.Strings(toolsIndexMetadata.ProductIds)
   181  	c.Assert(toolsIndexMetadata.ProductIds, gc.DeepEquals, expectedProductIds.SortedValues())
   182  
   183  	toolsMetadata := make([]*tools.ToolsMetadata, len(toolsMetadataMap))
   184  	for i, key := range toolsVersions.SortedValues() {
   185  		toolsMetadata[i] = toolsMetadataMap[key]
   186  	}
   187  
   188  	if expectMirrors {
   189  		r, err = stor.Get(path.Join("tools", simplestreams.UnsignedMirror("v1")))
   190  		c.Assert(err, jc.ErrorIsNil)
   191  		defer r.Close()
   192  		data, err = ioutil.ReadAll(r)
   193  		c.Assert(err, jc.ErrorIsNil)
   194  		c.Assert(string(data), jc.Contains, `"mirrors":`)
   195  		c.Assert(string(data), jc.Contains, tools.ToolsContentId(stream))
   196  		c.Assert(err, jc.ErrorIsNil)
   197  	}
   198  	return toolsMetadata
   199  }
   200  
   201  type metadataFile struct {
   202  	path string
   203  	data []byte
   204  }
   205  
   206  func generateMetadata(c *gc.C, stream string, versions ...version.Binary) []metadataFile {
   207  	var metadata = make([]*tools.ToolsMetadata, len(versions))
   208  	for i, vers := range versions {
   209  		basePath := fmt.Sprintf("%s/tools-%s.tar.gz", stream, vers.String())
   210  		metadata[i] = &tools.ToolsMetadata{
   211  			Release: vers.Series,
   212  			Version: vers.Number.String(),
   213  			Arch:    vers.Arch,
   214  			Path:    basePath,
   215  		}
   216  	}
   217  	var streamMetadata = map[string][]*tools.ToolsMetadata{
   218  		stream: metadata,
   219  	}
   220  	index, legacyIndex, products, err := tools.MarshalToolsMetadataJSON(streamMetadata, time.Now())
   221  	c.Assert(err, jc.ErrorIsNil)
   222  	objects := []metadataFile{
   223  		{simplestreams.UnsignedIndex("v1", 2), index},
   224  	}
   225  	if stream == "released" {
   226  		objects = append(objects, metadataFile{simplestreams.UnsignedIndex("v1", 1), legacyIndex})
   227  	}
   228  	for stream, metadata := range products {
   229  		objects = append(objects, metadataFile{tools.ProductMetadataPath(stream), metadata})
   230  	}
   231  	return objects
   232  }
   233  
   234  // UploadToStorage uploads tools and metadata for the specified versions to storage.
   235  func UploadToStorage(c *gc.C, stor storage.Storage, stream string, versions ...version.Binary) map[version.Binary]string {
   236  	uploaded := map[version.Binary]string{}
   237  	if len(versions) == 0 {
   238  		return uploaded
   239  	}
   240  	var err error
   241  	for _, vers := range versions {
   242  		filename := fmt.Sprintf("tools/%s/tools-%s.tar.gz", stream, vers.String())
   243  		// Put a file in images since the dummy storage provider requires a
   244  		// file to exist before the URL can be found. This is to ensure it behaves
   245  		// the same way as MAAS.
   246  		err = stor.Put(filename, strings.NewReader("dummy"), 5)
   247  		c.Assert(err, jc.ErrorIsNil)
   248  		uploaded[vers], err = stor.URL(filename)
   249  		c.Assert(err, jc.ErrorIsNil)
   250  	}
   251  	objects := generateMetadata(c, stream, versions...)
   252  	for _, object := range objects {
   253  		toolspath := path.Join("tools", object.path)
   254  		err = stor.Put(toolspath, bytes.NewReader(object.data), int64(len(object.data)))
   255  		c.Assert(err, jc.ErrorIsNil)
   256  	}
   257  	return uploaded
   258  }
   259  
   260  // UploadToDirectory uploads tools and metadata for the specified versions to dir.
   261  func UploadToDirectory(c *gc.C, stream, dir string, versions ...version.Binary) map[version.Binary]string {
   262  	uploaded := map[version.Binary]string{}
   263  	if len(versions) == 0 {
   264  		return uploaded
   265  	}
   266  	for _, vers := range versions {
   267  		basePath := fmt.Sprintf("%s/tools-%s.tar.gz", stream, vers.String())
   268  		uploaded[vers] = utils.MakeFileURL(fmt.Sprintf("%s/%s", dir, basePath))
   269  	}
   270  	objects := generateMetadata(c, stream, versions...)
   271  	for _, object := range objects {
   272  		path := filepath.Join(dir, object.path)
   273  		dir := filepath.Dir(path)
   274  		if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) {
   275  			c.Assert(err, jc.ErrorIsNil)
   276  		}
   277  		err := ioutil.WriteFile(path, object.data, 0644)
   278  		c.Assert(err, jc.ErrorIsNil)
   279  	}
   280  	return uploaded
   281  }