github.com/altoros/juju-vmware@v0.0.0-20150312064031-f19ae857ccca/provider/maas/storage_test.go (about)

     1  // Copyright 2013 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package maas
     5  
     6  import (
     7  	"bytes"
     8  	"encoding/base64"
     9  	"io/ioutil"
    10  	"math/rand"
    11  	"net/http"
    12  	"net/url"
    13  	"sync"
    14  
    15  	"github.com/juju/errors"
    16  	jc "github.com/juju/testing/checkers"
    17  	gc "gopkg.in/check.v1"
    18  	"launchpad.net/gomaasapi"
    19  
    20  	"github.com/juju/juju/environs/storage"
    21  )
    22  
    23  type storageSuite struct {
    24  	providerSuite
    25  }
    26  
    27  var _ = gc.Suite(&storageSuite{})
    28  
    29  // makeStorage creates a MAAS storage object for the running test.
    30  func (s *storageSuite) makeStorage(name string) *maasStorage {
    31  	maasobj := s.testMAASObject.MAASObject
    32  	env := s.makeEnviron()
    33  	env.name = name
    34  	env.maasClientUnlocked = &maasobj
    35  	return NewStorage(env).(*maasStorage)
    36  }
    37  
    38  // makeRandomBytes returns an array of arbitrary byte values.
    39  func makeRandomBytes(length int) []byte {
    40  	data := make([]byte, length)
    41  	for index := range data {
    42  		data[index] = byte(rand.Intn(256))
    43  	}
    44  	return data
    45  }
    46  
    47  // fakeStoredFile creates a file directly in the (simulated) MAAS file store.
    48  // It will contain an arbitrary amount of random data.  The contents are also
    49  // returned.
    50  //
    51  // If you want properly random data here, initialize the randomizer first.
    52  // Or don't, if you want consistent (and debuggable) results.
    53  func (s *storageSuite) fakeStoredFile(stor storage.Storage, name string) gomaasapi.MAASObject {
    54  	data := makeRandomBytes(rand.Intn(10))
    55  	// The filename must be prefixed with the private namespace as we're
    56  	// bypassing the Put() method that would normally do that.
    57  	prefixFilename := stor.(*maasStorage).prefixWithPrivateNamespace("") + name
    58  	return s.testMAASObject.TestServer.NewFile(prefixFilename, data)
    59  }
    60  
    61  func (s *storageSuite) TestGetSnapshotCreatesClone(c *gc.C) {
    62  	original := s.makeStorage("storage-name")
    63  	snapshot := original.getSnapshot()
    64  	c.Check(snapshot.environUnlocked, gc.Equals, original.environUnlocked)
    65  	c.Check(snapshot.maasClientUnlocked.URL().String(), gc.Equals, original.maasClientUnlocked.URL().String())
    66  	// Snapshotting locks the original internally, but does not leave
    67  	// either the original or the snapshot locked.
    68  	unlockedMutexValue := sync.Mutex{}
    69  	c.Check(original.Mutex, gc.Equals, unlockedMutexValue)
    70  	c.Check(snapshot.Mutex, gc.Equals, unlockedMutexValue)
    71  }
    72  
    73  func (s *storageSuite) TestGetRetrievesFile(c *gc.C) {
    74  	const filename = "stored-data"
    75  	stor := s.makeStorage("get-retrieves-file")
    76  	file := s.fakeStoredFile(stor, filename)
    77  	base64Content, err := file.GetField("content")
    78  	c.Assert(err, jc.ErrorIsNil)
    79  	content, err := base64.StdEncoding.DecodeString(base64Content)
    80  	c.Assert(err, jc.ErrorIsNil)
    81  
    82  	reader, err := storage.Get(stor, filename)
    83  	c.Assert(err, jc.ErrorIsNil)
    84  	defer reader.Close()
    85  
    86  	buf, err := ioutil.ReadAll(reader)
    87  	c.Assert(err, jc.ErrorIsNil)
    88  	c.Check(len(buf), gc.Equals, len(content))
    89  	c.Check(buf, gc.DeepEquals, content)
    90  }
    91  
    92  func (s *storageSuite) TestRetrieveFileObjectReturnsFileObject(c *gc.C) {
    93  	const filename = "myfile"
    94  	stor := s.makeStorage("rfo-test")
    95  	file := s.fakeStoredFile(stor, filename)
    96  	fileURI, err := file.GetField("anon_resource_uri")
    97  	c.Assert(err, jc.ErrorIsNil)
    98  	fileContent, err := file.GetField("content")
    99  	c.Assert(err, jc.ErrorIsNil)
   100  
   101  	prefixFilename := stor.prefixWithPrivateNamespace(filename)
   102  	obj, err := stor.retrieveFileObject(prefixFilename)
   103  	c.Assert(err, jc.ErrorIsNil)
   104  
   105  	uri, err := obj.GetField("anon_resource_uri")
   106  	c.Assert(err, jc.ErrorIsNil)
   107  	c.Check(uri, gc.Equals, fileURI)
   108  	content, err := obj.GetField("content")
   109  	c.Check(content, gc.Equals, fileContent)
   110  }
   111  
   112  func (s *storageSuite) TestRetrieveFileObjectReturnsNotFoundForMissingFile(c *gc.C) {
   113  	stor := s.makeStorage("rfo-test")
   114  	_, err := stor.retrieveFileObject("nonexistent-file")
   115  	c.Assert(err, gc.NotNil)
   116  	c.Check(err, jc.Satisfies, errors.IsNotFound)
   117  }
   118  
   119  func (s *storageSuite) TestRetrieveFileObjectEscapesName(c *gc.C) {
   120  	const filename = "#a?b c&d%e!"
   121  	data := []byte("File contents here")
   122  	stor := s.makeStorage("rfo-test")
   123  	err := stor.Put(filename, bytes.NewReader(data), int64(len(data)))
   124  	c.Assert(err, jc.ErrorIsNil)
   125  
   126  	prefixFilename := stor.prefixWithPrivateNamespace(filename)
   127  	obj, err := stor.retrieveFileObject(prefixFilename)
   128  	c.Assert(err, jc.ErrorIsNil)
   129  
   130  	base64Content, err := obj.GetField("content")
   131  	c.Assert(err, jc.ErrorIsNil)
   132  	content, err := base64.StdEncoding.DecodeString(base64Content)
   133  	c.Assert(err, jc.ErrorIsNil)
   134  	c.Check(content, gc.DeepEquals, data)
   135  }
   136  
   137  func (s *storageSuite) TestFileContentsAreBinary(c *gc.C) {
   138  	const filename = "myfile.bin"
   139  	data := []byte{0, 1, 255, 2, 254, 3}
   140  	stor := s.makeStorage("binary-test")
   141  
   142  	err := stor.Put(filename, bytes.NewReader(data), int64(len(data)))
   143  	c.Assert(err, jc.ErrorIsNil)
   144  	file, err := storage.Get(stor, filename)
   145  	c.Assert(err, jc.ErrorIsNil)
   146  	content, err := ioutil.ReadAll(file)
   147  	c.Assert(err, jc.ErrorIsNil)
   148  
   149  	c.Check(content, gc.DeepEquals, data)
   150  }
   151  
   152  func (s *storageSuite) TestGetReturnsNotFoundErrorIfNotFound(c *gc.C) {
   153  	const filename = "lost-data"
   154  	stor := NewStorage(s.makeEnviron())
   155  	_, err := storage.Get(stor, filename)
   156  	c.Assert(err, jc.Satisfies, errors.IsNotFound)
   157  }
   158  
   159  func (s *storageSuite) TestListReturnsEmptyIfNoFilesStored(c *gc.C) {
   160  	stor := NewStorage(s.makeEnviron())
   161  	listing, err := storage.List(stor, "")
   162  	c.Assert(err, jc.ErrorIsNil)
   163  	c.Check(listing, gc.DeepEquals, []string{})
   164  }
   165  
   166  func (s *storageSuite) TestListReturnsAllFilesIfPrefixEmpty(c *gc.C) {
   167  	stor := NewStorage(s.makeEnviron())
   168  	files := []string{"1a", "2b", "3c"}
   169  	for _, name := range files {
   170  		s.fakeStoredFile(stor, name)
   171  	}
   172  
   173  	listing, err := storage.List(stor, "")
   174  	c.Assert(err, jc.ErrorIsNil)
   175  	c.Check(listing, gc.DeepEquals, files)
   176  }
   177  
   178  func (s *storageSuite) TestListSortsResults(c *gc.C) {
   179  	stor := NewStorage(s.makeEnviron())
   180  	files := []string{"4d", "1a", "3c", "2b"}
   181  	for _, name := range files {
   182  		s.fakeStoredFile(stor, name)
   183  	}
   184  
   185  	listing, err := storage.List(stor, "")
   186  	c.Assert(err, jc.ErrorIsNil)
   187  	c.Check(listing, gc.DeepEquals, []string{"1a", "2b", "3c", "4d"})
   188  }
   189  
   190  func (s *storageSuite) TestListReturnsNoFilesIfNoFilesMatchPrefix(c *gc.C) {
   191  	stor := NewStorage(s.makeEnviron())
   192  	s.fakeStoredFile(stor, "foo")
   193  
   194  	listing, err := storage.List(stor, "bar")
   195  	c.Assert(err, jc.ErrorIsNil)
   196  	c.Check(listing, gc.DeepEquals, []string{})
   197  }
   198  
   199  func (s *storageSuite) TestListReturnsOnlyFilesWithMatchingPrefix(c *gc.C) {
   200  	stor := NewStorage(s.makeEnviron())
   201  	s.fakeStoredFile(stor, "abc")
   202  	s.fakeStoredFile(stor, "xyz")
   203  
   204  	listing, err := storage.List(stor, "x")
   205  	c.Assert(err, jc.ErrorIsNil)
   206  	c.Check(listing, gc.DeepEquals, []string{"xyz"})
   207  }
   208  
   209  func (s *storageSuite) TestListMatchesPrefixOnly(c *gc.C) {
   210  	stor := NewStorage(s.makeEnviron())
   211  	s.fakeStoredFile(stor, "abc")
   212  	s.fakeStoredFile(stor, "xabc")
   213  
   214  	listing, err := storage.List(stor, "a")
   215  	c.Assert(err, jc.ErrorIsNil)
   216  	c.Check(listing, gc.DeepEquals, []string{"abc"})
   217  }
   218  
   219  func (s *storageSuite) TestListOperatesOnFlatNamespace(c *gc.C) {
   220  	stor := NewStorage(s.makeEnviron())
   221  	s.fakeStoredFile(stor, "a/b/c/d")
   222  
   223  	listing, err := storage.List(stor, "a/b")
   224  	c.Assert(err, jc.ErrorIsNil)
   225  	c.Check(listing, gc.DeepEquals, []string{"a/b/c/d"})
   226  }
   227  
   228  // getFileAtURL requests, and returns, the file at the given URL.
   229  func getFileAtURL(fileURL string) ([]byte, error) {
   230  	response, err := http.Get(fileURL)
   231  	if err != nil {
   232  		return nil, err
   233  	}
   234  	body, err := ioutil.ReadAll(response.Body)
   235  	if err != nil {
   236  		return nil, err
   237  	}
   238  	return body, nil
   239  }
   240  
   241  func (s *storageSuite) TestURLReturnsURLCorrespondingToFile(c *gc.C) {
   242  	const filename = "my-file.txt"
   243  	stor := NewStorage(s.makeEnviron()).(*maasStorage)
   244  	file := s.fakeStoredFile(stor, filename)
   245  	// The file contains an anon_resource_uri, which lacks a network part
   246  	// (but will probably contain a query part).  anonURL will be the
   247  	// file's full URL.
   248  	anonURI, err := file.GetField("anon_resource_uri")
   249  	c.Assert(err, jc.ErrorIsNil)
   250  	parsedURI, err := url.Parse(anonURI)
   251  	c.Assert(err, jc.ErrorIsNil)
   252  	anonURL := stor.maasClientUnlocked.URL().ResolveReference(parsedURI)
   253  	c.Assert(err, jc.ErrorIsNil)
   254  
   255  	fileURL, err := stor.URL(filename)
   256  	c.Assert(err, jc.ErrorIsNil)
   257  
   258  	c.Check(fileURL, gc.NotNil)
   259  	c.Check(fileURL, gc.Equals, anonURL.String())
   260  }
   261  
   262  func (s *storageSuite) TestPutStoresRetrievableFile(c *gc.C) {
   263  	const filename = "broken-toaster.jpg"
   264  	contents := []byte("Contents here")
   265  	length := int64(len(contents))
   266  	stor := NewStorage(s.makeEnviron())
   267  
   268  	err := stor.Put(filename, bytes.NewReader(contents), length)
   269  
   270  	reader, err := storage.Get(stor, filename)
   271  	c.Assert(err, jc.ErrorIsNil)
   272  	defer reader.Close()
   273  
   274  	buf, err := ioutil.ReadAll(reader)
   275  	c.Assert(err, jc.ErrorIsNil)
   276  	c.Check(buf, gc.DeepEquals, contents)
   277  }
   278  
   279  func (s *storageSuite) TestPutOverwritesFile(c *gc.C) {
   280  	const filename = "foo.bar"
   281  	stor := NewStorage(s.makeEnviron())
   282  	s.fakeStoredFile(stor, filename)
   283  	newContents := []byte("Overwritten")
   284  
   285  	err := stor.Put(filename, bytes.NewReader(newContents), int64(len(newContents)))
   286  	c.Assert(err, jc.ErrorIsNil)
   287  
   288  	reader, err := storage.Get(stor, filename)
   289  	c.Assert(err, jc.ErrorIsNil)
   290  	defer reader.Close()
   291  
   292  	buf, err := ioutil.ReadAll(reader)
   293  	c.Assert(err, jc.ErrorIsNil)
   294  	c.Check(len(buf), gc.Equals, len(newContents))
   295  	c.Check(buf, gc.DeepEquals, newContents)
   296  }
   297  
   298  func (s *storageSuite) TestPutStopsAtGivenLength(c *gc.C) {
   299  	const filename = "xyzzyz.2.xls"
   300  	const length = 5
   301  	contents := []byte("abcdefghijklmnopqrstuvwxyz")
   302  	stor := NewStorage(s.makeEnviron())
   303  
   304  	err := stor.Put(filename, bytes.NewReader(contents), length)
   305  	c.Assert(err, jc.ErrorIsNil)
   306  
   307  	reader, err := storage.Get(stor, filename)
   308  	c.Assert(err, jc.ErrorIsNil)
   309  	defer reader.Close()
   310  
   311  	buf, err := ioutil.ReadAll(reader)
   312  	c.Assert(err, jc.ErrorIsNil)
   313  	c.Check(len(buf), gc.Equals, length)
   314  }
   315  
   316  func (s *storageSuite) TestPutToExistingFileTruncatesAtGivenLength(c *gc.C) {
   317  	const filename = "a-file-which-is-mine"
   318  	oldContents := []byte("abcdefghijklmnopqrstuvwxyz")
   319  	newContents := []byte("xyz")
   320  	stor := NewStorage(s.makeEnviron())
   321  	err := stor.Put(filename, bytes.NewReader(oldContents), int64(len(oldContents)))
   322  	c.Assert(err, jc.ErrorIsNil)
   323  
   324  	err = stor.Put(filename, bytes.NewReader(newContents), int64(len(newContents)))
   325  	c.Assert(err, jc.ErrorIsNil)
   326  
   327  	reader, err := storage.Get(stor, filename)
   328  	c.Assert(err, jc.ErrorIsNil)
   329  	defer reader.Close()
   330  
   331  	buf, err := ioutil.ReadAll(reader)
   332  	c.Assert(err, jc.ErrorIsNil)
   333  	c.Check(len(buf), gc.Equals, len(newContents))
   334  	c.Check(buf, gc.DeepEquals, newContents)
   335  }
   336  
   337  func (s *storageSuite) TestRemoveDeletesFile(c *gc.C) {
   338  	const filename = "doomed.txt"
   339  	stor := NewStorage(s.makeEnviron())
   340  	s.fakeStoredFile(stor, filename)
   341  
   342  	err := stor.Remove(filename)
   343  	c.Assert(err, jc.ErrorIsNil)
   344  
   345  	_, err = storage.Get(stor, filename)
   346  	c.Assert(err, jc.Satisfies, errors.IsNotFound)
   347  
   348  	listing, err := storage.List(stor, filename)
   349  	c.Assert(err, jc.ErrorIsNil)
   350  	c.Assert(listing, gc.DeepEquals, []string{})
   351  }
   352  
   353  func (s *storageSuite) TestRemoveIsIdempotent(c *gc.C) {
   354  	const filename = "half-a-file"
   355  	stor := NewStorage(s.makeEnviron())
   356  	s.fakeStoredFile(stor, filename)
   357  
   358  	err := stor.Remove(filename)
   359  	c.Assert(err, jc.ErrorIsNil)
   360  
   361  	err = stor.Remove(filename)
   362  	c.Assert(err, jc.ErrorIsNil)
   363  }
   364  
   365  func (s *storageSuite) TestNamesMayHaveSlashes(c *gc.C) {
   366  	const filename = "name/with/slashes"
   367  	content := []byte("File contents")
   368  	stor := NewStorage(s.makeEnviron())
   369  
   370  	err := stor.Put(filename, bytes.NewReader(content), int64(len(content)))
   371  	c.Assert(err, jc.ErrorIsNil)
   372  
   373  	// There's not much we can say about the anonymous URL, except that
   374  	// we get one.
   375  	anonURL, err := stor.URL(filename)
   376  	c.Assert(err, jc.ErrorIsNil)
   377  	c.Check(anonURL, gc.Matches, "http[s]*://.*")
   378  
   379  	reader, err := storage.Get(stor, filename)
   380  	c.Assert(err, jc.ErrorIsNil)
   381  	defer reader.Close()
   382  	data, err := ioutil.ReadAll(reader)
   383  	c.Assert(err, jc.ErrorIsNil)
   384  	c.Check(data, gc.DeepEquals, content)
   385  }
   386  
   387  func (s *storageSuite) TestRemoveAllDeletesAllFiles(c *gc.C) {
   388  	stor := s.makeStorage("get-retrieves-file")
   389  	const filename1 = "stored-data1"
   390  	s.fakeStoredFile(stor, filename1)
   391  	const filename2 = "stored-data2"
   392  	s.fakeStoredFile(stor, filename2)
   393  
   394  	err := stor.RemoveAll()
   395  	c.Assert(err, jc.ErrorIsNil)
   396  	listing, err := storage.List(stor, "")
   397  	c.Assert(err, jc.ErrorIsNil)
   398  	c.Assert(listing, gc.DeepEquals, []string{})
   399  }
   400  
   401  func (s *storageSuite) TestprefixWithPrivateNamespacePrefixesWithAgentName(c *gc.C) {
   402  	sstor := NewStorage(s.makeEnviron())
   403  	stor := sstor.(*maasStorage)
   404  	agentName := stor.environUnlocked.ecfg().maasAgentName()
   405  	c.Assert(agentName, gc.Not(gc.Equals), "")
   406  	expectedPrefix := agentName + "-"
   407  	const name = "myname"
   408  	expectedResult := expectedPrefix + name
   409  	c.Assert(stor.prefixWithPrivateNamespace(name), gc.Equals, expectedResult)
   410  }
   411  
   412  func (s *storageSuite) TesttprefixWithPrivateNamespaceIgnoresAgentName(c *gc.C) {
   413  	sstor := NewStorage(s.makeEnviron())
   414  	stor := sstor.(*maasStorage)
   415  	ecfg := stor.environUnlocked.ecfg()
   416  	ecfg.attrs["maas-agent-name"] = ""
   417  
   418  	const name = "myname"
   419  	c.Assert(stor.prefixWithPrivateNamespace(name), gc.Equals, name)
   420  }