github.com/mhilton/juju-juju@v0.0.0-20150901100907-a94dd2c73455/apiserver/highavailability/highavailability_test.go (about)

     1  // Copyright 2014 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package highavailability_test
     5  
     6  import (
     7  	stdtesting "testing"
     8  
     9  	"github.com/juju/errors"
    10  	jc "github.com/juju/testing/checkers"
    11  	gc "gopkg.in/check.v1"
    12  
    13  	"github.com/juju/juju/apiserver/common"
    14  	commontesting "github.com/juju/juju/apiserver/common/testing"
    15  	"github.com/juju/juju/apiserver/highavailability"
    16  	"github.com/juju/juju/apiserver/params"
    17  	apiservertesting "github.com/juju/juju/apiserver/testing"
    18  	"github.com/juju/juju/constraints"
    19  	"github.com/juju/juju/juju/testing"
    20  	"github.com/juju/juju/state"
    21  	"github.com/juju/juju/state/presence"
    22  	coretesting "github.com/juju/juju/testing"
    23  	"github.com/juju/juju/testing/factory"
    24  )
    25  
    26  func TestAll(t *stdtesting.T) {
    27  	coretesting.MgoTestPackage(t)
    28  }
    29  
    30  type clientSuite struct {
    31  	testing.JujuConnSuite
    32  
    33  	resources  *common.Resources
    34  	authoriser apiservertesting.FakeAuthorizer
    35  	haServer   *highavailability.HighAvailabilityAPI
    36  	pingers    []*presence.Pinger
    37  
    38  	commontesting.BlockHelper
    39  }
    40  
    41  type Killer interface {
    42  	Kill() error
    43  }
    44  
    45  var _ = gc.Suite(&clientSuite{})
    46  
    47  func assertKill(c *gc.C, killer Killer) {
    48  	c.Assert(killer.Kill(), gc.IsNil)
    49  }
    50  
    51  var (
    52  	emptyCons     = constraints.Value{}
    53  	defaultSeries = ""
    54  )
    55  
    56  func (s *clientSuite) SetUpTest(c *gc.C) {
    57  	s.JujuConnSuite.SetUpTest(c)
    58  	s.resources = common.NewResources()
    59  	s.AddCleanup(func(_ *gc.C) { s.resources.StopAll() })
    60  
    61  	s.authoriser = apiservertesting.FakeAuthorizer{
    62  		Tag:            s.AdminUserTag(c),
    63  		EnvironManager: true,
    64  	}
    65  
    66  	var err error
    67  	s.haServer, err = highavailability.NewHighAvailabilityAPI(s.State, s.resources, s.authoriser)
    68  	c.Assert(err, jc.ErrorIsNil)
    69  
    70  	_, err = s.State.AddMachine("quantal", state.JobManageEnviron)
    71  	c.Assert(err, jc.ErrorIsNil)
    72  	// We have to ensure the agents are alive, or EnsureAvailability will
    73  	// create more to replace them.
    74  	s.pingers = []*presence.Pinger{s.setAgentPresence(c, "0")}
    75  	s.BlockHelper = commontesting.NewBlockHelper(s.APIState)
    76  	s.AddCleanup(func(*gc.C) { s.BlockHelper.Close() })
    77  }
    78  
    79  func (s *clientSuite) TearDownTest(c *gc.C) {
    80  	for _, pinger := range s.pingers {
    81  		assertKill(c, pinger)
    82  	}
    83  	s.JujuConnSuite.TearDownTest(c)
    84  }
    85  
    86  func (s *clientSuite) setAgentPresence(c *gc.C, machineId string) *presence.Pinger {
    87  	m, err := s.State.Machine(machineId)
    88  	c.Assert(err, jc.ErrorIsNil)
    89  	pinger, err := m.SetAgentPresence()
    90  	c.Assert(err, jc.ErrorIsNil)
    91  	s.State.StartSync()
    92  	err = m.WaitAgentPresence(coretesting.LongWait)
    93  	c.Assert(err, jc.ErrorIsNil)
    94  	return pinger
    95  }
    96  
    97  func (s *clientSuite) ensureAvailability(
    98  	c *gc.C, numStateServers int, cons constraints.Value, series string, placement []string,
    99  ) (params.StateServersChanges, error) {
   100  	return ensureAvailability(c, s.haServer, numStateServers, cons, series, placement)
   101  }
   102  
   103  func ensureAvailability(
   104  	c *gc.C, haServer *highavailability.HighAvailabilityAPI, numStateServers int, cons constraints.Value, series string, placement []string,
   105  ) (params.StateServersChanges, error) {
   106  	arg := params.StateServersSpecs{
   107  		Specs: []params.StateServersSpec{{
   108  			NumStateServers: numStateServers,
   109  			Constraints:     cons,
   110  			Series:          series,
   111  			Placement:       placement,
   112  		}}}
   113  	results, err := haServer.EnsureAvailability(arg)
   114  	c.Assert(err, jc.ErrorIsNil)
   115  	c.Assert(results.Results, gc.HasLen, 1)
   116  	result := results.Results[0]
   117  	// We explicitly return nil here so we can do typed nil checking
   118  	// of the result like normal.
   119  	err = nil
   120  	if result.Error != nil {
   121  		err = result.Error
   122  	}
   123  	return result.Result, err
   124  }
   125  
   126  func (s *clientSuite) TestEnsureAvailabilitySeries(c *gc.C) {
   127  	machines, err := s.State.AllMachines()
   128  	c.Assert(err, jc.ErrorIsNil)
   129  	c.Assert(machines, gc.HasLen, 1)
   130  	c.Assert(machines[0].Series(), gc.Equals, "quantal")
   131  
   132  	ensureAvailabilityResult, err := s.ensureAvailability(c, 3, emptyCons, defaultSeries, nil)
   133  	c.Assert(err, jc.ErrorIsNil)
   134  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   135  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-1", "machine-2"})
   136  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   137  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   138  
   139  	machines, err = s.State.AllMachines()
   140  	c.Assert(err, jc.ErrorIsNil)
   141  	c.Assert(machines, gc.HasLen, 3)
   142  	c.Assert(machines[0].Series(), gc.Equals, "quantal")
   143  	c.Assert(machines[1].Series(), gc.Equals, "quantal")
   144  	c.Assert(machines[2].Series(), gc.Equals, "quantal")
   145  
   146  	pingerB := s.setAgentPresence(c, "1")
   147  	defer assertKill(c, pingerB)
   148  
   149  	pingerC := s.setAgentPresence(c, "2")
   150  	defer assertKill(c, pingerC)
   151  
   152  	ensureAvailabilityResult, err = s.ensureAvailability(c, 5, emptyCons, "non-default", nil)
   153  	c.Assert(err, jc.ErrorIsNil)
   154  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0", "machine-1", "machine-2"})
   155  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-3", "machine-4"})
   156  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   157  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   158  
   159  	c.Assert(err, jc.ErrorIsNil)
   160  	machines, err = s.State.AllMachines()
   161  	c.Assert(err, jc.ErrorIsNil)
   162  	c.Assert(machines, gc.HasLen, 5)
   163  	c.Assert(machines[0].Series(), gc.Equals, "quantal")
   164  	c.Assert(machines[1].Series(), gc.Equals, "quantal")
   165  	c.Assert(machines[2].Series(), gc.Equals, "quantal")
   166  	c.Assert(machines[3].Series(), gc.Equals, "non-default")
   167  	c.Assert(machines[4].Series(), gc.Equals, "non-default")
   168  }
   169  
   170  func (s *clientSuite) TestEnsureAvailabilityConstraints(c *gc.C) {
   171  	ensureAvailabilityResult, err := s.ensureAvailability(c, 3, constraints.MustParse("mem=4G"), defaultSeries, nil)
   172  	c.Assert(err, jc.ErrorIsNil)
   173  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   174  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-1", "machine-2"})
   175  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   176  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   177  
   178  	machines, err := s.State.AllMachines()
   179  	c.Assert(err, jc.ErrorIsNil)
   180  	c.Assert(machines, gc.HasLen, 3)
   181  	expectedCons := []constraints.Value{
   182  		{},
   183  		constraints.MustParse("mem=4G"),
   184  		constraints.MustParse("mem=4G"),
   185  	}
   186  	for i, m := range machines {
   187  		cons, err := m.Constraints()
   188  		c.Assert(err, jc.ErrorIsNil)
   189  		c.Check(cons, gc.DeepEquals, expectedCons[i])
   190  	}
   191  }
   192  
   193  func (s *clientSuite) TestBlockEnsureAvailability(c *gc.C) {
   194  	// Block all changes.
   195  	s.BlockAllChanges(c, "TestBlockEnsureAvailability")
   196  
   197  	ensureAvailabilityResult, err := s.ensureAvailability(c, 3, constraints.MustParse("mem=4G"), defaultSeries, nil)
   198  	s.AssertBlocked(c, err, "TestBlockEnsureAvailability")
   199  
   200  	c.Assert(ensureAvailabilityResult.Maintained, gc.HasLen, 0)
   201  	c.Assert(ensureAvailabilityResult.Added, gc.HasLen, 0)
   202  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   203  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   204  
   205  	machines, err := s.State.AllMachines()
   206  	c.Assert(err, jc.ErrorIsNil)
   207  	c.Assert(machines, gc.HasLen, 1)
   208  }
   209  
   210  func (s *clientSuite) TestEnsureAvailabilityPlacement(c *gc.C) {
   211  	placement := []string{"valid"}
   212  	ensureAvailabilityResult, err := s.ensureAvailability(c, 3, constraints.MustParse("mem=4G"), defaultSeries, placement)
   213  	c.Assert(err, jc.ErrorIsNil)
   214  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   215  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-1", "machine-2"})
   216  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   217  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   218  
   219  	machines, err := s.State.AllMachines()
   220  	c.Assert(err, jc.ErrorIsNil)
   221  	c.Assert(machines, gc.HasLen, 3)
   222  	expectedCons := []constraints.Value{
   223  		{},
   224  		constraints.MustParse("mem=4G"),
   225  		constraints.MustParse("mem=4G"),
   226  	}
   227  	expectedPlacement := []string{"", "valid", ""}
   228  	for i, m := range machines {
   229  		cons, err := m.Constraints()
   230  		c.Assert(err, jc.ErrorIsNil)
   231  		c.Check(cons, gc.DeepEquals, expectedCons[i])
   232  		c.Check(m.Placement(), gc.Equals, expectedPlacement[i])
   233  	}
   234  }
   235  
   236  func (s *clientSuite) TestEnsureAvailabilityPlacementTo(c *gc.C) {
   237  	_, err := s.State.AddMachine("quantal", state.JobHostUnits)
   238  	c.Assert(err, jc.ErrorIsNil)
   239  	s.pingers = append(s.pingers, s.setAgentPresence(c, "1"))
   240  
   241  	_, err = s.State.AddMachine("quantal", state.JobHostUnits)
   242  	c.Assert(err, jc.ErrorIsNil)
   243  	s.pingers = append(s.pingers, s.setAgentPresence(c, "2"))
   244  
   245  	placement := []string{"1", "2"}
   246  	ensureAvailabilityResult, err := s.ensureAvailability(c, 3, emptyCons, defaultSeries, placement)
   247  	c.Assert(err, jc.ErrorIsNil)
   248  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   249  	c.Assert(ensureAvailabilityResult.Added, gc.HasLen, 0)
   250  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   251  	c.Assert(ensureAvailabilityResult.Converted, gc.DeepEquals, []string{"machine-1", "machine-2"})
   252  
   253  	machines, err := s.State.AllMachines()
   254  	c.Assert(err, jc.ErrorIsNil)
   255  	c.Assert(machines, gc.HasLen, 3)
   256  	expectedCons := []constraints.Value{{}, {}, {}}
   257  	expectedPlacement := []string{"", "", ""}
   258  	for i, m := range machines {
   259  		cons, err := m.Constraints()
   260  		c.Assert(err, jc.ErrorIsNil)
   261  		c.Check(cons, gc.DeepEquals, expectedCons[i])
   262  		c.Check(m.Placement(), gc.Equals, expectedPlacement[i])
   263  	}
   264  }
   265  
   266  func (s *clientSuite) TestEnsureAvailability0Preserves(c *gc.C) {
   267  	// A value of 0 says either "if I'm not HA, make me HA" or "preserve my
   268  	// current HA settings".
   269  	ensureAvailabilityResult, err := s.ensureAvailability(c, 0, emptyCons, defaultSeries, nil)
   270  	c.Assert(err, jc.ErrorIsNil)
   271  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   272  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-1", "machine-2"})
   273  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   274  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   275  
   276  	machines, err := s.State.AllMachines()
   277  	c.Assert(machines, gc.HasLen, 3)
   278  
   279  	pingerB := s.setAgentPresence(c, "1")
   280  	defer assertKill(c, pingerB)
   281  
   282  	// Now, we keep agent 1 alive, but not agent 2, calling
   283  	// EnsureAvailability(0) again will cause us to start another machine
   284  	ensureAvailabilityResult, err = s.ensureAvailability(c, 0, emptyCons, defaultSeries, nil)
   285  	c.Assert(err, jc.ErrorIsNil)
   286  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0", "machine-1"})
   287  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-3"})
   288  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   289  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   290  
   291  	machines, err = s.State.AllMachines()
   292  	c.Assert(err, jc.ErrorIsNil)
   293  	c.Assert(machines, gc.HasLen, 4)
   294  }
   295  
   296  func (s *clientSuite) TestEnsureAvailability0Preserves5(c *gc.C) {
   297  	// Start off with 5 servers
   298  	ensureAvailabilityResult, err := s.ensureAvailability(c, 5, emptyCons, defaultSeries, nil)
   299  	c.Assert(err, jc.ErrorIsNil)
   300  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   301  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-1", "machine-2", "machine-3", "machine-4"})
   302  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   303  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   304  
   305  	machines, err := s.State.AllMachines()
   306  	c.Assert(machines, gc.HasLen, 5)
   307  	pingerB := s.setAgentPresence(c, "1")
   308  	defer assertKill(c, pingerB)
   309  
   310  	pingerC := s.setAgentPresence(c, "2")
   311  	defer assertKill(c, pingerC)
   312  
   313  	pingerD := s.setAgentPresence(c, "3")
   314  	defer assertKill(c, pingerD)
   315  	// Keeping all alive but one, will bring up 1 more server to preserve 5
   316  	ensureAvailabilityResult, err = s.ensureAvailability(c, 0, emptyCons, defaultSeries, nil)
   317  	c.Assert(err, jc.ErrorIsNil)
   318  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0", "machine-1",
   319  		"machine-2", "machine-3"})
   320  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-5"})
   321  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   322  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   323  
   324  	machines, err = s.State.AllMachines()
   325  	c.Assert(machines, gc.HasLen, 6)
   326  	c.Assert(err, jc.ErrorIsNil)
   327  }
   328  
   329  func (s *clientSuite) TestEnsureAvailabilityErrors(c *gc.C) {
   330  	ensureAvailabilityResult, err := s.ensureAvailability(c, -1, emptyCons, defaultSeries, nil)
   331  	c.Assert(err, gc.ErrorMatches, "number of state servers must be odd and non-negative")
   332  
   333  	ensureAvailabilityResult, err = s.ensureAvailability(c, 3, emptyCons, defaultSeries, nil)
   334  	c.Assert(err, jc.ErrorIsNil)
   335  	c.Assert(ensureAvailabilityResult.Maintained, gc.DeepEquals, []string{"machine-0"})
   336  	c.Assert(ensureAvailabilityResult.Added, gc.DeepEquals, []string{"machine-1", "machine-2"})
   337  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   338  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   339  
   340  	_, err = s.ensureAvailability(c, 1, emptyCons, defaultSeries, nil)
   341  	c.Assert(err, gc.ErrorMatches, "failed to create new state server machines: cannot reduce state server count")
   342  }
   343  
   344  func (s *clientSuite) TestEnsureAvailabilityHostedEnvErrors(c *gc.C) {
   345  	st2 := s.Factory.MakeEnvironment(c, &factory.EnvParams{ConfigAttrs: coretesting.Attrs{"state-server": false}})
   346  	defer st2.Close()
   347  
   348  	haServer, err := highavailability.NewHighAvailabilityAPI(st2, s.resources, s.authoriser)
   349  	c.Assert(err, jc.ErrorIsNil)
   350  
   351  	ensureAvailabilityResult, err := ensureAvailability(c, haServer, 3, constraints.MustParse("mem=4G"), defaultSeries, nil)
   352  	c.Assert(errors.Cause(err), gc.ErrorMatches, "unsupported with hosted environments")
   353  
   354  	c.Assert(ensureAvailabilityResult.Maintained, gc.HasLen, 0)
   355  	c.Assert(ensureAvailabilityResult.Added, gc.HasLen, 0)
   356  	c.Assert(ensureAvailabilityResult.Removed, gc.HasLen, 0)
   357  	c.Assert(ensureAvailabilityResult.Converted, gc.HasLen, 0)
   358  
   359  	machines, err := st2.AllMachines()
   360  	c.Assert(err, jc.ErrorIsNil)
   361  	c.Assert(machines, gc.HasLen, 0)
   362  }