github.com/axw/juju@v0.0.0-20161005053422-4bd6544d08d4/worker/peergrouper/worker.go (about)

     1  // Copyright 2014 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package peergrouper
     5  
     6  import (
     7  	"fmt"
     8  	"time"
     9  
    10  	"github.com/juju/errors"
    11  	"github.com/juju/loggo"
    12  	"github.com/juju/replicaset"
    13  
    14  	"github.com/juju/juju/instance"
    15  	"github.com/juju/juju/network"
    16  	"github.com/juju/juju/state"
    17  	"github.com/juju/juju/status"
    18  	"github.com/juju/juju/worker"
    19  	"github.com/juju/juju/worker/catacomb"
    20  )
    21  
    22  var logger = loggo.GetLogger("juju.worker.peergrouper")
    23  
    24  type stateInterface interface {
    25  	Machine(id string) (stateMachine, error)
    26  	WatchControllerInfo() state.NotifyWatcher
    27  	WatchControllerStatusChanges() state.StringsWatcher
    28  	ControllerInfo() (*state.ControllerInfo, error)
    29  	MongoSession() mongoSession
    30  	Space(id string) (SpaceReader, error)
    31  	SetOrGetMongoSpaceName(spaceName network.SpaceName) (network.SpaceName, error)
    32  	SetMongoSpaceState(mongoSpaceState state.MongoSpaceStates) error
    33  }
    34  
    35  type stateMachine interface {
    36  	Id() string
    37  	InstanceId() (instance.Id, error)
    38  	Status() (status.StatusInfo, error)
    39  	Refresh() error
    40  	Watch() state.NotifyWatcher
    41  	WantsVote() bool
    42  	HasVote() bool
    43  	SetHasVote(hasVote bool) error
    44  	APIHostPorts() []network.HostPort
    45  	MongoHostPorts() []network.HostPort
    46  }
    47  
    48  type mongoSession interface {
    49  	CurrentStatus() (*replicaset.Status, error)
    50  	CurrentMembers() ([]replicaset.Member, error)
    51  	Set([]replicaset.Member) error
    52  }
    53  
    54  type publisherInterface interface {
    55  	// publish publishes information about the given controllers
    56  	// to whomsoever it may concern. When it is called there
    57  	// is no guarantee that any of the information has actually changed.
    58  	publishAPIServers(apiServers [][]network.HostPort, instanceIds []instance.Id) error
    59  }
    60  
    61  var (
    62  	// If we fail to set the mongo replica set members,
    63  	// we start retrying with the following interval,
    64  	// before exponentially backing off with each further
    65  	// attempt.
    66  	initialRetryInterval = 2 * time.Second
    67  
    68  	// maxRetryInterval holds the maximum interval
    69  	// between retry attempts.
    70  	maxRetryInterval = 5 * time.Minute
    71  
    72  	// pollInterval holds the interval at which the replica set
    73  	// members will be updated even in the absence of changes
    74  	// to State. This enables us to make changes to members
    75  	// that are triggered by changes to member status.
    76  	pollInterval = 1 * time.Minute
    77  )
    78  
    79  // pgWorker is a worker which watches the controller machines in state
    80  // as well as the MongoDB replicaset configuration, adding and
    81  // removing controller machines as they change or are added and
    82  // removed.
    83  type pgWorker struct {
    84  	catacomb catacomb.Catacomb
    85  
    86  	// st represents the State. It is an interface so we can swap
    87  	// out the implementation during testing.
    88  	st stateInterface
    89  
    90  	// machineChanges receives events from the machineTrackers when
    91  	// controller machines change in ways that are relevant to the
    92  	// peergrouper.
    93  	machineChanges chan struct{}
    94  
    95  	// machineTrackers holds the workers which track the machines we
    96  	// are currently watching (all the controller machines).
    97  	machineTrackers map[string]*machineTracker
    98  
    99  	// publisher holds the implementation of the API
   100  	// address publisher.
   101  	publisher publisherInterface
   102  
   103  	providerSupportsSpaces bool
   104  }
   105  
   106  // New returns a new worker that maintains the mongo replica set
   107  // with respect to the given state.
   108  func New(st *state.State, supportsSpaces bool) (worker.Worker, error) {
   109  	cfg, err := st.ControllerConfig()
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  	shim := &stateShim{
   114  		State:     st,
   115  		mongoPort: cfg.StatePort(),
   116  		apiPort:   cfg.APIPort(),
   117  	}
   118  	return newWorker(shim, newPublisher(st), supportsSpaces)
   119  }
   120  
   121  func newWorker(st stateInterface, pub publisherInterface, supportsSpaces bool) (worker.Worker, error) {
   122  	w := &pgWorker{
   123  		st:                     st,
   124  		machineChanges:         make(chan struct{}),
   125  		machineTrackers:        make(map[string]*machineTracker),
   126  		publisher:              pub,
   127  		providerSupportsSpaces: supportsSpaces,
   128  	}
   129  	err := catacomb.Invoke(catacomb.Plan{
   130  		Site: &w.catacomb,
   131  		Work: w.loop,
   132  	})
   133  	if err != nil {
   134  		return nil, errors.Trace(err)
   135  	}
   136  	return w, nil
   137  }
   138  
   139  func (w *pgWorker) Kill() {
   140  	w.catacomb.Kill(nil)
   141  }
   142  
   143  func (w *pgWorker) Wait() error {
   144  	return w.catacomb.Wait()
   145  }
   146  
   147  func (w *pgWorker) loop() error {
   148  	controllerChanges, err := w.watchForControllerChanges()
   149  	if err != nil {
   150  		return errors.Trace(err)
   151  	}
   152  
   153  	var updateChan <-chan time.Time
   154  	retryInterval := initialRetryInterval
   155  
   156  	for {
   157  		select {
   158  		case <-w.catacomb.Dying():
   159  			return w.catacomb.ErrDying()
   160  		case <-controllerChanges:
   161  			changed, err := w.updateControllerMachines()
   162  			if err != nil {
   163  				return errors.Trace(err)
   164  			}
   165  			if changed {
   166  				// A controller machine was added or removed, update
   167  				// the replica set immediately.
   168  				// TODO(fwereade): 2016-03-17 lp:1558657
   169  				updateChan = time.After(0)
   170  			}
   171  
   172  		case <-w.machineChanges:
   173  			// One of the controller machines changed, update the
   174  			// replica set immediately.
   175  			// TODO(fwereade): 2016-03-17 lp:1558657
   176  			updateChan = time.After(0)
   177  
   178  		case <-updateChan:
   179  			ok := true
   180  			servers, instanceIds, err := w.apiPublishInfo()
   181  			if err != nil {
   182  				return fmt.Errorf("cannot get API server info: %v", err)
   183  			}
   184  			if err := w.publisher.publishAPIServers(servers, instanceIds); err != nil {
   185  				logger.Errorf("cannot publish API server addresses: %v", err)
   186  				ok = false
   187  			}
   188  			if err := w.updateReplicaset(); err != nil {
   189  				if _, isReplicaSetError := err.(*replicaSetError); !isReplicaSetError {
   190  					return err
   191  				}
   192  				logger.Errorf("cannot set replicaset: %v", err)
   193  				ok = false
   194  			}
   195  			if ok {
   196  				// Update the replica set members occasionally
   197  				// to keep them up to date with the current
   198  				// replica set member statuses.
   199  				// TODO(fwereade): 2016-03-17 lp:1558657
   200  				updateChan = time.After(pollInterval)
   201  				retryInterval = initialRetryInterval
   202  			} else {
   203  				// TODO(fwereade): 2016-03-17 lp:1558657
   204  				updateChan = time.After(retryInterval)
   205  				retryInterval *= 2
   206  				if retryInterval > maxRetryInterval {
   207  					retryInterval = maxRetryInterval
   208  				}
   209  			}
   210  
   211  		}
   212  	}
   213  }
   214  
   215  // watchForControllerChanges starts two watchers pertaining to changes
   216  // to the controllers, returning a channel which will receive events
   217  // if either watcher fires.
   218  func (w *pgWorker) watchForControllerChanges() (<-chan struct{}, error) {
   219  	controllerInfoWatcher := w.st.WatchControllerInfo()
   220  	if err := w.catacomb.Add(controllerInfoWatcher); err != nil {
   221  		return nil, errors.Trace(err)
   222  	}
   223  
   224  	controllerStatusWatcher := w.st.WatchControllerStatusChanges()
   225  	if err := w.catacomb.Add(controllerStatusWatcher); err != nil {
   226  		return nil, errors.Trace(err)
   227  	}
   228  
   229  	out := make(chan struct{})
   230  	go func() {
   231  		for {
   232  			select {
   233  			case <-w.catacomb.Dying():
   234  				return
   235  			case <-controllerInfoWatcher.Changes():
   236  				out <- struct{}{}
   237  			case <-controllerStatusWatcher.Changes():
   238  				out <- struct{}{}
   239  			}
   240  		}
   241  	}()
   242  	return out, nil
   243  }
   244  
   245  // updateControllerMachines updates the peergrouper's current list of
   246  // controller machines, as well as starting and stopping trackers for
   247  // them as they are added and removed.
   248  func (w *pgWorker) updateControllerMachines() (bool, error) {
   249  	info, err := w.st.ControllerInfo()
   250  	if err != nil {
   251  		return false, fmt.Errorf("cannot get controller info: %v", err)
   252  	}
   253  
   254  	logger.Debugf("controller machines in state: %#v", info.MachineIds)
   255  	changed := false
   256  
   257  	// Stop machine goroutines that no longer correspond to controller
   258  	// machines.
   259  	for _, m := range w.machineTrackers {
   260  		if !inStrings(m.Id(), info.MachineIds) {
   261  			worker.Stop(m)
   262  			delete(w.machineTrackers, m.Id())
   263  			changed = true
   264  		}
   265  	}
   266  
   267  	// Start machines with no watcher
   268  	for _, id := range info.MachineIds {
   269  		if _, ok := w.machineTrackers[id]; ok {
   270  			continue
   271  		}
   272  		logger.Debugf("found new machine %q", id)
   273  		stm, err := w.st.Machine(id)
   274  		if err != nil {
   275  			if errors.IsNotFound(err) {
   276  				// If the machine isn't found, it must have been
   277  				// removed and will soon enough be removed
   278  				// from the controller list. This will probably
   279  				// never happen, but we'll code defensively anyway.
   280  				logger.Warningf("machine %q from controller list not found", id)
   281  				continue
   282  			}
   283  			return false, fmt.Errorf("cannot get machine %q: %v", id, err)
   284  		}
   285  
   286  		// Don't add the machine unless it is "Started"
   287  		machineStatus, err := stm.Status()
   288  		if err != nil {
   289  			return false, errors.Annotatef(err, "cannot get status for machine %q", id)
   290  		}
   291  		if machineStatus.Status == status.Started {
   292  			logger.Debugf("machine %q has started, adding it to peergrouper list", id)
   293  			tracker, err := newMachineTracker(stm, w.machineChanges)
   294  			if err != nil {
   295  				return false, errors.Trace(err)
   296  			}
   297  			if err := w.catacomb.Add(tracker); err != nil {
   298  				return false, errors.Trace(err)
   299  			}
   300  			w.machineTrackers[id] = tracker
   301  			changed = true
   302  		} else {
   303  			logger.Debugf("machine %q not ready: %v", id, machineStatus.Status)
   304  		}
   305  
   306  	}
   307  	return changed, nil
   308  }
   309  
   310  func inStrings(t string, ss []string) bool {
   311  	for _, s := range ss {
   312  		if s == t {
   313  			return true
   314  		}
   315  	}
   316  	return false
   317  }
   318  
   319  func (w *pgWorker) apiPublishInfo() ([][]network.HostPort, []instance.Id, error) {
   320  	servers := make([][]network.HostPort, 0, len(w.machineTrackers))
   321  	instanceIds := make([]instance.Id, 0, len(w.machineTrackers))
   322  	for _, m := range w.machineTrackers {
   323  		if len(m.APIHostPorts()) == 0 {
   324  			continue
   325  		}
   326  		instanceId, err := m.stm.InstanceId()
   327  		if err != nil {
   328  			return nil, nil, err
   329  		}
   330  		instanceIds = append(instanceIds, instanceId)
   331  		servers = append(servers, m.APIHostPorts())
   332  
   333  	}
   334  	return servers, instanceIds, nil
   335  }
   336  
   337  // peerGroupInfo collates current session information about the
   338  // mongo peer group with information from state machines.
   339  func (w *pgWorker) peerGroupInfo() (*peerGroupInfo, error) {
   340  	session := w.st.MongoSession()
   341  	info := &peerGroupInfo{}
   342  	var err error
   343  	status, err := session.CurrentStatus()
   344  	if err != nil {
   345  		return nil, fmt.Errorf("cannot get replica set status: %v", err)
   346  	}
   347  	info.statuses = status.Members
   348  	info.members, err = session.CurrentMembers()
   349  	if err != nil {
   350  		return nil, fmt.Errorf("cannot get replica set members: %v", err)
   351  	}
   352  	info.machineTrackers = w.machineTrackers
   353  
   354  	spaceName, err := w.getMongoSpace(mongoAddresses(info.machineTrackers))
   355  	if err != nil {
   356  		return nil, err
   357  	}
   358  	info.mongoSpace = spaceName
   359  
   360  	return info, nil
   361  }
   362  
   363  func mongoAddresses(machines map[string]*machineTracker) [][]network.Address {
   364  	addresses := make([][]network.Address, len(machines))
   365  	i := 0
   366  	for _, m := range machines {
   367  		for _, hp := range m.MongoHostPorts() {
   368  			addresses[i] = append(addresses[i], hp.Address)
   369  		}
   370  		i++
   371  	}
   372  	return addresses
   373  }
   374  
   375  // getMongoSpace updates info with the space that Mongo servers should exist in.
   376  func (w *pgWorker) getMongoSpace(addrs [][]network.Address) (network.SpaceName, error) {
   377  	unset := network.SpaceName("")
   378  
   379  	stateInfo, err := w.st.ControllerInfo()
   380  	if err != nil {
   381  		return unset, errors.Annotate(err, "cannot get state server info")
   382  	}
   383  
   384  	switch stateInfo.MongoSpaceState {
   385  	case state.MongoSpaceUnknown:
   386  		if !w.providerSupportsSpaces {
   387  			err := w.st.SetMongoSpaceState(state.MongoSpaceUnsupported)
   388  			if err != nil {
   389  				return unset, errors.Annotate(err, "cannot set Mongo space state")
   390  			}
   391  			return unset, nil
   392  		}
   393  
   394  		// We want to find a space that contains all Mongo servers so we can
   395  		// use it to look up the IP address of each Mongo server to be used
   396  		// to set up the peer group.
   397  		spaceStats := generateSpaceStats(addrs)
   398  		if spaceStats.LargestSpaceContainsAll == false {
   399  			err := w.st.SetMongoSpaceState(state.MongoSpaceInvalid)
   400  			if err != nil {
   401  				return unset, errors.Annotate(err, "cannot set Mongo space state")
   402  			}
   403  			logger.Warningf("couldn't find a space containing all peer group machines")
   404  			return unset, nil
   405  		} else {
   406  			spaceName, err := w.st.SetOrGetMongoSpaceName(spaceStats.LargestSpace)
   407  			if err != nil {
   408  				return unset, errors.Annotate(err, "error setting/getting Mongo space")
   409  			}
   410  			return spaceName, nil
   411  		}
   412  
   413  	case state.MongoSpaceValid:
   414  		space, err := w.st.Space(stateInfo.MongoSpaceName)
   415  		if err != nil {
   416  			return unset, errors.Annotate(err, "looking up space")
   417  		}
   418  		return network.SpaceName(space.Name()), nil
   419  	}
   420  
   421  	return unset, nil
   422  }
   423  
   424  // replicaSetError holds an error returned as a result
   425  // of calling replicaset.Set. As this is expected to fail
   426  // in the normal course of things, it needs special treatment.
   427  type replicaSetError struct {
   428  	error
   429  }
   430  
   431  // updateReplicaset sets the current replica set members, and applies the
   432  // given voting status to machines in the state.
   433  func (w *pgWorker) updateReplicaset() error {
   434  	info, err := w.peerGroupInfo()
   435  	if err != nil {
   436  		return errors.Annotate(err, "cannot get peergrouper info")
   437  	}
   438  	members, voting, err := desiredPeerGroup(info)
   439  	if err != nil {
   440  		return fmt.Errorf("cannot compute desired peer group: %v", err)
   441  	}
   442  	if members != nil {
   443  		logger.Debugf("desired peer group members: %#v", members)
   444  	} else {
   445  		logger.Debugf("no change in desired peer group (voting %#v)", voting)
   446  	}
   447  
   448  	// We cannot change the HasVote flag of a machine in state at exactly
   449  	// the same moment as changing its voting status in the replica set.
   450  	//
   451  	// Thus we need to be careful that a machine which is actually a voting
   452  	// member is not seen to not have a vote, because otherwise
   453  	// there is nothing to prevent the machine being removed.
   454  	//
   455  	// To avoid this happening, we make sure when we call SetReplicaSet,
   456  	// that the voting status of machines is the union of both old
   457  	// and new voting machines - that is the set of HasVote machines
   458  	// is a superset of all the actual voting machines.
   459  	//
   460  	// Only after the call has taken place do we reset the voting status
   461  	// of the machines that have lost their vote.
   462  	//
   463  	// If there's a crash, the voting status may not reflect the
   464  	// actual voting status for a while, but when things come
   465  	// back on line, it will be sorted out, as desiredReplicaSet
   466  	// will return the actual voting status.
   467  	//
   468  	// Note that we potentially update the HasVote status of the machines even
   469  	// if the members have not changed.
   470  	var added, removed []*machineTracker
   471  	for m, hasVote := range voting {
   472  		switch {
   473  		case hasVote && !m.stm.HasVote():
   474  			added = append(added, m)
   475  		case !hasVote && m.stm.HasVote():
   476  			removed = append(removed, m)
   477  		}
   478  	}
   479  	if err := setHasVote(added, true); err != nil {
   480  		return errors.Annotate(err, "cannot set HasVote added")
   481  	}
   482  	if members != nil {
   483  		if err := w.st.MongoSession().Set(members); err != nil {
   484  			// We've failed to set the replica set, so revert back
   485  			// to the previous settings.
   486  			if err1 := setHasVote(added, false); err1 != nil {
   487  				logger.Errorf("cannot revert machine voting after failure to change replica set: %v", err1)
   488  			}
   489  			return &replicaSetError{err}
   490  		}
   491  		logger.Infof("successfully changed replica set to %#v", members)
   492  	}
   493  	if err := setHasVote(removed, false); err != nil {
   494  		return errors.Annotate(err, "cannot set HasVote removed")
   495  	}
   496  	return nil
   497  }
   498  
   499  // setHasVote sets the HasVote status of all the given
   500  // machines to hasVote.
   501  func setHasVote(ms []*machineTracker, hasVote bool) error {
   502  	if len(ms) == 0 {
   503  		return nil
   504  	}
   505  	logger.Infof("setting HasVote=%v on machines %v", hasVote, ms)
   506  	for _, m := range ms {
   507  		if err := m.stm.SetHasVote(hasVote); err != nil {
   508  			return fmt.Errorf("cannot set voting status of %q to %v: %v", m.Id(), hasVote, err)
   509  		}
   510  	}
   511  	return nil
   512  }
   513  
   514  // allSpaceStats holds a SpaceStats for both API and Mongo machines
   515  type allSpaceStats struct {
   516  	APIMachines   spaceStats
   517  	MongoMachines spaceStats
   518  }
   519  
   520  // SpaceStats holds information useful when choosing which space to pick an
   521  // address from.
   522  type spaceStats struct {
   523  	SpaceRefCount           map[network.SpaceName]int
   524  	LargestSpace            network.SpaceName
   525  	LargestSpaceSize        int
   526  	LargestSpaceContainsAll bool
   527  }
   528  
   529  // generateSpaceStats takes a list of machine addresses and returns information
   530  // about what spaces are referenced by those machines.
   531  func generateSpaceStats(addresses [][]network.Address) spaceStats {
   532  	var stats spaceStats
   533  	stats.SpaceRefCount = make(map[network.SpaceName]int)
   534  
   535  	for i := range addresses {
   536  		for _, addr := range addresses[i] {
   537  			v := stats.SpaceRefCount[addr.SpaceName]
   538  			v++
   539  			stats.SpaceRefCount[addr.SpaceName] = v
   540  
   541  			if v > stats.LargestSpaceSize {
   542  				stats.LargestSpace = addr.SpaceName
   543  				stats.LargestSpaceSize = v
   544  			}
   545  		}
   546  	}
   547  
   548  	stats.LargestSpaceContainsAll = stats.LargestSpaceSize == len(addresses)
   549  
   550  	return stats
   551  }